Rewrite of netgraph to start getting ready for SMP.

This version is functional and is aproaching solid..
notice I said APROACHING. There are many node types I cannot test
I have tested: echo hole ppp socket vjc iface tee bpf async tty
The rest compile and "Look" right.  More changes to follow.
DEBUGGING is enabled in this code to help if people have problems.

38 files changed, 3444 insertions, 1884 deletions

diff --git a/sys/dev/ar/if_ar.c b/sys/dev/ar/if_ar.c
index fe9abeb..722a133 100644
--- a/sys/dev/ar/if_ar.c
+++ b/sys/dev/ar/if_ar.c
@@ -267,7 +267,7 @@ static	void	ngar_init(void* ignored);
 
 static ng_constructor_t	ngar_constructor;
 static ng_rcvmsg_t	ngar_rcvmsg;
-static ng_shutdown_t	ngar_rmnode;
+static ng_shutdown_t	ngar_shutdown;
 static ng_newhook_t	ngar_newhook;
 /*static ng_findhook_t	ngar_findhook; */
 static ng_connect_t	ngar_connect;
@@ -280,7 +280,7 @@ static struct ng_type typestruct = {
 	NULL,
 	ngar_constructor,
 	ngar_rcvmsg,
-	ngar_rmnode,
+	ngar_shutdown,
 	ngar_newhook,
 	NULL,
 	ngar_connect,
@@ -509,18 +509,17 @@ arattach(struct ar_hardc *hc)
 		if (ngar_done_init == 0) ngar_init(NULL);
 		if (ng_make_node_common(&typestruct, &sc->node) != 0)
 			return (0);
+		sprintf(sc->nodename, "%s%d", NG_AR_NODE_TYPE, sc->unit);
+		if (ng_name_node(sc->node, sc->nodename)) {
+			ng_unref(sc->node); /* drop it again */
+			return (0);
+		}
 		sc->node->private = sc;
 		callout_handle_init(&sc->handle);
 		sc->xmitq.ifq_maxlen = IFQ_MAXLEN;
 		sc->xmitq_hipri.ifq_maxlen = IFQ_MAXLEN;
 		mtx_init(&sc->xmitq.ifq_mtx, "ar_xmitq", MTX_DEF);
 		mtx_init(&sc->xmitq_hipri.ifq_mtx, "ar_xmitq_hipri", MTX_DEF);
-		sprintf(sc->nodename, "%s%d", NG_AR_NODE_TYPE, sc->unit);
-		if (ng_name_node(sc->node, sc->nodename)) {
-			ng_rmnode(sc->node);
-			ng_unref(sc->node);
-			return (0);
-		}
 		sc->running = 0;
 #endif	/* NETGRAPH */
 	}
@@ -2173,7 +2172,7 @@ ngar_watchdog_frame(void * arg)
  * If the hardware exists, it will already have created it.
  */
 static	int
-ngar_constructor(node_p *nodep)
+ngar_constructor(node_p node)
 {
 	return (EINVAL);
 }
@@ -2216,13 +2215,14 @@ ngar_newhook(node_p node, hook_p hook, const char *name)
  * Just respond to the generic TEXT_STATUS message
  */
 static	int
-ngar_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
-				struct ng_mesg **rptr, hook_p lasthook)
+ngar_rcvmsg(node_p node, item_p item, hook_p lasthook)
 {
 	struct ar_softc *	sc;
 	struct ng_mesg *resp = NULL;
 	int error = 0;
+	struct ng_mesg *msg;
 
+	NGI_GET_MSG(item, msg);
 	sc = node->private;
 	switch (msg->header.typecookie) {
 	case	NG_AR_COOKIE: 
@@ -2268,13 +2268,8 @@ ngar_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
 		break;
 	}
 	/* Take care of synchronous response, if any */
-	if (rptr)
-		*rptr = resp;
-	else if (resp)
-		/* Should send the hard way */
-		FREE(resp, M_NETGRAPH);
-
-	free(msg, M_NETGRAPH);
+	NG_RESPOND_MSG(error, node, item, resp);
+	NG_FREE_MSG(msg);
 	return (error);
 }
 
@@ -2282,14 +2277,18 @@ ngar_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
  * get data from another node and transmit it to the correct channel
  */
 static	int
-ngar_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
+ngar_rcvdata(hook_p hook, item_p item)
 {
 	int s;
 	int error = 0;
 	struct ar_softc * sc = hook->node->private;
 	struct ifqueue	*xmitq_p;
+	struct mbuf *m;
+	meta_p meta;
 	
+	NGI_GET_M(item, m);
+	NGI_GET_META(item, meta);
+	NG_FREE_ITEM(item);
 	/*
 	 * data doesn't come in from just anywhere (e.g control hook)
 	 */
@@ -2326,7 +2325,8 @@ bad:
 	 * It was an error case.
 	 * check if we need to free the mbuf, and then return the error
 	 */
-	NG_FREE_DATA(m, meta);
+	NG_FREE_M(m);
+	NG_FREE_META(meta);
 	return (error);
 }
 
@@ -2336,13 +2336,27 @@ bad:
  * don't unref the node, or remove our name. just clear our links up.
  */
 static	int
-ngar_rmnode(node_p node)
+ngar_shutdown(node_p node)
 {
 	struct ar_softc * sc = node->private;
 
 	ar_down(sc);
-	ng_cutlinks(node);
-	node->flags &= ~NG_INVALID; /* bounce back to life */
+	ng_unref(node);
+	/* XXX need to drain the output queues! */
+
+	/* The node is dead, long live the node! */
+	/* stolen from the attach routine */
+	if (ng_make_node_common(&typestruct, &sc->node) != 0)
+		return (0);
+	sprintf(sc->nodename, "%s%d", NG_AR_NODE_TYPE, sc->unit);
+	if (ng_name_node(sc->node, sc->nodename)) {
+		sc->node = NULL;
+		printf("node naming failed\n");
+		ng_unref(sc->node); /* node dissappears */
+		return (0);
+	}
+	sc->node->private = sc;
+	sc->running = 0;
 	return (0);
 }
 
diff --git a/sys/dev/ar/if_ar_isa.c b/sys/dev/ar/if_ar_isa.c
index fe9abeb..722a133 100644
--- a/sys/dev/ar/if_ar_isa.c
+++ b/sys/dev/ar/if_ar_isa.c
@@ -267,7 +267,7 @@ static	void	ngar_init(void* ignored);
 
 static ng_constructor_t	ngar_constructor;
 static ng_rcvmsg_t	ngar_rcvmsg;
-static ng_shutdown_t	ngar_rmnode;
+static ng_shutdown_t	ngar_shutdown;
 static ng_newhook_t	ngar_newhook;
 /*static ng_findhook_t	ngar_findhook; */
 static ng_connect_t	ngar_connect;
@@ -280,7 +280,7 @@ static struct ng_type typestruct = {
 	NULL,
 	ngar_constructor,
 	ngar_rcvmsg,
-	ngar_rmnode,
+	ngar_shutdown,
 	ngar_newhook,
 	NULL,
 	ngar_connect,
@@ -509,18 +509,17 @@ arattach(struct ar_hardc *hc)
 		if (ngar_done_init == 0) ngar_init(NULL);
 		if (ng_make_node_common(&typestruct, &sc->node) != 0)
 			return (0);
+		sprintf(sc->nodename, "%s%d", NG_AR_NODE_TYPE, sc->unit);
+		if (ng_name_node(sc->node, sc->nodename)) {
+			ng_unref(sc->node); /* drop it again */
+			return (0);
+		}
 		sc->node->private = sc;
 		callout_handle_init(&sc->handle);
 		sc->xmitq.ifq_maxlen = IFQ_MAXLEN;
 		sc->xmitq_hipri.ifq_maxlen = IFQ_MAXLEN;
 		mtx_init(&sc->xmitq.ifq_mtx, "ar_xmitq", MTX_DEF);
 		mtx_init(&sc->xmitq_hipri.ifq_mtx, "ar_xmitq_hipri", MTX_DEF);
-		sprintf(sc->nodename, "%s%d", NG_AR_NODE_TYPE, sc->unit);
-		if (ng_name_node(sc->node, sc->nodename)) {
-			ng_rmnode(sc->node);
-			ng_unref(sc->node);
-			return (0);
-		}
 		sc->running = 0;
 #endif	/* NETGRAPH */
 	}
@@ -2173,7 +2172,7 @@ ngar_watchdog_frame(void * arg)
  * If the hardware exists, it will already have created it.
  */
 static	int
-ngar_constructor(node_p *nodep)
+ngar_constructor(node_p node)
 {
 	return (EINVAL);
 }
@@ -2216,13 +2215,14 @@ ngar_newhook(node_p node, hook_p hook, const char *name)
  * Just respond to the generic TEXT_STATUS message
  */
 static	int
-ngar_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
-				struct ng_mesg **rptr, hook_p lasthook)
+ngar_rcvmsg(node_p node, item_p item, hook_p lasthook)
 {
 	struct ar_softc *	sc;
 	struct ng_mesg *resp = NULL;
 	int error = 0;
+	struct ng_mesg *msg;
 
+	NGI_GET_MSG(item, msg);
 	sc = node->private;
 	switch (msg->header.typecookie) {
 	case	NG_AR_COOKIE: 
@@ -2268,13 +2268,8 @@ ngar_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
 		break;
 	}
 	/* Take care of synchronous response, if any */
-	if (rptr)
-		*rptr = resp;
-	else if (resp)
-		/* Should send the hard way */
-		FREE(resp, M_NETGRAPH);
-
-	free(msg, M_NETGRAPH);
+	NG_RESPOND_MSG(error, node, item, resp);
+	NG_FREE_MSG(msg);
 	return (error);
 }
 
@@ -2282,14 +2277,18 @@ ngar_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
  * get data from another node and transmit it to the correct channel
  */
 static	int
-ngar_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
+ngar_rcvdata(hook_p hook, item_p item)
 {
 	int s;
 	int error = 0;
 	struct ar_softc * sc = hook->node->private;
 	struct ifqueue	*xmitq_p;
+	struct mbuf *m;
+	meta_p meta;
 	
+	NGI_GET_M(item, m);
+	NGI_GET_META(item, meta);
+	NG_FREE_ITEM(item);
 	/*
 	 * data doesn't come in from just anywhere (e.g control hook)
 	 */
@@ -2326,7 +2325,8 @@ bad:
 	 * It was an error case.
 	 * check if we need to free the mbuf, and then return the error
 	 */
-	NG_FREE_DATA(m, meta);
+	NG_FREE_M(m);
+	NG_FREE_META(meta);
 	return (error);
 }
 
@@ -2336,13 +2336,27 @@ bad:
  * don't unref the node, or remove our name. just clear our links up.
  */
 static	int
-ngar_rmnode(node_p node)
+ngar_shutdown(node_p node)
 {
 	struct ar_softc * sc = node->private;
 
 	ar_down(sc);
-	ng_cutlinks(node);
-	node->flags &= ~NG_INVALID; /* bounce back to life */
+	ng_unref(node);
+	/* XXX need to drain the output queues! */
+
+	/* The node is dead, long live the node! */
+	/* stolen from the attach routine */
+	if (ng_make_node_common(&typestruct, &sc->node) != 0)
+		return (0);
+	sprintf(sc->nodename, "%s%d", NG_AR_NODE_TYPE, sc->unit);
+	if (ng_name_node(sc->node, sc->nodename)) {
+		sc->node = NULL;
+		printf("node naming failed\n");
+		ng_unref(sc->node); /* node dissappears */
+		return (0);
+	}
+	sc->node->private = sc;
+	sc->running = 0;
 	return (0);
 }
 
diff --git a/sys/dev/lmc/if_lmc.c b/sys/dev/lmc/if_lmc.c
index 0aa6ea6..bec558d 100644
--- a/sys/dev/lmc/if_lmc.c
+++ b/sys/dev/lmc/if_lmc.c
@@ -1130,18 +1130,17 @@ lmc_attach(lmc_softc_t * const sc)
 	if (ng_lmc_done_init == 0) ng_lmc_init(NULL);
 	if (ng_make_node_common(&typestruct, &sc->lmc_node) != 0)
 		return (0);
+	sprintf(sc->lmc_nodename, "%s%d", NG_LMC_NODE_TYPE, sc->lmc_unit);
+	if (ng_name_node(sc->lmc_node, sc->lmc_nodename)) {
+		ng_unref(sc->lmc_node); /* make it go away again */
+		return (0);
+	}
 	sc->lmc_node->private = sc;
 	callout_handle_init(&sc->lmc_handle);
 	sc->lmc_xmitq.ifq_maxlen = IFQ_MAXLEN;
 	sc->lmc_xmitq_hipri.ifq_maxlen = IFQ_MAXLEN;
 	mtx_init(&sc->lmc_xmitq.ifq_mtx, "lmc_xmitq", MTX_DEF);
 	mtx_init(&sc->lmc_xmitq_hipri.ifq_mtx, "lmc_xmitq_hipri", MTX_DEF);
-	sprintf(sc->lmc_nodename, "%s%d", NG_LMC_NODE_TYPE, sc->lmc_unit);
-	if (ng_name_node(sc->lmc_node, sc->lmc_nodename)) {
-		ng_rmnode(sc->lmc_node);
-		ng_unref(sc->lmc_node);
-		return (0);
-	}
 	sc->lmc_running = 0;
 
 	/*
@@ -1251,7 +1250,7 @@ ng_lmc_watchdog_frame(void *arg)
  * If the hardware exists, it will already have created it.
  */
 static  int
-ng_lmc_constructor(node_p *nodep)
+ng_lmc_constructor(node_p node)
 {
         return (EINVAL);
 }
@@ -1295,13 +1294,14 @@ ng_lmc_newhook(node_p node, hook_p hook, const char *name)
  * Just respond to the generic TEXT_STATUS message
  */
 static  int
-ng_lmc_rcvmsg(node_p node, struct ng_mesg *msg,
-	const char *retaddr, struct ng_mesg **rptr, hook_p lasthook)
+ng_lmc_rcvmsg(node_p node, item_p item, hook_p lasthook)
 {
 	lmc_softc_t *sc = (lmc_softc_t *) node->private;
 	struct ng_mesg *resp = NULL;
 	int error = 0;
+	struct ng_mesg *msg;
 
+	NGI_GET_MSG(item, msg);
 	switch (msg->header.typecookie) {
 	case NG_LMC_COOKIE:
 		switch (msg->header.cmd) {
@@ -1377,12 +1377,8 @@ ng_lmc_rcvmsg(node_p node, struct ng_mesg *msg,
 	}
 
 	/* Take care of synchronous response, if any */
-	if (rptr)
-		*rptr = resp;
-	else if (resp)
-		FREE(resp, M_NETGRAPH);
-
-	free(msg, M_NETGRAPH);
+	NG_RESPOND_MSG(error, node, item, resp);
+	NG_FREE_MSG(msg);
 	return (error);
 }
 
@@ -1390,13 +1386,19 @@ ng_lmc_rcvmsg(node_p node, struct ng_mesg *msg,
  * get data from another node and transmit it to the line
  */
 static  int
-ng_lmc_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-	struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
+ng_lmc_rcvdata(hook_p hook, item_p item)
 {
         int s;
         int error = 0;
         lmc_softc_t * sc = (lmc_softc_t *) hook->node->private;
         struct ifqueue  *xmitq_p;
+	struct mbuf *m;
+	meta_p meta;
+
+	/* Unpack all the data components */
+	NGI_GET_M(item, m);
+	NGI_GET_META(item, meta);
+	NG_FREE_ITEM(item);
 
         /*
          * data doesn't come in from just anywhere (e.g control hook)
@@ -1434,7 +1436,8 @@ bad:
          * It was an error case.
          * check if we need to free the mbuf, and then return the error
          */
-        NG_FREE_DATA(m, meta);
+        NG_FREE_M(m);
+        NG_FREE_META(meta);
         return (error);
 }
 
@@ -1449,8 +1452,35 @@ ng_lmc_rmnode(node_p node)
         lmc_softc_t * sc = (lmc_softc_t *) node->private;
 
         lmc_ifdown(sc);
-        ng_cutlinks(node);
-        node->flags &= ~NG_INVALID; /* bounce back to life */
+	
+	/*
+	 * Get rid of the old node.
+	 */
+	node->flags |= NG_INVALID;
+	node->private = NULL;
+	ng_unref(node);
+	
+	/*
+	 * Create a new node. This is basically what a device
+	 * driver would do in the attach routine. So let's just do that..
+	 * The node is dead, long live the node!
+	 */
+	if (ng_make_node_common(&typestruct, &sc->lmc_node) != 0)
+		return (0);
+	sprintf(sc->lmc_nodename, "%s%d", NG_LMC_NODE_TYPE, sc->lmc_unit);
+	if (ng_name_node(sc->lmc_node, sc->lmc_nodename)) {
+		sc->lmc_node = NULL; /* to be sure */
+		ng_unref(sc->lmc_node); /* make it go away */
+		return (0);
+	}
+	sc->lmc_node->private = sc;
+	callout_handle_init(&sc->lmc_handle);
+	sc->lmc_running = 0;
+	/*
+	 * turn off those LEDs...
+	 */
+	sc->lmc_miireg16 |= LMC_MII16_LED_ALL;
+	lmc_led_on(sc, LMC_MII16_LED0);
         return (0);
 }
 /* already linked */
diff --git a/sys/dev/musycc/musycc.c b/sys/dev/musycc/musycc.c
index 98d5334..e1817fa 100644
--- a/sys/dev/musycc/musycc.c
+++ b/sys/dev/musycc/musycc.c
@@ -834,7 +834,7 @@ musycc_intr1(void *arg)
  */
 
 static int
-musycc_constructor(node_p *nodep)
+musycc_constructor(node_p node)
 {
 
 	return (EINVAL);
@@ -929,21 +929,19 @@ barf:
 /*
  * Handle status and config enquiries.
  * Respond with a synchronous response.
- * [JRE] -this would be easier to read with the usual 'switch' structure-
  */
 static int
-musycc_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
-					struct ng_mesg **rptr, hook_p lasthook)
+musycc_rcvmsg(node_p node, item_p item, hook_p lasthook)
 {
 	struct softc *sc;
 	struct ng_mesg *resp = NULL;
 	char *s, *r;
 	int error = 0;
+	struct ng_mesg *msg;
 
+	
+	NGI_GET_MSG(item, msg);
 	sc = node->private;
-	if (rptr)
-		*rptr = NULL; /* default answer in case of errors */
-
 	if (msg->header.typecookie != NGM_GENERIC_COOKIE)
 		goto out;
 
@@ -957,7 +955,7 @@ musycc_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
 		s = (char *)resp->data;
 		status_8370(sc, s);
 		resp->header.arglen = strlen(s) + 1;
-		FREE(msg, M_NETGRAPH);
+		NG_FREE_MSG(msg);
 		
 	} else if (msg->header.cmd == NGM_TEXT_CONFIG) {
 		if (msg->header.arglen) {
@@ -976,19 +974,15 @@ musycc_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
 		*r = '\0';
 		musycc_config(node, s, r);
 		resp->header.arglen = strlen(r) + 1;
-		FREE(msg, M_NETGRAPH);
+		NG_FREE_MSG(msg);
 	} else {
 		error = EINVAL;
 	}
 
 out:
 	/* Take care of synchronous response, if any */
-	if (rptr)
-		*rptr = resp;
-	else if (resp)
-		FREE(resp, M_NETGRAPH); /* eventually 'send' the response */
-
-	FREE(msg, M_NETGRAPH);
+	NG_RESPOND_MSG(error, node, item, resp);
+	NG_FREE_MSG(msg);
 	return (error);
 }
 
@@ -1044,8 +1038,7 @@ musycc_newhook(node_p node, hook_p hook, const char *name)
 }
 
 static int
-musycc_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
+musycc_rcvdata(hook_p hook, item_p item)
 {
 
 	struct softc *sc;
@@ -1054,6 +1047,7 @@ musycc_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 	struct mdesc *md;
 	u_int32_t ch, u, len;
 	struct mbuf *m2;
+	struct mbuf *m;
 
 	sch = hook->private;
 	sc = sch->sc;
@@ -1062,24 +1056,26 @@ musycc_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 
 	if (csc->state != C_RUNNING) {
 		printf("csc->state = %d\n", csc->state);
-		NG_FREE_DATA(m, meta);
+		NG_FREE_ITEM(item);
 		return (0);
 	}
 
-	NG_FREE_META(meta);
-	meta = NULL;
 
 	if (sch->state != UP) {
 		printf("sch->state = %d\n", sch->state);
-		NG_FREE_DATA(m, meta);
+		NG_FREE_ITEM(item);
 		return (0);
 	} 
+	NGI_GET_M(item, m);
+	NG_FREE_ITEM(item);
 	if (sch->tx_pending + m->m_pkthdr.len > sch->tx_limit * maxlatency) {
-		printf("pend %ld len %d lim %ld\n", sch->tx_pending, m->m_pkthdr.len, sch->tx_limit * maxlatency);
-		NG_FREE_DATA(m, meta);
+		printf("pend %ld len %d lim %ld\n", sch->tx_pending,
+				m->m_pkthdr.len, sch->tx_limit * maxlatency);
+		NG_FREE_M(m);
 		return (0);
 	}
 
+
 	m2 = m;
 	len = m->m_pkthdr.len;
 	while (len) {
diff --git a/sys/dev/sr/if_sr.c b/sys/dev/sr/if_sr.c
index 9ca41d4..a7ba8e9 100644
--- a/sys/dev/sr/if_sr.c
+++ b/sys/dev/sr/if_sr.c
@@ -367,7 +367,7 @@ static	void	ngsr_init(void* ignored);
 
 static ng_constructor_t	ngsr_constructor;
 static ng_rcvmsg_t	ngsr_rcvmsg;
-static ng_shutdown_t	ngsr_rmnode;
+static ng_shutdown_t	ngsr_shutdown;
 static ng_newhook_t	ngsr_newhook;
 /*static ng_findhook_t	ngsr_findhook; */
 static ng_connect_t	ngsr_connect;
@@ -380,7 +380,7 @@ static struct ng_type typestruct = {
 	NULL,
 	ngsr_constructor,
 	ngsr_rcvmsg,
-	ngsr_rmnode,
+	ngsr_shutdown,
 	ngsr_newhook,
 	NULL,
 	ngsr_connect,
@@ -934,18 +934,17 @@ srattach(struct sr_hardc *hc)
 		if (ngsr_done_init == 0) ngsr_init(NULL);
 		if (ng_make_node_common(&typestruct, &sc->node) != 0)
 			return (0);
+		sprintf(sc->nodename, "%s%d", NG_SR_NODE_TYPE, sc->unit);
+		if (ng_name_node(sc->node, sc->nodename)) {
+			ng_unref(sc->node); /* make it go away again */
+			return (0);
+		}
 		sc->node->private = sc;
 		callout_handle_init(&sc->handle);
 		sc->xmitq.ifq_maxlen = IFQ_MAXLEN;
 		sc->xmitq_hipri.ifq_maxlen = IFQ_MAXLEN;
 		mtx_init(&sc->xmitq.ifq_mtx, "sr_xmitq", MTX_DEF);
 		mtx_init(&sc->xmitq_hipri.ifq_mtx, "sr_xmitq_hipri", MTX_DEF);
-		sprintf(sc->nodename, "%s%d", NG_SR_NODE_TYPE, sc->unit);
-		if (ng_name_node(sc->node, sc->nodename)) {
-			ng_rmnode(sc->node);
-			ng_unref(sc->node);
-			return (0);
-		}
 		sc->running = 0;
 #endif	/* NETGRAPH */
 	}
@@ -3126,7 +3125,7 @@ ngsr_watchdog_frame(void * arg)
  * If the hardware exists, it will already have created it.
  */
 static	int
-ngsr_constructor(node_p *nodep)
+ngsr_constructor(node_p node)
 {
 	return (EINVAL);
 }
@@ -3169,13 +3168,14 @@ ngsr_newhook(node_p node, hook_p hook, const char *name)
  * Just respond to the generic TEXT_STATUS message
  */
 static	int
-ngsr_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
-				struct ng_mesg **rptr, hook_p lasthook)
+ngsr_rcvmsg(node_p node, item_p item, hook_p lasthook)
 {
 	struct sr_softc *	sc;
 	struct ng_mesg *resp = NULL;
 	int error = 0;
+	struct ng_mesg *msg;
 
+	NGI_GET_MSG(item,msg);
 	sc = node->private;
 	switch (msg->header.typecookie) {
 	case	NG_SR_COOKIE: 
@@ -3223,12 +3223,8 @@ ngsr_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
 		break;
 	}
 	/* Take care of synchronous response, if any */
-	if (rptr)
-		*rptr = resp;
-	else if (resp)
-		FREE(resp, M_NETGRAPH);
-
-	free(msg, M_NETGRAPH);
+	NG_RESPOND_MSG(error, node, item, resp);
+	NG_FREE_MSG(msg);
 	return (error);
 }
 
@@ -3236,14 +3232,18 @@ ngsr_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
  * get data from another node and transmit it to the correct channel
  */
 static	int
-ngsr_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
+ngsr_rcvdata(hook_p hook, item_p item)
 {
 	int s;
 	int error = 0;
 	struct sr_softc * sc = hook->node->private;
 	struct ifqueue	*xmitq_p;
+	struct mbuf *m;
+	meta_p meta;
 	
+	NGI_GET_M(item, m);
+	NGI_GET_META(item, meta);
+	NG_FREE_ITEM(item);
 	/*
 	 * data doesn't come in from just anywhere (e.g control hook)
 	 */
@@ -3280,7 +3280,8 @@ bad:
 	 * It was an error case.
 	 * check if we need to free the mbuf, and then return the error
 	 */
-	NG_FREE_DATA(m, meta);
+	NG_FREE_M(m);
+	NG_FREE_META(meta);
 	return (error);
 }
 
@@ -3290,13 +3291,25 @@ bad:
  * don't unref the node, or remove our name. just clear our links up.
  */
 static	int
-ngsr_rmnode(node_p node)
+ngsr_shutdown(node_p node)
 {
 	struct sr_softc * sc = node->private;
 
 	sr_down(sc);
-	ng_cutlinks(node);
-	node->flags &= ~NG_INVALID; /* bounce back to life */
+	ng_unref(node);
+/* XXX should drain queues! */
+	if (ng_make_node_common(&typestruct, &sc->node) != 0)
+		return (0);
+	sprintf(sc->nodename, "%s%d", NG_SR_NODE_TYPE, sc->unit);
+	if (ng_name_node(sc->node, sc->nodename)) {
+		printf("node naming failed\n");
+		sc->node = NULL;
+		ng_unref(sc->node); /* drop it again */
+		return (0);
+	}
+	sc->node->private = sc;
+	callout_handle_init(&sc->handle); /* should kill timeout */
+	sc->running = 0;
 	return (0);
 }
 
diff --git a/sys/dev/sr/if_sr_isa.c b/sys/dev/sr/if_sr_isa.c
index 9ca41d4..a7ba8e9 100644
--- a/sys/dev/sr/if_sr_isa.c
+++ b/sys/dev/sr/if_sr_isa.c
@@ -367,7 +367,7 @@ static	void	ngsr_init(void* ignored);
 
 static ng_constructor_t	ngsr_constructor;
 static ng_rcvmsg_t	ngsr_rcvmsg;
-static ng_shutdown_t	ngsr_rmnode;
+static ng_shutdown_t	ngsr_shutdown;
 static ng_newhook_t	ngsr_newhook;
 /*static ng_findhook_t	ngsr_findhook; */
 static ng_connect_t	ngsr_connect;
@@ -380,7 +380,7 @@ static struct ng_type typestruct = {
 	NULL,
 	ngsr_constructor,
 	ngsr_rcvmsg,
-	ngsr_rmnode,
+	ngsr_shutdown,
 	ngsr_newhook,
 	NULL,
 	ngsr_connect,
@@ -934,18 +934,17 @@ srattach(struct sr_hardc *hc)
 		if (ngsr_done_init == 0) ngsr_init(NULL);
 		if (ng_make_node_common(&typestruct, &sc->node) != 0)
 			return (0);
+		sprintf(sc->nodename, "%s%d", NG_SR_NODE_TYPE, sc->unit);
+		if (ng_name_node(sc->node, sc->nodename)) {
+			ng_unref(sc->node); /* make it go away again */
+			return (0);
+		}
 		sc->node->private = sc;
 		callout_handle_init(&sc->handle);
 		sc->xmitq.ifq_maxlen = IFQ_MAXLEN;
 		sc->xmitq_hipri.ifq_maxlen = IFQ_MAXLEN;
 		mtx_init(&sc->xmitq.ifq_mtx, "sr_xmitq", MTX_DEF);
 		mtx_init(&sc->xmitq_hipri.ifq_mtx, "sr_xmitq_hipri", MTX_DEF);
-		sprintf(sc->nodename, "%s%d", NG_SR_NODE_TYPE, sc->unit);
-		if (ng_name_node(sc->node, sc->nodename)) {
-			ng_rmnode(sc->node);
-			ng_unref(sc->node);
-			return (0);
-		}
 		sc->running = 0;
 #endif	/* NETGRAPH */
 	}
@@ -3126,7 +3125,7 @@ ngsr_watchdog_frame(void * arg)
  * If the hardware exists, it will already have created it.
  */
 static	int
-ngsr_constructor(node_p *nodep)
+ngsr_constructor(node_p node)
 {
 	return (EINVAL);
 }
@@ -3169,13 +3168,14 @@ ngsr_newhook(node_p node, hook_p hook, const char *name)
  * Just respond to the generic TEXT_STATUS message
  */
 static	int
-ngsr_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
-				struct ng_mesg **rptr, hook_p lasthook)
+ngsr_rcvmsg(node_p node, item_p item, hook_p lasthook)
 {
 	struct sr_softc *	sc;
 	struct ng_mesg *resp = NULL;
 	int error = 0;
+	struct ng_mesg *msg;
 
+	NGI_GET_MSG(item,msg);
 	sc = node->private;
 	switch (msg->header.typecookie) {
 	case	NG_SR_COOKIE: 
@@ -3223,12 +3223,8 @@ ngsr_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
 		break;
 	}
 	/* Take care of synchronous response, if any */
-	if (rptr)
-		*rptr = resp;
-	else if (resp)
-		FREE(resp, M_NETGRAPH);
-
-	free(msg, M_NETGRAPH);
+	NG_RESPOND_MSG(error, node, item, resp);
+	NG_FREE_MSG(msg);
 	return (error);
 }
 
@@ -3236,14 +3232,18 @@ ngsr_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
  * get data from another node and transmit it to the correct channel
  */
 static	int
-ngsr_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
+ngsr_rcvdata(hook_p hook, item_p item)
 {
 	int s;
 	int error = 0;
 	struct sr_softc * sc = hook->node->private;
 	struct ifqueue	*xmitq_p;
+	struct mbuf *m;
+	meta_p meta;
 	
+	NGI_GET_M(item, m);
+	NGI_GET_META(item, meta);
+	NG_FREE_ITEM(item);
 	/*
 	 * data doesn't come in from just anywhere (e.g control hook)
 	 */
@@ -3280,7 +3280,8 @@ bad:
 	 * It was an error case.
 	 * check if we need to free the mbuf, and then return the error
 	 */
-	NG_FREE_DATA(m, meta);
+	NG_FREE_M(m);
+	NG_FREE_META(meta);
 	return (error);
 }
 
@@ -3290,13 +3291,25 @@ bad:
  * don't unref the node, or remove our name. just clear our links up.
  */
 static	int
-ngsr_rmnode(node_p node)
+ngsr_shutdown(node_p node)
 {
 	struct sr_softc * sc = node->private;
 
 	sr_down(sc);
-	ng_cutlinks(node);
-	node->flags &= ~NG_INVALID; /* bounce back to life */
+	ng_unref(node);
+/* XXX should drain queues! */
+	if (ng_make_node_common(&typestruct, &sc->node) != 0)
+		return (0);
+	sprintf(sc->nodename, "%s%d", NG_SR_NODE_TYPE, sc->unit);
+	if (ng_name_node(sc->node, sc->nodename)) {
+		printf("node naming failed\n");
+		sc->node = NULL;
+		ng_unref(sc->node); /* drop it again */
+		return (0);
+	}
+	sc->node->private = sc;
+	callout_handle_init(&sc->handle); /* should kill timeout */
+	sc->running = 0;
 	return (0);
 }
 
diff --git a/sys/dev/usb/udbp.c b/sys/dev/usb/udbp.c
index 29e203e..125558b 100644
--- a/sys/dev/usb/udbp.c
+++ b/sys/dev/usb/udbp.c
@@ -355,22 +355,20 @@ USB_ATTACH(udbp)
 
 	if ((err = ng_make_node_common(&ng_udbp_typestruct, &sc->node)) == 0) {
 		char	nodename[128];
-		sc->node->private = sc;
-		sc->xmitq.ifq_maxlen = IFQ_MAXLEN;
-		sc->xmitq_hipri.ifq_maxlen = IFQ_MAXLEN;
-		mtx_init(&sc->xmitq.ifq_mtx, "usb_xmitq", MTX_DEF);
-		mtx_init(&sc->xmitq_hipri.ifq_mtx, "usb_xmitq_hipri", MTX_DEF);
 		sprintf(nodename, "%s", USBDEVNAME(sc->sc_dev));
 		if ((err = ng_name_node(sc->node, nodename))) {
-			ng_rmnode(sc->node);
 			ng_unref(sc->node);
+			sc->node = NULL;
+			goto bad;
 		} else {
-			/* something to note it's done */
+			sc->node->private = sc;
+			sc->xmitq.ifq_maxlen = IFQ_MAXLEN;
+			sc->xmitq_hipri.ifq_maxlen = IFQ_MAXLEN;
+			mtx_init(&sc->xmitq.ifq_mtx, "usb_xmitq", MTX_DEF);
+			mtx_init(&sc->xmitq_hipri.ifq_mtx,
+					"usb_xmitq_hipri", MTX_DEF);
 		}
 	}
-	if (err) {
-		goto bad;
-	}
 	sc->flags = NETGRAPH_INITIALISED;
 	/* sc->flags &= ~DISCONNECTED; */ /* XXX */
 
@@ -612,7 +610,7 @@ DRIVER_MODULE(udbp, uhub, udbp_driver, udbp_devclass, usbd_driver_load, 0);
  * to create nodes that depend on hardware (unless you can add the hardware :)
  */
 Static int
-ng_udbp_constructor(node_p *nodep)
+ng_udbp_constructor(node_p node)
 {
 	return (EINVAL);
 }
@@ -641,7 +639,6 @@ ng_udbp_newhook(node_p node, hook_p hook, const char *name)
 #endif
 
 	if (strcmp(name, NG_UDBP_HOOK_NAME) == 0) {
-		/* do something specific to a debug connection */
 		sc->hook = hook;
 		hook->private = NULL;
 	} else {
@@ -662,14 +659,14 @@ ng_udbp_newhook(node_p node, hook_p hook, const char *name)
  * (so that old userland programs could continue to work).
  */
 Static int
-ng_udbp_rcvmsg(node_p node,
-	   struct ng_mesg *msg, const char *retaddr, struct ng_mesg **rptr,
-	   hook_p lasthook)
+ng_udbp_rcvmsg(node_p node, item_p item, hook_p lasthook)
 {
 	const udbp_p sc = node->private;
 	struct ng_mesg *resp = NULL;
 	int error = 0;
+	struct ng_mesg *msg;
 
+	NGI_GET_MSG(item, msg);
 	/* Deal with message according to cookie and command */
 	switch (msg->header.typecookie) {
 	case NGM_UDBP_COOKIE: 
@@ -706,13 +703,8 @@ ng_udbp_rcvmsg(node_p node,
 	}
 
 	/* Take care of synchronous response, if any */
-	if (rptr)
-		*rptr = resp;
-	else if (resp)
-		FREE(resp, M_NETGRAPH);
-
-	/* Free the message and return */
-	FREE(msg, M_NETGRAPH);
+	NG_RESPOND_MSG(error, node, item, resp);
+	NG_FREE_MSG(msg);
 	return(error);
 }
 
@@ -720,14 +712,18 @@ ng_udbp_rcvmsg(node_p node,
  * Accept data from the hook and queue it for output.
  */
 Static int
-ng_udbp_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
+ng_udbp_rcvdata(hook_p hook, item_p item)
 {
 	const udbp_p sc = hook->node->private;
 	int error;
 	struct ifqueue	*xmitq_p;
 	int	s;
+	struct mbuf *m;
+	meta_p meta;
 
+	NGI_GET_M(item, m);
+	NGI_GET_META(item, meta);
+	NG_FREE_ITEM(item);
 	/* 
 	 * Now queue the data for when it can be sent
 	 */
@@ -756,7 +752,8 @@ bad:	/*
          * It was an error case.
 	 * check if we need to free the mbuf, and then return the error
 	 */
-	NG_FREE_DATA(m, meta);
+	NG_FREE_M(m);
+	NG_FREE_META(meta);
 	return (error);
 }
 
@@ -769,9 +766,9 @@ Static int
 ng_udbp_rmnode(node_p node)
 {
 	const udbp_p sc = node->private;
+	int err;
 
 	node->flags |= NG_INVALID;
-	ng_cutlinks(node);
 
 	/* Drain the queues */
 	IF_DRAIN(&sc->xmitq_hipri);
@@ -779,8 +776,21 @@ ng_udbp_rmnode(node_p node)
 
 	sc->packets_in = 0;		/* reset stats */
 	sc->packets_out = 0;
-	node->flags &= ~NG_INVALID;		/* reset invalid flag */
-	return (0);
+	ng_unref(node);			/* forget it ever existed */
+
+	/* stolen from attach routine */
+	if ((err = ng_make_node_common(&ng_udbp_typestruct, &sc->node)) == 0) {
+		char	nodename[128];
+		sprintf(nodename, "%s", USBDEVNAME(sc->sc_dev));
+		if ((err = ng_name_node(sc->node, nodename))) {
+			ng_unref(sc->node); /* out damned spot! */
+			sc->flags &= ~NETGRAPH_INITIALISED;
+			sc->node = NULL;
+		} else {
+			sc->node->private = sc;
+		}
+	}
+	return (err);
 }
 
 /*
@@ -807,8 +817,9 @@ ng_udbp_disconnect(hook_p hook)
 	const udbp_p sc = hook->node->private;
 	sc->hook = NULL;
 
-	if (hook->node->numhooks == 0)
-		ng_rmnode(hook->node);
+	if ((hook->node->numhooks == 0)
+	&& ((hook->node->flags & NG_INVALID) == 0))
+		ng_rmnode_self(hook->node);
 	return (0);
 }
 
diff --git a/sys/i386/isa/if_ar.c b/sys/i386/isa/if_ar.c
index fe9abeb..722a133 100644
--- a/sys/i386/isa/if_ar.c
+++ b/sys/i386/isa/if_ar.c
@@ -267,7 +267,7 @@ static	void	ngar_init(void* ignored);
 
 static ng_constructor_t	ngar_constructor;
 static ng_rcvmsg_t	ngar_rcvmsg;
-static ng_shutdown_t	ngar_rmnode;
+static ng_shutdown_t	ngar_shutdown;
 static ng_newhook_t	ngar_newhook;
 /*static ng_findhook_t	ngar_findhook; */
 static ng_connect_t	ngar_connect;
@@ -280,7 +280,7 @@ static struct ng_type typestruct = {
 	NULL,
 	ngar_constructor,
 	ngar_rcvmsg,
-	ngar_rmnode,
+	ngar_shutdown,
 	ngar_newhook,
 	NULL,
 	ngar_connect,
@@ -509,18 +509,17 @@ arattach(struct ar_hardc *hc)
 		if (ngar_done_init == 0) ngar_init(NULL);
 		if (ng_make_node_common(&typestruct, &sc->node) != 0)
 			return (0);
+		sprintf(sc->nodename, "%s%d", NG_AR_NODE_TYPE, sc->unit);
+		if (ng_name_node(sc->node, sc->nodename)) {
+			ng_unref(sc->node); /* drop it again */
+			return (0);
+		}
 		sc->node->private = sc;
 		callout_handle_init(&sc->handle);
 		sc->xmitq.ifq_maxlen = IFQ_MAXLEN;
 		sc->xmitq_hipri.ifq_maxlen = IFQ_MAXLEN;
 		mtx_init(&sc->xmitq.ifq_mtx, "ar_xmitq", MTX_DEF);
 		mtx_init(&sc->xmitq_hipri.ifq_mtx, "ar_xmitq_hipri", MTX_DEF);
-		sprintf(sc->nodename, "%s%d", NG_AR_NODE_TYPE, sc->unit);
-		if (ng_name_node(sc->node, sc->nodename)) {
-			ng_rmnode(sc->node);
-			ng_unref(sc->node);
-			return (0);
-		}
 		sc->running = 0;
 #endif	/* NETGRAPH */
 	}
@@ -2173,7 +2172,7 @@ ngar_watchdog_frame(void * arg)
  * If the hardware exists, it will already have created it.
  */
 static	int
-ngar_constructor(node_p *nodep)
+ngar_constructor(node_p node)
 {
 	return (EINVAL);
 }
@@ -2216,13 +2215,14 @@ ngar_newhook(node_p node, hook_p hook, const char *name)
  * Just respond to the generic TEXT_STATUS message
  */
 static	int
-ngar_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
-				struct ng_mesg **rptr, hook_p lasthook)
+ngar_rcvmsg(node_p node, item_p item, hook_p lasthook)
 {
 	struct ar_softc *	sc;
 	struct ng_mesg *resp = NULL;
 	int error = 0;
+	struct ng_mesg *msg;
 
+	NGI_GET_MSG(item, msg);
 	sc = node->private;
 	switch (msg->header.typecookie) {
 	case	NG_AR_COOKIE: 
@@ -2268,13 +2268,8 @@ ngar_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
 		break;
 	}
 	/* Take care of synchronous response, if any */
-	if (rptr)
-		*rptr = resp;
-	else if (resp)
-		/* Should send the hard way */
-		FREE(resp, M_NETGRAPH);
-
-	free(msg, M_NETGRAPH);
+	NG_RESPOND_MSG(error, node, item, resp);
+	NG_FREE_MSG(msg);
 	return (error);
 }
 
@@ -2282,14 +2277,18 @@ ngar_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
  * get data from another node and transmit it to the correct channel
  */
 static	int
-ngar_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
+ngar_rcvdata(hook_p hook, item_p item)
 {
 	int s;
 	int error = 0;
 	struct ar_softc * sc = hook->node->private;
 	struct ifqueue	*xmitq_p;
+	struct mbuf *m;
+	meta_p meta;
 	
+	NGI_GET_M(item, m);
+	NGI_GET_META(item, meta);
+	NG_FREE_ITEM(item);
 	/*
 	 * data doesn't come in from just anywhere (e.g control hook)
 	 */
@@ -2326,7 +2325,8 @@ bad:
 	 * It was an error case.
 	 * check if we need to free the mbuf, and then return the error
 	 */
-	NG_FREE_DATA(m, meta);
+	NG_FREE_M(m);
+	NG_FREE_META(meta);
 	return (error);
 }
 
@@ -2336,13 +2336,27 @@ bad:
  * don't unref the node, or remove our name. just clear our links up.
  */
 static	int
-ngar_rmnode(node_p node)
+ngar_shutdown(node_p node)
 {
 	struct ar_softc * sc = node->private;
 
 	ar_down(sc);
-	ng_cutlinks(node);
-	node->flags &= ~NG_INVALID; /* bounce back to life */
+	ng_unref(node);
+	/* XXX need to drain the output queues! */
+
+	/* The node is dead, long live the node! */
+	/* stolen from the attach routine */
+	if (ng_make_node_common(&typestruct, &sc->node) != 0)
+		return (0);
+	sprintf(sc->nodename, "%s%d", NG_AR_NODE_TYPE, sc->unit);
+	if (ng_name_node(sc->node, sc->nodename)) {
+		sc->node = NULL;
+		printf("node naming failed\n");
+		ng_unref(sc->node); /* node dissappears */
+		return (0);
+	}
+	sc->node->private = sc;
+	sc->running = 0;
 	return (0);
 }
 
diff --git a/sys/i386/isa/if_sr.c b/sys/i386/isa/if_sr.c
index 9ca41d4..a7ba8e9 100644
--- a/sys/i386/isa/if_sr.c
+++ b/sys/i386/isa/if_sr.c
@@ -367,7 +367,7 @@ static	void	ngsr_init(void* ignored);
 
 static ng_constructor_t	ngsr_constructor;
 static ng_rcvmsg_t	ngsr_rcvmsg;
-static ng_shutdown_t	ngsr_rmnode;
+static ng_shutdown_t	ngsr_shutdown;
 static ng_newhook_t	ngsr_newhook;
 /*static ng_findhook_t	ngsr_findhook; */
 static ng_connect_t	ngsr_connect;
@@ -380,7 +380,7 @@ static struct ng_type typestruct = {
 	NULL,
 	ngsr_constructor,
 	ngsr_rcvmsg,
-	ngsr_rmnode,
+	ngsr_shutdown,
 	ngsr_newhook,
 	NULL,
 	ngsr_connect,
@@ -934,18 +934,17 @@ srattach(struct sr_hardc *hc)
 		if (ngsr_done_init == 0) ngsr_init(NULL);
 		if (ng_make_node_common(&typestruct, &sc->node) != 0)
 			return (0);
+		sprintf(sc->nodename, "%s%d", NG_SR_NODE_TYPE, sc->unit);
+		if (ng_name_node(sc->node, sc->nodename)) {
+			ng_unref(sc->node); /* make it go away again */
+			return (0);
+		}
 		sc->node->private = sc;
 		callout_handle_init(&sc->handle);
 		sc->xmitq.ifq_maxlen = IFQ_MAXLEN;
 		sc->xmitq_hipri.ifq_maxlen = IFQ_MAXLEN;
 		mtx_init(&sc->xmitq.ifq_mtx, "sr_xmitq", MTX_DEF);
 		mtx_init(&sc->xmitq_hipri.ifq_mtx, "sr_xmitq_hipri", MTX_DEF);
-		sprintf(sc->nodename, "%s%d", NG_SR_NODE_TYPE, sc->unit);
-		if (ng_name_node(sc->node, sc->nodename)) {
-			ng_rmnode(sc->node);
-			ng_unref(sc->node);
-			return (0);
-		}
 		sc->running = 0;
 #endif	/* NETGRAPH */
 	}
@@ -3126,7 +3125,7 @@ ngsr_watchdog_frame(void * arg)
  * If the hardware exists, it will already have created it.
  */
 static	int
-ngsr_constructor(node_p *nodep)
+ngsr_constructor(node_p node)
 {
 	return (EINVAL);
 }
@@ -3169,13 +3168,14 @@ ngsr_newhook(node_p node, hook_p hook, const char *name)
  * Just respond to the generic TEXT_STATUS message
  */
 static	int
-ngsr_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
-				struct ng_mesg **rptr, hook_p lasthook)
+ngsr_rcvmsg(node_p node, item_p item, hook_p lasthook)
 {
 	struct sr_softc *	sc;
 	struct ng_mesg *resp = NULL;
 	int error = 0;
+	struct ng_mesg *msg;
 
+	NGI_GET_MSG(item,msg);
 	sc = node->private;
 	switch (msg->header.typecookie) {
 	case	NG_SR_COOKIE: 
@@ -3223,12 +3223,8 @@ ngsr_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
 		break;
 	}
 	/* Take care of synchronous response, if any */
-	if (rptr)
-		*rptr = resp;
-	else if (resp)
-		FREE(resp, M_NETGRAPH);
-
-	free(msg, M_NETGRAPH);
+	NG_RESPOND_MSG(error, node, item, resp);
+	NG_FREE_MSG(msg);
 	return (error);
 }
 
@@ -3236,14 +3232,18 @@ ngsr_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
  * get data from another node and transmit it to the correct channel
  */
 static	int
-ngsr_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
+ngsr_rcvdata(hook_p hook, item_p item)
 {
 	int s;
 	int error = 0;
 	struct sr_softc * sc = hook->node->private;
 	struct ifqueue	*xmitq_p;
+	struct mbuf *m;
+	meta_p meta;
 	
+	NGI_GET_M(item, m);
+	NGI_GET_META(item, meta);
+	NG_FREE_ITEM(item);
 	/*
 	 * data doesn't come in from just anywhere (e.g control hook)
 	 */
@@ -3280,7 +3280,8 @@ bad:
 	 * It was an error case.
 	 * check if we need to free the mbuf, and then return the error
 	 */
-	NG_FREE_DATA(m, meta);
+	NG_FREE_M(m);
+	NG_FREE_META(meta);
 	return (error);
 }
 
@@ -3290,13 +3291,25 @@ bad:
  * don't unref the node, or remove our name. just clear our links up.
  */
 static	int
-ngsr_rmnode(node_p node)
+ngsr_shutdown(node_p node)
 {
 	struct sr_softc * sc = node->private;
 
 	sr_down(sc);
-	ng_cutlinks(node);
-	node->flags &= ~NG_INVALID; /* bounce back to life */
+	ng_unref(node);
+/* XXX should drain queues! */
+	if (ng_make_node_common(&typestruct, &sc->node) != 0)
+		return (0);
+	sprintf(sc->nodename, "%s%d", NG_SR_NODE_TYPE, sc->unit);
+	if (ng_name_node(sc->node, sc->nodename)) {
+		printf("node naming failed\n");
+		sc->node = NULL;
+		ng_unref(sc->node); /* drop it again */
+		return (0);
+	}
+	sc->node->private = sc;
+	callout_handle_init(&sc->handle); /* should kill timeout */
+	sc->running = 0;
 	return (0);
 }
 
diff --git a/sys/i4b/driver/i4b_ing.c b/sys/i4b/driver/i4b_ing.c
index 8807fce..e54360a 100644
--- a/sys/i4b/driver/i4b_ing.c
+++ b/sys/i4b/driver/i4b_ing.c
@@ -166,7 +166,7 @@ struct ngingstat {
 
 static ng_constructor_t	ng_ing_constructor;
 static ng_rcvmsg_t	ng_ing_rcvmsg;
-static ng_shutdown_t	ng_ing_rmnode;
+static ng_shutdown_t	ng_ing_shutdown;
 static ng_newhook_t	ng_ing_newhook;
 static ng_connect_t	ng_ing_connect;
 static ng_rcvdata_t	ng_ing_rcvdata;
@@ -208,7 +208,7 @@ static struct ng_type typestruct = {
 	NULL,
 	ng_ing_constructor,
 	ng_ing_rcvmsg,
-	ng_ing_rmnode,
+	ng_ing_shutdown,
 	ng_ing_newhook,
 	NULL,
 	ng_ing_connect,
@@ -274,22 +274,22 @@ i4bingattach(void *dummy)
 			printf("ing: ng_make_node_common, ret = %d\n!", ret);
 		}
 
-		sc->node->private = sc;
-
-		sc->xmitq.ifq_maxlen = IFQ_MAXLEN;
-		sc->xmitq_hipri.ifq_maxlen = IFQ_MAXLEN;
-		mtx_init(&sc->xmitq.ifq_mtx, "i4b_ing_xmitq", MTX_DEF);
-		mtx_init(&sc->xmitq_hipri.ifq_mtx, "i4b_ing_hipri", MTX_DEF);
-				
 		/* name the netgraph node */
 
 		sprintf(sc->nodename, "%s%d", NG_ING_NODE_TYPE, sc->sc_unit);
-
-		if(ng_name_node(sc->node, sc->nodename))
+		if((ret = ng_name_node(sc->node, sc->nodename)))
 		{
-			ng_rmnode(sc->node);
+			printf("ing: ng_name node, ret = %d\n!", ret);
 			ng_unref(sc->node);
+			break;
 		}
+
+		sc->node->private = sc;
+
+		sc->xmitq.ifq_maxlen = IFQ_MAXLEN;
+		sc->xmitq_hipri.ifq_maxlen = IFQ_MAXLEN;
+		mtx_init(&sc->xmitq.ifq_mtx, "i4b_ing_xmitq", MTX_DEF);
+		mtx_init(&sc->xmitq_hipri.ifq_mtx, "i4b_ing_hipri", MTX_DEF);
 	}
 }
 
@@ -575,7 +575,7 @@ ing_init_linktab(int unit)
  * If the hardware exists, it will already have created it.
  *---------------------------------------------------------------------------*/
 static int
-ng_ing_constructor(node_p *nodep)
+ng_ing_constructor(node_p node)
 {
 	return(EINVAL);
 }
@@ -621,8 +621,7 @@ ng_ing_newhook(node_p node, hook_p hook, const char *name)
  *---------------------------------------------------------------------------*/
 #if defined(__FreeBSD_version) && __FreeBSD_version >= 500000
 static int
-ng_ing_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
-		struct ng_mesg **rptr, hook_p lasthook)
+ng_ing_rcvmsg(node_p node, item_p item, hook_p lasthook)
 #else
 static int
 ng_ing_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
@@ -633,7 +632,11 @@ ng_ing_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
 
 	struct ng_mesg *resp = NULL;
 	int error = 0;
+#if defined(__FreeBSD_version) && __FreeBSD_version >= 500000
+	struct ng_mesg *msg;
 
+	NGI_GET_MSG(item, msg);
+#endif
 	if(msg->header.typecookie == NGM_GENERIC_COOKIE)
 	{
 		switch(msg->header.cmd)
@@ -726,15 +729,9 @@ ng_ing_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
 	}
 
 	/* Take care of synchronous response, if any */
-
-	if (rptr)
-		*rptr = resp;
-	else if (resp)
-		FREE(resp, M_NETGRAPH);
-
+	NG_RESPOND_MSG(error, node, item, resp);
 	/* Free the message and return */
-
-	FREE(msg, M_NETGRAPH);
+	NG_FREE_MSG(msg);
 	return(error);
 }
 
@@ -743,8 +740,7 @@ ng_ing_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
  *---------------------------------------------------------------------------*/
 #if defined(__FreeBSD_version) && __FreeBSD_version >= 500000
 static int
-ng_ing_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-                struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
+ng_ing_rcvdata(hook_p hook, item_p item)
 #else
 static int
 ng_ing_rcvdata(hook_p hook, struct mbuf *m, meta_p meta)
@@ -753,10 +749,18 @@ ng_ing_rcvdata(hook_p hook, struct mbuf *m, meta_p meta)
 	struct ing_softc *sc = hook->node->private;
 	struct ifqueue  *xmitq_p;
 	int s;
+#if defined(__FreeBSD_version) && __FreeBSD_version >= 500000
+	struct mbuf *m;
+	meta_p meta;
 	
+	NGI_GET_M(item, m);
+	NGI_GET_META(item, meta);
+	NG_FREE_ITEM(item);
+#endif
 	if(hook->private == NULL)
 	{
-		NG_FREE_DATA(m, meta);
+		NG_FREE_M(m);
+		NG_FREE_META(meta);
 		return(ENETDOWN);
 	}
 	
@@ -789,7 +793,8 @@ ng_ing_rcvdata(hook_p hook, struct mbuf *m, meta_p meta)
 		_IF_DROP(xmitq_p);
 		IF_UNLOCK(xmitq_p);
 		splx(s);
-		NG_FREE_DATA(m, meta);
+		NG_FREE_M(m);
+		NG_FREE_META(meta);
 		return(ENOBUFS);
 	}
 
@@ -808,17 +813,32 @@ ng_ing_rcvdata(hook_p hook, struct mbuf *m, meta_p meta)
  * we'd only remove our links and reset ourself.
  *---------------------------------------------------------------------------*/
 static int
-ng_ing_rmnode(node_p node)
+ng_ing_shutdown(node_p node)
 {
 	struct ing_softc *sc = node->private;
+	int	ret;
 
 	node->flags |= NG_INVALID;
-	ng_cutlinks(node);
+	ng_unref(node);
 
 	sc->packets_in = 0;		/* reset stats */
 	sc->packets_out = 0;
 
-	node->flags &= ~NG_INVALID;	/* reset invalid flag */
+	if ((ret = ng_make_node_common(&typestruct, &sc->node)))
+	{
+		printf("ing: ng_make_node_common, ret = %d\n!", ret);
+	}
+
+	/* name the netgraph node */
+	sprintf(sc->nodename, "%s%d", NG_ING_NODE_TYPE, sc->sc_unit);
+	if((ret = ng_name_node(sc->node, sc->nodename)))
+	{
+		printf("ing: ng_name node, ret = %d\n!", ret);
+		ng_unref(sc->node);
+		return (0);
+	}
+
+	sc->node->private = sc;
 
 	return (0);
 }
diff --git a/sys/netgraph/NOTES b/sys/netgraph/NOTES
index 69d8b71..0ddc02f 100644
--- a/sys/netgraph/NOTES
+++ b/sys/netgraph/NOTES
@@ -13,6 +13,8 @@ Archie's suggestions... :-)
 	- They allow conditional compilation which keeps
 	  statistics & counters on various memory allocation
 	  (or so it seems)
+	   - Now tend to have NG_FREE_XX() macros. they
+	     allow better debugging
 
  - In struct ng_mesg: at least make "header" into "hdr", if not
    getting rid of it altogether. It doesn't seem necessary and
@@ -34,9 +36,12 @@ Archie's suggestions... :-)
 	    #define NG_MSG_HDR_SIZE	(sizeof(struct ng_message) - 1)
 	  #endif
 
+	  - inertia rules :-b
+
+
  - Have a user level program to print out and manipulate nodes, etc.
 	- [DONE]
-		see ngctl
+		see ngctl, nghook
 
  - "Netgraph global" flags to turn on tracing, etc.
 
@@ -57,6 +62,8 @@ Archie's suggestions... :-)
    colon character at the end of the name. Note ngctl allows you
    to do it either way!
 	[DONE] (I think)
+	- bind on a control socket has been disabled
+	  it was a bad idea.
 
  - Need to implement passing meta information through socket nodes
    using sendmsg() and recvmsg().
@@ -64,14 +71,16 @@ Archie's suggestions... :-)
  - Stuff needing to be added to manual:
 
    - Awareness of SPL level, use ng_queue*() functions when necessary.
-   - Malloc all memory with type M_NETGRAPH.
+   - Malloc all memory with type M_NETGRAPH. -DONE
    - Write code so it can be an LKM or built into the kernel.. this means
      be careful with things like #ifdef INET.
    - All nodes assume that all data mbufs have the M_PKTHDR flag set!
      The ng_send_data() and related functions should have an
      #ifdef DIAGNOSTICS check to check this assumption for every mbuf.
+     -DONE with INVARIANTS. Framework should test this more.
    - More generally, netgraph code should make liberal use of the
      #ifdef DIAGNOSTICS definition.
+     -INVARIANTS.
    - Since data and messages are sent functionally, programmers need
      to watch out for infinite feedback loops. Should ng_base.c detect
      this automatically?
@@ -79,3 +88,16 @@ Archie's suggestions... :-)
 	which is set to the colour of the packet as you pass through.
 	hitting a node already of your colour would abort. Each packet
 	has another (incremented) colour.
+	-new 'item' type can hold a hopcount...
+
+NEW in 2001
+All piggyback responses have gone away.
+use the node ID in the return address field for quick response delivery.
+
+Every node has a queue, plus there is a list of nodes that have queued work.
+Extensive use of Mutexes. Getting in shape for SMP.
+
+Messages and data are deliverd in a new form. An Item now has
+all information needed to queue such a request and deliver it later, so 
+it is now the basis of all data transfer since any transfer may need to
+be queued.
diff --git a/sys/netgraph/netgraph.h b/sys/netgraph/netgraph.h
index 56cf7d6f..323aacc 100644
--- a/sys/netgraph/netgraph.h
+++ b/sys/netgraph/netgraph.h
@@ -51,11 +51,24 @@
 #error "This file should not be included in user level programs"
 #endif
 
+#include <sys/mutex.h>
+
+/* The structure for queueing across ISR switches */
+struct ng_item ; /* forward reference */
+typedef	struct ng_item *item_p;
+struct ng_queue {
+	u_long          q_flags;
+	struct mtx      q_mtx;
+	item_p queue;
+	item_p *last;
+	struct ng_node *q_node;		/* find the front of the node.. */
+};
+
 /*
  * Structure of a hook
  */
 struct ng_hook {
-	char   *name;		/* what this node knows this link as */
+	char   name[NG_HOOKLEN+1];/* what this node knows this link as */
 	void   *private;	/* node dependant ID for this hook */
 	int	flags;		/* info about this hook/link */
 	int	refs;		/* dont actually free this till 0 */
@@ -68,30 +81,32 @@ typedef struct ng_hook *hook_p;
 /* Flags for a hook */
 #define HK_INVALID		0x0001	/* don't trust it! */
 #define HK_QUEUE		0x0002	/* queue for later delivery */
+#define HK_FORCE_WRITER		0x0004	/* Incoming data queued as a writer */
 
 /*
  * Structure of a node
  */
 struct ng_node {
-	char   *name;		/* optional globally unique name */
+	char   name[NG_NODELEN+1]; /* optional globally unique name */
 	struct	ng_type *type;	/* the installed 'type' */
 	int	flags;		/* see below for bit definitions */
-	int	sleepers;	/* #procs sleeping on this node */
 	int	refs;		/* number of references to this node */
 	int	numhooks;	/* number of hooks */
-	int	colour;		/* for graph colouring algorithms */
 	void   *private;	/* node type dependant node ID */
 	ng_ID_t		ID;	/* Unique per node */
 	LIST_HEAD(hooks, ng_hook) hooks;	/* linked list of node hooks */
 	LIST_ENTRY(ng_node)	  nodes;	/* linked list of all nodes */
 	LIST_ENTRY(ng_node)	  idnodes;	/* ID hash collision list */
+	TAILQ_ENTRY(ng_node)	  work;		/* nodes with work to do */
+	struct	ng_queue	  input_queue;	/* input queue for locking */
 };
 typedef struct ng_node *node_p;
 
 /* Flags for a node */
-#define NG_INVALID	0x001	/* free when all sleepers and refs go to 0 */
-#define NG_BUSY		0x002	/* callers should sleep or wait */
-#define NG_TOUCHED	0x004	/* to avoid cycles when 'flooding' */
+#define NG_INVALID	0x00000001	/* free when refs go to 0 */
+#define NG_WORKQ	0x00000002	/* node is on the work queue */
+#define NG_FORCE_WRITER	0x00000004	/* Never multithread this node */
+#define NG_CLOSING	0x00000008	/* ng_rmnode() at work */
 #define NGF_TYPE1	0x10000000	/* reserved for type specific storage */
 #define NGF_TYPE2	0x20000000	/* reserved for type specific storage */
 #define NGF_TYPE3	0x40000000	/* reserved for type specific storage */
@@ -131,18 +146,15 @@ typedef struct ng_meta *meta_p;
 #define NGMF_TRACE	0x02	/* trace when handing this data to a node */
 
 /* node method definitions */
-typedef	int	ng_constructor_t(node_p *node);
-typedef	int	ng_rcvmsg_t(node_p node, struct ng_mesg *msg,
-			const char *retaddr, struct ng_mesg **resp,
-			hook_p lasthook);
+typedef	int	ng_constructor_t(node_p node);
+typedef	int	ng_rcvmsg_t(node_p node, item_p item, hook_p lasthook);
 typedef	int	ng_shutdown_t(node_p node);
 typedef	int	ng_newhook_t(node_p node, hook_p hook, const char *name);
 typedef	hook_p	ng_findhook_t(node_p node, const char *name);
 typedef	int	ng_connect_t(hook_p hook);
-typedef	int	ng_rcvdata_t(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp);
+typedef	int	ng_rcvdata_t(hook_p hook, item_p item);
 typedef	int	ng_disconnect_t(hook_p hook);
-
+typedef	int	ng_rcvitem (node_p node, hook_p hook, item_p item);
 /*
  * Command list -- each node type specifies the command that it knows
  * how to convert between ASCII and binary using an array of these.
@@ -191,73 +203,371 @@ struct ng_type {
 	int		    refs;		/* number of instances */
 };
 
+struct ng_item {
+	u_long	el_flags;
+	item_p	el_next;
+	node_p	el_dest; /* The node it will be applied against (or NULL) */
+	hook_p	el_hook; /* Entering hook. Optional in Control messages */
+	union {
+		struct {
+			struct mbuf	*da_m;
+			meta_p		da_meta;
+		} data;
+		struct {
+			struct ng_mesg	*msg_msg;
+			ng_ID_t		msg_retaddr;
+		} msg;
+	} body;
+#define ITEM_DEBUG 
+#ifdef	ITEM_DEBUG
+	char *lastfile;
+	int  lastline;
+	TAILQ_ENTRY(ng_item)	  all;		/* all existing items */
+#endif	/* ITEM_DEBUG */
+};
+#define NGQF_D_M	0x01		/* MASK of data/message */
+#define NGQF_DATA	0x01		/* the queue element is data */
+#define NGQF_MESG	0x00		/* the queue element is a message */
+#define NGQF_TYPE	0x02		/*  MASK for queue entry type */
+#define NGQF_READER	0x02		/* queued as a reader */
+#define NGQF_WRITER	0x00		/* queued as a writer */
+#define NGQF_FREE	0x04
+
 /*
  * This defines the in-kernel binary interface version.
  * It is possible to change this but leave the external message
  * API the same. Each type also has it's own cookies for versioning as well.
+ * Change it for ITEM_DEBUG version so we cannot mix debug and non debug
+ * modules.
  */
-#define NG_ABI_VERSION	5
+#define _NG_ABI_VERSION 5
+#ifdef	ITEM_DEBUG
+#define NG_ABI_VERSION	(_NG_ABI_VERSION + 0x10000)
+#else	/* ITEM_DEBUG */
+#define NG_ABI_VERSION	_NG_ABI_VERSION
+#endif	/* ITEM_DEBUG */
+
+/**********************************************************************
+* Queue item macros.  Peek and extract values.
+**********************************************************************/
+/*
+ * Get the mbuf (etc) out of an item.
+ * Sets the value in the item to NULL in case we need to call NG_FREE_ITEM()
+ * with it, (to avoid freeing the things twice).
+ * If you don't want to zero out the item then realise that the
+ * item still owns it.
+ * Retaddr is different. There are no references on that. It's just a number.
+ * The debug versions must be either all used everywhere or not at all.
+ */
+
+#define _NGI_M(i) ((i)->body.data.da_m)
+#define _NGI_META(i) ((i)->body.data.da_meta)
+#define _NGI_MSG(i) ((i)->body.msg.msg_msg)
+#define _NGI_RETADDR(i) ((i)->body.msg.msg_retaddr)
+
+#ifdef ITEM_DEBUG
+void	dumpitem(item_p item, char *file, int line);
+static __inline void _ngi_check(item_p item, char *file, int line) ;
+static __inline struct mbuf ** _ngi_m(item_p item, char *file, int line) ;
+static __inline meta_p * _ngi_meta(item_p item, char *file, int line) ;
+static __inline struct ng_mesg ** _ngi_msg(item_p item, char *file, int line) ;
+static __inline ng_ID_t * _ngi_retaddr(item_p item, char *file, int line) ;
+
+static __inline void
+_ngi_check(item_p item, char *file, int line) 
+{
+	if (item->el_flags & NGQF_FREE) {
+		dumpitem(item, file, line);
+		panic ("free item!");
+	}
+	(item)->lastline = line;
+	(item)->lastfile = file;
+}
+
+static __inline struct mbuf **
+_ngi_m(item_p item, char *file, int line) 
+{
+	_ngi_check(item, file, line);
+	return (&_NGI_M(item));
+}
+
+static __inline meta_p *
+_ngi_meta(item_p item, char *file, int line) 
+{
+	_ngi_check(item, file, line);
+	return (&_NGI_META(item));
+}
+
+static __inline struct ng_mesg **
+_ngi_msg(item_p item, char *file, int line) 
+{
+	_ngi_check(item, file, line);
+	return (&_NGI_MSG(item));
+}
+
+static __inline ng_ID_t *
+_ngi_retaddr(item_p item, char *file, int line) 
+{
+	_ngi_check(item, file, line);
+	return (&_NGI_RETADDR(item));
+}
+
+#define NGI_M(i) (*_ngi_m(i, __FILE__, __LINE__))
+
+#define NGI_META(i) (*_ngi_meta(i, __FILE__, __LINE__))
+
+#define NGI_MSG(i) (*_ngi_msg(i, __FILE__, __LINE__))
+
+#define NGI_RETADDR(i) (*_ngi_retaddr(i, __FILE__, __LINE__))
+
+#define NGI_GET_M(i,m)							\
+	do {								\
+		m = NGI_M(i);						\
+		_NGI_M(i) = NULL;					\
+	} while (0)
+
+#define NGI_GET_META(i,m)						\
+	do {								\
+		m = NGI_META(i);					\
+		_NGI_META(i) = NULL;					\
+	} while (0)
+
+#define NGI_GET_MSG(i,m)						\
+	do {								\
+		m = NGI_MSG(i);						\
+		_NGI_MSG(i) = NULL;					\
+	} while (0)
+
+#define NG_FREE_ITEM(item)						\
+	do {								\
+		_ngi_check(item, __FILE__, __LINE__);			\
+		ng_free_item((item));					\
+	} while (0)
 
-/* Send data packet with meta-data */
-#define NG_SEND_DATA(err, hook, m, meta)				\
+#define	SAVE_LINE(item)							\
 	do {								\
-		(err) = ng_send_data((hook), (m), (meta),	\
-						NULL, NULL, NULL);	\
+		(item)->lastline = __LINE__;				\
+		(item)->lastfile = __FILE__;				\
+	} while (0)
+
+#else	/* ITEM_DEBUG */
+
+
+#define NGI_M(i)	_NGI_M(i)
+#define NGI_META(i)	_NGI_META(i)
+#define NGI_MSG(i)	_NGI_MSG(i)
+#define NGI_RETADDR(i)	_NGI_RETADDR(i)
+
+#define NGI_GET_M(i,m)       do {m = NGI_M(i); NGI_M(i) = NULL;      } while (0)
+#define NGI_GET_META(i,m)    do {m = NGI_META(i); NGI_META(i) = NULL;} while (0)
+#define NGI_GET_MSG(i,m)     do {m = NGI_MSG(i); NGI_MSG(i) = NULL;  } while (0)
+
+#define	NG_FREE_ITEM(item)	ng_free_item((item))
+#define	SAVE_LINE(item)
+
+#endif	/* ITEM_DEBUG */
+	
+/**********************************************************************
+* Data macros.  Send, manipulate and free.
+**********************************************************************/
+/* Send data packet including a possible sync response pointer */
+#define NG_SEND_DATA(error, hook, m, meta)				\
+	do {								\
+		item_p item;						\
+		if ((item = ng_package_data((m), (meta)))) {		\
+			if (!((error) = ng_address_hook(NULL, item,	\
+							hook, NULL))) {	\
+				SAVE_LINE(item);			\
+				(error) = ng_snd_item((item), 0);	\
+			}						\
+		} else {						\
+			(error) = ENOMEM;				\
+		}							\
 		(m) = NULL;						\
 		(meta) = NULL;						\
 	} while (0)
 
-/* Send data packet with no meta-data */
-#define NG_SEND_DATA_ONLY(err, hook, m)					\
+#define NG_SEND_DATA_ONLY(error, hook, m)				\
 	do {								\
-		(err) = ng_send_data((hook), (m), NULL,	\
-						NULL, NULL, NULL);	\
+		item_p item;						\
+		if ((item = ng_package_data((m), NULL))) {		\
+			if (!((error) = ng_address_hook(NULL, item,	\
+							hook, NULL))) {	\
+				SAVE_LINE(item);			\
+				(error) = ng_snd_item((item), 0);	\
+			}						\
+		} else {						\
+			(error) = ENOMEM;				\
+		}							\
 		(m) = NULL;						\
 	} while (0)
 
-/* Send data packet including a possible sync response pointer */
-#define NG_SEND_DATA_RESP(err, hook, m, meta, rmsg)			\
+/*
+ * Forward a data packet with no new meta-data.
+ * old metadata is passed along without change.
+ * Mbuf pointer is updated to new value.
+ */
+#define NG_FWD_NEW_DATA(error, item, hook, m)				\
 	do {								\
-		(err) = ng_send_data((hook), (m), (meta),	\
-					NULL, NULL, rmsg);		\
+		NGI_M(item) = m;					\
+		if (!((error) = ng_address_hook(NULL, (item),		\
+						(hook), NULL))) {	\
+			SAVE_LINE(item);				\
+			(error) = ng_snd_item((item), 0);		\
+		}							\
+		(item) = NULL;						\
 		(m) = NULL;						\
-		(meta) = NULL;						\
 	} while (0)
 
 /*
- * Send data packet including a possible sync response pointer
- * Allow the callee to replace the data and pass it back
- * or to simply 'reject it' or 'keep it'
+ * Assuming the data is already ok, just set the new address and send
  */
-#define NG_SEND_DATA_RET(err, hook, m, meta, rmsg)			\
+#define NG_FWD_DATA(error, item, hook)					\
 	do {								\
-		struct mbuf *rm = NULL;					\
-		meta_p rmeta = NULL;					\
-		(err) = ng_send_data((hook), (m), (meta),	\
-					&rm, &rmeta, (rmsg));		\
-		(m) = rm;						\
-		(meta) = rmeta;						\
+		if (!((error) = ng_address_hook(NULL, (item),		\
+						(hook), NULL))) {	\
+			SAVE_LINE(item);				\
+			(error) = ng_snd_item((item), 0);		\
+		} else {						\
+			(error) = ENXIO;				\
+		}							\
+		(item) = NULL;						\
 	} while (0)
 
 
-/* Free metadata */
+/* Note that messages can be static (e.g. in ng_rmnode_self()) */
+/* XXX flag should not be user visible  */
+#define NG_FREE_MSG(msg)						\
+	do {								\
+		if ((msg)) {						\
+			if ((msg->header.flags & NGF_STATIC) == 0) {	\
+				FREE((msg), M_NETGRAPH_MSG);		\
+			}						\
+			(msg) = NULL;					\
+		}	 						\
+	} while (0)
+
 #define NG_FREE_META(meta)						\
 	do {								\
 		if ((meta)) {						\
-			FREE((meta), M_NETGRAPH);			\
-			meta = NULL;					\
-		}							\
+			FREE((meta), M_NETGRAPH_META);			\
+			(meta) = NULL;					\
+		}	 						\
 	} while (0)
 
-/* Free any data packet and/or meta-data */
-#define NG_FREE_DATA(m, meta)						\
+#define NG_FREE_M(m)							\
 	do {								\
 		if ((m)) {						\
 			m_freem((m));					\
-			m = NULL;					\
+			(m) = NULL;					\
 		}							\
+	} while (0)
+/* Free any data packet and/or meta-data */
+#define NG_FREE_DATAX(m, meta)						\
+	do {								\
+		NG_FREE_M((m));						\
 		NG_FREE_META((meta));					\
 	} while (0)
+/*****************************************
+* Message macros
+*****************************************/
+
+#define NG_SEND_MSG_HOOK(error, here, msg, hook, retaddr)		\
+	do {								\
+		item_p item;						\
+		if ((item = ng_package_msg(msg))			\
+		&& (ng_address_hook((here), (item),			\
+					(hook), (retaddr)) == 0)) {	\
+			SAVE_LINE(item);				\
+			(error) = ng_snd_item((item), 0);		\
+		} else {						\
+			(error) = EINVAL;				\
+		}							\
+		(msg) = NULL;						\
+	} while (0)
+
+#define NG_SEND_MSG_PATH(error, here, msg, path, retaddr)		\
+	do {								\
+		item_p item;						\
+		if ((item = ng_package_msg(msg))			\
+		&& (ng_address_path((here), (item),			\
+					(path), (retaddr)) == 0)) {	\
+			SAVE_LINE(item);				\
+			(error) = ng_snd_item((item), 0);		\
+		} else {						\
+			(error) = EINVAL;				\
+		}							\
+		(msg) = NULL;						\
+	} while (0)
+
+#define NG_SEND_MSG_ID(error, here, msg, ID, retaddr)			\
+	do {								\
+		item_p item;						\
+		if ((item = ng_package_msg(msg))			\
+		&& (ng_address_ID((here), (item),			\
+					(ID), (retaddr)) == 0)) {	\
+			SAVE_LINE(item);				\
+			(error) = ng_snd_item((item), 0);		\
+		} else {						\
+			(error) = EINVAL;				\
+		}							\
+		(msg) = NULL;						\
+	} while (0)
+
+#define NG_QUEUE_MSG(error, here, msg, path, retaddr)			\
+	do {								\
+		item_p item;						\
+		if ((item = ng_package_msg(msg))			\
+		&& (ng_address_path((here), (item),			\
+					(path), (retaddr)) == 0)) {	\
+			SAVE_LINE(item);				\
+			(error) = ng_snd_item((item), 0);		\
+		} else {						\
+			(error) = EINVAL;				\
+		}							\
+		(msg) = NULL;						\
+	} while (0)
+
+/*
+ * Redirect the message to the next hop using the given hook.
+ * ng_retarget_msg() frees the item if there is an error
+ * and returns an error code.  It returns 0 on success.
+ */
+#define NG_FWD_MSG_HOOK(error, here, item, hook, retaddr)		\
+	do {								\
+		if ((ng_address_hook((here), (item),			\
+					(hook), (retaddr))) == 0) {	\
+			SAVE_LINE(item);				\
+			(error) = ng_snd_item((item), 0);		\
+		} else {						\
+			(error) = EINVAL;				\
+		}							\
+		(item) = NULL;						\
+	} while (0)
+
+/*
+ * Send a queue item back to it's originator with a response message.
+ * Assume original message was removed and freed separatly.
+ */
+#define NG_RESPOND_MSG(error, here, item, resp)				\
+	do {								\
+		if (resp) {						\
+			ng_ID_t dest = NGI_RETADDR(item);		\
+			NGI_RETADDR(item) = NULL;			\
+			NGI_MSG(item) = resp;				\
+			if ((ng_address_ID((here), (item),		\
+					dest, NULL )) == 0) {		\
+				SAVE_LINE(item);			\
+				(error) = ng_snd_item((item), 1);	\
+			} else {					\
+				(error) = EINVAL;			\
+			}						\
+		} else {						\
+			NG_FREE_ITEM(item);				\
+		}							\
+		(item) = NULL;						\
+	} while (0)
+
 
 /*
  * Use the NETGRAPH_INIT() macro to link a node type into the
@@ -284,39 +594,35 @@ MODULE_DEPEND(ng_##typename, netgraph, 1, 1, 1)
 	NETGRAPH_INIT_ORDERED(tn, tp, SI_SUB_PSEUDO, SI_ORDER_ANY)
 
 /* Special malloc() type for netgraph structs and ctrl messages */
+/* Only these two types should be visible to nodes */ 
 MALLOC_DECLARE(M_NETGRAPH);
+MALLOC_DECLARE(M_NETGRAPH_MSG);
+MALLOC_DECLARE(M_NETGRAPH_META);
+
+
 
-int	ng_bypass(hook_p hook1, hook_p hook2);
-void	ng_cutlinks(node_p node);
-int	ng_con_nodes(node_p node,
-	     const char *name, node_p node2, const char *name2);
+/* Methods that should go away (or become private)*/
+/* Methods that should exist */
+int	ng_address_ID(node_p here, item_p item, ng_ID_t ID, ng_ID_t retaddr);
+int	ng_address_hook(node_p here, item_p item, hook_p hook, ng_ID_t retaddr);
+int	ng_address_path(node_p here, item_p item,
+	char *address, ng_ID_t retaddr);
 meta_p	ng_copy_meta(meta_p meta);
-void	ng_destroy_hook(hook_p hook);
 hook_p	ng_findhook(node_p node, const char *name);
-node_p	ng_findname(node_p node, const char *name);
-struct	ng_type *ng_findtype(const char *type);
-int	ng_make_node(const char *type, node_p *nodepp);
+void	ng_free_item(item_p item);
 int	ng_make_node_common(struct ng_type *typep, node_p *nodep);
-int	ng_mkpeer(node_p node, const char *name, const char *name2, char *type);
 int	ng_mod_event(module_t mod, int what, void *arg);
 int	ng_name_node(node_p node, const char *name);
 int	ng_newtype(struct ng_type *tp);
 ng_ID_t ng_node2ID(node_p node);
-int	ng_path2node(node_p here, const char *path,
-		node_p *dest, hook_p *lasthook);
-int	ng_path_parse(char *addr, char **node, char **path, char **hook);
-int	ng_queue_data(hook_p hook, struct mbuf *m, meta_p meta);
-int	ng_queue_msg(node_p here, struct ng_mesg *msg, const char *address,
-		hook_p hook, char *retaddr);
-void	ng_release_node(node_p node);
-void	ng_rmnode(node_p node);
-int	ng_send_data(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp);
-int	ng_send_msg(node_p here, struct ng_mesg *msg, const char *address,
-		hook_p hook, char *retaddr, struct ng_mesg **resp);
+item_p	ng_package_data(struct mbuf *m, meta_p meta);
+item_p	ng_package_msg(struct ng_mesg *msg);
+item_p	ng_package_msg_self(node_p here, hook_p hook, struct ng_mesg *msg);
+void	ng_replace_retaddr(node_p here, item_p item, ng_ID_t retaddr);
+int	ng_rmnode_self(node_p here);
+int	ng_snd_item(item_p item, int queue);
 void	ng_unname(node_p node);
 void	ng_unref(node_p node);
-int	ng_wait_node(node_p node, char *msg);
 
 #endif /* _NETGRAPH_NETGRAPH_H_ */
 
diff --git a/sys/netgraph/ng_UI.c b/sys/netgraph/ng_UI.c
index 0e2ef10..faf4f55 100644
--- a/sys/netgraph/ng_UI.c
+++ b/sys/netgraph/ng_UI.c
@@ -70,7 +70,7 @@ typedef struct ng_UI_private *priv_p;
 /* Netgraph node methods */
 static ng_constructor_t	ng_UI_constructor;
 static ng_rcvmsg_t	ng_UI_rcvmsg;
-static ng_shutdown_t	ng_UI_rmnode;
+static ng_shutdown_t	ng_UI_shutdown;
 static ng_newhook_t	ng_UI_newhook;
 static ng_rcvdata_t	ng_UI_rcvdata;
 static ng_disconnect_t	ng_UI_disconnect;
@@ -82,7 +82,7 @@ static struct ng_type typestruct = {
 	NULL,
 	ng_UI_constructor,
 	ng_UI_rcvmsg,
-	ng_UI_rmnode,
+	ng_UI_shutdown,
 	ng_UI_newhook,
 	NULL,
 	NULL,
@@ -101,24 +101,16 @@ NETGRAPH_INIT(UI, &typestruct);
  */
 
 static int
-ng_UI_constructor(node_p *nodep)
+ng_UI_constructor(node_p nodep)
 {
 	priv_p  priv;
-	int     error;
 
 	/* Allocate private structure */
 	MALLOC(priv, priv_p, sizeof(*priv), M_NETGRAPH, M_NOWAIT | M_ZERO);
-	if (priv == NULL)
+	if (priv == NULL) {
 		return (ENOMEM);
-
-	/* Call generic node constructor */
-	if ((error = ng_make_node_common(&typestruct, nodep))) {
-		FREE(priv, M_NETGRAPH);
-		return (error);
 	}
-	(*nodep)->private = priv;
-
-	/* Done */
+	nodep->private = priv;
 	return (0);
 }
 
@@ -147,26 +139,30 @@ ng_UI_newhook(node_p node, hook_p hook, const char *name)
  * Receive a control message
  */
 static int
-ng_UI_rcvmsg(node_p node, struct ng_mesg *msg,
-	     const char *raddr, struct ng_mesg **rp, hook_p lasthook)
+ng_UI_rcvmsg(node_p node, item_p item, hook_p lasthook)
 {
+	int	error;
 	const priv_p priv = node->private;
+	struct ng_mesg *msg;
 
+	msg = NGI_MSG(item); /* only peeking */
 	if ((msg->header.typecookie == NGM_FLOW_COOKIE) && lasthook)  {
 		if (lasthook == priv->downlink) {
 			if (priv->uplink) {
-				return (ng_send_msg(node, msg, NULL,
-					priv->uplink, raddr, rp));
+				NG_FWD_MSG_HOOK(error, node, item,
+					priv->uplink, 0);
+				return (error);
 			}
 		} else {
 			if (priv->downlink) {
-				return (ng_send_msg(node, msg, NULL,
-					priv->downlink, raddr, rp));
+				NG_FWD_MSG_HOOK(error, node, item, 
+					priv->downlink, 0);
+				return (error);
 			}
 		}
 	}
 		
-	FREE(msg, M_NETGRAPH);
+	NG_FREE_MSG(msg);
 	return (EINVAL);
 }
 
@@ -177,13 +173,14 @@ ng_UI_rcvmsg(node_p node, struct ng_mesg *msg,
  * Receive a data frame
  */
 static int
-ng_UI_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta,  struct ng_mesg **resp)
+ng_UI_rcvdata(hook_p hook, item_p item)
 {
 	const node_p node = hook->node;
 	const priv_p priv = node->private;
+	struct mbuf *m;
 	int error = 0;
 
+	NGI_GET_M(item, m);
 	if (hook == priv->downlink) {
 		u_char *start, *ptr;
 
@@ -197,18 +194,20 @@ ng_UI_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 			ERROUT(0);
 
 		m_adj(m, ptr - start);
-		NG_SEND_DATA(error, priv->uplink, m, meta);	/* m -> NULL */
+		NG_FWD_NEW_DATA(error, item, priv->uplink, m);	/* m -> NULL */
 	} else if (hook == priv->uplink) {
 		M_PREPEND(m, 1, M_DONTWAIT);	/* Prepend IP NLPID */
 		if (!m)
 			ERROUT(ENOBUFS);
 		mtod(m, u_char *)[0] = HDLC_UI;
-		NG_SEND_DATA(error, priv->downlink, m, meta);	/* m -> NULL */
+		NG_FWD_NEW_DATA(error, item, priv->downlink, m);	/* m -> NULL */
 	} else
 		panic(__FUNCTION__);
 
 done:
-	NG_FREE_DATA(m, meta);	/* does nothing if m == NULL */
+	NG_FREE_M(m);	/* does nothing if m == NULL */
+	if (item)
+		NG_FREE_ITEM(item);
 	return (error);
 }
 
@@ -216,15 +215,11 @@ done:
  * Shutdown node
  */
 static int
-ng_UI_rmnode(node_p node)
+ng_UI_shutdown(node_p node)
 {
 	const priv_p priv = node->private;
 
 	/* Take down netgraph node */
-	node->flags |= NG_INVALID;
-	ng_cutlinks(node);
-	ng_unname(node);
-	bzero(priv, sizeof(*priv));
 	FREE(priv, M_NETGRAPH);
 	node->private = NULL;
 	ng_unref(node);
@@ -239,14 +234,20 @@ ng_UI_disconnect(hook_p hook)
 {
 	const priv_p priv = hook->node->private;
 
-	if (hook->node->numhooks == 0)
-		ng_rmnode(hook->node);
-	else if (hook == priv->downlink)
+	if (hook == priv->downlink)
 		priv->downlink = NULL;
 	else if (hook == priv->uplink)
 		priv->uplink = NULL;
 	else
 		panic(__FUNCTION__);
+	/*
+	 * If we are not already shutting down,
+	 * and we have no more hooks, then DO shut down.
+	 */
+	if ((hook->node->numhooks == 0)
+	&& ((hook->node->flags & NG_INVALID) == 0)) {
+			ng_rmnode_self(hook->node);
+	}
 	return (0);
 }
 
diff --git a/sys/netgraph/ng_async.c b/sys/netgraph/ng_async.c
index cbc51bb..8f9e118 100644
--- a/sys/netgraph/ng_async.c
+++ b/sys/netgraph/ng_async.c
@@ -94,8 +94,8 @@ static ng_newhook_t		nga_newhook;
 static ng_disconnect_t		nga_disconnect;
 
 /* Helper stuff */
-static int	nga_rcv_sync(const sc_p sc, struct mbuf *m, meta_p meta);
-static int	nga_rcv_async(const sc_p sc, struct mbuf *m, meta_p meta);
+static int	nga_rcv_sync(const sc_p sc, item_p item);
+static int	nga_rcv_async(const sc_p sc, item_p item);
 
 /* Parse type for struct ng_async_cfg */
 static const struct ng_parse_struct_info
@@ -174,13 +174,10 @@ static const u_int16_t fcstab[];
  * Initialize a new node
  */
 static int
-nga_constructor(node_p *nodep)
+nga_constructor(node_p node)
 {
 	sc_p sc;
-	int error;
 
-	if ((error = ng_make_node_common(&typestruct, nodep)))
-		return (error);
 	MALLOC(sc, sc_p, sizeof(*sc), M_NETGRAPH, M_NOWAIT | M_ZERO);
 	if (sc == NULL)
 		return (ENOMEM);
@@ -200,8 +197,8 @@ fail:
 		FREE(sc, M_NETGRAPH);
 		return (ENOMEM);
 	}
-	(*nodep)->private = sc;
-	sc->node = *nodep;
+	node->private = sc;
+	sc->node = node;
 	return (0);
 }
 
@@ -214,13 +211,30 @@ nga_newhook(node_p node, hook_p hook, const char *name)
 	const sc_p sc = node->private;
 	hook_p *hookp;
 
-	if (!strcmp(name, NG_ASYNC_HOOK_ASYNC))
+	if (!strcmp(name, NG_ASYNC_HOOK_ASYNC)) {
+		/*
+		 * We use a static buffer here so only one packet
+		 * at a time can be allowed to travel in this direction.
+		 * Force Writer semantics.
+		 */
+		hook->flags |= HK_FORCE_WRITER;
+		hookp = &sc->async;
+	} else if (!strcmp(name, NG_ASYNC_HOOK_SYNC)) {
+		/*
+		 * We use a static state here so only one packet
+		 * at a time can be allowed to travel in this direction.
+		 * Force Writer semantics.
+		 * Since we set this for both directions
+		 * we might as well set it for the whole node
+		 * bit I haven;t done that (yet).
+		 */
+		hook->flags |= HK_FORCE_WRITER;
 		hookp = &sc->async;
-	else if (!strcmp(name, NG_ASYNC_HOOK_SYNC))
 		hookp = &sc->sync;
-	else
+	} else {
 		return (EINVAL);
-	if (*hookp)
+	}
+	if (*hookp) /* actually can't happen I think [JRE] */
 		return (EISCONN);
 	*hookp = hook;
 	return (0);
@@ -230,15 +244,17 @@ nga_newhook(node_p node, hook_p hook, const char *name)
  * Receive incoming data
  */
 static int
-nga_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
+nga_rcvdata(hook_p hook, item_p item)
 {
 	const sc_p sc = hook->node->private;
+#ifdef	ITEM_DEBUG
+	meta_p meta = NGI_META(item);
+#endif
 
 	if (hook == sc->sync)
-		return (nga_rcv_sync(sc, m, meta));
+		return (nga_rcv_sync(sc, item));
 	if (hook == sc->async)
-		return (nga_rcv_async(sc, m, meta));
+		return (nga_rcv_async(sc, item));
 	panic(__FUNCTION__);
 }
 
@@ -246,13 +262,15 @@ nga_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
  * Receive incoming control message
  */
 static int
-nga_rcvmsg(node_p node, struct ng_mesg *msg,
-	const char *rtn, struct ng_mesg **rptr, hook_p lasthook)
+nga_rcvmsg(node_p node, item_p item, hook_p lasthook)
 {
 	const sc_p sc = (sc_p) node->private;
 	struct ng_mesg *resp = NULL;
 	int error = 0;
+	struct ng_mesg *msg;
+	
 
+	NGI_GET_MSG(item, msg);
 	switch (msg->header.typecookie) {
 	case NGM_ASYNC_COOKIE:
 		switch (msg->header.cmd) {
@@ -317,13 +335,9 @@ nga_rcvmsg(node_p node, struct ng_mesg *msg,
 	default:
 		ERROUT(EINVAL);
 	}
-	if (rptr)
-		*rptr = resp;
-	else if (resp)
-		FREE(resp, M_NETGRAPH);
-
 done:
-	FREE(msg, M_NETGRAPH);
+	NG_RESPOND_MSG(error, node, item, resp);
+	NG_FREE_MSG(msg);
 	return (error);
 }
 
@@ -335,8 +349,6 @@ nga_shutdown(node_p node)
 {
 	const sc_p sc = node->private;
 
-	ng_cutlinks(node);
-	ng_unname(node);
 	FREE(sc->abuf, M_NETGRAPH);
 	FREE(sc->sbuf, M_NETGRAPH);
 	bzero(sc, sizeof(*sc));
@@ -366,8 +378,9 @@ nga_disconnect(hook_p hook)
 	*hookp = NULL;
 	bzero(&sc->stats, sizeof(sc->stats));
 	sc->lasttime = 0;
-	if (hook->node->numhooks == 0)
-		ng_rmnode(hook->node);
+	if ((hook->node->numhooks == 0)
+	&& ((hook->node->flags & NG_INVALID) == 0))
+		ng_rmnode_self(hook->node);
 	return (0);
 }
 
@@ -395,21 +408,26 @@ nga_async_add(const sc_p sc, u_int16_t *fcs, u_int32_t accm, int *len, u_char x)
  * Receive incoming synchronous data.
  */
 static int
-nga_rcv_sync(const sc_p sc, struct mbuf *m, meta_p meta)
+nga_rcv_sync(const sc_p sc, item_p item)
 {
-	struct ifnet *const rcvif = m->m_pkthdr.rcvif;
+	struct ifnet *rcvif;
 	int alen, error = 0;
 	struct timeval time;
 	u_int16_t fcs, fcs0;
 	u_int32_t accm;
+	struct mbuf *m;
+
 
 #define ADD_BYTE(x)	nga_async_add(sc, &fcs, accm, &alen, (x))
 
 	/* Check for bypass mode */
 	if (!sc->cfg.enabled) {
-		NG_SEND_DATA(error, sc->async, m, meta);
+		NG_FWD_DATA(error, item, sc->async );
 		return (error);
 	}
+	NGI_GET_M(item, m);
+
+	rcvif = m->m_pkthdr.rcvif;
 
 	/* Get ACCM; special case LCP frames, which use full ACCM */
 	accm = sc->cfg.accm;
@@ -430,7 +448,8 @@ nga_rcv_sync(const sc_p sc, struct mbuf *m, meta_p meta)
 	/* Check for overflow */
 	if (m->m_pkthdr.len > sc->cfg.smru) {
 		sc->stats.syncOverflows++;
-		NG_FREE_DATA(m, meta);
+		NG_FREE_M(m);
+		NG_FREE_ITEM(item);
 		return (EMSGSIZE);
 	}
 
@@ -470,10 +489,11 @@ nga_rcv_sync(const sc_p sc, struct mbuf *m, meta_p meta)
 
 	/* Put frame in an mbuf and ship it off */
 	if (!(m = m_devget(sc->abuf, alen, 0, rcvif, NULL))) {
-		NG_FREE_META(meta);
+		NG_FREE_ITEM(item);
 		error = ENOBUFS;
-	} else
-		NG_SEND_DATA(error, sc->async, m, meta);
+	} else {
+		NG_FWD_NEW_DATA(error, item, sc->async, m);
+	}
 	return (error);
 }
 
@@ -483,16 +503,18 @@ nga_rcv_sync(const sc_p sc, struct mbuf *m, meta_p meta)
  *     that are in our ACCM. Not sure if this is good or not.
  */
 static int
-nga_rcv_async(const sc_p sc, struct mbuf * m, meta_p meta)
+nga_rcv_async(const sc_p sc, item_p item)
 {
-	struct ifnet *const rcvif = m->m_pkthdr.rcvif;
+	struct ifnet *rcvif;
 	int error;
+	struct mbuf *m;
 
 	if (!sc->cfg.enabled) {
-		NG_SEND_DATA(error, sc->sync, m, meta);
+		NG_FWD_DATA(error, item,  sc->sync);
 		return (error);
 	}
-	NG_FREE_META(meta);
+	NGI_GET_M(item, m);
+	rcvif = m->m_pkthdr.rcvif;
 	while (m) {
 		struct mbuf *n;
 
@@ -531,8 +553,15 @@ nga_rcv_async(const sc_p sc, struct mbuf * m, meta_p meta)
 
 				/* OK, ship it out */
 				if ((n = m_devget(sc->sbuf + skip,
-					   sc->slen - skip, 0, rcvif, NULL)))
-					NG_SEND_DATA(error, sc->sync, n, meta);
+					   sc->slen - skip, 0, rcvif, NULL))) {
+					if (item) { /* sets NULL -> item */
+						NG_FWD_NEW_DATA(error, item,
+							sc->sync, n);
+					} else {
+						NG_SEND_DATA_ONLY(error,
+							sc->sync ,n);
+					}
+				}
 				sc->stats.asyncFrames++;
 reset:
 				sc->amode = MODE_NORMAL;
@@ -569,6 +598,8 @@ reset:
 		MFREE(m, n);
 		m = n;
 	}
+	if (item)
+		NG_FREE_ITEM(item);
 	return (0);
 }
 
diff --git a/sys/netgraph/ng_base.c b/sys/netgraph/ng_base.c
index d9b26f4..0cbd63d 100644
--- a/sys/netgraph/ng_base.c
+++ b/sys/netgraph/ng_base.c
@@ -1,10 +1,9 @@
-
 /*
  * ng_base.c
  *
  * Copyright (c) 1996-1999 Whistle Communications, Inc.
  * All rights reserved.
- * 
+ *
  * Subject to the following obligations and disclaimer of warranty, use and
  * redistribution of this software, in source or object code forms, with or
  * without modifications are expressly permitted by Whistle Communications;
@@ -15,7 +14,7 @@
  *    Communications, Inc. trademarks, including the mark "WHISTLE
  *    COMMUNICATIONS" on advertising, endorsements, or otherwise except as
  *    such appears in the above copyright notice or in the software.
- * 
+ *
  * THIS SOFTWARE IS BEING PROVIDED BY WHISTLE COMMUNICATIONS "AS IS", AND
  * TO THE MAXIMUM EXTENT PERMITTED BY LAW, WHISTLE COMMUNICATIONS MAKES NO
  * REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED, REGARDING THIS SOFTWARE,
@@ -51,6 +50,7 @@
 #include <sys/kernel.h>
 #include <sys/malloc.h>
 #include <sys/syslog.h>
+#include <sys/sysctl.h>
 #include <sys/linker.h>
 #include <sys/queue.h>
 #include <sys/mbuf.h>
@@ -66,32 +66,73 @@
 MODULE_VERSION(netgraph, 1);
 
 /* List of all nodes */
-static LIST_HEAD(, ng_node) nodelist;
+static LIST_HEAD(, ng_node) ng_nodelist;
+static struct mtx	ng_nodelist_mtx;
+
+/* NETISR queue */
+/* List nodes with unallocated work */
+static TAILQ_HEAD(, ng_node) ng_worklist = TAILQ_HEAD_INITIALIZER(ng_worklist);
+static struct mtx	ng_worklist_mtx;
 
 /* List of installed types */
-static LIST_HEAD(, ng_type) typelist;
+static LIST_HEAD(, ng_type) ng_typelist;
+static struct mtx	ng_typelist_mtx;
 
-/* Hash releted definitions */
-#define ID_HASH_SIZE 32 /* most systems wont need even this many */
-static LIST_HEAD(, ng_node) ID_hash[ID_HASH_SIZE];
+/* Hash related definitions */
 /* Don't nead to initialise them because it's a LIST */
+#define ID_HASH_SIZE 32 /* most systems wont need even this many */
+static LIST_HEAD(, ng_node) ng_ID_hash[ID_HASH_SIZE];
+static struct mtx	ng_idhash_mtx;
+
+/* Mutex that protects the free queue item list */
+static volatile item_p		ngqfree;	/* free ones */
+static struct mtx	ngq_mtx;
 
 /* Internal functions */
 static int	ng_add_hook(node_p node, const char *name, hook_p * hookp);
 static int	ng_connect(hook_p hook1, hook_p hook2);
 static void	ng_disconnect_hook(hook_p hook);
-static int	ng_generic_msg(node_p here, struct ng_mesg *msg,
-			const char *retaddr, struct ng_mesg ** resp,
-			hook_p hook);
+static int	ng_generic_msg(node_p here, item_p item, hook_p lasthook);
 static ng_ID_t	ng_decodeidname(const char *name);
 static int	ngb_mod_event(module_t mod, int event, void *data);
-static int	ng_send_data_dont_queue(hook_p hook, struct mbuf *m,
-		meta_p meta, struct mbuf **ret_m, meta_p *ret_meta,
-		struct ng_mesg **resp);
+static void	ng_worklist_remove(node_p node);
 static void	ngintr(void);
+static int	ng_apply_item(node_p node, item_p item);
+static void	ng_flush_input_queue(struct ng_queue * ngq);
+static void	ng_setisr(node_p node);
+static node_p	ng_ID2noderef(ng_ID_t ID);
+
+/* imported */
+int	ng_bypass(hook_p hook1, hook_p hook2);
+void	ng_cutlinks(node_p node);
+int	ng_con_nodes(node_p node, const char *name, node_p node2,
+	const char *name2);
+void	ng_destroy_hook(hook_p hook);
+node_p	ng_name2noderef(node_p node, const char *name);
+int	ng_path2noderef(node_p here, const char *path,
+	node_p *dest, hook_p *lasthook);
+struct	ng_type *ng_findtype(const char *type);
+int	ng_make_node(const char *type, node_p *nodepp);
+int	ng_mkpeer(node_p node, const char *name, const char *name2, char *type);
+int	ng_path_parse(char *addr, char **node, char **path, char **hook);
+void	ng_rmnode(node_p node);
+
 
 /* Our own netgraph malloc type */
 MALLOC_DEFINE(M_NETGRAPH, "netgraph", "netgraph structures and ctrl messages");
+MALLOC_DEFINE(M_NETGRAPH_HOOK, "netgraph_hook", "netgraph hook structures");
+MALLOC_DEFINE(M_NETGRAPH_NODE, "netgraph_node", "netgraph node structures");
+MALLOC_DEFINE(M_NETGRAPH_ITEM, "netgraph_item", "netgraph item structures");
+MALLOC_DEFINE(M_NETGRAPH_META, "netgraph_meta", "netgraph name storage");
+MALLOC_DEFINE(M_NETGRAPH_MSG, "netgraph_msg", "netgraph name storage");
+
+/* Should not be visible outside this file */
+#define NG_FREE_HOOK(hook) do { FREE((hook), M_NETGRAPH_HOOK); } while (0)
+#define NG_FREE_NODE(node) do { FREE((node), M_NETGRAPH_NODE); } while (0)
+#define NG_FREE_NAME(name) do { FREE((name), M_NETGRAPH_NAME); } while (0)
+/* Warning: Generally use NG_FREE_ITEM() instead */
+#define NG_FREE_ITEM_REAL(item) do { FREE((item), M_NETGRAPH_ITEM); } while (0)
+
 
 /* Set this to Debugger("X") to catch all errors as they occur */
 #ifndef TRAP_ERROR
@@ -310,6 +351,7 @@ int
 ng_make_node(const char *typename, node_p *nodepp)
 {
 	struct ng_type *type;
+	int	error;
 
 	/* Check that the type makes sense */
 	if (typename == NULL) {
@@ -335,15 +377,32 @@ ng_make_node(const char *typename, node_p *nodepp)
 			return (ENXIO);
 	}
 
-	/* Call the constructor */
-	if (type->constructor != NULL)
-		return ((*type->constructor)(nodepp));
-	else
-		return (ng_make_node_common(type, nodepp));
+	/*
+	 * If we have a constructor, then make the node and
+	 * call the constructor to do type specific initialisation.
+	 */
+	if (type->constructor != NULL) {
+		if ((error = ng_make_node_common(type, nodepp)) == 0) {
+			if ((error = ((*type->constructor)(*nodepp)) != 0)) {
+				ng_unref(*nodepp);
+			}
+		}
+	} else {
+		/*
+		 * Node has no constructor. We cannot ask for one
+		 * to be made. It must be brought into existance by
+		 * some external agency. The external acency should
+		 * call ng_make_node_common() directly to get the
+		 * netgraph part initialised.
+		 */
+		error = EINVAL;
+	}
+	return (error);
 }
 
 /*
- * Generic node creation. Called by node constructors.
+ * Generic node creation. Called by node initialisation for externally
+ * instantiated nodes (e.g. hardware, sockets, etc ).
  * The returned node has a reference count of 1.
  */
 int
@@ -358,7 +417,7 @@ ng_make_node_common(struct ng_type *type, node_p *nodepp)
 	}
 
 	/* Make a node and try attach it to the type */
-	MALLOC(node, node_p, sizeof(*node), M_NETGRAPH, M_NOWAIT | M_ZERO);
+	MALLOC(node, node_p, sizeof(*node), M_NETGRAPH_NODE, M_NOWAIT | M_ZERO);
 	if (node == NULL) {
 		TRAP_ERROR;
 		return (ENOMEM);
@@ -367,15 +426,36 @@ ng_make_node_common(struct ng_type *type, node_p *nodepp)
 	node->refs++;				/* note reference */
 	type->refs++;
 
-	/* Link us into the node linked list */
-	LIST_INSERT_HEAD(&nodelist, node, nodes);
+	mtx_init(&node->input_queue.q_mtx, "netgraph node mutex", 0);
+	node->input_queue.queue = NULL;
+	node->input_queue.last = &node->input_queue.queue;
+	node->input_queue.q_flags = 0;
+	node->input_queue.q_node = node;
 
 	/* Initialize hook list for new node */
 	LIST_INIT(&node->hooks);
 
+	/* Link us into the node linked list */
+	mtx_enter(&ng_nodelist_mtx, MTX_DEF);
+	LIST_INSERT_HEAD(&ng_nodelist, node, nodes);
+	mtx_exit(&ng_nodelist_mtx, MTX_DEF);
+
+
 	/* get an ID and put us in the hash chain */
-	node->ID = nextID++; /* 137 per second for 1 year before wrap */
-	LIST_INSERT_HEAD(&ID_hash[node->ID % ID_HASH_SIZE], node, idnodes);
+	mtx_enter(&ng_idhash_mtx, MTX_DEF);
+	do { /* wrap protection, even if silly */
+		node_p node2 = NULL;
+		node->ID = nextID++; /* 137 per second for 1 year before wrap */
+		if ((node->ID == 0) || (node2 = ng_ID2noderef(node->ID))) {
+			if (node2) {
+				ng_unref(node2);
+				node2 = NULL;
+			}
+			continue;	/* try again */
+		}
+	} while (0);
+	LIST_INSERT_HEAD(&ng_ID_hash[node->ID % ID_HASH_SIZE], node, idnodes);
+	mtx_exit(&ng_idhash_mtx, MTX_DEF);
 
 	/* Done */
 	*nodepp = node;
@@ -387,29 +467,43 @@ ng_make_node_common(struct ng_type *type, node_p *nodepp)
  * it's shutdown method, or do the default shutdown if there is
  * no type-specific method.
  *
- * Persistent nodes must have a type-specific method which
- * resets the NG_INVALID flag.
+ * We can only be called form a shutdown message, so we know we have
+ * a writer lock, and therefore exclusive access.
+ *
+ * Persistent node types must have a type-specific method which
+ * Allocates a new node. This one is irretrievably going away.
  */
 void
 ng_rmnode(node_p node)
 {
 	/* Check if it's already shutting down */
-	if ((node->flags & NG_INVALID) != 0)
+	if ((node->flags & NG_CLOSING) != 0)
 		return;
 
 	/* Add an extra reference so it doesn't go away during this */
 	node->refs++;
 
 	/* Mark it invalid so any newcomers know not to try use it */
-	node->flags |= NG_INVALID;
+	node->flags |= NG_INVALID|NG_CLOSING;
+
+	ng_unname(node);
+	ng_cutlinks(node);
+	/*
+	 * Drain the input queue forceably.
+	 */
+	ng_flush_input_queue(&node->input_queue);
+
+	/*
+	 * Take us off the work queue if we are there.
+	 */
+	ng_worklist_remove(node);
+
 
 	/* Ask the type if it has anything to do in this case */
-	if (node->type && node->type->shutdown)
+	if (node->type && node->type->shutdown) {
 		(*node->type->shutdown)(node);
-	else {				/* do the default thing */
-		ng_unname(node);
-		ng_cutlinks(node);
-		ng_unref(node);
+	} else {				/* do the default thing */
+		ng_unref(node); /* XXX hmmmmm check this */
 	}
 
 	/* Remove extra reference, possibly the last */
@@ -427,9 +521,12 @@ ng_cutlinks(node_p node)
 	/* Make sure that this is set to stop infinite loops */
 	node->flags |= NG_INVALID;
 
-	/* If we have sleepers, wake them up; they'll see NG_INVALID */
-	if (node->sleepers)
-		wakeup(node);
+	/*
+	 * Drain the input queue forceably.
+	 * We also do this in ng_rmnode
+	 * to make sure we get all code paths.
+	 */
+	ng_flush_input_queue(&node->input_queue);
 
 	/* Notify all remaining connected nodes to disconnect */
 	while ((hook = LIST_FIRST(&node->hooks)) != NULL)
@@ -445,83 +542,45 @@ ng_unref(node_p node)
 	int	s;
 
 	s = splhigh();
+/* XXX not atomic.. fix */
 	if (--node->refs <= 0) {
-		node->type->refs--;
+
+		mtx_enter(&ng_nodelist_mtx, MTX_DEF);
+		node->type->refs--; /* XXX maybe should get types lock? */
 		LIST_REMOVE(node, nodes);
-		LIST_REMOVE(node, idnodes);
-		FREE(node, M_NETGRAPH);
-	}
-	splx(s);
-}
+		mtx_exit(&ng_nodelist_mtx, MTX_DEF);
 
-/*
- * Wait for a node to come ready. Returns a node with a reference count;
- * don't forget to drop it when we are done with it using ng_release_node().
- */
-int
-ng_wait_node(node_p node, char *msg)
-{
-	int s, error = 0;
+		mtx_enter(&ng_idhash_mtx, MTX_DEF);
+		LIST_REMOVE(node, idnodes);
+		mtx_exit(&ng_idhash_mtx, MTX_DEF);
 
-	if (msg == NULL)
-		msg = "netgraph";
-	s = splnet();
-	node->sleepers++;
-	node->refs++;		/* the sleeping process counts as a reference */
-	while ((node->flags & (NG_BUSY | NG_INVALID)) == NG_BUSY)
-		error = tsleep(node, (PZERO + 1) | PCATCH, msg, 0);
-	node->sleepers--;
-	if (node->flags & NG_INVALID) {
-		TRAP_ERROR;
-		error = ENXIO;
-	} else {
-		KASSERT(node->refs > 1,
-		    ("%s: refs=%d", __FUNCTION__, node->refs));
-		node->flags |= NG_BUSY;
+		NG_FREE_NODE(node);
 	}
 	splx(s);
-
-	/* Release the reference we had on it */
-	if (error != 0)
-		ng_unref(node);
-	return error;
-}
-
-/*
- * Release a node acquired via ng_wait_node()
- */
-void
-ng_release_node(node_p node)
-{
-	/* Declare that we don't want it */
-	node->flags &= ~NG_BUSY;
-
-	/* If we have sleepers, then wake them up */
-	if (node->sleepers)
-		wakeup(node);
-
-	/* We also have a reference.. drop it too */
-	ng_unref(node);
 }
 
 /************************************************************************
 			Node ID handling
 ************************************************************************/
 static node_p
-ng_ID2node(ng_ID_t ID)
+ng_ID2noderef(ng_ID_t ID)
 {
 	node_p np;
-	LIST_FOREACH(np, &ID_hash[ID % ID_HASH_SIZE], idnodes) {
+	mtx_enter(&ng_idhash_mtx, MTX_DEF);
+	LIST_FOREACH(np, &ng_ID_hash[ID % ID_HASH_SIZE], idnodes) {
 		if (np->ID == ID)
 			break;
 	}
+	if(np)
+		np->refs++;
+	mtx_exit(&ng_idhash_mtx, MTX_DEF);
 	return(np);
 }
 
 ng_ID_t
 ng_node2ID(node_p node)
 {
-	return (node->ID);
+	return (node?node->ID:0);
 }
 
 /************************************************************************
@@ -535,6 +594,7 @@ int
 ng_name_node(node_p node, const char *name)
 {
 	int i;
+	node_p node2;
 
 	/* Check the name is valid */
 	for (i = 0; i < NG_NODELEN + 1; i++) {
@@ -550,28 +610,16 @@ ng_name_node(node_p node, const char *name)
 		return (EINVAL);
 	}
 
-	/* Check the node isn't already named */
-	if (node->name != NULL) {
-		TRAP_ERROR;
-		return (EISCONN);
-	}
-
 	/* Check the name isn't already being used */
-	if (ng_findname(node, name) != NULL) {
+	if ((node2 = ng_name2noderef(node, name)) != NULL) {
+		ng_unref(node2);
 		TRAP_ERROR;
 		return (EADDRINUSE);
 	}
 
-	/* Allocate space and copy it */
-	MALLOC(node->name, char *, strlen(name) + 1, M_NETGRAPH, M_NOWAIT);
-	if (node->name == NULL) {
-		TRAP_ERROR;
-		return (ENOMEM);
-	}
+	/* copy it */
 	strcpy(node->name, name);
 
-	/* The name counts as a reference */
-	node->refs++;
 	return (0);
 }
 
@@ -581,27 +629,35 @@ ng_name_node(node_p node, const char *name)
  * with ID (ie, at address) xxx".
  *
  * Returns the node if found, else NULL.
+ * Eventually should add something faster than a sequential search.
+ * Note it holds a reference on the node so you an be sure it's still there.
  */
 node_p
-ng_findname(node_p this, const char *name)
+ng_name2noderef(node_p here, const char *name)
 {
 	node_p node;
 	ng_ID_t temp;
 
 	/* "." means "this node" */
-	if (strcmp(name, ".") == 0)
-		return(this);
+	if (strcmp(name, ".") == 0) {
+		here->refs++;
+		return(here);
+	}
 
 	/* Check for name-by-ID */
 	if ((temp = ng_decodeidname(name)) != 0) {
-		return (ng_ID2node(temp));
+		return (ng_ID2noderef(temp));
 	}
 
 	/* Find node by name */
-	LIST_FOREACH(node, &nodelist, nodes) {
-		if (node->name != NULL && strcmp(node->name, name) == 0)
+	mtx_enter(&ng_nodelist_mtx, MTX_DEF);
+	LIST_FOREACH(node, &ng_nodelist, nodes) {
+		if (node->name[0] != '\0' && strcmp(node->name, name) == 0)
 			break;
 	}
+	if (node)
+		node->refs++;
+	mtx_exit(&ng_nodelist_mtx, MTX_DEF);
 	return (node);
 }
 
@@ -635,19 +691,13 @@ ng_decodeidname(const char *name)
 void
 ng_unname(node_p node)
 {
-	if (node->name) {
-		FREE(node->name, M_NETGRAPH);
-		node->name = NULL;
-		ng_unref(node);
-	}
+	bzero(node->name, NG_NODELEN);
 }
 
 /************************************************************************
 			Hook routines
-
  Names are not optional. Hooks are always connected, except for a
  brief moment within these routines.
-
 ************************************************************************/
 
 /*
@@ -659,8 +709,14 @@ ng_unref_hook(hook_p hook)
 	int	s;
 
 	s = splhigh();
-	if (--hook->refs == 0)
-		FREE(hook, M_NETGRAPH);
+/* XXX not atomic.. fix */
+	if (--hook->refs == 0) {
+		if (hook->node) {
+			ng_unref(hook->node);
+			hook->node = NULL;
+		}
+		NG_FREE_HOOK(hook);
+	}
 	splx(s);
 }
 
@@ -684,7 +740,7 @@ ng_add_hook(node_p node, const char *name, hook_p *hookp)
 	}
 
 	/* Allocate the hook and link it up */
-	MALLOC(hook, hook_p, sizeof(*hook), M_NETGRAPH, M_NOWAIT | M_ZERO);
+	MALLOC(hook, hook_p, sizeof(*hook), M_NETGRAPH_HOOK, M_NOWAIT | M_ZERO);
 	if (hook == NULL) {
 		TRAP_ERROR;
 		return (ENOMEM);
@@ -696,28 +752,20 @@ ng_add_hook(node_p node, const char *name, hook_p *hookp)
 
 	/* Check if the node type code has something to say about it */
 	if (node->type->newhook != NULL)
-		if ((error = (*node->type->newhook)(node, hook, name)) != 0)
-			goto fail;
+		if ((error = (*node->type->newhook)(node, hook, name)) != 0) {
+			ng_unref_hook(hook);	/* this frees the hook */
+			return (error);
+		}
 
 	/*
 	 * The 'type' agrees so far, so go ahead and link it in.
 	 * We'll ask again later when we actually connect the hooks.
+	 * The reference we have is for this linkage.
 	 */
 	LIST_INSERT_HEAD(&node->hooks, hook, hooks);
 	node->numhooks++;
 
 	/* Set hook name */
-	MALLOC(hook->name, char *, strlen(name) + 1, M_NETGRAPH, M_NOWAIT);
-	if (hook->name == NULL) {
-		error = ENOMEM;
-		LIST_REMOVE(hook, hooks);
-		node->numhooks--;
-fail:
-		hook->node = NULL;
-		ng_unref(node);
-		ng_unref_hook(hook);	/* this frees the hook */
-		return (error);
-	}
 	strcpy(hook->name, name);
 	if (hookp)
 		*hookp = hook;
@@ -767,7 +815,7 @@ ng_findhook(node_p node, const char *name)
 	if (node->type->findhook != NULL)
 		return (*node->type->findhook)(node, name);
 	LIST_FOREACH(hook, &node->hooks, hooks) {
-		if (hook->name != NULL && strcmp(hook->name, name) == 0)
+		if (strcmp(hook->name, name) == 0)
 			return (hook);
 	}
 	return (NULL);
@@ -820,10 +868,6 @@ ng_disconnect_hook(hook_p hook)
 		 */
 		(*node->type->disconnect) (hook);
 	}
-	ng_unref(node);		/* might be the last reference */
-	if (hook->name)
-		FREE(hook->name, M_NETGRAPH);
-	hook->node = NULL;	/* may still be referenced elsewhere */
 	ng_unref_hook(hook);
 }
 
@@ -869,9 +913,12 @@ ng_newtype(struct ng_type *tp)
 		return (EEXIST);
 	}
 
-	/* Link in new type */
-	LIST_INSERT_HEAD(&typelist, tp, types);
 	tp->refs = 0;
+
+	/* Link in new type */
+	mtx_enter(&ng_typelist_mtx, MTX_DEF);
+	LIST_INSERT_HEAD(&ng_typelist, tp, types);
+	mtx_exit(&ng_typelist_mtx, MTX_DEF);
 	return (0);
 }
 
@@ -883,14 +930,15 @@ ng_findtype(const char *typename)
 {
 	struct ng_type *type;
 
-	LIST_FOREACH(type, &typelist, types) {
+	mtx_enter(&ng_typelist_mtx, MTX_DEF);
+	LIST_FOREACH(type, &ng_typelist, types) {
 		if (strcmp(type->name, typename) == 0)
 			break;
 	}
+	mtx_exit(&ng_typelist_mtx, MTX_DEF);
 	return (type);
 }
 
-
 /************************************************************************
 			Composite routines
 ************************************************************************/
@@ -946,8 +994,64 @@ ng_con_nodes(node_p node, const char *name, node_p node2, const char *name2)
 	}
 	return (ng_connect(hook, hook2));
 }
-
+/************************************************************************
+		Utility routines to send self messages
+************************************************************************/
 /*
+ * Static version of shutdown message. we don't want to need resources
+ * to shut down (we may be doing it to release resources because we ran out.
+ */
+static struct	ng_mesg  ng_msg_shutdown = {
+	{NG_VERSION,		/* u_char */
+	0,			/* u_char spare */
+	0,			/* u_int16_t arglen */
+	NGF_STATIC,		/* u_int32_t flags */
+	0,			/* u_int32_t token */
+	NGM_GENERIC_COOKIE,	/* u_int32_t */
+	NGM_SHUTDOWN,		/* u_int32_t */
+	"shutdown"}		/* u_char[16] */
+};
+	
+int
+ng_rmnode_self(node_p here)
+{
+	item_p	item;
+	struct	ng_mesg	*msg;
+
+	/*
+	 * Use the static version to avoid needing
+	 * memory allocation to succeed.
+	 * The message is never written to and always the same.
+	 */
+	msg = &ng_msg_shutdown;
+
+	/*
+	 * Try get a queue item to send it with.
+	 * Hopefully since it has a reserve, we can get one.
+	 * If we can't we are screwed anyhow.
+	 * Increase the chances by flushing our queue first.
+	 * We may free an item, (if we were the hog).
+	 * Work in progress is allowed to complete.
+	 * We also pretty much ensure that we come straight
+	 * back in to do the shutdown. It may be a good idea
+	 * to hold a reference actually to stop it from all
+	 * going up in smoke.
+	 */
+/*	ng_flush_input_queue(&here->input_queue); will mask problem  */
+	item = ng_package_msg_self(here, NULL, msg);
+	if (item == NULL) { /* it would have freed the msg except static */
+		/* try again after flushing our queue */
+		ng_flush_input_queue(&here->input_queue);
+		item = ng_package_msg_self(here, NULL, msg);
+		if (item == NULL) {
+			printf("failed to free node 0x%x\n", ng_node2ID(here));
+			return (ENOMEM);
+		}
+	}
+	return (ng_snd_item(item, 0));
+}
+
+/***********************************************************************
  * Parse and verify a string of the form:  <NODE:><PATH>
  *
  * Such a string can refer to a specific node or a specific hook
@@ -961,8 +1065,7 @@ ng_con_nodes(node_p node, const char *name, node_p node2, const char *name2)
  * the final hook component of <PATH>, if any, otherwise NULL.
  *
  * This returns -1 if the path is malformed. The char ** are optional.
- */
-
+ ***********************************************************************/
 int
 ng_path_parse(char *addr, char **nodep, char **pathp, char **hookp)
 {
@@ -1030,14 +1133,15 @@ ng_path_parse(char *addr, char **nodep, char **pathp, char **hookp)
 
 /*
  * Given a path, which may be absolute or relative, and a starting node,
- * return the destination node. Compute the "return address" if desired.
+ * return the destination node.
  */
 int
-ng_path2node(node_p here, const char *address, node_p *destp, hook_p *lasthook)
+ng_path2noderef(node_p here, const char *address,
+				node_p *destp, hook_p *lasthook)
 {
 	char    fullpath[NG_PATHLEN + 1];
 	char   *nodename, *path, pbuf[2];
-	node_p  node;
+	node_p  node, oldnode;
 	char   *cp;
 	hook_p hook = NULL;
 
@@ -1061,17 +1165,39 @@ ng_path2node(node_p here, const char *address, node_p *destp, hook_p *lasthook)
 		path = pbuf;
 	}
 
-	/* For an absolute address, jump to the starting node */
+	/*
+	 * For an absolute address, jump to the starting node.
+	 * Note that this holds a reference on the node for us.
+	 * Don't forget to drop the reference if we don't need it.
+	 */
 	if (nodename) {
-		node = ng_findname(here, nodename);
+		node = ng_name2noderef(here, nodename);
 		if (node == NULL) {
 			TRAP_ERROR;
 			return (ENOENT);
 		}
-	} else
+	} else {
+		if (here == NULL) {
+			TRAP_ERROR
+			return (EINVAL);
+		}
 		node = here;
+		node->refs++;
+	}
 
-	/* Now follow the sequence of hooks */
+	/*
+	 * Now follow the sequence of hooks 
+	 * XXX
+	 * We actually cannot guarantee that the sequence
+	 * is not being demolished as we crawl along it
+	 * without extra-ordinary locking etc.
+	 * So this is a bit dodgy to say the least.
+	 * We can probably hold up some things by holding
+	 * the nodelist mutex for the time of this
+	 * crawl if we wanted.. At least that way we wouldn't have to
+	 * worry about the nodes dissappearing, but the hooks would still
+	 * be a problem.
+	 */
 	for (cp = path; node != NULL && *cp != '\0'; ) {
 		char *segment;
 
@@ -1097,13 +1223,28 @@ ng_path2node(node_p here, const char *address, node_p *destp, hook_p *lasthook)
 		/* Can't get there from here... */
 		if (hook == NULL
 		    || hook->peer == NULL
-		    || (hook->flags & HK_INVALID) != 0) {
+		    || (hook->flags & HK_INVALID) != 0
+		    || (hook->peer->flags & HK_INVALID) != 0) {
 			TRAP_ERROR;
+			ng_unref(node);
 			return (ENOENT);
 		}
 
-		/* Hop on over to the next node */
-		node = hook->peer->node;
+		/*
+		 * Hop on over to the next node 
+		 * XXX
+		 * Big race conditions here as hooks and nodes go away 
+		 * *** Idea.. store an ng_ID_t in each hook and use that
+		 * instead of the direct hook in this crawl?
+		 */
+		oldnode = node;
+		if ((node = hook->peer->node))
+			node->refs++;	/* XXX RACE */
+		ng_unref(oldnode);	/* XXX another race */
+		if (node->flags & NG_INVALID) {
+			ng_unref(node);	/* XXX more races */
+			node = NULL;
+		}
 	}
 
 	/* If node somehow missing, fail here (probably this is not needed) */
@@ -1115,116 +1256,740 @@ ng_path2node(node_p here, const char *address, node_p *destp, hook_p *lasthook)
 	/* Done */
 	*destp = node;
 	if (lasthook != NULL)
-		*lasthook = hook ? hook->peer : NULL;
+		*lasthook = (hook ? hook->peer : NULL);
 	return (0);
 }
 
+/***************************************************************\
+* Input queue handling.
+* All activities are submitted to the node via the input queue
+* which implements a multiple-reader/single-writer gate.
+* Items which cannot be handled immeditly are queued.
+*
+* read-write queue locking inline functions			*
+\***************************************************************/
+
+static __inline item_p ng_dequeue(struct ng_queue * ngq);
+static __inline item_p ng_acquire_read(struct ng_queue * ngq,
+					item_p  item);
+static __inline item_p ng_acquire_write(struct ng_queue * ngq,
+					item_p  item);
+static __inline void	ng_leave_read(struct ng_queue * ngq);
+static __inline void	ng_leave_write(struct ng_queue * ngq);
+static __inline void	ng_queue_rw(struct ng_queue * ngq,
+					item_p  item, int rw);
+
 /*
- * Call the appropriate message handler for the object.
- * It is up to the message handler to free the message.
- * If it's a generic message, handle it generically, otherwise
- * call the type's message handler (if it exists)
- * XXX (race). Remember that a queued message may reference a node
- * or hook that has just been invalidated. It will exist
- * as the queue code is holding a reference, but..
+ * Definition of the bits fields in the ng_queue flag word.
+ * Defined here rather than in netgraph.h because no-one should fiddle
+ * with them.
+ *
+ * The ordering here is important! don't shuffle these. If you add
+ * READ_PENDING to the word when it has READ_PENDING already set, you
+ * generate a carry into the reader count, this you atomically add a reader,
+ * and remove the pending reader count! Similarly for the pending writer
+ * flag, adding WRITE_PENDING generates a carry and sets the WRITER_ACTIVE
+ * flag, while clearing WRITE_PENDING. When 'SINGLE_THREAD_ONLY' is set, then
+ * it is only permitted to do WRITER operations. Reader operations will
+ * result in errors.
+ * But that "hack" is unnecessary: "cpp" can do the math for us!
  */
+/*-
+ Safety Barrier--------+ (adjustable to suit taste) (not used yet)
+                       |
+                       V
++-------+-------+-------+-------+-------+-------+-------+-------+
+| | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | |
+|A|c|t|i|v|e| |R|e|a|d|e|r| |C|o|u|n|t| | | | | | | | | |R|A|W|S|
+| | | | | | | | | | | | | | | | | | | | | | | | | | | | |P|W|P|T|
++-------+-------+-------+-------+-------+-------+-------+-------+
+\_________________________ ____________________________/ | | | |
+                          V                              | | | |
+                [active reader count]                    | | | |
+                                                         | | | |
+        Read Pending ------------------------------------+ | | |
+                                                           | | |
+        Active Writer -------------------------------------+ | |
+                                                             | |
+        Write Pending ---------------------------------------+ |
+                                                               |
+        Single Threading Only ---------------------------------+
+*/
+#define	SINGLE_THREAD_ONLY 0x00000001	/* if set, even reads single thread */
+#define WRITE_PENDING	0x00000002
+#define	WRITER_ACTIVE	0x00000004
+#define READ_PENDING	0x00000008
+#define	READER_INCREMENT 0x00000010
+#define	READER_MASK	0xfffffff0	/* Not valid if WRITER_ACTIVE is set */
+#define SAFETY_BARRIER	0x00100000	/* 64K items queued should be enough */
+/*
+ * Taking into account the current state of the queue and node, possibly take
+ * the next entry off the queue and return it. Return NULL if there was
+ * nothing we could return, either because there really was nothing there, or
+ * because the node was in a state where it cannot yet process the next item
+ * on the queue.
+ *
+ * This MUST MUST MUST be called with the mutex held.
+ */
+static __inline item_p
+ng_dequeue(struct ng_queue *ngq)
+{
+	item_p item;
+	u_int		add_arg;
+	/*
+	 * If there is a writer, then the answer is "no". Everything else
+	 * stops when there is a WRITER.
+	 */
+	if (ngq->q_flags & WRITER_ACTIVE) {
+		return (NULL);
+	}
+	/* Now take a look at what's on the queue and what's running */
+	if ((ngq->q_flags & ~(READER_MASK | SINGLE_THREAD_ONLY)) == READ_PENDING) {
+		/*
+		 * It was a reader and we have no write active. We don't care
+		 * how many readers are already active. Adjust the count for
+		 * the item we are about to dequeue. Adding READ_PENDING to
+		 * the exisiting READ_PENDING clears it and generates a carry
+		 * into the reader count.
+		 */
+		add_arg = READ_PENDING;
+	} else if ((ngq->q_flags & ~SINGLE_THREAD_ONLY) == WRITE_PENDING) {
+		/*
+		 * There is a pending write, no readers and no active writer.
+		 * This means we can go ahead with the pending writer. Note
+		 * the fact that we now have a writer, ready for when we take
+		 * it off the queue.
+		 *
+		 * We don't need to worry about a possible collision with the
+		 * fasttrack reader.
+		 *
+		 * The fasttrack thread may take a long time to discover that we
+		 * are running so we would have an inconsistent state in the
+		 * flags for a while. Since we ignore the reader count
+		 * entirely when the WRITER_ACTIVE flag is set, this should
+		 * not matter (in fact it is defined that way). If it tests
+		 * the flag before this operation, the WRITE_PENDING flag
+		 * will make it fail, and if it tests it later, the
+		 * ACTIVE_WRITER flag will do the same. If it is SO slow that
+		 * we have actually completed the operation, and neither flag
+		 * is set (nor the READ_PENDING) by the time that it tests
+		 * the flags, then it is actually ok for it to continue. If
+		 * it completes and we've finished and the read pending is
+		 * set it still fails.
+		 *
+		 * So we can just ignore it,  as long as we can ensure that the
+		 * transition from WRITE_PENDING state to the WRITER_ACTIVE
+		 * state is atomic.
+		 *
+		 * After failing, first it will be held back by the mutex, then
+		 * when it can proceed, it will queue its request, then it
+		 * would arrive at this function. Usually it will have to
+		 * leave empty handed because the ACTIVE WRITER bit wil be
+		 * set.
+		 */
+		/*
+		 * Adjust the flags for the item we are about to dequeue.
+		 * Adding WRITE_PENDING to the exisiting WRITE_PENDING clears
+		 * it and generates a carry into the WRITER_ACTIVE flag, all
+		 * atomically.
+		 */
+		add_arg = WRITE_PENDING;
+		/*
+		 * We want to write "active writer, no readers " Now go make
+		 * it true. In fact there may be a number in the readers
+		 * count but we know it is not true and will be fixed soon.
+		 * We will fix the flags for the next pending entry in a
+		 * moment.
+		 */
+	} else {
+		/*
+		 * We can't dequeue anything.. return and say so. Probably we
+		 * have a write pending and the readers count is non zero. If
+		 * we got here because a reader hit us just at the wrong
+		 * moment with the fasttrack code, and put us in a strange
+		 * state, then it will be through in just a moment, (as soon
+		 * as we release the mutex) and keep things moving.
+		 */
+		return (0);
+	}
 
-#define CALL_MSG_HANDLER(error, node, msg, retaddr, resp, hook)		\
-do {									\
-	if((msg)->header.typecookie == NGM_GENERIC_COOKIE) {		\
-		(error) = ng_generic_msg((node), (msg),			\
-			(retaddr), (resp), (hook));			\
-	} else {							\
-		if ((node)->type->rcvmsg != NULL) {			\
-			(error) = (*(node)->type->rcvmsg)((node),	\
-				(msg), (retaddr), (resp), (hook));	\
-		} else {						\
-			TRAP_ERROR;					\
-			FREE((msg), M_NETGRAPH);			\
-			(error) = EINVAL;				\
-		}							\
-	}								\
-} while (0)
+	/*
+	 * Now we dequeue the request (whatever it may be) and correct the
+	 * pending flags and the next and last pointers.
+	 */
+	item = ngq->queue;
+	ngq->queue = item->el_next;
+	if (ngq->last == &(item->el_next)) {
+		/*
+		 * that was the last entry in the queue so set the 'last
+		 * pointer up correctly and make sure the pending flags are
+		 * clear.
+		 */
+		ngq->last = &(ngq->queue);
+		/*
+		 * Whatever flag was set is cleared and the carry sets the
+		 * correct new active state/count. So we don't need to change
+		 * add_arg.
+		 */
+	} else {
+		if ((ngq->queue->el_flags & NGQF_TYPE) == NGQF_READER) {
+			/*
+			 * If we had a READ_PENDING and have another one, we
+			 * just want to add READ_PENDING twice (the same as
+			 * adding READER_INCREMENT). If we had WRITE_PENDING,
+			 * we want to add READ_PENDING + WRITE_PENDING to
+			 * clear the old WRITE_PENDING, set ACTIVE_WRITER,
+			 * and set READ_PENDING. Either way we just add
+			 * READ_PENDING to whatever we already had.
+			 */
+			add_arg += READ_PENDING;
+		} else {
+			/*
+			 * If we had a WRITE_PENDING and have another one, we
+			 * just want to add WRITE_PENDING twice (the same as
+			 * adding ACTIVE_WRITER). If we had READ_PENDING, we
+			 * want to add READ_PENDING + WRITE_PENDING to clear
+			 * the old READ_PENDING, increment the readers, and
+			 * set WRITE_PENDING. Either way we just add
+			 * WRITE_PENDING to whatever we already had.
+			 */
+			add_arg += WRITE_PENDING;
+		}
+	}
+	atomic_add_long(&ngq->q_flags, add_arg);
+	/*
+	 * We have successfully cleared the old pending flag, set the new one
+	 * if it is needed, and incremented the appropriate active field.
+	 * (all in one atomic addition.. wow)
+	 */
+	return (item);
+}
+
+/*
+ * Queue a packet to be picked up by someone else.
+ * We really don't care who, but we can't or don't want to hang around
+ * to process it ourselves. We are probably an interrupt routine..
+ * 1 = writer, 0 = reader
+ * We should set something to indicate NETISR requested
+ * If it's the first item queued.
+ */
+#define NGQRW_R 0
+#define NGQRW_W 1
+static __inline void
+ng_queue_rw(struct ng_queue * ngq, item_p  item, int rw)
+{
+	item->el_next = NULL;	/* maybe not needed */
+	*ngq->last = item;
+	/*
+	 * If it was the first item in the queue then we need to
+	 * set the last pointer and the type flags.
+	 */
+	if (ngq->last == &(ngq->queue)) {
+		/*
+		 * When called with constants for rw, the optimiser will
+		 * remove the unneeded branch below.
+		 */
+		if (rw == NGQRW_W) {
+			atomic_add_long(&ngq->q_flags, WRITE_PENDING);
+		} else {
+			atomic_add_long(&ngq->q_flags, READ_PENDING);
+		}
+	}
+	ngq->last = &(item->el_next);
+}
 
 
 /*
- * Send a control message to a node.
- * If hook is supplied, use it in preference to the address.
- * If the return address is not supplied it will be set to this node.
+ * This function 'cheats' in that it first tries to 'grab' the use of the
+ * node, without going through the mutex. We can do this becasue of the
+ * semantics of the lock. The semantics include a clause that says that the
+ * value of the readers count is invalid if the WRITER_ACTIVE flag is set. It
+ * also says that the WRITER_ACTIVE flag cannot be set if the readers count
+ * is not zero. Note that this talks about what is valid to SET the
+ * WRITER_ACTIVE flag, because from the moment it is set, the value if the
+ * reader count is immaterial, and not valid. The two 'pending' flags have a
+ * similar effect, in that If they are orthogonal to the two active fields in
+ * how they are set, but if either is set, the attempted 'grab' need to be
+ * backed out because there is earlier work, and we maintain ordering in the
+ * queue. The result of this is that the reader request can try obtain use of
+ * the node with only a single atomic addition, and without any of the mutex
+ * overhead. If this fails the operation degenerates to the same as for other
+ * cases.
+ *
  */
-int
-ng_send_msg(node_p here, struct ng_mesg *msg, const char *address,
-		hook_p hook, char *retaddr, struct ng_mesg **rptr)
+static __inline item_p
+ng_acquire_read(struct ng_queue *ngq, item_p item)
 {
-	node_p  dest = NULL;
-	int     error;
-	hook_p	lasthook;
 
+	/* ######### Hack alert ######### */
+	atomic_add_long(&ngq->q_flags, READER_INCREMENT);
+	if ((ngq->q_flags & (~READER_MASK)) == 0) {
+		/* Successfully grabbed node */
+		return (item);
+	}
+	/* undo the damage if we didn't succeed */
+	atomic_subtract_long(&ngq->q_flags, READER_INCREMENT);
+
+	/* ######### End Hack alert ######### */
+	mtx_enter((&ngq->q_mtx), MTX_SPIN);
 	/*
-	 * Find the target node.
-	 * If there is a HOOK argument, then use that in preference
-	 * to the address.
+	 * Try again. Another processor (or interrupt for that matter) may
+	 * have removed the last queued item that was stopping us from
+	 * running, between the previous test, and the moment that we took
+	 * the mutex. (Or maybe a writer completed.)
 	 */
-	if (hook) {
-		lasthook = hook->peer;
-		dest = lasthook->node;
-	} else {
-		error = ng_path2node(here, address, &dest, &lasthook);
-		if (error) {
-			FREE(msg, M_NETGRAPH);
-			return (error);
+	if ((ngq->q_flags & (~READER_MASK)) == 0) {
+		atomic_add_long(&ngq->q_flags, READER_INCREMENT);
+		mtx_exit((&ngq->q_mtx), MTX_SPIN);
+		return (item);
+	}
+
+	/*
+	 * Quick check that we are doing things right.
+	 */
+	if (ngq->q_flags & SINGLE_THREAD_ONLY) {
+		panic("Calling single treaded queue incorrectly");
+	}
+
+	/*
+	 * and queue the request for later.
+	 */
+	item->el_flags |= NGQF_TYPE;
+	ng_queue_rw(ngq, item, NGQRW_R);
+
+	/*
+	 * Ok, so that's the item successfully queued for later. So now we
+	 * see if we can dequeue something to run instead.
+	 */
+	item = ng_dequeue(ngq);
+	mtx_exit(&(ngq->q_mtx), MTX_SPIN);
+	return (item);
+}
+
+static __inline item_p
+ng_acquire_write(struct ng_queue *ngq, item_p item)
+{
+restart:
+	mtx_enter(&(ngq->q_mtx), MTX_SPIN);
+	/*
+	 * If there are no readers, no writer, and no pending packets, then
+	 * we can just go ahead. In all other situations we need to queue the
+	 * request
+	 */
+	if ((ngq->q_flags & (~SINGLE_THREAD_ONLY)) == 0) {
+		atomic_add_long(&ngq->q_flags, WRITER_ACTIVE);
+		mtx_exit((&ngq->q_mtx), MTX_SPIN);
+		if (ngq->q_flags & READER_MASK) {
+			/* Collision with fast-track reader */
+			atomic_add_long(&ngq->q_flags, -WRITER_ACTIVE);
+			goto restart;
+		}
+
+		return (item);
+	}
+
+	/*
+	 * and queue the request for later.
+	 */
+	item->el_flags &= ~NGQF_TYPE;
+	ng_queue_rw(ngq, item, NGQRW_W);
+
+	/*
+	 * Ok, so that's the item successfully queued for later. So now we
+	 * see if we can dequeue something to run instead.
+	 */
+	item = ng_dequeue(ngq);
+	mtx_exit(&(ngq->q_mtx), MTX_SPIN);
+	return (item);
+}
+
+static __inline void
+ng_leave_read(struct ng_queue *ngq)
+{
+	atomic_subtract_long(&ngq->q_flags, READER_INCREMENT);
+}
+
+static __inline void
+ng_leave_write(struct ng_queue *ngq)
+{
+	atomic_subtract_long(&ngq->q_flags, WRITER_ACTIVE);
+}
+
+static void
+ng_flush_input_queue(struct ng_queue * ngq)
+{
+	item_p item;
+	u_int		add_arg;
+	mtx_enter(&ngq->q_mtx, MTX_SPIN);
+	for (;;) {
+		/* Now take a look at what's on the queue */
+		if (ngq->q_flags & READ_PENDING) {
+			add_arg = -READ_PENDING;
+		} else if (ngq->q_flags & WRITE_PENDING) {
+			add_arg = -WRITE_PENDING;
+		} else {
+			break;
+		}
+
+		item = ngq->queue;
+		ngq->queue = item->el_next;
+		if (ngq->last == &(item->el_next)) {
+			ngq->last = &(ngq->queue);
+		} else {
+			if ((ngq->queue->el_flags & NGQF_TYPE) == NGQF_READER) {
+				add_arg += READ_PENDING;
+			} else {
+				add_arg += WRITE_PENDING;
+			}
 		}
+		atomic_add_long(&ngq->q_flags, add_arg);
+
+		mtx_exit(&ngq->q_mtx, MTX_SPIN);
+		NG_FREE_ITEM(item);
+		mtx_enter(&ngq->q_mtx, MTX_SPIN);
 	}
+	mtx_exit(&ngq->q_mtx, MTX_SPIN);
+}
+
+/***********************************************************************
+* Externally visible method for sending or queueing messages or data.
+***********************************************************************/
+
+/*
+ * MACRO WILL DO THE JOB OF CALLING ng_package_msg IN CALLER
+ * before we are called. The user code should have filled out the item
+ * correctly by this stage:
+ * Common:
+ *    reference to destination node.
+ *    Reference to destination rcv hook if relevant.
+ * Data:
+ *    pointer to mbuf
+ *    pointer to metadata
+ * Control_Message:
+ *    pointer to msg.
+ *    ID of original sender node. (return address)
+ *
+ * The nodes have several routines and macros to help with this task:
+ * ng_package_msg()
+ * ng_package_data() do much of the work.
+ * ng_retarget_msg
+ * ng_retarget_data
+ */
+
+int
+ng_snd_item(item_p item, int queue)
+{
+	hook_p hook = item->el_hook;
+	node_p dest = item->el_dest;
+	int rw;
+	int error = 0, ierror;
+	item_p	oitem;
+	struct ng_queue * ngq = &dest->input_queue;
+
+#ifdef	ITEM_DEBUG
+        _ngi_check(item, __FILE__, __LINE__);
+#endif
 
-	/* If the user didn't supply a return addres, assume it's "here". */
-	if (retaddr == NULL) {
+	if (item == NULL) {
+		return (EINVAL);	/* failed to get queue element */
+	}
+	if (dest == NULL) {
+		NG_FREE_ITEM(item);
+		return (EINVAL);	/* No address */
+	}
+	if ((item->el_flags & NGQF_D_M) == NGQF_DATA) {
 		/*
-		 * Now fill out the return address,
-		 * i.e. the name/ID of the sender. (If we didn't get one)
+		 * DATA MESSAGE
+		 * Delivered to a node via a non-optional hook.
+		 * Both should be present in the item even though
+		 * the node is derivable from the hook.
+		 * References are held on both by the item.
 		 */
-		MALLOC(retaddr, char *, NG_NODELEN + 2, M_NETGRAPH, M_NOWAIT);
-		if (retaddr == NULL) {
+#ifdef	ITEM_DEBUG
+        _ngi_check(item, __FILE__, __LINE__);
+#endif
+		CHECK_DATA_MBUF(NGI_M(item));
+		if (hook == NULL) {
+			NG_FREE_ITEM(item);
+			return(EINVAL);
+		}
+		if (((hook->flags & HK_INVALID) != 0)
+		|| ((hook->node->flags & NG_INVALID) != 0)) {
 			TRAP_ERROR;
-			return (ENOMEM);
+			NG_FREE_ITEM(item);
+			return (ENOTCONN);
 		}
-		if (here->name != NULL)
-			sprintf(retaddr, "%s:", here->name);
-		else
-			sprintf(retaddr, "[%x]:", ng_node2ID(here));
+		if ((hook->flags & HK_QUEUE)) {
+			queue = 1;
+		}
+		/* By default data is a reader in the locking scheme */
+		item->el_flags |= NGQF_READER;
+		rw = NGQRW_R;
+	} else {
+		/*
+		 * CONTROL MESSAGE
+		 * Delivered to a node.
+		 * Hook is optional.
+		 * References are held by the item on the node and
+		 * the hook if it is present.
+		 */
+		if (hook && (hook->flags & HK_QUEUE)) {
+			queue = 1;
+		}
+		/* Data messages count as writers unles explicitly exempted */
+		if (NGI_MSG(item)->header.cmd & NGM_READONLY) {
+			item->el_flags |= NGQF_READER;
+			rw = NGQRW_R;
+		} else {
+			item->el_flags &= ~NGQF_TYPE;
+			rw = NGQRW_W;
+		}
+	}
+	/*
+	 * If the node specifies single threading, force writer semantics
+	 * Similarly the node may say one hook always produces writers.
+	 * These are over-rides.
+	 */
+	if ((ngq->q_flags & SINGLE_THREAD_ONLY)
+	|| (dest->flags & NG_FORCE_WRITER)
+	|| (hook && (hook->flags & HK_FORCE_WRITER))) {
+			rw = NGQRW_W;
+			item->el_flags &= ~NGQF_TYPE;
+	}
+	if (queue) {
+		/* Put it on the queue for that node*/
+#ifdef	ITEM_DEBUG
+        _ngi_check(item, __FILE__, __LINE__);
+#endif
+		mtx_enter(&(ngq->q_mtx), MTX_SPIN);
+		ng_queue_rw(ngq, item, rw);
+		mtx_exit(&(ngq->q_mtx), MTX_SPIN);
+		/*
+		 * If there are active elements then we can rely on
+		 * them. if not we should not rely on another packet
+		 * coming here by another path,
+		 * so it is best to put us in the netisr list.
+		 */
+		if ((ngq->q_flags & (READER_MASK|WRITER_ACTIVE)) == 0) {
+			ng_setisr(ngq->q_node);
+		}
+		return (0);
 	}
+	/*
+	 * Take a queue item and a node and see if we can apply the item to
+	 * the node. We may end up getting a different item to apply instead.
+	 * Will allow for a piggyback reply only in the case where
+	 * there is no queueing.
+	 */
 
-	/* Make sure the resp field is null before we start */
-	if (rptr != NULL)
-		*rptr = NULL;
+	oitem = item;
+	/*
+	 * We already decided how we will be queueud or treated.
+	 * Try get the appropriate operating permission.
+	 */
+ 	if (rw == NGQRW_R) {
+		item = ng_acquire_read(ngq, item);
+	} else {
+		item = ng_acquire_write(ngq, item);
+	}
 
-	CALL_MSG_HANDLER(error, dest, msg, retaddr, rptr, lasthook);
+	/*
+	 * May have come back with a different item.
+	 * or maybe none at all. The one we started with will
+	 * have been queued in thises cases.
+	 */
+	if (item == NULL) {
+		return (0);
+	}
 
-	/* Make sure that if there is a response, it has the RESP bit set */
-	if ((error == 0) && rptr && *rptr)
-		(*rptr)->header.flags |= NGF_RESP;
+#ifdef	ITEM_DEBUG
+        _ngi_check(item, __FILE__, __LINE__);
+#endif
+	ierror = ng_apply_item(dest, item); /* drops r/w lock when done */
+
+	/* only return an error if it was our initial item.. (compat hack) */
+	if (oitem == item) {
+		error = ierror;
+	}
 
 	/*
-	 * If we had a return address it is up to us to free it. They should
-	 * have taken a copy if they needed to make a delayed response.
+	 * Now we've handled the packet we brought, (or a friend of it) let's
+	 * look for any other packets that may have been queued up. We hold
+	 * no locks, so if someone puts something in the queue after
+	 * we check that it is empty, it is their problem
+	 * to ensure it is processed. If we have the netisr thread cme in here
+	 * while we still say we have stuff to do, we may get a boost
+	 * in SMP systems. :-)
 	 */
-	if (retaddr)
-		FREE(retaddr, M_NETGRAPH);
-	return (error);
+	for (;;) {
+		/* quick hack to save all that mutex stuff */
+		if ((ngq->q_flags & (WRITE_PENDING | READ_PENDING)) == 0) {
+			if (dest->flags & NG_WORKQ)
+				ng_worklist_remove(dest);
+			return (0);
+		}
+		/*
+		 * dequeue acquires and adjusts the input_queue as it dequeues
+		 * packets. It acquires the rw lock as needed.
+		 */
+		mtx_enter(&ngq->q_mtx, MTX_SPIN);
+		item = ng_dequeue(ngq);
+		mtx_exit(&ngq->q_mtx, MTX_SPIN);
+		if (!item) {
+			/*
+			 * If we have no work to do
+			 * then we certainly don't need to be
+			 * on the worklist.
+			 */
+			if (dest->flags & NG_WORKQ)
+				ng_worklist_remove(dest);
+			return (0);
+		}
+#ifdef	ITEM_DEBUG
+        _ngi_check(item, __FILE__, __LINE__);
+#endif
+
+		/*
+		 * We have the appropriate lock, so run the item.
+		 * When finished it will drop the lock accordingly
+		 */
+
+		ierror = ng_apply_item(dest, item);
+		/*
+		 * only return an error if it was our initial
+		 * item.. (compat hack)
+		 */
+		if (oitem == item) {
+			error = ierror;
+		}
+	}
+	return (0);
 }
 
 /*
- * Implement the 'generic' control messages
+ * We have an item that was possibly queued somewhere.
+ * It should contain all the information needed
+ * to run it on the appropriate node/hook.
  */
 static int
-ng_generic_msg(node_p here, struct ng_mesg *msg, const char *retaddr,
-	       struct ng_mesg **resp, hook_p lasthook)
+ng_apply_item(node_p node, item_p item)
+{
+	hook_p  hook;
+	int was_reader = ((item->el_flags & NGQF_TYPE));
+	int error = 0;
+	ng_rcvdata_t *rcvdata;
+
+	hook = item->el_hook;
+	item->el_hook = NULL;	/* so NG_FREE_ITEM doesn't ng_unref_hook() */
+	/* We already have the node.. assume responsibility */
+	/* And the reference */
+	/* node = item->el_dest; */
+	item->el_dest = NULL;	/* same as for the hook above */
+#ifdef	ITEM_DEBUG
+        _ngi_check(item, __FILE__, __LINE__);
+#endif
+
+	switch (item->el_flags & NGQF_D_M) {
+	case NGQF_DATA:
+		/*
+		 * Check things are still ok as when we were queued.
+		 */
+
+		if ((hook == NULL)
+		|| ((hook->flags & HK_INVALID) != 0)
+		|| ((hook->node->flags & NG_INVALID) != 0)
+		|| ((rcvdata = hook->node->type->rcvdata) == NULL)) {
+			error = EIO;
+			NG_FREE_ITEM(item);
+		} else {
+			error = (*rcvdata)(hook, item);
+		}
+		break;
+	case NGQF_MESG:
+
+		if (hook) {
+			item->el_hook = NULL;
+			if ((hook->flags & HK_INVALID) != 0) {
+			/*
+			 * If the hook has been zapped then we can't use it.
+			 * Immediatly drop its reference.
+			 * The message may not need it.
+			 */
+				ng_unref_hook(hook);
+				hook = NULL;
+			}
+		}
+		/*
+		 * Similarly, if the node is a zombie there is
+		 * nothing we can do with it, drop everything.
+		 */
+		if (node->flags & NG_INVALID) {
+			error = EINVAL;
+			NG_FREE_ITEM(item);
+		} else {
+			/*
+			 * Call the appropriate message handler for the object.
+			 * It is up to the message handler to free the message.
+			 * If it's a generic message, handle it generically,
+			 * otherwise call the type's message handler
+			 * (if it exists)
+			 * XXX (race). Remember that a queued message may
+			 * reference a node or hook that has just been
+			 * invalidated. It will exist as the queue code
+			 * is holding a reference, but..
+			 */
+
+			struct ng_mesg *msg = NGI_MSG(item);
+
+			if ((msg->header.typecookie == NGM_GENERIC_COOKIE)
+			&& ((msg->header.flags & NGF_RESP) == 0)) {
+				error = ng_generic_msg(node, item, hook);
+			} else {
+				if ((node)->type->rcvmsg != NULL) {
+					error = (*(node)->type->rcvmsg)((node),
+						(item), (hook));
+				} else {
+					TRAP_ERROR;
+					error = EINVAL; /* XXX */
+					NG_FREE_ITEM(item);
+				}
+			}
+			/* item is now invalid */
+		}
+		break;
+	}
+	/*
+	 * We held references on some of the resources
+	 * that we took from the item. Now that we have
+	 * finished doing everything, drop those references.
+	 */
+	if (hook) {
+		ng_unref_hook(hook);
+	}
+
+	if (was_reader) {
+		ng_leave_read(&node->input_queue);
+	} else {
+		ng_leave_write(&node->input_queue);
+	}
+	ng_unref(node);
+	return (error);
+}
+
+/***********************************************************************
+ * Implement the 'generic' control messages
+ ***********************************************************************/
+static int
+ng_generic_msg(node_p here, item_p item, hook_p lasthook)
 {
 	int error = 0;
+	struct ng_mesg *msg;
+	struct ng_mesg *resp = NULL;
 
+	NGI_GET_MSG(item, msg);
 	if (msg->header.typecookie != NGM_GENERIC_COOKIE) {
-		TRAP_ERROR;
-		FREE(msg, M_NETGRAPH);
-		return (EINVAL);
+		error = EINVAL;
+		goto out;
 	}
 	switch (msg->header.cmd) {
 	case NGM_SHUTDOWN:
@@ -1235,8 +2000,8 @@ ng_generic_msg(node_p here, struct ng_mesg *msg, const char *retaddr,
 		struct ngm_mkpeer *const mkp = (struct ngm_mkpeer *) msg->data;
 
 		if (msg->header.arglen != sizeof(*mkp)) {
-			TRAP_ERROR;
-			return (EINVAL);
+			error = EINVAL;
+			break;
 		}
 		mkp->type[sizeof(mkp->type) - 1] = '\0';
 		mkp->ourhook[sizeof(mkp->ourhook) - 1] = '\0';
@@ -1251,16 +2016,18 @@ ng_generic_msg(node_p here, struct ng_mesg *msg, const char *retaddr,
 		node_p node2;
 
 		if (msg->header.arglen != sizeof(*con)) {
-			TRAP_ERROR;
-			return (EINVAL);
+			error = EINVAL;
+			break;
 		}
 		con->path[sizeof(con->path) - 1] = '\0';
 		con->ourhook[sizeof(con->ourhook) - 1] = '\0';
 		con->peerhook[sizeof(con->peerhook) - 1] = '\0';
-		error = ng_path2node(here, con->path, &node2, NULL);
+		/* Don't forget we get a reference.. */
+		error = ng_path2noderef(here, con->path, &node2, NULL);
 		if (error)
 			break;
 		error = ng_con_nodes(here, con->ourhook, node2, con->peerhook);
+		ng_unref(node2);
 		break;
 	    }
 	case NGM_NAME:
@@ -1268,8 +2035,8 @@ ng_generic_msg(node_p here, struct ng_mesg *msg, const char *retaddr,
 		struct ngm_name *const nam = (struct ngm_name *) msg->data;
 
 		if (msg->header.arglen != sizeof(*nam)) {
-			TRAP_ERROR;
-			return (EINVAL);
+			error = EINVAL;
+			break;
 		}
 		nam->name[sizeof(nam->name) - 1] = '\0';
 		error = ng_name_node(here, nam->name);
@@ -1281,8 +2048,8 @@ ng_generic_msg(node_p here, struct ng_mesg *msg, const char *retaddr,
 		hook_p hook;
 
 		if (msg->header.arglen != sizeof(*rmh)) {
-			TRAP_ERROR;
-			return (EINVAL);
+			error = EINVAL;
+			break;
 		}
 		rmh->ourhook[sizeof(rmh->ourhook) - 1] = '\0';
 		if ((hook = ng_findhook(here, rmh->ourhook)) != NULL)
@@ -1292,27 +2059,20 @@ ng_generic_msg(node_p here, struct ng_mesg *msg, const char *retaddr,
 	case NGM_NODEINFO:
 	    {
 		struct nodeinfo *ni;
-		struct ng_mesg *rp;
 
-		/* Get response struct */
+		NG_MKRESPONSE(resp, msg, sizeof(*ni), M_NOWAIT);
 		if (resp == NULL) {
-			error = EINVAL;
-			break;
-		}
-		NG_MKRESPONSE(rp, msg, sizeof(*ni), M_NOWAIT);
-		if (rp == NULL) {
 			error = ENOMEM;
 			break;
 		}
 
 		/* Fill in node info */
-		ni = (struct nodeinfo *) rp->data;
+		ni = (struct nodeinfo *) resp->data;
 		if (here->name != NULL)
 			strncpy(ni->name, here->name, NG_NODELEN);
 		strncpy(ni->type, here->type->name, NG_TYPELEN);
 		ni->id = ng_node2ID(here);
 		ni->hooks = here->numhooks;
-		*resp = rp;
 		break;
 	    }
 	case NGM_LISTHOOKS:
@@ -1320,21 +2080,16 @@ ng_generic_msg(node_p here, struct ng_mesg *msg, const char *retaddr,
 		const int nhooks = here->numhooks;
 		struct hooklist *hl;
 		struct nodeinfo *ni;
-		struct ng_mesg *rp;
 		hook_p hook;
 
 		/* Get response struct */
-		if (resp == NULL) {
-			error = EINVAL;
-			break;
-		}
-		NG_MKRESPONSE(rp, msg, sizeof(*hl)
+		NG_MKRESPONSE(resp, msg, sizeof(*hl)
 		    + (nhooks * sizeof(struct linkinfo)), M_NOWAIT);
-		if (rp == NULL) {
+		if (resp == NULL) {
 			error = ENOMEM;
 			break;
 		}
-		hl = (struct hooklist *) rp->data;
+		hl = (struct hooklist *) resp->data;
 		ni = &hl->nodeinfo;
 
 		/* Fill in node info */
@@ -1357,7 +2112,7 @@ ng_generic_msg(node_p here, struct ng_mesg *msg, const char *retaddr,
 				continue;
 			strncpy(link->ourhook, hook->name, NG_HOOKLEN);
 			strncpy(link->peerhook, hook->peer->name, NG_HOOKLEN);
-			if (hook->peer->node->name != NULL)
+			if (hook->peer->node->name[0] != '\0')
 				strncpy(link->nodeinfo.name,
 				    hook->peer->node->name, NG_NODELEN);
 			strncpy(link->nodeinfo.type,
@@ -1366,7 +2121,6 @@ ng_generic_msg(node_p here, struct ng_mesg *msg, const char *retaddr,
 			link->nodeinfo.hooks = hook->peer->node->numhooks;
 			ni->hooks++;
 		}
-		*resp = rp;
 		break;
 	    }
 
@@ -1375,37 +2129,30 @@ ng_generic_msg(node_p here, struct ng_mesg *msg, const char *retaddr,
 	    {
 		const int unnamed = (msg->header.cmd == NGM_LISTNODES);
 		struct namelist *nl;
-		struct ng_mesg *rp;
 		node_p node;
 		int num = 0;
 
-		if (resp == NULL) {
-			error = EINVAL;
-			break;
-		}
-
+		mtx_enter(&ng_nodelist_mtx, MTX_DEF);
 		/* Count number of nodes */
-		LIST_FOREACH(node, &nodelist, nodes) {
-			if (unnamed || node->name != NULL)
+		LIST_FOREACH(node, &ng_nodelist, nodes) {
+			if (unnamed || node->name[0] != '\0')
 				num++;
 		}
+		mtx_exit(&ng_nodelist_mtx, MTX_DEF);
 
 		/* Get response struct */
-		if (resp == NULL) {
-			error = EINVAL;
-			break;
-		}
-		NG_MKRESPONSE(rp, msg, sizeof(*nl)
+		NG_MKRESPONSE(resp, msg, sizeof(*nl)
 		    + (num * sizeof(struct nodeinfo)), M_NOWAIT);
-		if (rp == NULL) {
+		if (resp == NULL) {
 			error = ENOMEM;
 			break;
 		}
-		nl = (struct namelist *) rp->data;
+		nl = (struct namelist *) resp->data;
 
 		/* Cycle through the linked list of nodes */
 		nl->numnames = 0;
-		LIST_FOREACH(node, &nodelist, nodes) {
+		mtx_enter(&ng_nodelist_mtx, MTX_DEF);
+		LIST_FOREACH(node, &ng_nodelist, nodes) {
 			struct nodeinfo *const np = &nl->nodeinfo[nl->numnames];
 
 			if (nl->numnames >= num) {
@@ -1415,51 +2162,44 @@ ng_generic_msg(node_p here, struct ng_mesg *msg, const char *retaddr,
 			}
 			if ((node->flags & NG_INVALID) != 0)
 				continue;
-			if (!unnamed && node->name == NULL)
+			if (!unnamed && node->name[0] == '\0')
 				continue;
-			if (node->name != NULL)
+			if (node->name[0] != '\0')
 				strncpy(np->name, node->name, NG_NODELEN);
 			strncpy(np->type, node->type->name, NG_TYPELEN);
 			np->id = ng_node2ID(node);
 			np->hooks = node->numhooks;
 			nl->numnames++;
 		}
-		*resp = rp;
+		mtx_exit(&ng_nodelist_mtx, MTX_DEF);
 		break;
 	    }
 
 	case NGM_LISTTYPES:
 	    {
 		struct typelist *tl;
-		struct ng_mesg *rp;
 		struct ng_type *type;
 		int num = 0;
 
-		if (resp == NULL) {
-			error = EINVAL;
-			break;
-		}
-
+		mtx_enter(&ng_typelist_mtx, MTX_DEF);
 		/* Count number of types */
-		LIST_FOREACH(type, &typelist, types)
+		LIST_FOREACH(type, &ng_typelist, types)
 			num++;
+		mtx_exit(&ng_typelist_mtx, MTX_DEF);
 
 		/* Get response struct */
-		if (resp == NULL) {
-			error = EINVAL;
-			break;
-		}
-		NG_MKRESPONSE(rp, msg, sizeof(*tl)
+		NG_MKRESPONSE(resp, msg, sizeof(*tl)
 		    + (num * sizeof(struct typeinfo)), M_NOWAIT);
-		if (rp == NULL) {
+		if (resp == NULL) {
 			error = ENOMEM;
 			break;
 		}
-		tl = (struct typelist *) rp->data;
+		tl = (struct typelist *) resp->data;
 
 		/* Cycle through the linked list of types */
 		tl->numtypes = 0;
-		LIST_FOREACH(type, &typelist, types) {
+		mtx_enter(&ng_typelist_mtx, MTX_DEF);
+		LIST_FOREACH(type, &ng_typelist, types) {
 			struct typeinfo *const tp = &tl->typeinfo[tl->numtypes];
 
 			if (tl->numtypes >= num) {
@@ -1471,7 +2211,7 @@ ng_generic_msg(node_p here, struct ng_mesg *msg, const char *retaddr,
 			tp->numnodes = type->refs;
 			tl->numtypes++;
 		}
-		*resp = rp;
+		mtx_exit(&ng_typelist_mtx, MTX_DEF);
 		break;
 	    }
 
@@ -1480,30 +2220,24 @@ ng_generic_msg(node_p here, struct ng_mesg *msg, const char *retaddr,
 		int bufSize = 20 * 1024;	/* XXX hard coded constant */
 		const struct ng_parse_type *argstype;
 		const struct ng_cmdlist *c;
-		struct ng_mesg *rp, *binary, *ascii;
+		struct ng_mesg *binary, *ascii;
 
 		/* Data area must contain a valid netgraph message */
 		binary = (struct ng_mesg *)msg->data;
 		if (msg->header.arglen < sizeof(struct ng_mesg)
-		    || msg->header.arglen - sizeof(struct ng_mesg) 
+		    || msg->header.arglen - sizeof(struct ng_mesg)
 		      < binary->header.arglen) {
 			error = EINVAL;
 			break;
 		}
 
-		/* Check response pointer */
-		if (resp == NULL) {
-			error = EINVAL;
-			break;
-		}
-
 		/* Get a response message with lots of room */
-		NG_MKRESPONSE(rp, msg, sizeof(*ascii) + bufSize, M_NOWAIT);
-		if (rp == NULL) {
+		NG_MKRESPONSE(resp, msg, sizeof(*ascii) + bufSize, M_NOWAIT);
+		if (resp == NULL) {
 			error = ENOMEM;
 			break;
 		}
-		ascii = (struct ng_mesg *)rp->data;
+		ascii = (struct ng_mesg *)resp->data;
 
 		/* Copy binary message header to response message payload */
 		bcopy(binary, ascii, sizeof(*binary));
@@ -1522,7 +2256,7 @@ ng_generic_msg(node_p here, struct ng_mesg *msg, const char *retaddr,
 					break;
 			}
 			if (c->name == NULL) {
-				FREE(rp, M_NETGRAPH);
+				NG_FREE_MSG(resp);
 				error = ENOSYS;
 				break;
 			}
@@ -1541,7 +2275,7 @@ ng_generic_msg(node_p here, struct ng_mesg *msg, const char *retaddr,
 			if ((error = ng_unparse(argstype,
 			    (u_char *)binary->data,
 			    ascii->data, bufSize)) != 0) {
-				FREE(rp, M_NETGRAPH);
+				NG_FREE_MSG(resp);
 				break;
 			}
 		}
@@ -1549,8 +2283,7 @@ ng_generic_msg(node_p here, struct ng_mesg *msg, const char *retaddr,
 		/* Return the result as struct ng_mesg plus ASCII string */
 		bufSize = strlen(ascii->data) + 1;
 		ascii->header.arglen = bufSize;
-		rp->header.arglen = sizeof(*ascii) + bufSize;
-		*resp = rp;
+		resp->header.arglen = sizeof(*ascii) + bufSize;
 		break;
 	    }
 
@@ -1559,7 +2292,7 @@ ng_generic_msg(node_p here, struct ng_mesg *msg, const char *retaddr,
 		int bufSize = 2000;	/* XXX hard coded constant */
 		const struct ng_cmdlist *c;
 		const struct ng_parse_type *argstype;
-		struct ng_mesg *rp, *ascii, *binary;
+		struct ng_mesg *ascii, *binary;
 		int off = 0;
 
 		/* Data area must contain at least a struct ng_mesg + '\0' */
@@ -1573,19 +2306,13 @@ ng_generic_msg(node_p here, struct ng_mesg *msg, const char *retaddr,
 		}
 		ascii->data[ascii->header.arglen - 1] = '\0';
 
-		/* Check response pointer */
-		if (resp == NULL) {
-			error = EINVAL;
-			break;
-		}
-
 		/* Get a response message with lots of room */
-		NG_MKRESPONSE(rp, msg, sizeof(*binary) + bufSize, M_NOWAIT);
-		if (rp == NULL) {
+		NG_MKRESPONSE(resp, msg, sizeof(*binary) + bufSize, M_NOWAIT);
+		if (resp == NULL) {
 			error = ENOMEM;
 			break;
 		}
-		binary = (struct ng_mesg *)rp->data;
+		binary = (struct ng_mesg *)resp->data;
 
 		/* Copy ASCII message header to response message payload */
 		bcopy(ascii, binary, sizeof(*ascii));
@@ -1602,7 +2329,7 @@ ng_generic_msg(node_p here, struct ng_mesg *msg, const char *retaddr,
 					break;
 			}
 			if (c->name == NULL) {
-				FREE(rp, M_NETGRAPH);
+				NG_FREE_MSG(resp);
 				error = ENOSYS;
 				break;
 			}
@@ -1620,15 +2347,14 @@ ng_generic_msg(node_p here, struct ng_mesg *msg, const char *retaddr,
 		else {
 			if ((error = ng_parse(argstype, ascii->data,
 			    &off, (u_char *)binary->data, &bufSize)) != 0) {
-				FREE(rp, M_NETGRAPH);
+				NG_FREE_MSG(resp);
 				break;
 			}
 		}
 
 		/* Return the result */
 		binary->header.arglen = bufSize;
-		rp->header.arglen = sizeof(*binary) + bufSize;
-		*resp = rp;
+		resp->header.arglen = sizeof(*binary) + bufSize;
 		break;
 	    }
 
@@ -1637,79 +2363,34 @@ ng_generic_msg(node_p here, struct ng_mesg *msg, const char *retaddr,
 		/*
 		 * This one is tricky as it passes the command down to the
 		 * actual node, even though it is a generic type command.
-		 * This means we must assume that the msg is already freed
+		 * This means we must assume that the item/msg is already freed
 		 * when control passes back to us.
 		 */
-		if (resp == NULL) {
-			error = EINVAL;
-			break;
+		if (here->type->rcvmsg != NULL) {
+			NGI_MSG(item) = msg; /* put it back as we found it */
+			return((*here->type->rcvmsg)(here, item, lasthook));
 		}
-		if (here->type->rcvmsg != NULL)
-			return((*here->type->rcvmsg)(here, msg, retaddr,
-			resp, lasthook));
 		/* Fall through if rcvmsg not supported */
 	default:
 		TRAP_ERROR;
 		error = EINVAL;
 	}
-	FREE(msg, M_NETGRAPH);
+	/*
+	 * Sometimes a generic message may be statically allocated
+	 * to avoid problems with allocating when in tight memeory situations.
+	 * Don't free it if it is so.
+	 * I break them appart here, because erros may cause a free if the item
+	 * in which case we'd be doing it twice.
+	 * they are kept together above, to simplify freeing.
+	 */
+out:
+	NG_RESPOND_MSG(error, here, item, resp);
+	if ( msg && ((msg->header.flags & NGF_STATIC) == 0))
+		NG_FREE_MSG(msg);
 	return (error);
 }
 
 /*
- * Send a data packet to a node. If the recipient has no
- * 'receive data' method, then silently discard the packet.
- * The receiving node may elect to put the data onto the netgraph
- * NETISR queue for later delivery. It may do this because it knows there
- * is some recursion and wishes to unwind the stack, or because it has
- * some suspicion that it is being called at (say) splimp instead of
- * splnet.
- */
-int 
-ng_send_data(hook_p hook, struct mbuf *m, meta_p meta,
-	struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
-{
-	ng_rcvdata_t *rcvdata;
-
-	CHECK_DATA_MBUF(m);
-	if ((hook == NULL)
-	|| ((hook->flags & HK_INVALID) != 0)
-	|| ((rcvdata = hook->peer->node->type->rcvdata) == NULL)) {
-		TRAP_ERROR;
-		NG_FREE_DATA(m, meta);
-		return (ENOTCONN);
-	}
-	if (hook->peer->flags & HK_QUEUE) {
-		return (ng_queue_data(hook, m, meta));
-	}
-	return ( (*rcvdata)(hook->peer, m, meta, ret_m, ret_meta, resp));
-}
-
-/*
- * Send a queued data packet to a node. 
- *
- * This is meant for data that is being dequeued and should therefore NOT
- * be queued again. It ignores the queue flag and should NOT be called
- * outside of this file. (thus it is static)
- */
-static int 
-ng_send_data_dont_queue(hook_p hook, struct mbuf *m, meta_p meta,
-	struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
-{
-	ng_rcvdata_t *rcvdata;
-
-	CHECK_DATA_MBUF(m);
-	if ((hook == NULL)
-	|| ((hook->flags & HK_INVALID) != 0)
-	|| ((rcvdata = hook->peer->node->type->rcvdata) == NULL)) {
-		TRAP_ERROR;
-		NG_FREE_DATA(m, meta);
-		return (ENOTCONN);
-	} 
-	return ((*rcvdata)(hook->peer, m, meta, ret_m, ret_meta, resp));
-}
-
-/*
  * Copy a 'meta'.
  *
  * Returns new meta, or NULL if original meta is NULL or ENOMEM.
@@ -1721,7 +2402,7 @@ ng_copy_meta(meta_p meta)
 
 	if (meta == NULL)
 		return (NULL);
-	MALLOC(meta2, meta_p, meta->used_len, M_NETGRAPH, M_NOWAIT);
+	MALLOC(meta2, meta_p, meta->used_len, M_NETGRAPH_META, M_NOWAIT);
 	if (meta2 == NULL)
 		return (NULL);
 	meta2->allocated_len = meta->used_len;
@@ -1754,8 +2435,11 @@ ng_mod_event(module_t mod, int event, void *data)
 
 		/* Call type specific code */
 		if (type->mod_event != NULL)
-			if ((error = (*type->mod_event)(mod, event, data)) != 0)
+			if ((error = (*type->mod_event)(mod, event, data))) {
+				mtx_enter(&ng_typelist_mtx, MTX_DEF);
 				LIST_REMOVE(type, types);
+				mtx_exit(&ng_typelist_mtx, MTX_DEF);
+			}
 		splx(s);
 		break;
 
@@ -1771,7 +2455,9 @@ ng_mod_event(module_t mod, int event, void *data)
 					break;
 				}
 			}
+			mtx_enter(&ng_typelist_mtx, MTX_DEF);
 			LIST_REMOVE(type, types);
+			mtx_exit(&ng_typelist_mtx, MTX_DEF);
 		}
 		splx(s);
 		break;
@@ -1798,6 +2484,11 @@ ngb_mod_event(module_t mod, int event, void *data)
 	switch (event) {
 	case MOD_LOAD:
 		/* Register line discipline */
+		mtx_init(&ng_worklist_mtx, "netgraph worklist mutex", 0);
+		mtx_init(&ng_typelist_mtx, "netgraph types mutex", 0);
+		mtx_init(&ng_nodelist_mtx, "netgraph nodelist mutex", 0);
+		mtx_init(&ng_idhash_mtx, "netgraph idhash mutex", 0);
+		mtx_init(&ngq_mtx, "netgraph netisr mutex", 0);
 		s = splimp();
 		error = register_netisr(NETISR_NETGRAPH, ngintr);
 		splx(s);
@@ -1821,276 +2512,561 @@ static moduledata_t netgraph_mod = {
 DECLARE_MODULE(netgraph, netgraph_mod, SI_SUB_DRIVERS, SI_ORDER_MIDDLE);
 
 /************************************************************************
-			Queueing routines
+			Queue element get/free routines
 ************************************************************************/
 
-/* The structure for queueing across ISR switches */
-struct ng_queue_entry {
-	u_long	flags;
-	struct ng_queue_entry *next;
-	union {
-		struct {
-			hook_p		da_hook;	/*  target hook */
-			struct mbuf	*da_m;
-			meta_p		da_meta;
-		} data;
-		struct {
-			struct ng_mesg	*msg_msg;
-			node_p		msg_node;
-			hook_p		msg_lasthook;
-			char		*msg_retaddr;
-		} msg;
-	} body;
-};
-#define NGQF_DATA	0x01		/* the queue element is data */
-#define NGQF_MESG	0x02		/* the queue element is a message */
-
-static struct ng_queue_entry   *ngqbase;	/* items to be unqueued */
-static struct ng_queue_entry   *ngqlast;	/* last item queued */
-static const int		ngqroom = 64;	/* max items to queue */
-static int			ngqsize;	/* number of items in queue */
-
-static struct ng_queue_entry   *ngqfree;	/* free ones */
-static const int		ngqfreemax = 16;/* cache at most this many */
-static int			ngqfreesize;	/* number of cached entries */
 
+static int			allocated;	/* number of items malloc'd */
+static int			maxalloc = 128;	/* limit the damage of a leak */
+static const int		ngqfreemax = 64;/* cache at most this many */
+static const int		ngqfreelow = 4; /* try malloc if free < this */
+static volatile int		ngqfreesize;	/* number of cached entries */
+#ifdef	ITEM_DEBUG
+static TAILQ_HEAD(, ng_item) ng_itemlist = TAILQ_HEAD_INITIALIZER(ng_itemlist);
+#endif
 /*
  * Get a queue entry
+ * This is usually called when a packet first enters netgraph.
+ * By definition, this is usually from an interrupt, or from a user.
+ * Users are not so important, but try be quick for the times that it's
+ * an interrupt. Use atomic operations to cope with collisions
+ * with interrupts and other processors. Assumes MALLOC is SMP safe.
+ * XXX If reserve is low, we should try to get 2 from malloc as this
+ * would indicate it often fails.
  */
-static struct ng_queue_entry *
+static item_p
 ng_getqblk(void)
 {
-	register struct ng_queue_entry *q;
-	int s;
+	item_p item = NULL;
 
-	/* Could be guarding against tty ints or whatever */
-	s = splhigh();
+	/*
+	 * Try get a cached queue block, or else allocate a new one
+	 * If we are less than our reserve, try malloc. If malloc
+	 * fails, then that's what the reserve is for...
+	 * Don't completely trust ngqfreesize, as it is subject
+	 * to races.. (it'll eventually catch up but may be out by one or two
+	 * for brief moments(under SMP or interrupts).
+	 * ngqfree is the final arbiter. We have our little reserve
+	 * because we use M_NOWAIT for malloc. This just helps us
+	 * avoid dropping packets while not increasing the time
+	 * we take to service the interrupt (on average) (we hope).
+	 */
+	for (;;) {
+		if ((ngqfreesize < ngqfreelow) || (ngqfree == NULL)) {
+			if (allocated < maxalloc) {  /* don't leak forever */
+				MALLOC(item, item_p ,
+				    sizeof(*item), M_NETGRAPH_ITEM,
+				    (M_NOWAIT | M_ZERO));
+				if (item) {
+#ifdef	ITEM_DEBUG
+					TAILQ_INSERT_TAIL(&ng_itemlist,
+								item, all);
+#endif	/* ITEM_DEBUG */
+					atomic_add_int(&allocated, 1);
+					break;
+				}
+			}
+		}
 
-	/* Try get a cached queue block, or else allocate a new one */
-	if ((q = ngqfree) == NULL) {
-		splx(s);
-		if (ngqsize < ngqroom) {	/* don't worry about races */
-			MALLOC(q, struct ng_queue_entry *,
-			    sizeof(*q), M_NETGRAPH, M_NOWAIT);
+		/*
+		 * We didn't or couldn't malloc.
+		 * try get one from our cache.
+		 * item must be NULL to get here.
+		 */
+		if ((item = ngqfree) != NULL) {
+			/*
+			 * Atomically try grab the first item
+			 * and put it's successor in its place.
+			 * If we fail, just try again.. someone else
+			 * beat us to this one or freed one.
+			 * Don't worry about races with ngqfreesize.
+			 * Close enough is good enough..
+			 */
+			if (atomic_cmpset_ptr(&ngqfree, item, item->el_next)) {
+				atomic_subtract_int(&ngqfreesize, 1);
+				break;
+			}
+			item = NULL;
+		} else {
+			/* We really ran out */
+			break;
 		}
-	} else {
-		ngqfree = q->next;
-		ngqfreesize--;
-		splx(s);
 	}
-	return (q);
+	item->el_flags &= ~NGQF_FREE;
+	return (item);
 }
 
 /*
  * Release a queue entry
  */
-#define RETURN_QBLK(q)							\
-do {									\
-	int s;								\
-	if (ngqfreesize < ngqfreemax) { /* don't worry about races */ 	\
-		s = splhigh();						\
-		(q)->next = ngqfree;					\
-		ngqfree = (q);						\
-		ngqfreesize++;						\
-		splx(s);						\
-	} else {							\
-		FREE((q), M_NETGRAPH);					\
-	}								\
-} while (0)
-
-/*
- * Running at a raised (but we don't know which) processor priority level,
- * put the data onto a queue to be picked up by another PPL (probably splnet)
- */
-int
-ng_queue_data(hook_p hook, struct mbuf *m, meta_p meta)
+void
+ng_free_item(item_p item)
 {
-	struct ng_queue_entry *q;
-	int s;
 
-	if (hook == NULL) {
-		NG_FREE_DATA(m, meta);
-		return (0);
+	/*
+	 * The item may hold resources on it's own. We need to free
+	 * these before we can free the item. What they are depends upon
+	 * what kind of item it is. it is important that nodes zero
+	 * out pointers to resources that they remove from the item
+	 * or we release them again here.
+	 */
+	if (item->el_flags & NGQF_FREE) {
+		panic(" Freeing free queue item");
+	}
+	switch (item->el_flags & NGQF_D_M) {
+	case NGQF_DATA:
+		/* If we have an mbuf and metadata still attached.. */
+		NG_FREE_M(_NGI_M(item));
+		NG_FREE_META(_NGI_META(item));
+		break;
+	case NGQF_MESG:
+		_NGI_RETADDR(item) = NULL;
+		NG_FREE_MSG(_NGI_MSG(item));
+		break;
 	}
-	if ((q = ng_getqblk()) == NULL) {
-		NG_FREE_DATA(m, meta);
-		return (ENOBUFS);
+		/* If we still have a node or hook referenced... */
+	if (item->el_dest) {
+		ng_unref(item->el_dest);
+		item->el_dest = NULL;
 	}
+	if (item->el_hook) {
+		ng_unref_hook(item->el_hook);
+		item->el_hook = NULL;
+	}
+	item->el_flags |= NGQF_FREE;
 
-	/* Fill out the contents */
-	q->flags = NGQF_DATA;
-	q->next = NULL;
-	q->body.data.da_hook = hook;
-	q->body.data.da_m = m;
-	q->body.data.da_meta = meta;
-	s = splhigh();		/* protect refs and queue */
-	hook->refs++;		/* don't let it go away while on the queue */
-
-	/* Put it on the queue */
-	if (ngqbase) {
-		ngqlast->next = q;
+	/*
+	 * We have freed any resources held by the item.
+	 * now we can free the item itself.
+	 */
+	if (ngqfreesize < ngqfreemax) { /* don't worry about races */
+		for (;;) {
+			item->el_next = ngqfree;
+			if (atomic_cmpset_ptr(&ngqfree, item->el_next, item)) {
+				break;
+			}
+		}
+		atomic_add_int(&ngqfreesize, 1);
 	} else {
-		ngqbase = q;
+		/* This is the only place that should use this Macro */
+#ifdef	ITEM_DEBUG
+		TAILQ_REMOVE(&ng_itemlist, item, all);
+#endif	/* ITEM_DEBUG */
+		NG_FREE_ITEM_REAL(item);
+		atomic_subtract_int(&allocated, 1);
 	}
-	ngqlast = q;
-	ngqsize++;
-	splx(s);
+}
 
-	/* Schedule software interrupt to handle it later */
-	schednetisr(NETISR_NETGRAPH);
-	return (0);
+#ifdef	ITEM_DEBUG
+void
+dumpitem(item_p item, char *file, int line)
+{
+	if (item->el_flags & NGQF_FREE) {
+		printf(" Free item, freed at %s, line %d\n",
+			item->lastfile, item->lastline);
+	} else {
+		printf(" ACTIVE item, last used at %s, line %d",
+			item->lastfile, item->lastline);
+		if ((item->el_flags & NGQF_D_M) == NGQF_MESG) {
+			printf(" - retaddr[%d]:\n", _NGI_RETADDR(item));
+		} else {
+			printf(" - [data]\n");
+		}
+	}
+	printf(" problem discovered at file %s, line %d\n", file, line);
+	if (item->el_dest)
+		printf("node %X ([%x])\n",
+			item->el_dest, ng_node2ID(item->el_dest));
 }
 
+static int
+sysctl_debug_ng_dump_items(SYSCTL_HANDLER_ARGS)
+{
+	int error;
+	int val;
+	item_p item;
+	int i;
+
+	val = allocated;
+	i = 1;
+	error = sysctl_handle_int(oidp, &val, sizeof(int), req);
+	TAILQ_FOREACH(item, &ng_itemlist, all) {
+		if (item->el_flags & NGQF_FREE) {
+			printf("[%d] free item, freed at %s, line %d\n",
+				i++, item->lastfile, item->lastline);
+		} else {
+			printf("[%d] ACTIVE item, last used at %s, line %d",
+				i++, item->lastfile, item->lastline);
+			if ((item->el_flags & NGQF_D_M) == NGQF_MESG) {
+				printf(" - retaddr[%d]:\n", _NGI_RETADDR(item));
+			} else {
+				printf(" - [data]\n");
+			}
+		}
+		if (item->el_dest) {
+			printf("node %X ([%x])",
+				item->el_dest, ng_node2ID(item->el_dest));
+			printf("<%X>\n",item->el_dest->input_queue.q_flags);
+		} 
+	}
+	return error;
+}
+
+SYSCTL_PROC(_debug, OID_AUTO, ng_dump_items, CTLTYPE_INT | CTLFLAG_RD,
+    0, 0, sysctl_debug_ng_dump_items, "I", "Number of allocated items");
+#endif	/* ITEM_DEBUG */
+
+
+/***********************************************************************
+* Worklist routines
+**********************************************************************/
+/* NETISR thread enters here */
 /*
- * Running at a raised (but we don't know which) processor priority level,
- * put the msg onto a queue to be picked up by another PPL (probably splnet)
- * Either specify an address, or a hook to traverse.
- * The return address can be specified, or it will be pointed at this node.
+ * Pick a node off the list of nodes with work,
+ * try get an item to process off it.
+ * If there are no more, remove the node from the list.
  */
-int
-ng_queue_msg(node_p here, struct ng_mesg *msg, const char *address, hook_p hook,char *retaddr)
+static void
+ngintr(void)
 {
-	register struct ng_queue_entry *q;
-	int     s;
-	node_p  dest = NULL;
-	int     error;
-	hook_p	lasthook = NULL;
-
-	/*
-	 * Find the target node.
-	 * If there is a HOOK argument, then use that in preference
-	 * to the address.
-	 */
-	if (hook) {
-		lasthook = hook->peer;
-		dest = lasthook->node;
-	} else {
-		error = ng_path2node(here, address, &dest, &lasthook);
-		if (error) {
-			FREE(msg, M_NETGRAPH);
-			return (error);
+	item_p item;
+	node_p  node = NULL;
+
+	for (;;) {
+		mtx_enter(&ng_worklist_mtx, MTX_SPIN);
+		node = TAILQ_FIRST(&ng_worklist);
+		if (!node) {
+			mtx_exit(&ng_worklist_mtx, MTX_SPIN);
+			break;
+		}
+		TAILQ_REMOVE(&ng_worklist, node, work);
+		mtx_exit(&ng_worklist_mtx, MTX_SPIN);
+		/*
+		 * We have the node. We also take over the reference
+		 * that the list had on it.
+		 * Now process as much as you can, until it won't
+		 * let you have another item off the queue.
+		 * All this time, keep the reference
+		 * that lets us be sure that the node still exists.
+		 * Let the reference go at the last minute.
+		 */
+		for (;;) {
+			mtx_enter(&node->input_queue.q_mtx, MTX_SPIN);
+			item = ng_dequeue(&node->input_queue);
+			if (item == NULL) {
+				/*
+				 * Say we are on the queue as long as
+				 * we are processing it here.
+				 * it probably wouldn't come here while we
+				 * are processing anyhow.
+				 */
+				node->flags &= ~NG_WORKQ;	
+				mtx_exit(&node->input_queue.q_mtx, MTX_SPIN);
+				ng_unref(node);
+				break; /* go look for another node */
+			} else {
+				mtx_exit(&node->input_queue.q_mtx, MTX_SPIN);
+#ifdef	ITEM_DEBUG
+        _ngi_check(item, __FILE__, __LINE__);
+#endif
+				ng_apply_item(node, item);
+			}
 		}
 	}
+}
+
+static void
+ng_worklist_remove(node_p node)
+{
+	mtx_enter(&ng_worklist_mtx, MTX_SPIN);
+	if (node->flags & NG_WORKQ) {
+		TAILQ_REMOVE(&ng_worklist, node, work);
+		ng_unref(node);
+	}
+	node->flags &= ~NG_WORKQ;
+	mtx_exit(&ng_worklist_mtx, MTX_SPIN);
+}
 
-	if (retaddr == NULL) {
+static void
+ng_setisr(node_p node)
+{
+	mtx_enter(&ng_worklist_mtx, MTX_SPIN);
+	if ((node->flags & NG_WORKQ) == 0) {
 		/*
-		 * Now fill out the return address,
-		 * i.e. the name/ID of the sender. (If we didn't get one)
+		 * If we are not already on the work queue,
+		 * then put us on.
 		 */
-		MALLOC(retaddr, char *, NG_NODELEN + 2, M_NETGRAPH, M_NOWAIT);
-		if (retaddr == NULL) {
-			TRAP_ERROR;
-			return (ENOMEM);
+		node->flags |= NG_WORKQ;
+		TAILQ_INSERT_TAIL(&ng_worklist, node, work);
+		node->refs++;
+	}
+	mtx_exit(&ng_worklist_mtx, MTX_SPIN);
+	schednetisr(NETISR_NETGRAPH);
+}
+
+
+/***********************************************************************
+* Externally useable functions to set up a queue item ready for sending
+***********************************************************************/
+
+#ifdef	ITEM_DEBUG
+#define	DEBUG_CHECKS							\
+	do {								\
+		if (item->el_dest ) {					\
+			printf("item already has node");		\
+			Debugger("has node");				\
+			ng_unref(item->el_dest);			\
+			item->el_dest = NULL;				\
+		}							\
+		if (item->el_hook ) {					\
+			printf("item already has hook");		\
+			Debugger("has hook");				\
+			ng_unref_hook(item->el_hook);			\
+			item->el_hook = NULL;				\
+		}							\
+	} while (0)
+#else
+#define DEBUG_CHECKS
+#endif
+
+/*
+ * Put elements into the item.
+ * Hook and node references will be removed when the item is dequeued.
+ * (or equivalent)
+ * (XXX) Unsafe because no reference held by peer on remote node.
+ * remote node might go away in this timescale.
+ * We know the hooks can't go away because that would require getting
+ * a writer item on both nodes and we must have at least a  reader
+ * here to eb able to do this.
+ * Note that the hook loaded is the REMOTE hook.
+ *
+ * This is possibly in the critical path for new data.
+ */
+item_p
+ng_package_data(struct mbuf *m, meta_p meta)
+{
+	item_p item;
+
+	if ((item = ng_getqblk()) == NULL) {
+		NG_FREE_M(m);
+		NG_FREE_META(meta);
+		return (NULL);
+	}
+	DEBUG_CHECKS;
+	item->el_flags = NGQF_DATA;
+	item->el_next = NULL;
+	NGI_M(item) = m;
+	NGI_META(item) = meta;
+	return (item);
+}
+
+/*
+ * Allocate a queue item and put items into it..
+ * Evaluate the address as this will be needed to queue it and
+ * to work out what some of the fields should be.
+ * Hook and node references will be removed when the item is dequeued.
+ * (or equivalent)
+ */
+item_p
+ng_package_msg(struct ng_mesg *msg)
+{
+	item_p item;
+
+	if ((item = ng_getqblk()) == NULL) {
+		if ((msg->header.flags & NGF_STATIC) == 0) {
+			NG_FREE_MSG(msg);
 		}
-		if (here->name != NULL)
-			sprintf(retaddr, "%s:", here->name);
-		else
-			sprintf(retaddr, "[%x]:", ng_node2ID(here));
+		return (NULL);
 	}
+	DEBUG_CHECKS;
+	item->el_flags = NGQF_MESG;
+	item->el_next = NULL;
+	/*
+	 * Set the current lasthook into the queue item
+	 */
+	NGI_MSG(item) = msg;
+	NGI_RETADDR(item) = NULL;
+	return (item);
+}
+
+
+
+#define SET_RETADDR							\
+	do {	/* Data items don't have retaddrs */			\
+		if ((item->el_flags & NGQF_D_M) == NGQF_MESG) {		\
+			if (retaddr) {					\
+				NGI_RETADDR(item) = retaddr;		\
+			} else {					\
+				/*					\
+				 * The old return address should be ok.	\
+				 * If there isn't one, use the address	\
+				 * here.				\
+				 */					\
+				if (NGI_RETADDR(item) == 0) {		\
+					NGI_RETADDR(item)		\
+						= ng_node2ID(here);	\
+				}					\
+			}						\
+		}							\
+	} while (0)
 
-	if ((q = ng_getqblk()) == NULL) {
-		FREE(msg, M_NETGRAPH);
-		if (retaddr)
-			FREE(retaddr, M_NETGRAPH);
-		return (ENOBUFS);
+int
+ng_address_hook(node_p here, item_p item, hook_p hook, ng_ID_t retaddr)
+{
+	DEBUG_CHECKS;
+	/*
+	 * Quick sanity check..
+	 */
+	if ((hook == NULL)
+	|| ((hook->flags & HK_INVALID) != 0)
+	|| (hook->peer == NULL)
+	|| ((hook->peer->flags & HK_INVALID) != 0)
+	|| ((hook->peer->node->flags & NG_INVALID) != 0)) {
+		NG_FREE_ITEM(item);
+		return (EINVAL);
 	}
 
-	/* Fill out the contents */
-	q->flags = NGQF_MESG;
-	q->next = NULL;
-	q->body.msg.msg_node = dest;
-	q->body.msg.msg_msg = msg;
-	q->body.msg.msg_retaddr = retaddr; /* XXX malloc'd, give it away */
-	q->body.msg.msg_lasthook = lasthook; /* XXX needs reference */
-	s = splhigh();		/* protect refs and queue */
-	dest->refs++;		/* don't let it go away while on the queue */
-	if (lasthook)
-		lasthook->refs++; /* same for the hook */
-
-	/* Put it on the queue */
-	if (ngqbase) {
-		ngqlast->next = q;
-	} else {
-		ngqbase = q;
+	/*
+	 * Transfer our interest to the other (peer) end.
+	 * note sleazy use of 'hook'.
+	 */
+	item->el_hook = hook->peer;
+	item->el_hook->refs++;	/* don't let it go away while on the queue */
+	item->el_dest = hook->peer->node; /* sleaze */
+	item->el_dest->refs++; /* XXX dangerous, not atomic */
+	SET_RETADDR;
+	return (0);
+}
+
+int
+ng_address_path(node_p here, item_p item, char *address, ng_ID_t retaddr) 
+{
+	node_p  dest = NULL;
+	hook_p	hook = NULL;
+	int     error;
+
+	DEBUG_CHECKS;
+	/*
+	 * Note that ng_path2noderef increments the reference count
+	 * on the node for us if it finds one. So we don't have to.
+	 */
+	error = ng_path2noderef(here, address, &dest, &hook);
+	if (error) {
+		NG_FREE_ITEM(item);
+		return (EINVAL);
 	}
-	ngqlast = q;
-	ngqsize++;
-	splx(s);
+	item->el_dest = dest;
+	if (( item->el_hook = hook))
+		hook->refs++;	/* don't let it go away while on the queue */
+	SET_RETADDR;
+	return (0);
+}
 
-	/* Schedule software interrupt to handle it later */
-	schednetisr(NETISR_NETGRAPH);
+int
+ng_address_ID(node_p here, item_p item, ng_ID_t ID, ng_ID_t retaddr)
+{
+	node_p dest;
+
+	DEBUG_CHECKS;
+	/*
+	 * Find the target node.
+	 */
+	dest = ng_ID2noderef(ID); /* GETS REFERENCE! */
+	if (dest == NULL) {
+		NG_FREE_ITEM(item);
+		return(EINVAL);
+	}
+	/* Fill out the contents */
+	item->el_flags = NGQF_MESG;
+	item->el_next = NULL;
+	item->el_dest = dest;
+	item->el_hook = NULL;
+	/* NGI_RETADDR(item) = ng_node2ID(here); not sure why its here XXX */
+	SET_RETADDR;
 	return (0);
 }
 
 /*
- * Pick an item off the queue, process it, and dispose of the queue entry.
- * Should be running at splnet.
+ * special case to send a message to self (e.g. destroy node)
+ * Possibly indicate an arrival hook too.
+ * Useful for removing that hook :-)
  */
-static void
-ngintr(void)
+item_p
+ng_package_msg_self(node_p here, hook_p hook, struct ng_mesg *msg)
 {
-	hook_p  hook;
-	struct ng_queue_entry *ngq;
-	struct mbuf *m;
-	meta_p  meta;
-	void   *retaddr;
-	struct ng_mesg *msg;
-	node_p  node;
-	int     error = 0;
-	int     s;
+	item_p item;
 
-	while (1) {
-		s = splhigh();
-		if ((ngq = ngqbase)) {
-			ngqbase = ngq->next;
-			ngqsize--;
-		}
-		splx(s);
-		if (ngq == NULL)
-			return;
-		switch (ngq->flags) {
-		case NGQF_DATA:
-			hook = ngq->body.data.da_hook;
-			m = ngq->body.data.da_m;
-			meta = ngq->body.data.da_meta;
-			RETURN_QBLK(ngq);
-			ng_send_data_dont_queue(hook, m, meta,
-				NULL, NULL, NULL);
-			m = NULL;
-			meta = NULL;
-			ng_unref_hook(hook);
-			break;
-		case NGQF_MESG:
-			node = ngq->body.msg.msg_node;
-			msg = ngq->body.msg.msg_msg;
-			retaddr = ngq->body.msg.msg_retaddr;
-			hook = ngq->body.msg.msg_lasthook;
-			RETURN_QBLK(ngq);
-			if (hook) {
-				if ((hook->flags & HK_INVALID) != 0) {
-				/* If the hook has been zapped
-					then we can't use it */
-					ng_unref_hook(hook);
-					hook = NULL;
-				}
-			}
-			/* similarly, if the node is a zombie.. */
-			if (node->flags & NG_INVALID) {
-				FREE(msg, M_NETGRAPH);
-			} else {
-				CALL_MSG_HANDLER(error, node, msg,
-						 retaddr, NULL, hook);
-			}
-			if (hook)
-				ng_unref_hook(hook);
-			ng_unref(node);
-			if (retaddr)
-				FREE(retaddr, M_NETGRAPH);
-			break;
-		default:
-			RETURN_QBLK(ngq);
+	/*
+	 * Find the target node.
+	 * If there is a HOOK argument, then use that in preference
+	 * to the address.
+	 */
+	if ((item = ng_getqblk()) == NULL) {
+		if ((msg->header.flags & NGF_STATIC) == 0) {
+			NG_FREE_MSG(msg);
 		}
+		return (NULL);
 	}
+
+	/* Fill out the contents */
+	item->el_flags = NGQF_MESG;
+	item->el_next = NULL;
+	item->el_dest = here;
+	here->refs++; /* XXX  not atomic, + May have other races */
+	item->el_hook = hook;
+	if (hook)
+		hook->refs++;
+	NGI_MSG(item) = msg;
+	NGI_RETADDR(item) = ng_node2ID(here);
+	return (item);
 }
 
+/*
+ * Set the address, if none given, give the node here.
+ */
+void
+ng_replace_retaddr(node_p here, item_p item, ng_ID_t retaddr)
+{
+	if (retaddr) {
+		NGI_RETADDR(item) = retaddr;
+	} else {
+		/*
+		 * The old return address should be ok.
+		 * If there isn't one, use the address here.
+		 */
+		NGI_RETADDR(item) = ng_node2ID(here);
+	}
+}
+
+#define TESTING
+#ifdef TESTING
+/* just test all the macros */
+void
+ng_macro_test(item_p item);
+void
+ng_macro_test(item_p item)
+{
+	node_p node = NULL;
+	hook_p hook = NULL;
+	struct mbuf *m;
+	meta_p meta;
+	struct ng_mesg *msg;
+	ng_ID_t retaddr;
+	int	error;
+
+	NGI_GET_M(item, m);
+	NGI_GET_META(item, meta);
+	NGI_GET_MSG(item, msg);
+	retaddr = NGI_RETADDR(item);
+	NG_SEND_DATA(error, hook, m, meta);
+	NG_SEND_DATA_ONLY(error, hook, m);
+	NG_FWD_NEW_DATA(error, item, hook, m);
+	NG_FWD_DATA(error, item, hook);
+	NG_SEND_MSG_HOOK(error, node, msg, hook, retaddr);
+	NG_SEND_MSG_ID(error, node, msg, retaddr, retaddr);
+	NG_SEND_MSG_PATH(error, node, msg, ".:", retaddr);
+	NG_QUEUE_MSG(error, node, msg, ".:", retaddr);
+	NG_FWD_MSG_HOOK(error, node, item, hook, retaddr);
+}
+#endif /* TESTING */
 
diff --git a/sys/netgraph/ng_bpf.c b/sys/netgraph/ng_bpf.c
index 6d19550..33b8384 100644
--- a/sys/netgraph/ng_bpf.c
+++ b/sys/netgraph/ng_bpf.c
@@ -83,7 +83,7 @@ typedef struct ng_bpf_hookinfo *hinfo_p;
 /* Netgraph methods */
 static ng_constructor_t	ng_bpf_constructor;
 static ng_rcvmsg_t	ng_bpf_rcvmsg;
-static ng_shutdown_t	ng_bpf_rmnode;
+static ng_shutdown_t	ng_bpf_shutdown;
 static ng_newhook_t	ng_bpf_newhook;
 static ng_rcvdata_t	ng_bpf_rcvdata;
 static ng_disconnect_t	ng_bpf_disconnect;
@@ -191,7 +191,7 @@ static struct ng_type typestruct = {
 	NULL,
 	ng_bpf_constructor,
 	ng_bpf_rcvmsg,
-	ng_bpf_rmnode,
+	ng_bpf_shutdown,
 	ng_bpf_newhook,
 	NULL,
 	NULL,
@@ -214,15 +214,12 @@ static const struct ng_bpf_hookprog ng_bpf_default_prog = {
  * Node constructor
  *
  * We don't keep any per-node private data
+ * We go via the hooks.
  */
 static int
-ng_bpf_constructor(node_p *nodep)
+ng_bpf_constructor(node_p node)
 {
-	int error = 0;
-
-	if ((error = ng_make_node_common(&typestruct, nodep)))
-		return (error);
-	(*nodep)->private = NULL;
+	node->private = NULL;
 	return (0);
 }
 
@@ -260,12 +257,13 @@ ng_bpf_newhook(node_p node, hook_p hook, const char *name)
  * Receive a control message
  */
 static int
-ng_bpf_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
-	   struct ng_mesg **rptr, hook_p lasthook)
+ng_bpf_rcvmsg(node_p node, item_p item, hook_p lasthook)
 {
+	struct ng_mesg *msg;
 	struct ng_mesg *resp = NULL;
 	int error = 0;
 
+	NGI_GET_MSG(item, msg);
 	switch (msg->header.typecookie) {
 	case NGM_BPF_COOKIE:
 		switch (msg->header.cmd) {
@@ -357,13 +355,11 @@ ng_bpf_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
 		error = EINVAL;
 		break;
 	}
-	if (rptr)
-		*rptr = resp;
-	else if (resp)
-		FREE(resp, M_NETGRAPH);
-
+	NG_RESPOND_MSG(error, node, item, resp);
 done:
-	FREE(msg, M_NETGRAPH);
+	if (item)
+		NG_FREE_ITEM(item);
+	NG_FREE_MSG(msg);
 	return (error);
 }
 
@@ -373,19 +369,23 @@ done:
  * Apply the filter, and then drop or forward packet as appropriate.
  */
 static int
-ng_bpf_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
+ng_bpf_rcvdata(hook_p hook, item_p item)
 {
 	const hinfo_p hip = hook->private;
-	int totlen = m->m_pkthdr.len;
+	int totlen;
 	int needfree = 0, error = 0;
 	u_char *data, buf[256];
 	hinfo_p dhip;
 	hook_p dest;
 	u_int len;
-
-	/* Update stats on incoming hook */
-	hip->stats.recvFrames++;
+	struct mbuf *m;
+
+	m = NGI_M(item);	/* 'item' still owns it.. we are peeking */ 
+	totlen = m->m_pkthdr.len;
+	/* Update stats on incoming hook. XXX Can we do 64 bits atomically? */
+	/* atomic_add_int64(&hip->stats.recvFrames, 1); */
+	/* atomic_add_int64(&hip->stats.recvOctets, totlen); */
+	hip->stats.recvFrames++; 
 	hip->stats.recvOctets += totlen;
 
 	/* Need to put packet in contiguous memory for bpf */
@@ -393,7 +393,7 @@ ng_bpf_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 		if (totlen > sizeof(buf)) {
 			MALLOC(data, u_char *, totlen, M_NETGRAPH, M_NOWAIT);
 			if (data == NULL) {
-				NG_FREE_DATA(m, meta);
+				NG_FREE_ITEM(item);
 				return (ENOMEM);
 			}
 			needfree = 1;
@@ -412,10 +412,12 @@ ng_bpf_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 	if (len > 0) {
 
 		/* Update stats */
+		/* XXX atomically? */
 		hip->stats.recvMatchFrames++;
 		hip->stats.recvMatchOctets += totlen;
 
 		/* Truncate packet length if required by the filter */
+		/* Assume this never changes m */
 		if (len < totlen) {
 			m_adj(m, -(totlen - len));
 			totlen -= len;
@@ -424,7 +426,7 @@ ng_bpf_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 	} else
 		dest = ng_findhook(hip->node, hip->prog->ifNotMatch);
 	if (dest == NULL) {
-		NG_FREE_DATA(m, meta);
+		NG_FREE_ITEM(item);
 		return (0);
 	}
 
@@ -432,7 +434,7 @@ ng_bpf_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 	dhip = (hinfo_p)dest->private;
 	dhip->stats.xmitOctets += totlen;
 	dhip->stats.xmitFrames++;
-	NG_SEND_DATA(error, dest, m, meta);
+	NG_FWD_DATA(error, item, dest);
 	return (error);
 }
 
@@ -440,11 +442,9 @@ ng_bpf_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
  * Shutdown processing
  */
 static int
-ng_bpf_rmnode(node_p node)
+ng_bpf_shutdown(node_p node)
 {
 	node->flags |= NG_INVALID;
-	ng_cutlinks(node);
-	ng_unname(node);
 	ng_unref(node);
 	return (0);
 }
@@ -462,8 +462,10 @@ ng_bpf_disconnect(hook_p hook)
 	bzero(hip, sizeof(*hip));
 	FREE(hip, M_NETGRAPH);
 	hook->private = NULL;			/* for good measure */
-	if (hook->node->numhooks == 0)
-		ng_rmnode(hook->node);
+	if ((hook->node->numhooks == 0)
+	&& ((hook->node->flags && NG_INVALID) == 0)) {
+		ng_rmnode_self(hook->node);
+	}
 	return (0);
 }
 
diff --git a/sys/netgraph/ng_bridge.c b/sys/netgraph/ng_bridge.c
index 59ba0cc..9539757 100644
--- a/sys/netgraph/ng_bridge.c
+++ b/sys/netgraph/ng_bridge.c
@@ -112,7 +112,7 @@ SLIST_HEAD(ng_bridge_bucket, ng_bridge_hent);
 /* Netgraph node methods */
 static ng_constructor_t	ng_bridge_constructor;
 static ng_rcvmsg_t	ng_bridge_rcvmsg;
-static ng_shutdown_t	ng_bridge_rmnode;
+static ng_shutdown_t	ng_bridge_shutdown;
 static ng_newhook_t	ng_bridge_newhook;
 static ng_rcvdata_t	ng_bridge_rcvdata;
 static ng_disconnect_t	ng_bridge_disconnect;
@@ -271,7 +271,7 @@ static struct ng_type ng_bridge_typestruct = {
 	NULL,
 	ng_bridge_constructor,
 	ng_bridge_rcvmsg,
-	ng_bridge_rmnode,
+	ng_bridge_shutdown,
 	ng_bridge_newhook,
 	NULL,
 	NULL,
@@ -292,10 +292,9 @@ MODULE_DEPEND(ng_bridge, ng_ether, 1, 1, 1);
  * Node constructor
  */
 static int
-ng_bridge_constructor(node_p *nodep)
+ng_bridge_constructor(node_p node)
 {
 	priv_p priv;
-	int error;
 
 	/* Allocate and initialize private info */
 	MALLOC(priv, priv_p, sizeof(*priv), M_NETGRAPH, M_NOWAIT | M_ZERO);
@@ -317,17 +316,21 @@ ng_bridge_constructor(node_p *nodep)
 	priv->conf.maxStaleness = DEFAULT_MAX_STALENESS;
 	priv->conf.minStableAge = DEFAULT_MIN_STABLE_AGE;
 
-	/* Call superclass constructor */
-	if ((error = ng_make_node_common(&ng_bridge_typestruct, nodep))) {
-		FREE(priv, M_NETGRAPH);
-		return (error);
-	}
-	(*nodep)->private = priv;
-	priv->node = *nodep;
+	/*
+	 * This node has all kinds of stuff that could be screwed by SMP.
+	 * Until it gets it's own internal protection, we go through in 
+	 * single file. This could hurt a machine bridging beteen two 
+	 * GB ethernets so it should be fixed. 
+	 * When it's fixed the process SHOULD NOT SLEEP, spinlocks please!
+	 * (and atomic ops )
+	 */
+	node->flags |= NG_FORCE_WRITER;
+	node->private = priv;
+	priv->node = node;
 
 	/* Start timer by faking a timeout event */
-	(*nodep)->refs++;
-	ng_bridge_timeout(*nodep);
+	node->refs++; /* XXX ????  because of the timeout?*/
+	ng_bridge_timeout(node);
 	return (0);
 }
 
@@ -372,13 +375,14 @@ ng_bridge_newhook(node_p node, hook_p hook, const char *name)
  * Receive a control message
  */
 static int
-ng_bridge_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
-		struct ng_mesg **rptr, hook_p lasthook)
+ng_bridge_rcvmsg(node_p node, item_p item, hook_p lasthook)
 {
 	const priv_p priv = node->private;
 	struct ng_mesg *resp = NULL;
 	int error = 0;
+	struct ng_mesg *msg;
 
+	NGI_GET_MSG(item, msg);
 	switch (msg->header.typecookie) {
 	case NGM_BRIDGE_COOKIE:
 		switch (msg->header.cmd) {
@@ -497,11 +501,8 @@ ng_bridge_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
 	}
 
 	/* Done */
-	if (rptr)
-		*rptr = resp;
-	else if (resp != NULL)
-		FREE(resp, M_NETGRAPH);
-	FREE(msg, M_NETGRAPH);
+	NG_RESPOND_MSG(error, node, item, resp);
+	NG_FREE_MSG(msg);
 	return (error);
 }
 
@@ -509,8 +510,7 @@ ng_bridge_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
  * Receive data on a hook
  */
 static int
-ng_bridge_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
+ng_bridge_rcvdata(hook_p hook, item_p item)
 {
 	const node_p node = hook->node;
 	const priv_p priv = node->private;
@@ -518,8 +518,12 @@ ng_bridge_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 	struct ng_bridge_link *link;
 	struct ether_header *eh;
 	int error = 0, linkNum;
-	int i, manycast;
+	int manycast;
+	struct mbuf *m;
+	meta_p meta;
+	struct ng_bridge_link *firstLink;
 
+	NGI_GET_M(item, m);
 	/* Get link number */
 	linkNum = LINK_NUM(hook);
 	KASSERT(linkNum >= 0 && linkNum < NG_BRIDGE_MAX_LINKS,
@@ -530,25 +534,28 @@ ng_bridge_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 	/* Sanity check packet and pull up header */
 	if (m->m_pkthdr.len < ETHER_HDR_LEN) {
 		link->stats.recvRunts++;
-		NG_FREE_DATA(m, meta);
+		NG_FREE_ITEM(item);
+		NG_FREE_M(m);
 		return (EINVAL);
 	}
 	if (m->m_len < ETHER_HDR_LEN && !(m = m_pullup(m, ETHER_HDR_LEN))) {
 		link->stats.memoryFailures++;
-		NG_FREE_META(meta);
+		NG_FREE_ITEM(item);
 		return (ENOBUFS);
 	}
 	eh = mtod(m, struct ether_header *);
 	if ((eh->ether_shost[0] & 1) != 0) {
 		link->stats.recvInvalid++;
-		NG_FREE_DATA(m, meta);
+		NG_FREE_ITEM(item);
+		NG_FREE_M(m);
 		return (EINVAL);
 	}
 
 	/* Is link disabled due to a loopback condition? */
 	if (link->loopCount != 0) {
 		link->stats.loopDrops++;
-		NG_FREE_DATA(m, meta);
+		NG_FREE_ITEM(item);
+		NG_FREE_M(m);
 		return (ELOOP);		/* XXX is this an appropriate error? */
 	}
 
@@ -605,7 +612,8 @@ ng_bridge_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 
 				/* Drop packet */
 				link->stats.loopDrops++;
-				NG_FREE_DATA(m, meta);
+				NG_FREE_ITEM(item);
+				NG_FREE_M(m);
 				return (ELOOP);		/* XXX appropriate? */
 			}
 
@@ -616,7 +624,8 @@ ng_bridge_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 	} else {
 		if (!ng_bridge_put(priv, eh->ether_shost, linkNum)) {
 			link->stats.memoryFailures++;
-			NG_FREE_DATA(m, meta);
+			NG_FREE_ITEM(item);
+			NG_FREE_M(m);
 			return (ENOMEM);
 		}
 	}
@@ -641,14 +650,15 @@ ng_bridge_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 			KASSERT(destLink != NULL,
 			    ("%s: link%d null", __FUNCTION__, host->linkNum));
 			if (destLink == link) {
-				NG_FREE_DATA(m, meta);
+				NG_FREE_ITEM(item);
+				NG_FREE_M(m);
 				return (0);
 			}
 
 			/* Deliver packet out the destination link */
 			destLink->stats.xmitPackets++;
 			destLink->stats.xmitOctets += m->m_pkthdr.len;
-			NG_SEND_DATA(error, destLink->hook, m, meta);
+			NG_FWD_NEW_DATA(error, item, destLink->hook, m);
 			return (error);
 		}
 
@@ -657,31 +667,58 @@ ng_bridge_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 	}
 
 	/* Distribute unknown, multicast, broadcast pkts to all other links */
-	for (linkNum = i = 0; i < priv->numLinks - 1; linkNum++) {
-		struct ng_bridge_link *const destLink = priv->links[linkNum];
+	meta = NGI_META(item); /* peek.. */
+	firstLink = NULL;
+	for (linkNum = 0; linkNum <= priv->numLinks; linkNum++) {
+		struct ng_bridge_link *destLink;
 		meta_p meta2 = NULL;
-		struct mbuf *m2;
-
-		/* Skip incoming link and disconnected links */
-		if (destLink == NULL || destLink == link)
-			continue;
+		struct mbuf *m2 = NULL;
+
+		/*
+		 * If we have checked all the links then now
+		 * send the original on its reserved link
+		 */
+		if (linkNum == priv->numLinks) {
+			/* If we never saw a good link, leave. */
+			if (firstLink == NULL) {
+				NG_FREE_ITEM(item);
+				NG_FREE_M(m);
+				return (0);
+			}	
+			destLink = firstLink;
+		} else {
+			destLink = priv->links[linkNum];
+			/* Skip incoming link and disconnected links */
+			if (destLink == NULL || destLink == link) {
+				continue;
+			}
+			if (firstLink == NULL) {
+				/*
+				 * This is the first usable link we have found.
+				 * Reserve it for the originals.
+				 * If we never find another we save a copy.
+				 */
+				firstLink = destLink;
+				continue;
+			}
 
-		/* Copy mbuf and meta info */
-		if (++i == priv->numLinks - 1) {		/* last link */
-			m2 = m;
-			meta2 = meta;
-		}  else {
+			/*
+			 * It's usable link but not the reserved (first) one.
+			 * Copy mbuf and meta info for sending.
+			 */
 			m2 = m_dup(m, M_NOWAIT);	/* XXX m_copypacket() */
 			if (m2 == NULL) {
 				link->stats.memoryFailures++;
-				NG_FREE_DATA(m, meta);
+				NG_FREE_ITEM(item);
+				NG_FREE_M(m);
 				return (ENOBUFS);
 			}
 			if (meta != NULL
 			    && (meta2 = ng_copy_meta(meta)) == NULL) {
 				link->stats.memoryFailures++;
 				m_freem(m2);
-				NG_FREE_DATA(m, meta);
+				NG_FREE_ITEM(item);
+				NG_FREE_M(m);
 				return (ENOMEM);
 			}
 		}
@@ -701,7 +738,16 @@ ng_bridge_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 		}
 
 		/* Send packet */
-		NG_SEND_DATA(error, destLink->hook, m2, meta2);
+		if (destLink == firstLink) { 
+			/*
+			 * If we've sent all the others, send the original
+			 * on the first link we found.
+			 */
+			NG_FWD_NEW_DATA(error, item, destLink->hook, m);
+			break; /* always done last - not really needed. */
+		} else {
+			NG_SEND_DATA(error, destLink->hook, m2, meta2);
+		}
 	}
 	return (error);
 }
@@ -710,12 +756,10 @@ ng_bridge_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
  * Shutdown node
  */
 static int
-ng_bridge_rmnode(node_p node)
+ng_bridge_shutdown(node_p node)
 {
 	const priv_p priv = node->private;
 
-	ng_unname(node);
-	ng_cutlinks(node);		/* frees all link and host info */
 	KASSERT(priv->numLinks == 0 && priv->numHosts == 0,
 	    ("%s: numLinks=%d numHosts=%d",
 	    __FUNCTION__, priv->numLinks, priv->numHosts));
@@ -750,8 +794,9 @@ ng_bridge_disconnect(hook_p hook)
 	priv->numLinks--;
 
 	/* If no more hooks, go away */
-	if (hook->node->numhooks == 0)
-		ng_rmnode(hook->node);
+	if ((hook->node->numhooks == 0)
+	&& (( hook->node->flags & NG_INVALID) == 0))
+		ng_rmnode_self(hook->node);
 	return (0);
 }
 
diff --git a/sys/netgraph/ng_cisco.c b/sys/netgraph/ng_cisco.c
index 539e684..a926156 100644
--- a/sys/netgraph/ng_cisco.c
+++ b/sys/netgraph/ng_cisco.c
@@ -117,13 +117,13 @@ typedef struct cisco_priv *sc_p;
 /* Netgraph methods */
 static ng_constructor_t		cisco_constructor;
 static ng_rcvmsg_t		cisco_rcvmsg;
-static ng_shutdown_t		cisco_rmnode;
+static ng_shutdown_t		cisco_shutdown;
 static ng_newhook_t		cisco_newhook;
 static ng_rcvdata_t		cisco_rcvdata;
 static ng_disconnect_t		cisco_disconnect;
 
 /* Other functions */
-static int	cisco_input(sc_p sc, struct mbuf *m, meta_p meta);
+static int	cisco_input(sc_p sc, item_p item);
 static void	cisco_keepalive(void *arg);
 static int	cisco_send(sc_p sc, int type, long par1, long par2);
 
@@ -176,7 +176,7 @@ static struct ng_type typestruct = {
 	NULL,
 	cisco_constructor,
 	cisco_rcvmsg,
-	cisco_rmnode,
+	cisco_shutdown,
 	cisco_newhook,
 	NULL,
 	NULL,
@@ -190,22 +190,17 @@ NETGRAPH_INIT(cisco, &typestruct);
  * Node constructor
  */
 static int
-cisco_constructor(node_p *nodep)
+cisco_constructor(node_p node)
 {
 	sc_p sc;
-	int error = 0;
 
 	MALLOC(sc, sc_p, sizeof(*sc), M_NETGRAPH, M_NOWAIT | M_ZERO);
 	if (sc == NULL)
 		return (ENOMEM);
 
 	callout_handle_init(&sc->handle);
-	if ((error = ng_make_node_common(&typestruct, nodep))) {
-		FREE(sc, M_NETGRAPH);
-		return (error);
-	}
-	(*nodep)->private = sc;
-	sc->node = *nodep;
+	node->private = sc;
+	sc->node = node;
 
 	/* Initialise the varous protocol hook holders */
 	sc->downstream.af = 0xffff;
@@ -250,13 +245,14 @@ cisco_newhook(node_p node, hook_p hook, const char *name)
  * Receive control message.
  */
 static int
-cisco_rcvmsg(node_p node, struct ng_mesg *msg,
-	const char *retaddr, struct ng_mesg **rptr, hook_p lasthook)
+cisco_rcvmsg(node_p node, item_p item, hook_p lasthook)
 {
+	struct ng_mesg *msg;
 	const sc_p sc = node->private;
 	struct ng_mesg *resp = NULL;
 	int error = 0;
 
+	NGI_GET_MSG(item, msg);
 	switch (msg->header.typecookie) {
 	case NGM_GENERIC_COOKIE:
 		switch (msg->header.cmd) {
@@ -337,11 +333,8 @@ cisco_rcvmsg(node_p node, struct ng_mesg *msg,
 		error = EINVAL;
 		break;
 	}
-	if (rptr)
-		*rptr = resp;
-	else if (resp)
-		FREE(resp, M_NETGRAPH);
-	FREE(msg, M_NETGRAPH);
+	NG_RESPOND_MSG(error, node, item, resp);
+	NG_FREE_MSG(msg);
 	return (error);
 }
 
@@ -349,23 +342,24 @@ cisco_rcvmsg(node_p node, struct ng_mesg *msg,
  * Receive data
  */
 static int
-cisco_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
+cisco_rcvdata(hook_p hook, item_p item)
 {
 	const sc_p sc = hook->node->private;
 	struct protoent *pep;
 	struct cisco_header *h;
 	int error = 0;
+	struct mbuf *m;
 
 	if ((pep = hook->private) == NULL)
 		goto out;
 
 	/* If it came from our downlink, deal with it separately */
 	if (pep->af == 0xffff)
-		return (cisco_input(sc, m, meta));
+		return (cisco_input(sc, item));
 
 	/* OK so it came from a protocol, heading out. Prepend general data
 	   packet header. For now, IP,IPX only  */
+	m = NGI_M(item); /* still associated with item */
 	M_PREPEND(m, CISCO_HEADER_LEN, M_DONTWAIT);
 	if (!m) {
 		error = ENOBUFS;
@@ -394,11 +388,11 @@ cisco_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 	}
 
 	/* Send it */
-	NG_SEND_DATA(error, sc->downstream.hook, m, meta);
+	NG_FWD_NEW_DATA(error, item,  sc->downstream.hook, m);
 	return (error);
 
 out:
-	NG_FREE_DATA(m, meta);
+	NG_FREE_ITEM(item);
 	return (error);
 }
 
@@ -406,13 +400,11 @@ out:
  * Shutdown node
  */
 static int
-cisco_rmnode(node_p node)
+cisco_shutdown(node_p node)
 {
 	const sc_p sc = node->private;
 
 	node->flags |= NG_INVALID;
-	ng_cutlinks(node);
-	ng_unname(node);
 	node->private = NULL;
 	ng_unref(sc->node);
 	FREE(sc, M_NETGRAPH);
@@ -440,8 +432,9 @@ cisco_disconnect(hook_p hook)
 	}
 
 	/* If no more hooks, remove the node */
-	if (hook->node->numhooks == 0)
-		ng_rmnode(hook->node);
+	if ((hook->node->numhooks == 0)
+	&& ((hook->node->flags & NG_INVALID) == 0))
+		ng_rmnode_self(hook->node);
 	return (0);
 }
 
@@ -449,13 +442,15 @@ cisco_disconnect(hook_p hook)
  * Receive data
  */
 static int
-cisco_input(sc_p sc, struct mbuf *m, meta_p meta)
+cisco_input(sc_p sc, item_p item)
 {
 	struct cisco_header *h;
 	struct cisco_packet *p;
 	struct protoent *pep;
 	int error = 0;
+	struct mbuf *m;
 
+	m = NGI_M(item);
 	if (m->m_pkthdr.len <= CISCO_HEADER_LEN)
 		goto drop;
 
@@ -477,7 +472,7 @@ cisco_input(sc_p sc, struct mbuf *m, meta_p meta)
 			switch (ntohl(p->type)) {
 			default:
 				log(LOG_WARNING,
-				    "cisco: unknown cisco packet type: 0x%lx\n",
+				    "cisco: unknown cisco packet type: 0x%x\n",
 				       ntohl(p->type));
 				break;
 			case CISCO_ADDR_REPLY:
@@ -492,7 +487,8 @@ cisco_input(sc_p sc, struct mbuf *m, meta_p meta)
 				break;
 			case CISCO_ADDR_REQ:
 			    {
-				struct ng_mesg *msg, *resp;
+				struct ng_mesg *msg;
+				int dummy_error = 0;
 
 				/* Ask inet peer for IP address information */
 				if (sc->inet.hook == NULL)
@@ -501,12 +497,13 @@ cisco_input(sc_p sc, struct mbuf *m, meta_p meta)
 				    NGM_CISCO_GET_IPADDR, 0, M_NOWAIT);
 				if (msg == NULL)
 					goto nomsg;
-				ng_send_msg(sc->node, msg, NULL,
-					sc->inet.hook, NULL, &resp);
-				if (resp != NULL)
-					cisco_rcvmsg(sc->node, resp, ".",
-								NULL, NULL);
-
+				NG_SEND_MSG_HOOK(dummy_error, sc->node, msg,
+					sc->inet.hook, NULL);
+		/*
+		 * XXX Now maybe we should set a flag telling
+		 * our receiver to send this message when the response comes in
+		 * instead of now when the data may be bad.
+		 */
 		nomsg:
 				/* Send reply to peer device */
 				error = cisco_send(sc, CISCO_ADDR_REPLY,
@@ -535,11 +532,11 @@ cisco_input(sc_p sc, struct mbuf *m, meta_p meta)
 	/* Send it on */
 	if (pep->hook == NULL)
 		goto drop;
-	NG_SEND_DATA(error, pep->hook, m, meta);
+	NG_FWD_NEW_DATA(error, item, pep->hook, m);
 	return (error);
 
 drop:
-	NG_FREE_DATA(m, meta);
+	NG_FREE_ITEM(item);
 	return (error);
 }
 
@@ -570,7 +567,6 @@ cisco_send(sc_p sc, int type, long par1, long par2)
 	struct mbuf *m;
 	u_long  t;
 	int     error = 0;
-	meta_p  meta = NULL;
 	struct timeval time;
 
 	getmicrotime(&time);
@@ -596,6 +592,6 @@ cisco_send(sc_p sc, int type, long par1, long par2)
 	ch->time0 = htons((u_short) (t >> 16));
 	ch->time1 = htons((u_short) t);
 
-	NG_SEND_DATA(error, sc->downstream.hook, m, meta);
+	NG_SEND_DATA_ONLY(error, sc->downstream.hook, m);
 	return (error);
 }
diff --git a/sys/netgraph/ng_echo.c b/sys/netgraph/ng_echo.c
index 6c46341..337df72 100644
--- a/sys/netgraph/ng_echo.c
+++ b/sys/netgraph/ng_echo.c
@@ -55,6 +55,7 @@
 #include <netgraph/ng_echo.h>
 
 /* Netgraph methods */
+static ng_constructor_t	nge_cons;
 static ng_rcvmsg_t	nge_rcvmsg;
 static ng_rcvdata_t	nge_rcvdata;
 static ng_disconnect_t	nge_disconnect;
@@ -64,7 +65,7 @@ static struct ng_type typestruct = {
 	NG_ABI_VERSION,
 	NG_ECHO_NODE_TYPE,
 	NULL,
-	NULL,
+	nge_cons,
 	nge_rcvmsg,
 	NULL,
 	NULL,
@@ -76,33 +77,37 @@ static struct ng_type typestruct = {
 };
 NETGRAPH_INIT(echo, &typestruct);
 
+static int
+nge_cons(node_p node)
+{
+	return (0);
+}
+
 /*
  * Receive control message. We just bounce it back as a reply.
  */
 static int
-nge_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
-	   struct ng_mesg **rptr, hook_p lasthook)
+nge_rcvmsg(node_p node, item_p item, hook_p lasthook)
 {
-	if (rptr) {
-		msg->header.flags |= NGF_RESP;
-		*rptr = msg;
-	} else {
-		FREE(msg, M_NETGRAPH);
-	}
-	return (0);
+	struct ng_mesg *msg;
+	int error = 0;
+
+	NGI_GET_MSG(item, msg);
+	msg->header.flags |= NGF_RESP;
+	NG_RESPOND_MSG(error, node, item, msg);
+	return (error);
 }
 
 /*
  * Receive data
  */
 static int
-nge_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
+nge_rcvdata(hook_p hook, item_p item)
 {
 	int error = 0;
 
-	NG_SEND_DATA(error, hook, m, meta);
-	return (error);
+	NG_FWD_DATA(error, item, hook);
+	return (0);
 }
 
 /*
@@ -111,8 +116,10 @@ nge_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 static int
 nge_disconnect(hook_p hook)
 {
-	if (hook->node->numhooks == 0)
-		ng_rmnode(hook->node);
+	if ((hook->node->numhooks == 0)
+	&& ((hook->node->flags & NG_INVALID) == 0)) {
+		ng_rmnode_self(hook->node);
+	}
 	return (0);
 }
 
diff --git a/sys/netgraph/ng_ether.c b/sys/netgraph/ng_ether.c
index 194b5ee..1433ada 100644
--- a/sys/netgraph/ng_ether.c
+++ b/sys/netgraph/ng_ether.c
@@ -98,7 +98,7 @@ static int	ng_ether_rcv_upper(node_p node, struct mbuf *m, meta_p meta);
 /* Netgraph node methods */
 static ng_constructor_t	ng_ether_constructor;
 static ng_rcvmsg_t	ng_ether_rcvmsg;
-static ng_shutdown_t	ng_ether_rmnode;
+static ng_shutdown_t	ng_ether_shutdown;
 static ng_newhook_t	ng_ether_newhook;
 static ng_connect_t	ng_ether_connect;
 static ng_rcvdata_t	ng_ether_rcvdata;
@@ -185,7 +185,7 @@ static struct ng_type ng_ether_typestruct = {
 	ng_ether_mod_event,
 	ng_ether_constructor,
 	ng_ether_rcvmsg,
-	ng_ether_rmnode,
+	ng_ether_shutdown,
 	ng_ether_newhook,
 	NULL,
 	ng_ether_connect,
@@ -272,7 +272,6 @@ ng_ether_output(struct ifnet *ifp, struct mbuf **mp)
 {
 	const node_p node = IFP2NG(ifp);
 	const priv_p priv = node->private;
-	meta_p meta = NULL;
 	int error = 0;
 
 	/* If "upper" hook not connected, let packet continue */
@@ -280,13 +279,7 @@ ng_ether_output(struct ifnet *ifp, struct mbuf **mp)
 		return (0);
 
 	/* Send it out "upper" hook */
-	NG_SEND_DATA_RET(error, priv->upper, *mp, meta, NULL);
-
-	/* If we got a reflected packet back, handle it */
-	if (error == 0 && *mp != NULL) {
-		error = ng_ether_rcv_upper(node, *mp, meta);
-		*mp = NULL;
-	}
+	NG_SEND_DATA_ONLY(error, priv->upper, *mp);
 	return (error);
 }
 
@@ -342,9 +335,8 @@ ng_ether_detach(struct ifnet *ifp)
 
 	if (node == NULL)		/* no node (why not?), ignore */
 		return;
-	ng_rmnode(node);		/* break all links to other nodes */
+	ng_rmnode_self(node);		/* break all links to other nodes */
 	node->flags |= NG_INVALID;
-	ng_unname(node);		/* free name (and its reference) */
 	IFP2NG(ifp) = NULL;		/* detach node from interface */
 	priv = node->private;		/* free node private info */
 	bzero(priv, sizeof(*priv));
@@ -438,7 +430,7 @@ done:
  * this node type's KLD is loaded).
  */
 static int
-ng_ether_constructor(node_p *nodep)
+ng_ether_constructor(node_p node)
 {
 	return (EINVAL);
 }
@@ -495,13 +487,14 @@ ng_ether_connect(hook_p hook)
  * Receive an incoming control message.
  */
 static int
-ng_ether_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
-		struct ng_mesg **rptr, hook_p lasthook)
+ng_ether_rcvmsg(node_p node, item_p item, hook_p lasthook)
 {
 	const priv_p priv = node->private;
 	struct ng_mesg *resp = NULL;
 	int error = 0;
+	struct ng_mesg *msg;
 
+	NGI_GET_MSG(item, msg);
 	switch (msg->header.typecookie) {
 	case NGM_ETHER_COOKIE:
 		switch (msg->header.cmd) {
@@ -589,11 +582,8 @@ ng_ether_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
 		error = EINVAL;
 		break;
 	}
-	if (rptr)
-		*rptr = resp;
-	else if (resp != NULL)
-		FREE(resp, M_NETGRAPH);
-	FREE(msg, M_NETGRAPH);
+	NG_RESPOND_MSG(error, node, item, resp);
+	NG_FREE_MSG(msg);
 	return (error);
 }
 
@@ -601,12 +591,16 @@ ng_ether_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
  * Receive data on a hook.
  */
 static int
-ng_ether_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
+ng_ether_rcvdata(hook_p hook, item_p item)
 {
 	const node_p node = hook->node;
 	const priv_p priv = node->private;
+	struct mbuf *m;
+	meta_p meta;
 
+	NGI_GET_M(item, m);
+	NGI_GET_META(item, meta);
+	NG_FREE_ITEM(item);
 	if (hook == priv->lower)
 		return ng_ether_rcv_lower(node, m, meta);
 	if (hook == priv->upper)
@@ -624,7 +618,8 @@ ng_ether_rcv_lower(node_p node, struct mbuf *m, meta_p meta)
 
 	/* Make sure header is fully pulled up */
 	if (m->m_pkthdr.len < sizeof(struct ether_header)) {
-		NG_FREE_DATA(m, meta);
+		NG_FREE_M(m);
+		NG_FREE_META(meta);
 		return (EINVAL);
 	}
 	if (m->m_len < sizeof(struct ether_header)
@@ -656,7 +651,8 @@ ng_ether_rcv_upper(node_p node, struct mbuf *m, meta_p meta)
 
 	/* Check length and pull off header */
 	if (m->m_pkthdr.len < sizeof(*eh)) {
-		NG_FREE_DATA(m, meta);
+		NG_FREE_M(m);
+		NG_FREE_META(meta);
 		return (EINVAL);
 	}
 	if (m->m_len < sizeof(*eh) && (m = m_pullup(m, sizeof(*eh))) == NULL) {
@@ -677,19 +673,38 @@ ng_ether_rcv_upper(node_p node, struct mbuf *m, meta_p meta)
 
 /*
  * Shutdown node. This resets the node but does not remove it.
+ * Actually it produces a new node. XXX The problem  is what to do when 
+ * the node really DOES need to go away,
+ * or if our re-make of the node fails.
  */
 static int
-ng_ether_rmnode(node_p node)
+ng_ether_shutdown(node_p node)
 {
+	char name[IFNAMSIZ + 1];
 	const priv_p priv = node->private;
 
-	ng_cutlinks(node);
-	node->flags &= ~NG_INVALID;	/* bounce back to life */
 	if (priv->promisc) {		/* disable promiscuous mode */
 		(void)ifpromisc(priv->ifp, 0);
 		priv->promisc = 0;
 	}
+	ng_unref(node);
+	snprintf(name, sizeof(name), "%s%d", priv->ifp->if_name, priv->ifp->if_unit);
+	if (ng_make_node_common(&ng_ether_typestruct, &node) != 0) {
+		log(LOG_ERR, "%s: can't %s for %s\n",
+		    __FUNCTION__, "create node", name);
+		return (ENOMEM);
+	}
+
+	/* Allocate private data */
+	node->private = priv;
+	IFP2NG(priv->ifp) = node;
 	priv->autoSrcAddr = 1;		/* reset auto-src-addr flag */
+
+	/* Try to give the node the same name as the interface */
+	if (ng_name_node(node, name) != 0) {
+		log(LOG_WARNING, "%s: can't name node %s\n",
+		    __FUNCTION__, name);
+	}
 	return (0);
 }
 
@@ -708,8 +723,9 @@ ng_ether_disconnect(hook_p hook)
 		priv->lowerOrphan = 0;
 	} else
 		panic("%s: weird hook", __FUNCTION__);
-	if (hook->node->numhooks == 0)
-		ng_rmnode(hook->node);	/* reset node */
+	if ((hook->node->numhooks == 0)
+	&& ((hook->node->flags & NG_INVALID) == 0))
+		ng_rmnode_self(hook->node);	/* reset node */
 	return (0);
 }
 
diff --git a/sys/netgraph/ng_frame_relay.c b/sys/netgraph/ng_frame_relay.c
index 96d56bf..9887284 100644
--- a/sys/netgraph/ng_frame_relay.c
+++ b/sys/netgraph/ng_frame_relay.c
@@ -34,7 +34,7 @@
  * THIS SOFTWARE, EVEN IF WHISTLE COMMUNICATIONS IS ADVISED OF THE POSSIBILITY
  * OF SUCH DAMAGE.
  *
- * Author: Julian Elisher <julian@freebsd.org>
+ * Author: Julian Elischer <julian@freebsd.org>
  *
  * $FreeBSD$
  * $Whistle: ng_frame_relay.c,v 1.20 1999/11/01 09:24:51 julian Exp $
@@ -125,13 +125,13 @@ static struct segment {
 
 /* Netgraph methods */
 static ng_constructor_t	ngfrm_constructor;
-static ng_shutdown_t	ngfrm_rmnode;
+static ng_shutdown_t	ngfrm_shutdown;
 static ng_newhook_t	ngfrm_newhook;
 static ng_rcvdata_t	ngfrm_rcvdata;
 static ng_disconnect_t	ngfrm_disconnect;
 
 /* Other internal functions */
-static int ngfrm_decode(node_p node, struct mbuf * m, meta_p meta);
+static int ngfrm_decode(node_p node, item_p item);
 static int ngfrm_addrlen(char *hdr);
 static int ngfrm_allocate_CTX(sc_p sc, int dlci);
 
@@ -142,7 +142,7 @@ static struct ng_type typestruct = {
 	NULL,
 	ngfrm_constructor,
 	NULL,
-	ngfrm_rmnode,
+	ngfrm_shutdown,
 	ngfrm_newhook,
 	NULL,
 	NULL,
@@ -212,23 +212,18 @@ ngfrm_allocate_CTX(sc_p sc, int dlci)
  * Node constructor
  */
 static int
-ngfrm_constructor(node_p *nodep)
+ngfrm_constructor(node_p node)
 {
 	sc_p sc;
-	int error = 0;
 
 	MALLOC(sc, sc_p, sizeof(*sc), M_NETGRAPH, M_NOWAIT | M_ZERO);
 	if (!sc)
 		return (ENOMEM);
-	if ((error = ng_make_node_common(&typestruct, nodep))) {
-		FREE(sc, M_NETGRAPH);
-		return (error);
-	}
 	sc->addrlen = 2;	/* default */
 
 	/* Link the node and our private info */
-	(*nodep)->private = sc;
-	sc->node = *nodep;
+	node->private = sc;
+	sc->node = node;
 	return (0);
 }
 
@@ -335,8 +330,7 @@ ngfrm_addrlen(char *hdr)
  * Receive data packet
  */
 static int
-ngfrm_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
+ngfrm_rcvdata(hook_p hook, item_p item)
 {
 	struct	ctxinfo *const ctxp = hook->private;
 	int     error = 0;
@@ -344,6 +338,7 @@ ngfrm_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 	sc_p    sc;
 	int     alen;
 	char   *data;
+	struct mbuf *m;
 
 	/* Data doesn't come in from just anywhere (e.g debug hook) */
 	if (ctxp == NULL) {
@@ -354,8 +349,9 @@ ngfrm_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 	/* If coming from downstream, decode it to a channel */
 	dlci = ctxp->dlci;
 	if (dlci == -1)
-		return (ngfrm_decode(hook->node, m, meta));
+		return (ngfrm_decode(hook->node, item));
 
+	NGI_GET_M(item, m);
 	/* Derive the softc we will need */
 	sc = hook->node->private;
 
@@ -408,11 +404,12 @@ ngfrm_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 	}
 
 	/* Send it */
-	NG_SEND_DATA(error, sc->downstream.hook, m, meta);
+	NG_FWD_NEW_DATA(error, item, sc->downstream.hook, m);
 	return (error);
 
 bad:
-	NG_FREE_DATA(m, meta);
+	NG_FREE_ITEM(item);
+	NG_FREE_M(m);
 	return (error);
 }
 
@@ -420,7 +417,7 @@ bad:
  * Decode an incoming frame coming from the switch
  */
 static int
-ngfrm_decode(node_p node, struct mbuf *m, meta_p meta)
+ngfrm_decode(node_p node, item_p item)
 {
 	const sc_p  sc = node->private;
 	char       *data;
@@ -428,7 +425,9 @@ ngfrm_decode(node_p node, struct mbuf *m, meta_p meta)
 	u_int	    dlci = 0;
 	int	    error = 0;
 	int	    ctxnum;
+	struct mbuf *m;
 
+	NGI_GET_M(item, m);
 	if (m->m_len < 4 && (m = m_pullup(m, 4)) == NULL) {
 		error = ENOBUFS;
 		goto out;
@@ -466,13 +465,14 @@ ngfrm_decode(node_p node, struct mbuf *m, meta_p meta)
 	if ((ctxnum & CTX_VALID) && sc->channel[ctxnum &= CTX_VALUE].hook) {
 		/* Send it */
 		m_adj(m, alen);
-		NG_SEND_DATA(error, sc->channel[ctxnum].hook, m, meta);
+		NG_FWD_NEW_DATA(error, item, sc->channel[ctxnum].hook, m);
 		return (error);
 	} else {
 		error = ENETDOWN;
 	}
 out:
-	NG_FREE_DATA(m, meta);
+	NG_FREE_ITEM(item);
+	NG_FREE_M(m);
 	return (error);
 }
 
@@ -480,13 +480,11 @@ out:
  * Shutdown node
  */
 static int
-ngfrm_rmnode(node_p node)
+ngfrm_shutdown(node_p node)
 {
 	const sc_p sc = node->private;
 
 	node->flags |= NG_INVALID;
-	ng_cutlinks(node);
-	ng_unname(node);
 	node->private = NULL;
 	FREE(sc, M_NETGRAPH);
 	ng_unref(node);
@@ -515,7 +513,8 @@ ngfrm_disconnect(hook_p hook)
 		cp->flags = 0;
 		sc->datahooks--;
 	}
-	if (hook->node->numhooks == 0)
-		ng_rmnode(hook->node);
+	if ((hook->node->numhooks == 0)
+	&& ((hook->node->flags & NG_INVALID) == 0))
+		ng_rmnode_self(hook->node);
 	return (0);
 }
diff --git a/sys/netgraph/ng_hole.c b/sys/netgraph/ng_hole.c
index 1da47b6..92fa603 100644
--- a/sys/netgraph/ng_hole.c
+++ b/sys/netgraph/ng_hole.c
@@ -53,33 +53,42 @@
 #include <netgraph/ng_hole.h>
 
 /* Netgraph methods */
+static ng_constructor_t	ngh_cons;
 static ng_rcvdata_t	ngh_rcvdata;
 static ng_disconnect_t	ngh_disconnect;
 
 static struct ng_type typestruct = {
 	NG_ABI_VERSION,
 	NG_HOLE_NODE_TYPE,
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-	ngh_rcvdata,
-	ngh_disconnect,
-	NULL
+	NULL,		/* modeventhand_t */
+	ngh_cons,	/* ng_constructor_t */
+	NULL,		/* ng_rcvmsg_t */
+	NULL,		/* ng_shutdown_t */
+	NULL,		/* ng_newhook_t */
+	NULL,		/* ng_findhook_t */
+	NULL,		/* ng_connect_t */
+	ngh_rcvdata,	/* ng_rcvdata_t */
+	ngh_disconnect,	/* ng_disconnect_t */
+	NULL		/* ng_cmdlist */
 };
 NETGRAPH_INIT(hole, &typestruct);
 
+/* 
+ * Be obliging. but no work to do.
+ */
+static int
+ngh_cons(node_p node)
+{
+	return(0);
+}
+
 /*
  * Receive data
  */
 static int
-ngh_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
+ngh_rcvdata(hook_p hook, item_p item)
 {
-	NG_FREE_DATA(m, meta);
+	NG_FREE_ITEM(item);
 	return 0;
 }
 
@@ -90,6 +99,6 @@ static int
 ngh_disconnect(hook_p hook)
 {
 	if (hook->node->numhooks == 0)
-		ng_rmnode(hook->node);
+		ng_rmnode_self(hook->node);
 	return (0);
 }
diff --git a/sys/netgraph/ng_iface.c b/sys/netgraph/ng_iface.c
index 3df564a..42aa1ba 100644
--- a/sys/netgraph/ng_iface.c
+++ b/sys/netgraph/ng_iface.c
@@ -118,7 +118,7 @@ static void	ng_iface_print_ioctl(struct ifnet *ifp, int cmd, caddr_t data);
 /* Netgraph methods */
 static ng_constructor_t	ng_iface_constructor;
 static ng_rcvmsg_t	ng_iface_rcvmsg;
-static ng_shutdown_t	ng_iface_rmnode;
+static ng_shutdown_t	ng_iface_shutdown;
 static ng_newhook_t	ng_iface_newhook;
 static ng_rcvdata_t	ng_iface_rcvdata;
 static ng_disconnect_t	ng_iface_disconnect;
@@ -186,7 +186,7 @@ static struct ng_type typestruct = {
 	NULL,
 	ng_iface_constructor,
 	ng_iface_rcvmsg,
-	ng_iface_rmnode,
+	ng_iface_shutdown,
 	ng_iface_newhook,
 	NULL,
 	NULL,
@@ -521,11 +521,10 @@ ng_iface_print_ioctl(struct ifnet *ifp, int command, caddr_t data)
  * Constructor for a node
  */
 static int
-ng_iface_constructor(node_p *nodep)
+ng_iface_constructor(node_p node)
 {
 	char ifname[NG_IFACE_IFACE_NAME_MAX + 1];
 	struct ifnet *ifp;
-	node_p node;
 	priv_p priv;
 	int error = 0;
 
@@ -550,15 +549,6 @@ ng_iface_constructor(node_p *nodep)
 		return (error);
 	}
 
-	/* Call generic node constructor */
-	if ((error = ng_make_node_common(&typestruct, nodep)) != 0) {
-		ng_iface_free_unit(priv->unit);
-		FREE(ifp, M_NETGRAPH);
-		FREE(priv, M_NETGRAPH);
-		return (error);
-	}
-	node = *nodep;
-
 	/* Link together node and private info */
 	node->private = priv;
 	priv->node = node;
@@ -615,14 +605,15 @@ ng_iface_newhook(node_p node, hook_p hook, const char *name)
  * Receive a control message
  */
 static int
-ng_iface_rcvmsg(node_p node, struct ng_mesg *msg,
-		const char *retaddr, struct ng_mesg **rptr, hook_p lasthook)
+ng_iface_rcvmsg(node_p node, item_p item, hook_p lasthook)
 {
 	const priv_p priv = node->private;
 	struct ifnet *const ifp = priv->ifp;
 	struct ng_mesg *resp = NULL;
 	int error = 0;
+	struct ng_mesg *msg;
 
+	NGI_GET_MSG(item, msg);
 	switch (msg->header.typecookie) {
 	case NGM_IFACE_COOKIE:
 		switch (msg->header.cmd) {
@@ -707,11 +698,8 @@ ng_iface_rcvmsg(node_p node, struct ng_mesg *msg,
 		error = EINVAL;
 		break;
 	}
-	if (rptr)
-		*rptr = resp;
-	else if (resp)
-		FREE(resp, M_NETGRAPH);
-	FREE(msg, M_NETGRAPH);
+	NG_RESPOND_MSG(error, node, item, resp);
+	NG_FREE_MSG(msg);
 	return (error);
 }
 
@@ -719,20 +707,22 @@ ng_iface_rcvmsg(node_p node, struct ng_mesg *msg,
  * Recive data from a hook. Pass the packet to the correct input routine.
  */
 static int
-ng_iface_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
+ng_iface_rcvdata(hook_p hook, item_p item)
 {
 	const priv_p priv = hook->node->private;
 	const iffam_p iffam = get_iffam_from_hook(priv, hook);
 	struct ifnet *const ifp = priv->ifp;
+	struct mbuf *m;
 
+	NGI_GET_M(item, m);
+	NG_FREE_ITEM(item);
 	/* Sanity checks */
 	KASSERT(iffam != NULL, ("%s: iffam", __FUNCTION__));
 	KASSERT(m->m_flags & M_PKTHDR, ("%s: not pkthdr", __FUNCTION__));
 	if (m == NULL)
 		return (EINVAL);
 	if ((ifp->if_flags & IFF_UP) == 0) {
-		NG_FREE_DATA(m, meta);
+		NG_FREE_M(m);
 		return (ENETDOWN);
 	}
 
@@ -746,9 +736,6 @@ ng_iface_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 	/* Berkeley packet filter */
 	ng_iface_bpftap(ifp, m, iffam->family);
 
-	/* Ignore any meta-data */
-	NG_FREE_META(meta);
-
 	/* Send packet */
 	return family_enqueue(iffam->family, m);
 }
@@ -757,12 +744,10 @@ ng_iface_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
  * Shutdown and remove the node and its associated interface.
  */
 static int
-ng_iface_rmnode(node_p node)
+ng_iface_shutdown(node_p node)
 {
 	const priv_p priv = node->private;
 
-	ng_cutlinks(node);
-	ng_unname(node);
 	bpfdetach(priv->ifp);
 	if_detach(priv->ifp);
 	priv->ifp = NULL;
diff --git a/sys/netgraph/ng_ksocket.c b/sys/netgraph/ng_ksocket.c
index 9898da9..5162da5 100644
--- a/sys/netgraph/ng_ksocket.c
+++ b/sys/netgraph/ng_ksocket.c
@@ -80,7 +80,7 @@ typedef struct ng_ksocket_private *priv_p;
 /* Netgraph node methods */
 static ng_constructor_t	ng_ksocket_constructor;
 static ng_rcvmsg_t	ng_ksocket_rcvmsg;
-static ng_shutdown_t	ng_ksocket_rmnode;
+static ng_shutdown_t	ng_ksocket_shutdown;
 static ng_newhook_t	ng_ksocket_newhook;
 static ng_rcvdata_t	ng_ksocket_rcvdata;
 static ng_disconnect_t	ng_ksocket_disconnect;
@@ -464,7 +464,7 @@ static struct ng_type ng_ksocket_typestruct = {
 	NULL,
 	ng_ksocket_constructor,
 	ng_ksocket_rcvmsg,
-	ng_ksocket_rmnode,
+	ng_ksocket_shutdown,
 	ng_ksocket_newhook,
 	NULL,
 	NULL,
@@ -484,22 +484,16 @@ NETGRAPH_INIT(ksocket, &ng_ksocket_typestruct);
  * Node type constructor
  */
 static int
-ng_ksocket_constructor(node_p *nodep)
+ng_ksocket_constructor(node_p node)
 {
 	priv_p priv;
-	int error;
 
 	/* Allocate private structure */
 	MALLOC(priv, priv_p, sizeof(*priv), M_NETGRAPH, M_NOWAIT | M_ZERO);
 	if (priv == NULL)
 		return (ENOMEM);
 
-	/* Call generic node constructor */
-	if ((error = ng_make_node_common(&ng_ksocket_typestruct, nodep))) {
-		FREE(priv, M_NETGRAPH);
-		return (error);
-	}
-	(*nodep)->private = priv;
+	node->private = priv;
 
 	/* Done */
 	return (0);
@@ -563,15 +557,16 @@ ng_ksocket_newhook(node_p node, hook_p hook, const char *name0)
  * Receive a control message
  */
 static int
-ng_ksocket_rcvmsg(node_p node, struct ng_mesg *msg,
-	      const char *raddr, struct ng_mesg **rptr, hook_p lasthook)
+ng_ksocket_rcvmsg(node_p node, item_p item, hook_p lasthook)
 {
 	struct proc *p = curproc ? curproc : &proc0;	/* XXX broken */
 	const priv_p priv = node->private;
 	struct socket *const so = priv->so;
 	struct ng_mesg *resp = NULL;
 	int error = 0;
+	struct ng_mesg *msg;
 
+	NGI_GET_MSG(item, msg);
 	switch (msg->header.typecookie) {
 	case NGM_KSOCKET_COOKIE:
 		switch (msg->header.cmd) {
@@ -723,7 +718,7 @@ ng_ksocket_rcvmsg(node_p node, struct ng_mesg *msg,
 			ksopt = (struct ng_ksocket_sockopt *)resp->data;
 			sopt.sopt_val = ksopt->value;
 			if ((error = sogetopt(so, &sopt)) != 0) {
-				FREE(resp, M_NETGRAPH);
+				NG_FREE_MSG(resp);
 				break;
 			}
 
@@ -766,13 +761,9 @@ ng_ksocket_rcvmsg(node_p node, struct ng_mesg *msg,
 		error = EINVAL;
 		break;
 	}
-	if (rptr)
-		*rptr = resp;
-	else if (resp)
-		FREE(resp, M_NETGRAPH);
-
 done:
-	FREE(msg, M_NETGRAPH);
+	NG_RESPOND_MSG(error, node, item, resp);
+	NG_FREE_MSG(msg);
 	return (error);
 }
 
@@ -780,16 +771,17 @@ done:
  * Receive incoming data on our hook.  Send it out the socket.
  */
 static int
-ng_ksocket_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
+ng_ksocket_rcvdata(hook_p hook, item_p item)
 {
 	struct proc *p = curproc ? curproc : &proc0;	/* XXX broken */
 	const node_p node = hook->node;
 	const priv_p priv = node->private;
 	struct socket *const so = priv->so;
 	int error;
+	struct mbuf *m;
 
-	NG_FREE_META(meta);
+	NGI_GET_M(item, m);
+	NG_FREE_ITEM(item);
 	error = (*so->so_proto->pr_usrreqs->pru_sosend)(so, 0, 0, m, 0, 0, p);
 	return (error);
 }
@@ -798,7 +790,7 @@ ng_ksocket_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
  * Destroy node
  */
 static int
-ng_ksocket_rmnode(node_p node)
+ng_ksocket_shutdown(node_p node)
 {
 	const priv_p priv = node->private;
 
@@ -813,8 +805,6 @@ ng_ksocket_rmnode(node_p node)
 
 	/* Take down netgraph node */
 	node->flags |= NG_INVALID;
-	ng_cutlinks(node);
-	ng_unname(node);
 	bzero(priv, sizeof(*priv));
 	FREE(priv, M_NETGRAPH);
 	node->private = NULL;
@@ -830,7 +820,8 @@ ng_ksocket_disconnect(hook_p hook)
 {
 	KASSERT(hook->node->numhooks == 0,
 	    ("%s: numhooks=%d?", __FUNCTION__, hook->node->numhooks));
-	ng_rmnode(hook->node);
+	if ((hook->node->flags & NG_INVALID) == 0)
+		ng_rmnode_self(hook->node);
 	return (0);
 }
 
@@ -846,7 +837,6 @@ ng_ksocket_incoming(struct socket *so, void *arg, int waitflag)
 {
 	const node_p node = arg;
 	const priv_p priv = node->private;
-	meta_p meta = NULL;
 	struct mbuf *m;
 	struct uio auio;
 	int s, flags, error;
@@ -876,7 +866,7 @@ ng_ksocket_incoming(struct socket *so, void *arg, int waitflag)
 			   packet header and length correct (eg. kern/15175) */
 			for (n = m, m->m_pkthdr.len = 0; n; n = n->m_next)
 				m->m_pkthdr.len += n->m_len;
-			NG_SEND_DATA(error, priv->hook, m, meta);
+			NG_SEND_DATA_ONLY(error, priv->hook, m);
 		}
 	} while (error == 0 && m != NULL);
 	splx(s);
diff --git a/sys/netgraph/ng_lmi.c b/sys/netgraph/ng_lmi.c
index 9bede71..c088680 100644
--- a/sys/netgraph/ng_lmi.c
+++ b/sys/netgraph/ng_lmi.c
@@ -91,11 +91,11 @@
  */
 static ng_constructor_t	nglmi_constructor;
 static ng_rcvmsg_t	nglmi_rcvmsg;
-static ng_shutdown_t	nglmi_rmnode;
+static ng_shutdown_t	nglmi_shutdown;
 static ng_newhook_t	nglmi_newhook;
 static ng_rcvdata_t	nglmi_rcvdata;
 static ng_disconnect_t	nglmi_disconnect;
-static int	nglmi_checkdata(hook_p hook, struct mbuf *m, meta_p meta);
+static int	nglmi_checkdata(hook_p hook, struct mbuf *m);
 
 static struct ng_type typestruct = {
 	NG_ABI_VERSION,
@@ -103,7 +103,7 @@ static struct ng_type typestruct = {
 	NULL,
 	nglmi_constructor,
 	nglmi_rcvmsg,
-	nglmi_rmnode,
+	nglmi_shutdown,
 	nglmi_newhook,
 	NULL,
 	NULL,
@@ -184,23 +184,17 @@ do {									\
  * Node constructor
  */
 static int
-nglmi_constructor(node_p *nodep)
+nglmi_constructor(node_p node)
 {
 	sc_p sc;
-	int error = 0;
 
 	MALLOC(sc, sc_p, sizeof(*sc), M_NETGRAPH, M_NOWAIT | M_ZERO);
 	if (sc == NULL)
 		return (ENOMEM);
-
 	callout_handle_init(&sc->handle);
-	if ((error = ng_make_node_common(&typestruct, nodep))) {
-		FREE(sc, M_NETGRAPH);
-		return (error);
-	}
-	(*nodep)->private = sc;
+	node->private = sc;
 	sc->protoname = NAME_NONE;
-	sc->node = *nodep;
+	sc->node = node;
 	sc->liv_per_full = NG_LMI_SEQ_PER_FULL;	/* make this dynamic */
 	sc->liv_rate = NG_LMI_KEEPALIVE_RATE;
 	return (0);
@@ -449,13 +443,14 @@ ngauto_state_machine(sc_p sc)
  * Receive a netgraph control message.
  */
 static int
-nglmi_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
-	     struct ng_mesg **rptr, hook_p lasthook)
+nglmi_rcvmsg(node_p node, item_p item, hook_p lasthook)
 {
 	sc_p    sc = node->private;
 	struct ng_mesg *resp = NULL;
 	int     error = 0;
+	struct ng_mesg *msg;
 
+	NGI_GET_MSG(item, msg);
 	switch (msg->header.typecookie) {
 	case NGM_GENERIC_COOKIE:
 		switch (msg->header.cmd) {
@@ -544,12 +539,8 @@ nglmi_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
 		break;
 	}
 
-	if (rptr)
-		*rptr = resp;
-	else if (resp != NULL)
-		FREE(resp, M_NETGRAPH);
-
-	FREE(msg, M_NETGRAPH);
+	NG_RESPOND_MSG(error, node, item, resp);
+	NG_FREE_MSG(msg);
 	return (error);
 }
 
@@ -564,8 +555,7 @@ nglmi_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
  * Anything coming in on the debug port is discarded.
  */
 static int
-nglmi_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
+nglmi_rcvdata(hook_p hook, item_p item)
 {
 	sc_p    sc = hook->node->private;
 	u_char *data;
@@ -573,7 +563,10 @@ nglmi_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 	u_short packetlen;
 	int     resptype_seen = 0;
 	int     seq_seen = 0;
+	struct mbuf *m;
 
+	NGI_GET_M(item, m);
+	NG_FREE_ITEM(item);
 	if (hook->private == NULL) {
 		goto drop;
 	}
@@ -587,10 +580,9 @@ nglmi_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 	if (m->m_len < packetlen && (m = m_pullup(m, packetlen)) == NULL) {
 		log(LOG_WARNING,
 		    "nglmi: m_pullup failed for %d bytes\n", packetlen);
-		NG_FREE_META(meta);
 		return (0);
 	}
-	if (nglmi_checkdata(hook, m, meta) == 0)
+	if (nglmi_checkdata(hook, m) == 0)
 		return (0);
 
 	/* pass the first 4 bytes (already checked in the nglmi_checkdata()) */
@@ -734,11 +726,11 @@ nglmi_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 nextIE:
 		STEPBY(segsize + 2);
 	}
-	NG_FREE_DATA(m, meta);
+	NG_FREE_M(m);
 	return (0);
 
 drop:
-	NG_FREE_DATA(m, meta);
+	NG_FREE_M(m);
 	return (EINVAL);
 }
 
@@ -748,7 +740,7 @@ drop:
  * All data is discarded if a 0 is returned.
  */
 static int
-nglmi_checkdata(hook_p hook, struct mbuf *m, meta_p meta)
+nglmi_checkdata(hook_p hook, struct mbuf *m)
 {
 	sc_p    sc = hook->node->private;
 	u_char *data;
@@ -1052,7 +1044,7 @@ reject:
 			i++;
 		}
 	}
-	NG_FREE_DATA(m, meta);
+	NG_FREE_M(m);
 	return (0);
 }
 
@@ -1061,13 +1053,11 @@ reject:
  * Cut any remaining links and free our local resources.
  */
 static int
-nglmi_rmnode(node_p node)
+nglmi_shutdown(node_p node)
 {
 	const sc_p sc = node->private;
 
 	node->flags |= NG_INVALID;
-	ng_cutlinks(node);
-	ng_unname(node);
 	node->private = NULL;
 	ng_unref(sc->node);
 	FREE(sc, M_NETGRAPH);
@@ -1092,7 +1082,8 @@ nglmi_disconnect(hook_p hook)
 		untimeout(LMI_ticker, sc, sc->handle);
 
 	/* Self-destruct */
-	ng_rmnode(hook->node);
+	if ((hook->node->flags & NG_INVALID) == 0)
+		ng_rmnode_self(hook->node);
 	return (0);
 }
 
diff --git a/sys/netgraph/ng_message.h b/sys/netgraph/ng_message.h
index 993768d..0532780 100644
--- a/sys/netgraph/ng_message.h
+++ b/sys/netgraph/ng_message.h
@@ -66,6 +66,8 @@ struct ng_mesg {
 	char	data[0];		/* placeholder for actual data */
 };
 
+/* this command is guaranteed to not alter data or'd into the command */
+#define NGM_READONLY	0x10000000
 
 /* Keep this in sync with the above structure definition */
 #define NG_GENERIC_NG_MESG_INFO(dtype)	{			\
@@ -88,12 +90,13 @@ struct ng_mesg {
  * Interfaces within the kernel are defined by a different 
  * value (see NG_ABI_VERSION in netgraph.g)
  */
-#define NG_VERSION	4
+#define NG_VERSION	5
 
 /* Flags field flags */
-#define NGF_ORIG	0x0000		/* the msg is the original request */
-#define NGF_RESP	0x0001		/* the message is a response */
-
+#define NGF_ORIG	0x00000000	/* the msg is the original request */
+#define NGF_RESP	0x00000001	/* the message is a response */
+#define NGF_STATIC	0x00000002	/* Not malloc'd. Don't FREE */
+					/* Only checked in generic message */
 /* Type of a unique node ID */
 #define ng_ID_t unsigned int
 
@@ -104,7 +107,7 @@ struct ng_mesg {
  */
 
 /* Generic message type cookie */
-#define NGM_GENERIC_COOKIE	851672668
+#define NGM_GENERIC_COOKIE	977674408
 
 /* Generic messages defined for this type cookie */
 #define	NGM_SHUTDOWN		1	/* shut down node */
@@ -112,14 +115,14 @@ struct ng_mesg {
 #define NGM_CONNECT		3	/* connect two nodes */
 #define NGM_NAME		4	/* give a node a name */
 #define NGM_RMHOOK		5	/* break a connection btw. two nodes */
-#define	NGM_NODEINFO		6	/* get nodeinfo for the target */
-#define	NGM_LISTHOOKS		7	/* get list of hooks on node */
-#define	NGM_LISTNAMES		8	/* list all globally named nodes */
-#define	NGM_LISTNODES		9	/* list all nodes, named and unnamed */
-#define	NGM_LISTTYPES		10	/* list all installed node types */
-#define	NGM_TEXT_STATUS		11	/* (optional) get text status report */
-#define	NGM_BINARY2ASCII	12	/* convert struct ng_mesg to ascii */
-#define	NGM_ASCII2BINARY	13	/* convert ascii to struct ng_mesg */
+#define	NGM_NODEINFO		(6|NGM_READONLY)/* get nodeinfo for target */
+#define	NGM_LISTHOOKS		(7|NGM_READONLY)/* get list of hooks on node */
+#define	NGM_LISTNAMES		(8|NGM_READONLY)/* list globally named nodes */
+#define	NGM_LISTNODES		(9|NGM_READONLY)/* list nodes, named & not */
+#define	NGM_LISTTYPES		(10|NGM_READONLY)/* list installed node types */
+#define	NGM_TEXT_STATUS		(11|NGM_READONLY)/* (optional) get txt status */
+#define	NGM_BINARY2ASCII	(12|NGM_READONLY)/* convert ng_mesg to ascii */
+#define	NGM_ASCII2BINARY	(13|NGM_READONLY)/* convert ascii to ng_mesg */
 #define	NGM_TEXT_CONFIG		14	/* (optional) get/set text config */
 
 /*
@@ -143,13 +146,13 @@ struct ng_mesg {
 /* Downstream messages */
 #define NGM_DROP_LINK		41	/* drop DTR, etc. - stay in the graph */
 #define NGM_RAISE LINK		42	/* if you previously dropped it */
-#define NGM_FLUSH_QUEUE		43	/* no data */
-#define NGM_GET_BANDWIDTH	44	/* either real or measured */
-#define NGM_SET_XMIT_Q_LIMITS	45	/* includes queue state */
-#define NGM_GET_XMIT_Q_LIMITS	46	/* returns queue state */
-#define NGM_MICROMANAGE		47	/* We want sync. queue state reply 
-					   for each packet sent down */
-#define NGM_SET_FLOW_MANAGER	48	/* send flow control here */ 
+#define NGM_FLUSH_QUEUE		43		/* no data */
+#define NGM_GET_BANDWIDTH	(44|NGM_READONLY)	/* either real or measured */
+#define NGM_SET_XMIT_Q_LIMITS	45		/* includes queue state */
+#define NGM_GET_XMIT_Q_LIMITS	(46|NGM_READONLY)	/* returns queue state */
+#define NGM_MICROMANAGE		47		/* We want sync. queue state
+						reply for each packet sent */
+#define NGM_SET_FLOW_MANAGER	48		/* send flow control here */ 
 /* Structure used for NGM_MKPEER */
 struct ngm_mkpeer {
 	char	type[NG_TYPELEN + 1];			/* peer type */
@@ -388,7 +391,7 @@ struct flow_manager {
 #define NG_MKMESSAGE(msg, cookie, cmdid, len, how)			\
 	do {								\
 	  MALLOC((msg), struct ng_mesg *, sizeof(struct ng_mesg)	\
-	    + (len), M_NETGRAPH, (how) | M_ZERO);			\
+	    + (len), M_NETGRAPH_MSG, (how) | M_ZERO);			\
 	  if ((msg) == NULL)						\
 	    break;							\
 	  (msg)->header.version = NG_VERSION;				\
@@ -406,7 +409,7 @@ struct flow_manager {
 #define NG_MKRESPONSE(rsp, msg, len, how)				\
 	do {								\
 	  MALLOC((rsp), struct ng_mesg *, sizeof(struct ng_mesg)	\
-	    + (len), M_NETGRAPH, (how) | M_ZERO);			\
+	    + (len), M_NETGRAPH_MSG, (how) | M_ZERO);			\
 	  if ((rsp) == NULL)						\
 	    break;							\
 	  (rsp)->header.version = NG_VERSION;				\
diff --git a/sys/netgraph/ng_mppc.c b/sys/netgraph/ng_mppc.c
index d9c5a31..93cdc8c 100644
--- a/sys/netgraph/ng_mppc.c
+++ b/sys/netgraph/ng_mppc.c
@@ -119,14 +119,14 @@ struct ng_mppc_dir {
 struct ng_mppc_private {
 	struct ng_mppc_dir	xmit;		/* compress/encrypt config */
 	struct ng_mppc_dir	recv;		/* decompress/decrypt config */
-	char			*ctrlpath;	/* path to controlling node */
+	ng_ID_t			ctrlnode;	/* path to controlling node */
 };
 typedef struct ng_mppc_private *priv_p;
 
 /* Netgraph node methods */
 static ng_constructor_t	ng_mppc_constructor;
 static ng_rcvmsg_t	ng_mppc_rcvmsg;
-static ng_shutdown_t	ng_mppc_rmnode;
+static ng_shutdown_t	ng_mppc_shutdown;
 static ng_newhook_t	ng_mppc_newhook;
 static ng_rcvdata_t	ng_mppc_rcvdata;
 static ng_disconnect_t	ng_mppc_disconnect;
@@ -148,7 +148,7 @@ static struct ng_type ng_mppc_typestruct = {
 	NULL,
 	ng_mppc_constructor,
 	ng_mppc_rcvmsg,
-	ng_mppc_rmnode,
+	ng_mppc_shutdown,
 	ng_mppc_newhook,
 	NULL,
 	NULL,
@@ -171,22 +171,16 @@ static const u_char ng_mppe_weakenkey[3] = { 0xd1, 0x26, 0x9e };
  * Node type constructor
  */
 static int
-ng_mppc_constructor(node_p *nodep)
+ng_mppc_constructor(node_p node)
 {
 	priv_p priv;
-	int error;
 
 	/* Allocate private structure */
 	MALLOC(priv, priv_p, sizeof(*priv), M_NETGRAPH, M_NOWAIT | M_ZERO);
 	if (priv == NULL)
 		return (ENOMEM);
 
-	/* Call generic node constructor */
-	if ((error = ng_make_node_common(&ng_mppc_typestruct, nodep))) {
-		FREE(priv, M_NETGRAPH);
-		return (error);
-	}
-	(*nodep)->private = priv;
+	node->private = priv;
 
 	/* Done */
 	return (0);
@@ -222,13 +216,14 @@ ng_mppc_newhook(node_p node, hook_p hook, const char *name)
  * Receive a control message
  */
 static int
-ng_mppc_rcvmsg(node_p node, struct ng_mesg *msg,
-	      const char *raddr, struct ng_mesg **rptr, hook_p lasthook)
+ng_mppc_rcvmsg(node_p node, item_p item, hook_p lasthook)
 {
 	const priv_p priv = node->private;
 	struct ng_mesg *resp = NULL;
 	int error = 0;
+	struct ng_mesg *msg;
 
+	NGI_GET_MSG(item, msg);
 	switch (msg->header.typecookie) {
 	case NGM_MPPC_COOKIE:
 		switch (msg->header.cmd) {
@@ -260,17 +255,7 @@ ng_mppc_rcvmsg(node_p node, struct ng_mesg *msg,
 				cfg->bits = 0;
 
 			/* Save return address so we can send reset-req's */
-			if (priv->ctrlpath != NULL) {
-				FREE(priv->ctrlpath, M_NETGRAPH);
-				priv->ctrlpath = NULL;
-			}
-			if (!isComp && raddr != NULL) {
-				MALLOC(priv->ctrlpath, char *,
-				    strlen(raddr) + 1, M_NETGRAPH, M_NOWAIT);
-				if (priv->ctrlpath == NULL)
-					ERROUT(ENOMEM);
-				strcpy(priv->ctrlpath, raddr);
-			}
+			priv->ctrlnode = NGI_RETADDR(item);
 
 			/* Configuration is OK, reset to it */
 			d->cfg = *cfg;
@@ -331,13 +316,9 @@ ng_mppc_rcvmsg(node_p node, struct ng_mesg *msg,
 		error = EINVAL;
 		break;
 	}
-	if (rptr)
-		*rptr = resp;
-	else if (resp)
-		FREE(resp, M_NETGRAPH);
-
 done:
-	FREE(msg, M_NETGRAPH);
+	NG_RESPOND_MSG(error, node, item, resp);
+	NG_FREE_MSG(msg);
 	return (error);
 }
 
@@ -345,38 +326,43 @@ done:
  * Receive incoming data on our hook.
  */
 static int
-ng_mppc_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
+ng_mppc_rcvdata(hook_p hook, item_p item)
 {
 	const node_p node = hook->node;
 	const priv_p priv = node->private;
 	struct mbuf *out;
 	int error;
+	struct mbuf *m;
 
+	NGI_GET_M(item, m);
 	/* Compress and/or encrypt */
 	if (hook == priv->xmit.hook) {
 		if (!priv->xmit.cfg.enable) {
-			NG_FREE_DATA(m, meta);
+			NG_FREE_M(m);
+			NG_FREE_ITEM(item);
 			return (ENXIO);
 		}
 		if ((error = ng_mppc_compress(node, m, &out)) != 0) {
-			NG_FREE_DATA(m, meta);
+			NG_FREE_M(m);
+			NG_FREE_ITEM(item);
 			return(error);
 		}
-		m_freem(m);
-		NG_SEND_DATA(error, priv->xmit.hook, out, meta);
+		NG_FREE_M(m);
+		NG_FWD_NEW_DATA(error, item, priv->xmit.hook, out);
 		return (error);
 	}
 
 	/* Decompress and/or decrypt */
 	if (hook == priv->recv.hook) {
 		if (!priv->recv.cfg.enable) {
-			NG_FREE_DATA(m, meta);
+			NG_FREE_M(m);
+			NG_FREE_ITEM(item);
 			return (ENXIO);
 		}
 		if ((error = ng_mppc_decompress(node, m, &out)) != 0) {
-			NG_FREE_DATA(m, meta);
-			if (error == EINVAL && priv->ctrlpath != NULL) {
+			NG_FREE_M(m);
+			NG_FREE_ITEM(item);
+			if (error == EINVAL && priv->ctrlnode != NULL) {
 				struct ng_mesg *msg;
 
 				/* Need to send a reset-request */
@@ -384,14 +370,13 @@ ng_mppc_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 				    NGM_MPPC_RESETREQ, 0, M_NOWAIT);
 				if (msg == NULL)
 					return (error);
-				/* XXX can we use a hook instead of ctrlpath? */
-				ng_send_msg(node, msg, priv->ctrlpath,
-					NULL, NULL, NULL); 
+				NG_SEND_MSG_ID(error, node, msg,
+					priv->ctrlnode, NULL); 
 			}
 			return (error);
 		}
-		m_freem(m);
-		NG_SEND_DATA(error, priv->recv.hook, out, meta);
+		NG_FREE_M(m);
+		NG_FWD_NEW_DATA(error, item, priv->recv.hook, out);
 		return (error);
 	}
 
@@ -403,16 +388,12 @@ ng_mppc_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
  * Destroy node
  */
 static int
-ng_mppc_rmnode(node_p node)
+ng_mppc_shutdown(node_p node)
 {
 	const priv_p priv = node->private;
 
 	/* Take down netgraph node */
 	node->flags |= NG_INVALID;
-	ng_cutlinks(node);
-	ng_unname(node);
-	if (priv->ctrlpath != NULL)
-		FREE(priv->ctrlpath, M_NETGRAPH);
 #ifdef NETGRAPH_MPPC_COMPRESSION
 	if (priv->xmit.history != NULL)
 		FREE(priv->xmit.history, M_NETGRAPH);
@@ -442,8 +423,9 @@ ng_mppc_disconnect(hook_p hook)
 		priv->recv.hook = NULL;
 
 	/* Go away if no longer connected */
-	if (node->numhooks == 0)
-		ng_rmnode(node);
+	if ((node->numhooks == 0)
+	&& ((node->flags & NG_INVALID) == 0))
+		ng_rmnode_self(node);
 	return (0);
 }
 
diff --git a/sys/netgraph/ng_one2many.c b/sys/netgraph/ng_one2many.c
index ad0f90f..bf800a8 100644
--- a/sys/netgraph/ng_one2many.c
+++ b/sys/netgraph/ng_one2many.c
@@ -80,7 +80,7 @@ typedef struct ng_one2many_private *priv_p;
 /* Netgraph node methods */
 static ng_constructor_t	ng_one2many_constructor;
 static ng_rcvmsg_t	ng_one2many_rcvmsg;
-static ng_shutdown_t	ng_one2many_rmnode;
+static ng_shutdown_t	ng_one2many_shutdown;
 static ng_newhook_t	ng_one2many_newhook;
 static ng_rcvdata_t	ng_one2many_rcvdata;
 static ng_disconnect_t	ng_one2many_disconnect;
@@ -167,7 +167,7 @@ static struct ng_type ng_one2many_typestruct = {
 	NULL,
 	ng_one2many_constructor,
 	ng_one2many_rcvmsg,
-	ng_one2many_rmnode,
+	ng_one2many_shutdown,
 	ng_one2many_newhook,
 	NULL,
 	NULL,
@@ -185,10 +185,9 @@ NETGRAPH_INIT(one2many, &ng_one2many_typestruct);
  * Node constructor
  */
 static int
-ng_one2many_constructor(node_p *nodep)
+ng_one2many_constructor(node_p node)
 {
 	priv_p priv;
-	int error;
 
 	/* Allocate and initialize private info */
 	MALLOC(priv, priv_p, sizeof(*priv), M_NETGRAPH, M_NOWAIT | M_ZERO);
@@ -197,12 +196,7 @@ ng_one2many_constructor(node_p *nodep)
 	priv->conf.xmitAlg = NG_ONE2MANY_XMIT_ROUNDROBIN;
 	priv->conf.failAlg = NG_ONE2MANY_FAIL_MANUAL;
 
-	/* Call superclass constructor */
-	if ((error = ng_make_node_common(&ng_one2many_typestruct, nodep))) {
-		FREE(priv, M_NETGRAPH);
-		return (error);
-	}
-	(*nodep)->private = priv;
+	node->private = priv;
 
 	/* Done */
 	return (0);
@@ -260,13 +254,14 @@ ng_one2many_newhook(node_p node, hook_p hook, const char *name)
  * Receive a control message
  */
 static int
-ng_one2many_rcvmsg(node_p node, struct ng_mesg *msg,
-	const char *retaddr, struct ng_mesg **rptr, hook_p lasthook)
+ng_one2many_rcvmsg(node_p node, item_p item, hook_p lasthook)
 {
 	const priv_p priv = node->private;
 	struct ng_mesg *resp = NULL;
 	int error = 0;
+	struct ng_mesg *msg;
 
+	NGI_GET_MSG(item, msg);
 	switch (msg->header.typecookie) {
 	case NGM_ONE2MANY_COOKIE:
 		switch (msg->header.cmd) {
@@ -369,11 +364,8 @@ ng_one2many_rcvmsg(node_p node, struct ng_mesg *msg,
 	}
 
 	/* Done */
-	if (rptr)
-		*rptr = resp;
-	else if (resp != NULL)
-		FREE(resp, M_NETGRAPH);
-	FREE(msg, M_NETGRAPH);
+	NG_RESPOND_MSG(error, node, item, resp);
+	NG_FREE_MSG(msg);
 	return (error);
 }
 
@@ -381,8 +373,7 @@ ng_one2many_rcvmsg(node_p node, struct ng_mesg *msg,
  * Receive data on a hook
  */
 static int
-ng_one2many_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-	struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
+ng_one2many_rcvdata(hook_p hook, item_p item)
 {
 	const node_p node = hook->node;
 	const priv_p priv = node->private;
@@ -390,7 +381,9 @@ ng_one2many_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 	struct ng_one2many_link *dst;
 	int error = 0;
 	int linkNum;
+	struct mbuf *m;
 
+	m = NGI_M(item); /* just peaking, mbuf still owned by item */
 	/* Get link number */
 	linkNum = LINK_NUM(hook);
 	KASSERT(linkNum == NG_ONE2MANY_ONE_LINKNUM
@@ -409,7 +402,7 @@ ng_one2many_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 	/* Figure out destination link */
 	if (linkNum == NG_ONE2MANY_ONE_LINKNUM) {
 		if (priv->numActiveMany == 0) {
-			NG_FREE_DATA(m, meta);
+			NG_FREE_ITEM(item);
 			return (ENOTCONN);
 		}
 		dst = &priv->many[priv->activeMany[priv->nextMany]];
@@ -422,7 +415,7 @@ ng_one2many_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 	dst->stats.xmitOctets += m->m_pkthdr.len;
 
 	/* Deliver packet */
-	NG_SEND_DATA(error, dst->hook, m, meta);
+	NG_FWD_DATA(error, item, dst->hook);
 	return (error);
 }
 
@@ -430,12 +423,10 @@ ng_one2many_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
  * Shutdown node
  */
 static int
-ng_one2many_rmnode(node_p node)
+ng_one2many_shutdown(node_p node)
 {
 	const priv_p priv = node->private;
 
-	ng_unname(node);
-	ng_cutlinks(node);
 	KASSERT(priv->numActiveMany == 0,
 	    ("%s: numActiveMany=%d", __FUNCTION__, priv->numActiveMany));
 	FREE(priv, M_NETGRAPH);
@@ -469,8 +460,9 @@ ng_one2many_disconnect(hook_p hook)
 	}
 
 	/* If no hooks left, go away */
-	if (hook->node->numhooks == 0)
-		ng_rmnode(hook->node);
+	if ((hook->node->numhooks == 0)
+	&& ((hook->node->flags & NG_INVALID) == 0))
+		ng_rmnode_self(hook->node);
 	return (0);
 }
 
diff --git a/sys/netgraph/ng_ppp.c b/sys/netgraph/ng_ppp.c
index a2fea53..2587207 100644
--- a/sys/netgraph/ng_ppp.c
+++ b/sys/netgraph/ng_ppp.c
@@ -207,18 +207,17 @@ typedef struct ng_ppp_private *priv_p;
 /* Netgraph node methods */
 static ng_constructor_t	ng_ppp_constructor;
 static ng_rcvmsg_t	ng_ppp_rcvmsg;
-static ng_shutdown_t	ng_ppp_rmnode;
+static ng_shutdown_t	ng_ppp_shutdown;
 static ng_newhook_t	ng_ppp_newhook;
 static ng_rcvdata_t	ng_ppp_rcvdata;
 static ng_disconnect_t	ng_ppp_disconnect;
 
 /* Helper functions */
 static int	ng_ppp_input(node_p node, int bypass,
-			int linkNum, struct mbuf *m, meta_p meta);
+			int linkNum, item_p item);
 static int	ng_ppp_output(node_p node, int bypass, int proto,
-			int linkNum, struct mbuf *m, meta_p meta);
-static int	ng_ppp_mp_input(node_p node, int linkNum,
-			struct mbuf *m, meta_p meta);
+			int linkNum, item_p item);
+static int	ng_ppp_mp_input(node_p node, int linkNum, item_p item);
 static int	ng_ppp_check_packet(node_p node);
 static void	ng_ppp_get_packet(node_p node, struct mbuf **mp, meta_p *metap);
 static int	ng_ppp_frag_process(node_p node);
@@ -347,7 +346,7 @@ static struct ng_type ng_ppp_typestruct = {
 	NULL,
 	ng_ppp_constructor,
 	ng_ppp_rcvmsg,
-	ng_ppp_rmnode,
+	ng_ppp_shutdown,
 	ng_ppp_newhook,
 	NULL,
 	NULL,
@@ -375,22 +374,17 @@ static const struct timeval ng_ppp_max_staleness = { 2, 0 };	/* 2 seconds */
  * Node type constructor
  */
 static int
-ng_ppp_constructor(node_p *nodep)
+ng_ppp_constructor(node_p node)
 {
 	priv_p priv;
-	int i, error;
+	int i;
 
 	/* Allocate private structure */
 	MALLOC(priv, priv_p, sizeof(*priv), M_NETGRAPH, M_NOWAIT | M_ZERO);
 	if (priv == NULL)
 		return (ENOMEM);
 
-	/* Call generic node constructor */
-	if ((error = ng_make_node_common(&ng_ppp_typestruct, nodep))) {
-		FREE(priv, M_NETGRAPH);
-		return (error);
-	}
-	(*nodep)->private = priv;
+	node->private = priv;
 
 	/* Initialize state */
 	TAILQ_INIT(&priv->frags);
@@ -461,13 +455,14 @@ ng_ppp_newhook(node_p node, hook_p hook, const char *name)
  * Receive a control message
  */
 static int
-ng_ppp_rcvmsg(node_p node, struct ng_mesg *msg,
-	      const char *raddr, struct ng_mesg **rptr, hook_p lasthook)
+ng_ppp_rcvmsg(node_p node, item_p item, hook_p lasthook)
 {
 	const priv_p priv = node->private;
 	struct ng_mesg *resp = NULL;
 	int error = 0;
+	struct ng_mesg *msg;
 
+	NGI_GET_MSG(item, msg);
 	switch (msg->header.typecookie) {
 	case NGM_PPP_COOKIE:
 		switch (msg->header.cmd) {
@@ -555,26 +550,24 @@ ng_ppp_rcvmsg(node_p node, struct ng_mesg *msg,
 		break;
 	case NGM_VJC_COOKIE:
 	    {
-		char path[NG_PATHLEN + 1];
-		node_p origNode;
-
-		if ((error = ng_path2node(node, raddr, &origNode, NULL)) != 0)
-			ERROUT(error);
-		snprintf(path, sizeof(path), "[%lx]:%s",
-		    (long)node->ID, NG_PPP_HOOK_VJC_IP);
-		return ng_send_msg(origNode, msg, path, NULL, NULL, rptr);
+		/*
+		 * Forward it to the vjc node. leave the 
+		 * old return address alone.
+		 */
+		NGI_MSG(item) = msg;	/* put it back in the item */
+		if (priv->links[HOOK_INDEX_VJC_IP].hook) {
+			NG_FWD_MSG_HOOK(error, NULL, item,
+				priv->links[HOOK_INDEX_VJC_IP].hook, NULL);
+		}
+		return (error);
 	    }
 	default:
 		error = EINVAL;
 		break;
 	}
-	if (rptr)
-		*rptr = resp;
-	else if (resp)
-		FREE(resp, M_NETGRAPH);
-
 done:
-	FREE(msg, M_NETGRAPH);
+	NG_RESPOND_MSG(error, node, item, resp);
+	NG_FREE_MSG(msg);
 	return (error);
 }
 
@@ -582,8 +575,7 @@ done:
  * Receive data on a hook
  */
 static int
-ng_ppp_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
+ng_ppp_rcvdata(hook_p hook, item_p item)
 {
 	const node_p node = hook->node;
 	const priv_p priv = node->private;
@@ -591,7 +583,9 @@ ng_ppp_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 	u_int16_t linkNum = NG_PPP_BUNDLE_LINKNUM;
 	hook_p outHook = NULL;
 	int proto = 0, error;
+	struct mbuf *m;
 
+	NGI_GET_M(item, m);
 	/* Did it come from a link hook? */
 	if (index < 0) {
 		struct ng_ppp_link *link;
@@ -609,7 +603,7 @@ ng_ppp_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 		/* Strip address and control fields, if present */
 		if (m->m_pkthdr.len >= 2) {
 			if (m->m_len < 2 && (m = m_pullup(m, 2)) == NULL) {
-				NG_FREE_DATA(m, meta);
+				NG_FREE_ITEM(item);
 				return (ENOBUFS);
 			}
 			if (bcmp(mtod(m, u_char *), &ng_ppp_acf, 2) == 0)
@@ -617,31 +611,33 @@ ng_ppp_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 		}
 
 		/* Dispatch incoming frame (if not enabled, to bypass) */
+		NGI_M(item) = m; 	/* put changed m back in item */
 		return ng_ppp_input(node,
-		    !link->conf.enableLink, linkNum, m, meta);
+		    !link->conf.enableLink, linkNum, item);
 	}
 
 	/* Get protocol & check if data allowed from this hook */
+	NGI_M(item) = m; 	/* put possibly changed m back in item */
 	switch (index) {
 
 	/* Outgoing data */
 	case HOOK_INDEX_ATALK:
 		if (!priv->conf.enableAtalk) {
-			NG_FREE_DATA(m, meta);
+			NG_FREE_ITEM(item);
 			return (ENXIO);
 		}
 		proto = PROT_APPLETALK;
 		break;
 	case HOOK_INDEX_IPX:
 		if (!priv->conf.enableIPX) {
-			NG_FREE_DATA(m, meta);
+			NG_FREE_ITEM(item);
 			return (ENXIO);
 		}
 		proto = PROT_IPX;
 		break;
 	case HOOK_INDEX_IPV6:
 		if (!priv->conf.enableIPv6) {
-			NG_FREE_DATA(m, meta);
+			NG_FREE_ITEM(item);
 			return (ENXIO);
 		}
 		proto = PROT_IPV6;
@@ -649,54 +645,56 @@ ng_ppp_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 	case HOOK_INDEX_INET:
 	case HOOK_INDEX_VJC_VJIP:
 		if (!priv->conf.enableIP) {
-			NG_FREE_DATA(m, meta);
+			NG_FREE_ITEM(item);
 			return (ENXIO);
 		}
 		proto = PROT_IP;
 		break;
 	case HOOK_INDEX_VJC_COMP:
 		if (!priv->conf.enableVJCompression) {
-			NG_FREE_DATA(m, meta);
+			NG_FREE_ITEM(item);
 			return (ENXIO);
 		}
 		proto = PROT_VJCOMP;
 		break;
 	case HOOK_INDEX_VJC_UNCOMP:
 		if (!priv->conf.enableVJCompression) {
-			NG_FREE_DATA(m, meta);
+			NG_FREE_ITEM(item);
 			return (ENXIO);
 		}
 		proto = PROT_VJUNCOMP;
 		break;
 	case HOOK_INDEX_COMPRESS:
 		if (!priv->conf.enableCompression) {
-			NG_FREE_DATA(m, meta);
+			NG_FREE_ITEM(item);
 			return (ENXIO);
 		}
 		proto = PROT_COMPD;
 		break;
 	case HOOK_INDEX_ENCRYPT:
 		if (!priv->conf.enableEncryption) {
-			NG_FREE_DATA(m, meta);
+			NG_FREE_ITEM(item);
 			return (ENXIO);
 		}
 		proto = PROT_CRYPTD;
 		break;
 	case HOOK_INDEX_BYPASS:
 		if (m->m_pkthdr.len < 4) {
-			NG_FREE_DATA(m, meta);
+			NG_FREE_ITEM(item);
 			return (EINVAL);
 		}
 		if (m->m_len < 4 && (m = m_pullup(m, 4)) == NULL) {
-			NG_FREE_META(meta);
+			NGI_M(item) = NULL; /* don't free twice */
+			NG_FREE_ITEM(item);
 			return (ENOBUFS);
 		}
+		NGI_M(item) = m; /* m may have changed */
 		linkNum = ntohs(mtod(m, u_int16_t *)[0]);
 		proto = ntohs(mtod(m, u_int16_t *)[1]);
 		m_adj(m, 4);
 		if (linkNum >= NG_PPP_MAX_LINKS
 		    && linkNum != NG_PPP_BUNDLE_LINKNUM) {
-			NG_FREE_DATA(m, meta);
+			NG_FREE_ITEM(item);
 			return (EINVAL);
 		}
 		break;
@@ -704,19 +702,19 @@ ng_ppp_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 	/* Incoming data */
 	case HOOK_INDEX_VJC_IP:
 		if (!priv->conf.enableIP || !priv->conf.enableVJDecompression) {
-			NG_FREE_DATA(m, meta);
+			NG_FREE_ITEM(item);
 			return (ENXIO);
 		}
 		break;
 	case HOOK_INDEX_DECOMPRESS:
 		if (!priv->conf.enableDecompression) {
-			NG_FREE_DATA(m, meta);
+			NG_FREE_ITEM(item);
 			return (ENXIO);
 		}
 		break;
 	case HOOK_INDEX_DECRYPT:
 		if (!priv->conf.enableDecryption) {
-			NG_FREE_DATA(m, meta);
+			NG_FREE_ITEM(item);
 			return (ENXIO);
 		}
 		break;
@@ -743,9 +741,11 @@ ng_ppp_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 		if (priv->conf.enableCompression
 		    && priv->hooks[HOOK_INDEX_COMPRESS] != NULL) {
 			if ((m = ng_ppp_addproto(m, proto, 1)) == NULL) {
-				NG_FREE_META(meta);
+				NGI_M(item) = NULL;
+				NG_FREE_ITEM(item);
 				return (ENOBUFS);
 			}
+			NGI_M(item) = m; /* m may have changed */
 			outHook = priv->hooks[HOOK_INDEX_COMPRESS];
 			break;
 		}
@@ -754,24 +754,25 @@ ng_ppp_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 		if (priv->conf.enableEncryption
 		    && priv->hooks[HOOK_INDEX_ENCRYPT] != NULL) {
 			if ((m = ng_ppp_addproto(m, proto, 1)) == NULL) {
-				NG_FREE_META(meta);
+				NGI_M(item) = NULL;
+				NG_FREE_ITEM(item);
 				return (ENOBUFS);
 			}
+			NGI_M(item) = m; /* m may have changed */
 			outHook = priv->hooks[HOOK_INDEX_ENCRYPT];
 			break;
 		}
 		/* FALLTHROUGH */
 	case HOOK_INDEX_ENCRYPT:
-		return ng_ppp_output(node, 0,
-		    proto, NG_PPP_BUNDLE_LINKNUM, m, meta);
+		return ng_ppp_output(node, 0, proto, NG_PPP_BUNDLE_LINKNUM, item);
 
 	case HOOK_INDEX_BYPASS:
-		return ng_ppp_output(node, 1, proto, linkNum, m, meta);
+		return ng_ppp_output(node, 1, proto, linkNum, item);
 
 	/* Incoming data */
 	case HOOK_INDEX_DECRYPT:
 	case HOOK_INDEX_DECOMPRESS:
-		return ng_ppp_input(node, 0, NG_PPP_BUNDLE_LINKNUM, m, meta);
+		return ng_ppp_input(node, 0, NG_PPP_BUNDLE_LINKNUM, item);
 
 	case HOOK_INDEX_VJC_IP:
 		outHook = priv->hooks[HOOK_INDEX_INET];
@@ -779,9 +780,13 @@ ng_ppp_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 	}
 
 	/* Send packet out hook */
-	NG_SEND_DATA_RET(error, outHook, m, meta, resp);
-	if (m != NULL || meta != NULL)
-		return ng_ppp_rcvdata(outHook, m, meta, NULL, NULL, resp);
+	NG_FWD_DATA(error, item, outHook);
+#if 0
+	/* help archie... what's going on? */
+	/* Looks like you were acrually USING the stub functions
+	(now gone again) */
+		return ng_ppp_rcvdata(outHook, item);
+#endif
 	return (error);
 }
 
@@ -789,7 +794,7 @@ ng_ppp_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
  * Destroy node
  */
 static int
-ng_ppp_rmnode(node_p node)
+ng_ppp_shutdown(node_p node)
 {
 	const priv_p priv = node->private;
 
@@ -798,8 +803,6 @@ ng_ppp_rmnode(node_p node)
 
 	/* Take down netgraph node */
 	node->flags |= NG_INVALID;
-	ng_cutlinks(node);
-	ng_unname(node);
 	ng_ppp_frag_reset(node);
 	bzero(priv, sizeof(*priv));
 	FREE(priv, M_NETGRAPH);
@@ -825,10 +828,13 @@ ng_ppp_disconnect(hook_p hook)
 		priv->hooks[index] = NULL;
 
 	/* Update derived info (or go away if no hooks left) */
-	if (node->numhooks > 0)
+	if (node->numhooks > 0) {
 		ng_ppp_update(node, 0);
-	else
-		ng_rmnode(node);
+	} else {
+		if ((node->flags & NG_INVALID) == 0) {
+			ng_rmnode_self(node);
+		}
+	}
 	return (0);
 }
 
@@ -841,16 +847,19 @@ ng_ppp_disconnect(hook_p hook)
  * and dispatch accordingly.
  */
 static int
-ng_ppp_input(node_p node, int bypass, int linkNum, struct mbuf *m, meta_p meta)
+ng_ppp_input(node_p node, int bypass, int linkNum, item_p item)
 {
 	const priv_p priv = node->private;
 	hook_p outHook = NULL;
 	int proto, error;
+	struct mbuf *m;
+
 
+	NGI_GET_M(item, m);
 	/* Extract protocol number */
 	for (proto = 0; !PROT_VALID(proto) && m->m_pkthdr.len > 0; ) {
 		if (m->m_len < 1 && (m = m_pullup(m, 1)) == NULL) {
-			NG_FREE_META(meta);
+			NG_FREE_ITEM(item);
 			return (ENOBUFS);
 		}
 		proto = (proto << 8) + *mtod(m, u_char *);
@@ -861,7 +870,8 @@ ng_ppp_input(node_p node, int bypass, int linkNum, struct mbuf *m, meta_p meta)
 			priv->bundleStats.badProtos++;
 		else
 			priv->links[linkNum].stats.badProtos++;
-		NG_FREE_DATA(m, meta);
+		NG_FREE_ITEM(item);
+		NG_FREE_M(m);
 		return (EINVAL);
 	}
 
@@ -890,7 +900,7 @@ ng_ppp_input(node_p node, int bypass, int linkNum, struct mbuf *m, meta_p meta)
 	case PROT_MP:
 		if (priv->conf.enableMultilink
 		    && linkNum != NG_PPP_BUNDLE_LINKNUM)
-			return ng_ppp_mp_input(node, linkNum, m, meta);
+			return ng_ppp_mp_input(node, linkNum, item);
 		break;
 	case PROT_APPLETALK:
 		if (priv->conf.enableAtalk)
@@ -918,14 +928,14 @@ bypass:
 		hdr[0] = htons(linkNum);
 		hdr[1] = htons((u_int16_t)proto);
 		if ((m = ng_ppp_prepend(m, &hdr, 4)) == NULL) {
-			NG_FREE_META(meta);
+			NG_FREE_ITEM(item);
 			return (ENOBUFS);
 		}
 		outHook = priv->hooks[HOOK_INDEX_BYPASS];
 	}
 
 	/* Forward frame */
-	NG_SEND_DATA(error, outHook, m, meta);
+	NG_FWD_NEW_DATA(error, item, outHook, m);
 	return (error);
 }
 
@@ -935,12 +945,14 @@ bypass:
  */
 static int
 ng_ppp_output(node_p node, int bypass,
-	int proto, int linkNum, struct mbuf *m, meta_p meta)
+	int proto, int linkNum, item_p item)
 {
 	const priv_p priv = node->private;
 	struct ng_ppp_link *link;
 	int len, error;
+	struct mbuf *m;
 
+	 NGI_GET_M(item, m); 	/* separate them for a while */
 	/* If not doing MP, map bundle virtual link to (the only) link */
 	if (linkNum == NG_PPP_BUNDLE_LINKNUM && !priv->conf.enableMultilink)
 		linkNum = priv->activeLinks[0];
@@ -952,11 +964,13 @@ ng_ppp_output(node_p node, int bypass,
 	/* Check link status (if real) */
 	if (linkNum != NG_PPP_BUNDLE_LINKNUM) {
 		if (!bypass && !link->conf.enableLink) {
-			NG_FREE_DATA(m, meta);
+			NG_FREE_M(m);
+			NG_FREE_ITEM(item);
 			return (ENXIO);
 		}
 		if (link->hook == NULL) {
-			NG_FREE_DATA(m, meta);
+			NG_FREE_M(m);
+			NG_FREE_ITEM(item);
 			return (ENETDOWN);
 		}
 	}
@@ -965,25 +979,31 @@ ng_ppp_output(node_p node, int bypass,
 	if ((m = ng_ppp_addproto(m, proto,
 	    linkNum == NG_PPP_BUNDLE_LINKNUM
 	      || link->conf.enableProtoComp)) == NULL) {
-		NG_FREE_META(meta);
+		NG_FREE_ITEM(item);
 		return (ENOBUFS);
 	}
 
 	/* Special handling for the MP virtual link */
-	if (linkNum == NG_PPP_BUNDLE_LINKNUM)
+	if (linkNum == NG_PPP_BUNDLE_LINKNUM) {
+		meta_p meta;
+
+		/* strip off and discard the queue item */
+		NGI_GET_META(item, meta);
+		NG_FREE_ITEM(item);
 		return ng_ppp_mp_output(node, m, meta);
+	}
 
 	/* Prepend address and control field (unless compressed) */
 	if (proto == PROT_LCP || !link->conf.enableACFComp) {
 		if ((m = ng_ppp_prepend(m, &ng_ppp_acf, 2)) == NULL) {
-			NG_FREE_META(meta);
+			NG_FREE_ITEM(item);
 			return (ENOBUFS);
 		}
 	}
 
 	/* Deliver frame */
 	len = m->m_pkthdr.len;
-	NG_SEND_DATA(error, link->hook, m, meta);
+	NG_FWD_NEW_DATA(error, item,  link->hook, m);
 
 	/* Update stats and 'bytes in queue' counter */
 	if (error == 0) {
@@ -1048,14 +1068,19 @@ ng_ppp_output(node_p node, int bypass,
  * This assumes linkNum != NG_PPP_BUNDLE_LINKNUM.
  */
 static int
-ng_ppp_mp_input(node_p node, int linkNum, struct mbuf *m, meta_p meta)
+ng_ppp_mp_input(node_p node, int linkNum, item_p item)
 {
 	const priv_p priv = node->private;
 	struct ng_ppp_link *const link = &priv->links[linkNum];
 	struct ng_ppp_frag frag0, *frag = &frag0;
 	struct ng_ppp_frag *qent;
 	int i, diff, inserted;
+	struct mbuf *m;
+	meta_p meta;
 
+	NGI_GET_M(item, m);
+	NGI_GET_META(item, meta);
+	NG_FREE_ITEM(item);
 	/* Stats */
 	priv->bundleStats.recvFrames++;
 	priv->bundleStats.recvOctets += m->m_pkthdr.len;
@@ -1066,7 +1091,8 @@ ng_ppp_mp_input(node_p node, int linkNum, struct mbuf *m, meta_p meta)
 
 		if (m->m_pkthdr.len < 2) {
 			link->stats.runts++;
-			NG_FREE_DATA(m, meta);
+			NG_FREE_M(m);
+			NG_FREE_META(meta);
 			return (EINVAL);
 		}
 		if (m->m_len < 2 && (m = m_pullup(m, 2)) == NULL) {
@@ -1084,7 +1110,8 @@ ng_ppp_mp_input(node_p node, int linkNum, struct mbuf *m, meta_p meta)
 
 		if (m->m_pkthdr.len < 4) {
 			link->stats.runts++;
-			NG_FREE_DATA(m, meta);
+			NG_FREE_M(m);
+			NG_FREE_META(meta);
 			return (EINVAL);
 		}
 		if (m->m_len < 4 && (m = m_pullup(m, 4)) == NULL) {
@@ -1106,7 +1133,8 @@ ng_ppp_mp_input(node_p node, int linkNum, struct mbuf *m, meta_p meta)
 	   fragment as lost, so we have no choice now but to drop it */
 	if (diff < 0) {
 		link->stats.dropFragments++;
-		NG_FREE_DATA(m, meta);
+		NG_FREE_M(m);
+		NG_FREE_META(meta);
 		return (0);
 	}
 
@@ -1123,7 +1151,8 @@ ng_ppp_mp_input(node_p node, int linkNum, struct mbuf *m, meta_p meta)
 	/* Allocate a new frag struct for the queue */
 	MALLOC(frag, struct ng_ppp_frag *, sizeof(*frag), M_NETGRAPH, M_NOWAIT);
 	if (frag == NULL) {
-		NG_FREE_DATA(m, meta);
+		NG_FREE_M(m);
+		NG_FREE_META(meta);
 		ng_ppp_frag_process(node);
 		return (ENOMEM);
 	}
@@ -1139,7 +1168,8 @@ ng_ppp_mp_input(node_p node, int linkNum, struct mbuf *m, meta_p meta)
 			break;
 		} else if (diff == 0) {	     /* should never happen! */
 			link->stats.dupFragments++;
-			NG_FREE_DATA(frag->data, frag->meta);
+			NG_FREE_M(frag->data);
+			NG_FREE_META(frag->meta);
 			FREE(frag, M_NETGRAPH);
 			return (EINVAL);
 		}
@@ -1265,7 +1295,8 @@ ng_ppp_frag_trim(node_p node)
 			    ("%s: empty q", __FUNCTION__));
 			priv->bundleStats.dropFragments++;
 			TAILQ_REMOVE(&priv->frags, qent, f_qent);
-			NG_FREE_DATA(qent->data, qent->meta);
+			NG_FREE_M(qent->data);
+			NG_FREE_META(qent->meta);
 			FREE(qent, M_NETGRAPH);
 			priv->qlen--;
 			removed = 1;
@@ -1283,18 +1314,21 @@ ng_ppp_frag_process(node_p node)
 	const priv_p priv = node->private;
 	struct mbuf *m;
 	meta_p meta;
+	item_p item;
 
 	/* Deliver any deliverable packets */
 	while (ng_ppp_check_packet(node)) {
 		ng_ppp_get_packet(node, &m, &meta);
-		ng_ppp_input(node, 0, NG_PPP_BUNDLE_LINKNUM, m, meta);
+		item = ng_package_data(m, meta);
+		ng_ppp_input(node, 0, NG_PPP_BUNDLE_LINKNUM, item);
 	}
 
 	/* Delete dead fragments and try again */
 	if (ng_ppp_frag_trim(node)) {
 		while (ng_ppp_check_packet(node)) {
 			ng_ppp_get_packet(node, &m, &meta);
-			ng_ppp_input(node, 0, NG_PPP_BUNDLE_LINKNUM, m, meta);
+			item = ng_package_data(m, meta);
+			ng_ppp_input(node, 0, NG_PPP_BUNDLE_LINKNUM, item);
 		}
 	}
 
@@ -1327,7 +1361,8 @@ ng_ppp_frag_process(node_p node)
 		/* Drop it */
 		priv->bundleStats.dropFragments++;
 		TAILQ_REMOVE(&priv->frags, qent, f_qent);
-		NG_FREE_DATA(qent->data, qent->meta);
+		NG_FREE_M(qent->data);
+		NG_FREE_META(qent->meta);
 		FREE(qent, M_NETGRAPH);
 		priv->qlen--;
 
@@ -1360,6 +1395,7 @@ ng_ppp_frag_checkstale(node_p node)
 	struct mbuf *m;
 	meta_p meta;
 	int i, seq;
+	item_p item;
 
 	now.tv_sec = 0;			/* uninitialized state */
 	while (1) {
@@ -1403,7 +1439,8 @@ ng_ppp_frag_checkstale(node_p node)
 			    ("%s: empty q", __FUNCTION__));
 			priv->bundleStats.dropFragments++;
 			TAILQ_REMOVE(&priv->frags, qent, f_qent);
-			NG_FREE_DATA(qent->data, qent->meta);
+			NG_FREE_M(qent->data);
+			NG_FREE_META(qent->meta);
 			FREE(qent, M_NETGRAPH);
 			priv->qlen--;
 		}
@@ -1425,7 +1462,8 @@ ng_ppp_frag_checkstale(node_p node)
 		}
 
 		/* Deliver packet */
-		ng_ppp_input(node, 0, NG_PPP_BUNDLE_LINKNUM, m, meta);
+		item = ng_package_data(m, meta);
+		ng_ppp_input(node, 0, NG_PPP_BUNDLE_LINKNUM, item);
 	}
 }
 
@@ -1471,10 +1509,12 @@ ng_ppp_mp_output(node_p node, struct mbuf *m, meta_p meta)
 	int distrib[NG_PPP_MAX_LINKS];
 	int firstFragment;
 	int activeLinkNum;
+	item_p item;
 
 	/* At least one link must be active */
 	if (priv->numActiveLinks == 0) {
-		NG_FREE_DATA(m, meta);
+		NG_FREE_M(m);
+		NG_FREE_META(meta);
 		return (ENETDOWN);
 	}
 
@@ -1535,7 +1575,8 @@ deliver:
 				struct mbuf *n = m_split(m, len, M_NOWAIT);
 
 				if (n == NULL) {
-					NG_FREE_DATA(m, meta);
+					NG_FREE_M(m);
+					NG_FREE_META(meta);
 					return (ENOMEM);
 				}
 				m = n;
@@ -1578,11 +1619,13 @@ deliver:
 			meta2 = lastFragment ? meta : ng_copy_meta(meta);
 
 			/* Send fragment */
-			error = ng_ppp_output(node, 0,
-			    PROT_MP, linkNum, m2, meta2);
+			item = ng_package_data(m2, meta2);
+			error = ng_ppp_output(node, 0, PROT_MP, linkNum, item);
 			if (error != 0) {
-				if (!lastFragment)
-					NG_FREE_DATA(m, meta);
+				if (!lastFragment) {
+					NG_FREE_M(m);
+					NG_FREE_META(meta);
+				}
 				return (error);
 			}
 		}
@@ -1991,7 +2034,8 @@ ng_ppp_frag_reset(node_p node)
 
 	for (qent = TAILQ_FIRST(&priv->frags); qent; qent = qnext) {
 		qnext = TAILQ_NEXT(qent, f_qent);
-		NG_FREE_DATA(qent->data, qent->meta);
+		NG_FREE_M(qent->data);
+		NG_FREE_META(qent->meta);
 		FREE(qent, M_NETGRAPH);
 	}
 	TAILQ_INIT(&priv->frags);
diff --git a/sys/netgraph/ng_pppoe.c b/sys/netgraph/ng_pppoe.c
index 14bf650..4b90f9f 100644
--- a/sys/netgraph/ng_pppoe.c
+++ b/sys/netgraph/ng_pppoe.c
@@ -70,7 +70,7 @@
 
 static ng_constructor_t	ng_pppoe_constructor;
 static ng_rcvmsg_t	ng_pppoe_rcvmsg;
-static ng_shutdown_t	ng_pppoe_rmnode;
+static ng_shutdown_t	ng_pppoe_shutdown;
 static ng_newhook_t	ng_pppoe_newhook;
 static ng_connect_t	ng_pppoe_connect;
 static ng_rcvdata_t	ng_pppoe_rcvdata;
@@ -153,7 +153,7 @@ static struct ng_type typestruct = {
 	NULL,
 	ng_pppoe_constructor,
 	ng_pppoe_rcvmsg,
-	ng_pppoe_rmnode,
+	ng_pppoe_shutdown,
 	ng_pppoe_newhook,
 	NULL,
 	ng_pppoe_connect,
@@ -209,7 +209,7 @@ struct sess_con {
 	hook_p  		hook;
 	u_int16_t		Session_ID;
 	enum state		state;
-	char			creator[NG_NODELEN + 1]; /* who to notify */
+	ng_ID_t			creator;		/* who to notify */
 	struct pppoe_full_hdr	pkt_hdr;	/* used when connected */
 	negp			neg;		/* used when negotiating */
 	/*struct sess_con	*hash_next;*/	/* not yet used */
@@ -532,10 +532,9 @@ AAA
  * unref the node so it gets freed too.
  */
 static int
-ng_pppoe_constructor(node_p *nodep)
+ng_pppoe_constructor(node_p node)
 {
 	priv_p privdata;
-	int error;
 
 AAA
 	/* Initialize private descriptor */
@@ -544,15 +543,9 @@ AAA
 	if (privdata == NULL)
 		return (ENOMEM);
 
-	/* Call the 'generic' (ie, superclass) node constructor */
-	if ((error = ng_make_node_common(&typestruct, nodep))) {
-		FREE(privdata, M_NETGRAPH);
-		return (error);
-	}
-
 	/* Link structs together; this counts as our one reference to *nodep */
-	(*nodep)->private = privdata;
-	privdata->node = *nodep;
+	node->private = privdata;
+	privdata->node = node;
 	return (0);
 }
 
@@ -602,8 +595,7 @@ AAA
  * Always free the message.
  */
 static int
-ng_pppoe_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
-		struct ng_mesg **rptr, hook_p lasthook)
+ng_pppoe_rcvmsg(node_p node, item_p item, hook_p lasthook)
 {
 	priv_p privp = node->private;
 	struct ngpppoe_init_data *ourmsg = NULL;
@@ -612,8 +604,10 @@ ng_pppoe_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
 	hook_p hook = NULL;
 	sessp sp = NULL;
 	negp neg = NULL;
+	struct ng_mesg *msg;
 
 AAA
+	NGI_GET_MSG(item, msg);
 	/* Deal with message according to cookie and command */
 	switch (msg->header.typecookie) {
 	case NGM_PPPOE_COOKIE: 
@@ -704,8 +698,7 @@ AAA
 			neg->pkt->pkt_header.ph.sid = 0x0000;
 			neg->timeout = 0;
 
-			strncpy(sp->creator, retaddr, NG_NODELEN);
-			sp->creator[NG_NODELEN] = '\0';
+			sp->creator = NGI_RETADDR(item);
 		}
 		switch (msg->header.cmd) {
 		case NGM_PPPOE_GET_STATUS:
@@ -810,14 +803,10 @@ AAA
 	}
 
 	/* Take care of synchronous response, if any */
-	if (rptr)
-		*rptr = resp;
-	else if (resp)
-		FREE(resp, M_NETGRAPH);
-
-	/* Free the message and return */
 quit:
-	FREE(msg, M_NETGRAPH);
+	NG_RESPOND_MSG(error, node, item, resp);
+	/* Free the message and return */
+	NG_FREE_MSG(msg);
 	return(error);
 }
 
@@ -857,8 +846,7 @@ AAA
  * if we use up this data or abort we must free BOTH of these.
  */
 static int
-ng_pppoe_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
+ng_pppoe_rcvdata(hook_p hook, item_p item)
 {
 	node_p			node = hook->node;
 	const priv_p		privp = node->private;
@@ -876,14 +864,16 @@ ng_pppoe_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 		union	uniq	data;
 	} uniqtag;
 	negp			neg = NULL;
+	struct mbuf		*m;
 
 AAA
+	NGI_GET_M(item, m);
 	if (hook->private == &privp->debug_hook) {
 		/*
 		 * Data from the debug hook gets sent without modification
 		 * straight to the ethernet. 
 		 */
-		NG_SEND_DATA( error, privp->ethernet_hook, m, meta);
+		NG_FWD_DATA( error, item, privp->ethernet_hook);
 	 	privp->packets_out++;
 	} else if (hook->private == &privp->ethernet_hook) {
 		/*
@@ -901,10 +891,7 @@ AAA
 			}
 		}
 		wh = mtod(m, struct pppoe_full_hdr *);
-		ph = &wh->ph;
-		session = ntohs(wh->ph.sid);
 		length = ntohs(wh->ph.length);
-		code = wh->ph.code; 
 		switch(wh->eh.ether_type) {
 		case	ETHERTYPE_PPPOE_DISC:
 			/*
@@ -913,8 +900,6 @@ AAA
 			 * of a buffer and make a mess. 
 			 * (Linux wouldn't have this problem).
 			 */
-/*XXX fix this mess */
-			
 			if (m->m_pkthdr.len <= MHLEN) {
 				if( m->m_len < m->m_pkthdr.len) {
 					m = m_pullup(m, m->m_pkthdr.len);
@@ -928,10 +913,29 @@ AAA
 				/*
 				 * It's not all in one piece.
 				 * We need to do extra work.
+				 * Put it into a cluster.
 				 */
-				printf("packet fragmented\n");
-				LEAVE(EMSGSIZE);
+				struct mbuf *n;
+				n = m_dup(m, M_DONTWAIT);
+				m_freem(m);
+				m = n;
+				if (m) {
+					/* just check we got a cluster */
+					if (m->m_len != m->m_pkthdr.len) {
+						m_freem(m);
+						m = NULL;
+					}
+				}
+				if (m == NULL) {
+					printf("packet fragmented\n");
+					LEAVE(EMSGSIZE);
+				}
 			}
+			wh = mtod(m, struct pppoe_full_hdr *);
+			length = ntohs(wh->ph.length);
+			ph = &wh->ph;
+			session = ntohs(wh->ph.sid);
+			code = wh->ph.code; 
 
 			switch(code) {
 			case	PADI_CODE:
@@ -951,7 +955,8 @@ AAA
 				sendhook = pppoe_match_svc(hook->node,
 			    		tag->tag_data, ntohs(tag->tag_len));
 				if (sendhook) {
-					NG_SEND_DATA(error, sendhook, m, meta);
+					NG_FWD_NEW_DATA(error, item,
+								sendhook, m);
 				} else {
 					printf("no such service\n");
 					LEAVE(ENETUNREACH);
@@ -1149,7 +1154,6 @@ AAA
 				 * Find matching peer/session combination.
 				 */
 				sendhook = pppoe_findsession(node, wh);
-				NG_FREE_DATA(m, meta); /* no longer needed */
 				if (sendhook == NULL) {
 					LEAVE(ENETUNREACH);
 				}
@@ -1202,7 +1206,7 @@ AAA
 					break;
 				}
 			}
-			NG_SEND_DATA( error, sendhook, m, meta);
+			NG_FWD_NEW_DATA( error, item, sendhook, m);
 			break;
 		default:
 			LEAVE(EPFNOSUPPORT);
@@ -1247,7 +1251,7 @@ AAA
 			}
 			wh = mtod(m, struct pppoe_full_hdr *);
 			bcopy(&sp->pkt_hdr, wh, sizeof(*wh));
-			NG_SEND_DATA( error, privp->ethernet_hook, m, meta);
+			NG_FWD_NEW_DATA( error, item, privp->ethernet_hook, m);
 			privp->packets_out++;
 			break;
 			}
@@ -1332,7 +1336,8 @@ AAA
 		}
 	}
 quit:
-	NG_FREE_DATA(m, meta);
+	NG_FREE_ITEM(item);
+	NG_FREE_M(m);
 	return error;
 }
 
@@ -1342,14 +1347,12 @@ quit:
  * we'd only remove our links and reset ourself.
  */
 static int
-ng_pppoe_rmnode(node_p node)
+ng_pppoe_shutdown(node_p node)
 {
 	const priv_p privdata = node->private;
 
 AAA
 	node->flags |= NG_INVALID;
-	ng_cutlinks(node);
-	ng_unname(node);
 	node->private = NULL;
 	ng_unref(privdata->node);
 	FREE(privdata, M_NETGRAPH);
@@ -1387,7 +1390,8 @@ AAA
 		privp->debug_hook = NULL;
 	} else if (hook->private == &privp->ethernet_hook) {
 		privp->ethernet_hook = NULL;
-		ng_rmnode(node);
+		if ((node->flags & NG_INVALID) == 0)
+			ng_rmnode_self(node);
 	} else {
 		sp = hook->private;
 		if (sp->state != PPPOE_SNONE ) {
@@ -1405,7 +1409,6 @@ AAA
 			struct pppoe_full_hdr *wh;
 			struct pppoe_tag *tag;
 			int	msglen = strlen(SIGNOFF);
-			void *dummy = NULL;
 			int error = 0;
 
 			/* revert the stored header to DISC/PADT mode */
@@ -1434,8 +1437,8 @@ AAA
 				m->m_pkthdr.len = (m->m_len += sizeof(*tag) +
 				    msglen);
 				wh->ph.length = htons(sizeof(*tag) + msglen);
-				NG_SEND_DATA(error, privp->ethernet_hook, m,
-				    dummy);
+				NG_SEND_DATA_ONLY(error,
+					privp->ethernet_hook, m);
 			}
 		}
 		/*
@@ -1455,8 +1458,9 @@ AAA
 		if (privp->ethernet_hook) hooks -= 1;
 		if (privp->debug_hook) hooks -= 1;
 	}
-	if (node->numhooks == 0)
-		ng_rmnode(node);
+	if ((node->numhooks == 0)
+	&& ((node->flags & NG_INVALID) == 0))
+		ng_rmnode_self(node);
 	return (0);
 }
 
@@ -1473,7 +1477,6 @@ pppoe_ticker(void *arg)
 	int	error = 0;
 	struct mbuf *m0 = NULL;
 	priv_p privp = hook->node->private;
-	meta_p dummy = NULL;
 
 AAA
 	switch(sp->state) {
@@ -1486,7 +1489,7 @@ AAA
 	case	PPPOE_SREQ:
 		/* timeouts on these produce resends */
 		m0 = m_copypacket(sp->neg->m, M_DONTWAIT);
-		NG_SEND_DATA( error, privp->ethernet_hook, m0, dummy);
+		NG_SEND_DATA_ONLY( error, privp->ethernet_hook, m0);
 		neg->timeout_handle = timeout(pppoe_ticker,
 					hook, neg->timeout * hz);
 		if ((neg->timeout <<= 1) > PPPOE_TIMEOUT_LIMIT) {
@@ -1519,7 +1522,6 @@ sendpacket(sessp sp)
 	hook_p hook = sp->hook;
 	negp	neg = sp->neg;
 	priv_p	privp = hook->node->private;
-	meta_p dummy = NULL;
 
 AAA
 	switch(sp->state) {
@@ -1533,7 +1535,7 @@ AAA
 	case	PPPOE_NEWCONNECTED:
 		/* send the PADS without a timeout - we're now connected */
 		m0 = m_copypacket(sp->neg->m, M_DONTWAIT);
-		NG_SEND_DATA( error, privp->ethernet_hook, m0, dummy);
+		NG_SEND_DATA_ONLY( error, privp->ethernet_hook, m0);
 		break;
 
 	case	PPPOE_PRIMED:
@@ -1548,7 +1550,7 @@ AAA
 		 * in PPPOE_OFFER_TIMEOUT seconds, forget about it.
 		 */
 		m0 = m_copypacket(sp->neg->m, M_DONTWAIT);
-		NG_SEND_DATA( error, privp->ethernet_hook, m0, dummy);
+		NG_SEND_DATA_ONLY( error, privp->ethernet_hook, m0);
 		neg->timeout_handle = timeout(pppoe_ticker,
 					hook, PPPOE_OFFER_TIMEOUT * hz);
 		break;
@@ -1556,7 +1558,7 @@ AAA
 	case	PPPOE_SINIT:
 	case	PPPOE_SREQ:
 		m0 = m_copypacket(sp->neg->m, M_DONTWAIT);
-		NG_SEND_DATA( error, privp->ethernet_hook, m0, dummy);
+		NG_SEND_DATA_ONLY( error, privp->ethernet_hook, m0);
 		neg->timeout_handle = timeout(pppoe_ticker, hook,
 					(hz * PPPOE_INITIAL_TIMEOUT));
 		neg->timeout = PPPOE_INITIAL_TIMEOUT * 2;
@@ -1627,6 +1629,6 @@ AAA
 		return (ENOMEM);
 	sts = (struct ngpppoe_sts *)msg->data;
 	strncpy(sts->hook, sp->hook->name, NG_HOOKLEN + 1);
-	error = ng_send_msg(sp->hook->node, msg, sp->creator, NULL, NULL, NULL);
+	NG_SEND_MSG_ID(error, sp->hook->node, msg, sp->creator, NULL);
 	return (error);
 }
diff --git a/sys/netgraph/ng_pptpgre.c b/sys/netgraph/ng_pptpgre.c
index c5306b0..c126d9d 100644
--- a/sys/netgraph/ng_pptpgre.c
+++ b/sys/netgraph/ng_pptpgre.c
@@ -172,14 +172,14 @@ typedef struct ng_pptpgre_private *priv_p;
 /* Netgraph node methods */
 static ng_constructor_t	ng_pptpgre_constructor;
 static ng_rcvmsg_t	ng_pptpgre_rcvmsg;
-static ng_shutdown_t	ng_pptpgre_rmnode;
+static ng_shutdown_t	ng_pptpgre_shutdown;
 static ng_newhook_t	ng_pptpgre_newhook;
 static ng_rcvdata_t	ng_pptpgre_rcvdata;
 static ng_disconnect_t	ng_pptpgre_disconnect;
 
 /* Helper functions */
-static int	ng_pptpgre_xmit(node_p node, struct mbuf *m, meta_p meta);
-static int	ng_pptpgre_recv(node_p node, struct mbuf *m, meta_p meta);
+static int	ng_pptpgre_xmit(node_p node, item_p item);
+static int	ng_pptpgre_recv(node_p node, item_p item);
 static void	ng_pptpgre_start_send_ack_timer(node_p node, int ackTimeout);
 static void	ng_pptpgre_start_recv_ack_timer(node_p node);
 static void	ng_pptpgre_recv_ack_timeout(void *arg);
@@ -250,7 +250,7 @@ static struct ng_type ng_pptpgre_typestruct = {
 	NULL,
 	ng_pptpgre_constructor,
 	ng_pptpgre_rcvmsg,
-	ng_pptpgre_rmnode,
+	ng_pptpgre_shutdown,
 	ng_pptpgre_newhook,
 	NULL,
 	NULL,
@@ -270,22 +270,16 @@ NETGRAPH_INIT(pptpgre, &ng_pptpgre_typestruct);
  * Node type constructor
  */
 static int
-ng_pptpgre_constructor(node_p *nodep)
+ng_pptpgre_constructor(node_p node)
 {
 	priv_p priv;
-	int error;
 
 	/* Allocate private structure */
 	MALLOC(priv, priv_p, sizeof(*priv), M_NETGRAPH, M_NOWAIT | M_ZERO);
 	if (priv == NULL)
 		return (ENOMEM);
 
-	/* Call generic node constructor */
-	if ((error = ng_make_node_common(&ng_pptpgre_typestruct, nodep))) {
-		FREE(priv, M_NETGRAPH);
-		return (error);
-	}
-	(*nodep)->private = priv;
+	node->private = priv;
 
 	/* Initialize state */
 	callout_handle_init(&priv->ackp.sackTimer);
@@ -325,13 +319,14 @@ ng_pptpgre_newhook(node_p node, hook_p hook, const char *name)
  * Receive a control message.
  */
 static int
-ng_pptpgre_rcvmsg(node_p node, struct ng_mesg *msg,
-	      const char *raddr, struct ng_mesg **rptr, hook_p lasthook)
+ng_pptpgre_rcvmsg(node_p node, item_p item, hook_p lasthook)
 {
 	const priv_p priv = node->private;
 	struct ng_mesg *resp = NULL;
 	int error = 0;
+	struct ng_mesg *msg;
 
+	NGI_GET_MSG(item, msg);
 	switch (msg->header.typecookie) {
 	case NGM_PPTPGRE_COOKIE:
 		switch (msg->header.cmd) {
@@ -379,12 +374,8 @@ ng_pptpgre_rcvmsg(node_p node, struct ng_mesg *msg,
 		break;
 	}
 done:
-	if (rptr)
-		*rptr = resp;
-	else if (resp)
-		FREE(resp, M_NETGRAPH);
-
-	FREE(msg, M_NETGRAPH);
+	NG_RESPOND_MSG(error, node, item, resp);
+	NG_FREE_MSG(msg);
 	return (error);
 }
 
@@ -392,23 +383,22 @@ done:
  * Receive incoming data on a hook.
  */
 static int
-ng_pptpgre_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
+ng_pptpgre_rcvdata(hook_p hook, item_p item)
 {
 	const node_p node = hook->node;
 	const priv_p priv = node->private;
 
 	/* If not configured, reject */
 	if (!priv->conf.enabled) {
-		NG_FREE_DATA(m, meta);
+		NG_FREE_ITEM(item);
 		return (ENXIO);
 	}
 
 	/* Treat as xmit or recv data */
 	if (hook == priv->upper)
-		return ng_pptpgre_xmit(node, m, meta);
+		return ng_pptpgre_xmit(node, item);
 	if (hook == priv->lower)
-		return ng_pptpgre_recv(node, m, meta);
+		return ng_pptpgre_recv(node, item);
 	panic("%s: weird hook", __FUNCTION__);
 }
 
@@ -416,7 +406,7 @@ ng_pptpgre_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
  * Destroy node
  */
 static int
-ng_pptpgre_rmnode(node_p node)
+ng_pptpgre_shutdown(node_p node)
 {
 	const priv_p priv = node->private;
 
@@ -425,8 +415,6 @@ ng_pptpgre_rmnode(node_p node)
 
 	/* Take down netgraph node */
 	node->flags |= NG_INVALID;
-	ng_cutlinks(node);
-	ng_unname(node);
 	bzero(priv, sizeof(*priv));
 	FREE(priv, M_NETGRAPH);
 	node->private = NULL;
@@ -452,8 +440,9 @@ ng_pptpgre_disconnect(hook_p hook)
 		panic("%s: unknown hook", __FUNCTION__);
 
 	/* Go away if no longer connected to anything */
-	if (node->numhooks == 0)
-		ng_rmnode(node);
+	if ((node->numhooks == 0)
+	&& ((node->flags & NG_INVALID) == 0))
+		ng_rmnode_self(node);
 	return (0);
 }
 
@@ -465,14 +454,20 @@ ng_pptpgre_disconnect(hook_p hook)
  * Transmit an outgoing frame, or just an ack if m is NULL.
  */
 static int
-ng_pptpgre_xmit(node_p node, struct mbuf *m, meta_p meta)
+ng_pptpgre_xmit(node_p node, item_p item)
 {
 	const priv_p priv = node->private;
 	struct ng_pptpgre_ackp *const a = &priv->ackp;
 	u_char buf[sizeof(struct greheader) + 2 * sizeof(u_int32_t)];
 	struct greheader *const gre = (struct greheader *)buf;
 	int grelen, error;
+	struct mbuf *m;
 
+	if (item) {
+		NGI_GET_M(item, m);
+	} else {
+		m = NULL;
+	}
 	/* Check if there's data */
 	if (m != NULL) {
 
@@ -480,18 +475,21 @@ ng_pptpgre_xmit(node_p node, struct mbuf *m, meta_p meta)
 		if ((u_int32_t)PPTP_SEQ_DIFF(priv->xmitSeq, priv->recvAck)
 		      >= a->xmitWin) {
 			priv->stats.xmitDrops++;
-			NG_FREE_DATA(m, meta);
+			NG_FREE_M(m);
+			NG_FREE_ITEM(item);
 			return (ENOBUFS);
 		}
 
 		/* Sanity check frame length */
 		if (m != NULL && m->m_pkthdr.len > PPTP_MAX_PAYLOAD) {
 			priv->stats.xmitTooBig++;
-			NG_FREE_DATA(m, meta);
+			NG_FREE_M(m);
+			NG_FREE_ITEM(item);
 			return (EMSGSIZE);
 		}
-	} else
+	} else {
 		priv->stats.xmitLoneAcks++;
+	}
 
 	/* Build GRE header */
 	((u_int32_t *)gre)[0] = htonl(PPTP_INIT_VALUE);
@@ -521,7 +519,8 @@ ng_pptpgre_xmit(node_p node, struct mbuf *m, meta_p meta)
 		MGETHDR(m, M_DONTWAIT, MT_DATA);
 		if (m == NULL) {
 			priv->stats.memoryFailures++;
-			NG_FREE_META(meta);
+			if (item)
+				NG_FREE_ITEM(item);
 			return (ENOBUFS);
 		}
 		m->m_len = m->m_pkthdr.len = grelen;
@@ -531,7 +530,8 @@ ng_pptpgre_xmit(node_p node, struct mbuf *m, meta_p meta)
 		if (m == NULL || (m->m_len < grelen
 		    && (m = m_pullup(m, grelen)) == NULL)) {
 			priv->stats.memoryFailures++;
-			NG_FREE_META(meta);
+			if (item)
+				NG_FREE_ITEM(item);
 			return (ENOBUFS);
 		}
 	}
@@ -542,7 +542,12 @@ ng_pptpgre_xmit(node_p node, struct mbuf *m, meta_p meta)
 	priv->stats.xmitOctets += m->m_pkthdr.len;
 
 	/* Deliver packet */
-	NG_SEND_DATA(error, priv->lower, m, meta);
+	if (item) {
+		NG_FWD_NEW_DATA(error, item, priv->lower, m);
+	} else {
+		NG_SEND_DATA_ONLY(error, priv->lower, m);
+	}
+
 
 	/* Start receive ACK timer if data was sent and not already running */
 	if (error == 0 && gre->hasSeq && priv->xmitSeq == priv->recvAck + 1)
@@ -554,14 +559,16 @@ ng_pptpgre_xmit(node_p node, struct mbuf *m, meta_p meta)
  * Handle an incoming packet.  The packet includes the IP header.
  */
 static int
-ng_pptpgre_recv(node_p node, struct mbuf *m, meta_p meta)
+ng_pptpgre_recv(node_p node, item_p item)
 {
 	const priv_p priv = node->private;
 	int iphlen, grelen, extralen;
 	struct greheader *gre;
 	struct ip *ip;
 	int error = 0;
+	struct mbuf *m;
 
+	NGI_GET_M(item, m);
 	/* Update stats */
 	priv->stats.recvPackets++;
 	priv->stats.recvOctets += m->m_pkthdr.len;
@@ -570,7 +577,8 @@ ng_pptpgre_recv(node_p node, struct mbuf *m, meta_p meta)
 	if (m->m_pkthdr.len < sizeof(*ip) + sizeof(*gre)) {
 		priv->stats.recvRunts++;
 bad:
-		NG_FREE_DATA(m, meta);
+		NG_FREE_M(m);
+		NG_FREE_ITEM(item);
 		return (EINVAL);
 	}
 
@@ -578,7 +586,7 @@ bad:
 	if (m->m_len < sizeof(*ip) + sizeof(*gre)
 	    && (m = m_pullup(m, sizeof(*ip) + sizeof(*gre))) == NULL) {
 		priv->stats.memoryFailures++;
-		NG_FREE_META(meta);
+		NG_FREE_ITEM(item);
 		return (ENOBUFS);
 	}
 	ip = mtod(m, struct ip *);
@@ -586,7 +594,7 @@ bad:
 	if (m->m_len < iphlen + sizeof(*gre)) {
 		if ((m = m_pullup(m, iphlen + sizeof(*gre))) == NULL) {
 			priv->stats.memoryFailures++;
-			NG_FREE_META(meta);
+			NG_FREE_ITEM(item);
 			return (ENOBUFS);
 		}
 		ip = mtod(m, struct ip *);
@@ -600,7 +608,7 @@ bad:
 	if (m->m_len < iphlen + grelen) {
 		if ((m = m_pullup(m, iphlen + grelen)) == NULL) {
 			priv->stats.memoryFailures++;
-			NG_FREE_META(meta);
+			NG_FREE_ITEM(item);
 			return (ENOBUFS);
 		}
 		ip = mtod(m, struct ip *);
@@ -695,7 +703,7 @@ badAck:
 			/* If delayed ACK is disabled, send it now */
 			if (!priv->conf.enableDelayedAck
 			    || maxWait < PPTP_MIN_ACK_DELAY)
-				ng_pptpgre_xmit(node, NULL, NULL);
+				ng_pptpgre_xmit(node, NULL);
 			else {			/* send the ack later */
 				if (maxWait > PPTP_MAX_ACK_DELAY)
 					maxWait = PPTP_MAX_ACK_DELAY;
@@ -709,10 +717,11 @@ badAck:
 			m_adj(m, -extralen);
 
 		/* Deliver frame to upper layers */
-		NG_SEND_DATA(error, priv->upper, m, meta);
+		NG_FWD_NEW_DATA(error, item, priv->upper, m);
 	} else {
 		priv->stats.recvLoneAcks++;
-		NG_FREE_DATA(m, meta);		/* no data to deliver */
+		NG_FREE_ITEM(item);
+		NG_FREE_M(m);		/* no data to deliver */
 	}
 	return (error);
 }
@@ -868,7 +877,7 @@ ng_pptpgre_send_ack_timeout(void *arg)
 	a->sackTimerPtr = NULL;
 
 	/* Send a frame with an ack but no payload */
-  	ng_pptpgre_xmit(node, NULL, NULL);
+  	ng_pptpgre_xmit(node, NULL);
 	splx(s);
 }
 
diff --git a/sys/netgraph/ng_rfc1490.c b/sys/netgraph/ng_rfc1490.c
index b31fdca..52ee66c 100644
--- a/sys/netgraph/ng_rfc1490.c
+++ b/sys/netgraph/ng_rfc1490.c
@@ -89,7 +89,7 @@ typedef struct ng_rfc1490_private *priv_p;
 /* Netgraph node methods */
 static ng_constructor_t	ng_rfc1490_constructor;
 static ng_rcvmsg_t	ng_rfc1490_rcvmsg;
-static ng_shutdown_t	ng_rfc1490_rmnode;
+static ng_shutdown_t	ng_rfc1490_shutdown;
 static ng_newhook_t	ng_rfc1490_newhook;
 static ng_rcvdata_t	ng_rfc1490_rcvdata;
 static ng_disconnect_t	ng_rfc1490_disconnect;
@@ -101,7 +101,7 @@ static struct ng_type typestruct = {
 	NULL,
 	ng_rfc1490_constructor,
 	ng_rfc1490_rcvmsg,
-	ng_rfc1490_rmnode,
+	ng_rfc1490_shutdown,
 	ng_rfc1490_newhook,
 	NULL,
 	NULL,
@@ -119,22 +119,16 @@ NETGRAPH_INIT(rfc1490, &typestruct);
  * Node constructor
  */
 static int
-ng_rfc1490_constructor(node_p *nodep)
+ng_rfc1490_constructor(node_p node)
 {
 	priv_p priv;
-	int error;
 
 	/* Allocate private structure */
 	MALLOC(priv, priv_p, sizeof(*priv), M_NETGRAPH, M_NOWAIT | M_ZERO);
 	if (priv == NULL)
 		return (ENOMEM);
 
-	/* Call generic node constructor */
-	if ((error = ng_make_node_common(&typestruct, nodep))) {
-		FREE(priv, M_NETGRAPH);
-		return (error);
-	}
-	(*nodep)->private = priv;
+	node->private = priv;
 
 	/* Done */
 	return (0);
@@ -169,10 +163,9 @@ ng_rfc1490_newhook(node_p node, hook_p hook, const char *name)
  * Receive a control message. We don't support any special ones.
  */
 static int
-ng_rfc1490_rcvmsg(node_p node, struct ng_mesg *msg,
-		  const char *raddr, struct ng_mesg **rp, hook_p lasthook)
+ng_rfc1490_rcvmsg(node_p node, item_p item, hook_p lasthook)
 {
-	FREE(msg, M_NETGRAPH);
+	NG_FREE_ITEM(item);
 	return (EINVAL);
 }
 
@@ -213,13 +206,14 @@ ng_rfc1490_rcvmsg(node_p node, struct ng_mesg *msg,
 #define OUICMP(P,A,B,C)	((P)[0]==(A) && (P)[1]==(B) && (P)[2]==(C))
 
 static int
-ng_rfc1490_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
+ng_rfc1490_rcvdata(hook_p hook, item_p item)
 {
 	const node_p node = hook->node;
 	const priv_p priv = node->private;
 	int error = 0;
+	struct mbuf *m;
 
+	NGI_GET_M(item, m);
 	if (hook == priv->downlink) {
 		u_char *start, *ptr;
 
@@ -248,8 +242,8 @@ ng_rfc1490_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 				m_adj(m, ptr - start);
 				switch (etype) {
 				case ETHERTYPE_IP:
-					NG_SEND_DATA(error,
-					    priv->inet, m, meta);
+					NG_FWD_NEW_DATA(error, item,
+					    priv->inet, m);
 					break;
 				case ETHERTYPE_ARP:
 				case ETHERTYPE_REVARP:
@@ -263,11 +257,11 @@ ng_rfc1490_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 			break;
 		case NLPID_IP:
 			m_adj(m, ptr - start);
-			NG_SEND_DATA(error, priv->inet, m, meta);
+			NG_FWD_NEW_DATA(error, item, priv->inet, m);
 			break;
 		case NLPID_PPP:
 			m_adj(m, ptr - start);
-			NG_SEND_DATA(error, priv->ppp, m, meta);
+			NG_FWD_NEW_DATA(error, item, priv->ppp, m);
 			break;
 		case NLPID_Q933:
 		case NLPID_CLNP:
@@ -279,7 +273,7 @@ ng_rfc1490_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 			if ((*ptr & 0x01) == 0x01)
 				ERROUT(0);
 			m_adj(m, ptr - start);
-			NG_SEND_DATA(error, priv->ppp, m, meta);
+			NG_FWD_NEW_DATA(error, item, priv->ppp, m);
 			break;
 		}
 	} else if (hook == priv->ppp) {
@@ -288,19 +282,21 @@ ng_rfc1490_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 			ERROUT(ENOBUFS);
 		mtod(m, u_char *)[0] = HDLC_UI;
 		mtod(m, u_char *)[1] = NLPID_PPP;
-		NG_SEND_DATA(error, priv->downlink, m, meta);
+		NG_FWD_NEW_DATA(error, item, priv->downlink, m);
 	} else if (hook == priv->inet) {
 		M_PREPEND(m, 2, M_DONTWAIT);	/* Prepend IP NLPID */
 		if (!m)
 			ERROUT(ENOBUFS);
 		mtod(m, u_char *)[0] = HDLC_UI;
 		mtod(m, u_char *)[1] = NLPID_IP;
-		NG_SEND_DATA(error, priv->downlink, m, meta);
+		NG_FWD_NEW_DATA(error, item, priv->downlink, m);
 	} else
 		panic(__FUNCTION__);
 
 done:
-	NG_FREE_DATA(m, meta);
+	if (item)
+		NG_FREE_ITEM(item);
+	NG_FREE_M(m);
 	return (error);
 }
 
@@ -308,14 +304,12 @@ done:
  * Nuke node
  */
 static int
-ng_rfc1490_rmnode(node_p node)
+ng_rfc1490_shutdown(node_p node)
 {
 	const priv_p priv = node->private;
 
 	/* Take down netgraph node */
 	node->flags |= NG_INVALID;
-	ng_cutlinks(node);
-	ng_unname(node);
 	bzero(priv, sizeof(*priv));
 	node->private = NULL;
 	ng_unref(node);		/* let the node escape */
@@ -330,8 +324,9 @@ ng_rfc1490_disconnect(hook_p hook)
 {
 	const priv_p priv = hook->node->private;
 
-	if (hook->node->numhooks == 0)
-		ng_rmnode(hook->node);
+	if ((hook->node->numhooks == 0)
+	&& ((hook->node->flags & NG_INVALID) == 0))
+		ng_rmnode_self(hook->node);
 	else if (hook == priv->downlink)
 		priv->downlink = NULL;
 	else if (hook == priv->inet)
diff --git a/sys/netgraph/ng_sample.c b/sys/netgraph/ng_sample.c
index 831e070..76558f3 100644
--- a/sys/netgraph/ng_sample.c
+++ b/sys/netgraph/ng_sample.c
@@ -61,7 +61,7 @@
 
 static ng_constructor_t	ng_xxx_constructor;
 static ng_rcvmsg_t	ng_xxx_rcvmsg;
-static ng_shutdown_t	ng_xxx_rmnode;
+static ng_shutdown_t	ng_xxx_shutdown;
 static ng_newhook_t	ng_xxx_newhook;
 static ng_connect_t	ng_xxx_connect;
 static ng_rcvdata_t	ng_xxx_rcvdata;	 /* note these are both ng_rcvdata_t */
@@ -101,7 +101,7 @@ static struct ng_type typestruct = {
 	NULL,
 	ng_xxx_constructor,
 	ng_xxx_rcvmsg,
-	ng_xxx_rmnode,
+	ng_xxx_shutdown,
 	ng_xxx_newhook,
 	NULL,
 	ng_xxx_connect,
@@ -131,20 +131,16 @@ struct XXX {
 typedef struct XXX *xxx_p;
 
 /*
- * Allocate the private data structure and the generic node
- * and link them together.
- *
- * ng_make_node_common() returns with a generic node struct
- * with a single reference for us.. we transfer it to the
- * private structure.. when we free the private struct we must
- * unref the node so it gets freed too.
+ * Allocate the private data structure. The generic node has already
+ * been created. Link them together. We arrive with a reference to the node
+ * i.e. the reference count is incremented for us already.
  *
  * If this were a device node than this work would be done in the attach()
  * routine and the constructor would return EINVAL as you should not be able
  * to creatednodes that depend on hardware (unless you can add the hardware :)
  */
 static int
-ng_xxx_constructor(node_p *nodep)
+ng_xxx_constructor(node_p nodep)
 {
 	xxx_p privdata;
 	int i, error;
@@ -159,12 +155,6 @@ ng_xxx_constructor(node_p *nodep)
 		privdata->channel[i].channel = i;
 	}
 
-	/* Call the 'generic' (ie, superclass) node constructor */
-	if ((error = ng_make_node_common(&typestruct, nodep))) {
-		FREE(privdata, M_NETGRAPH);
-		return (error);
-	}
-
 	/* Link structs together; this counts as our one reference to *nodep */
 	(*nodep)->private = privdata;
 	privdata->node = *nodep;
@@ -245,6 +235,11 @@ ng_xxx_newhook(node_p node, hook_p hook, const char *name)
 
 /*
  * Get a netgraph control message.
+ * We actually recieve a queue item that has a pointer to the message.
+ * If we free the item, the message will be freed too, unless we remove
+ * it from the item using NGI_GET_MSG();
+ * The return address is also stored in the item, as an ng_ID_t,
+ * accessible as NGI_RETADDR(item);
  * Check it is one we understand. If needed, send a response.
  * We could save the address for an async action later, but don't here.
  * Always free the message.
@@ -255,13 +250,14 @@ ng_xxx_newhook(node_p node, hook_p hook, const char *name)
  * (so that old userland programs could continue to work).
  */
 static int
-ng_xxx_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
-		struct ng_mesg **rptr, hook_p lasthook)
+ng_xxx_rcvmsg(node_p node, item_p item, hook_p lasthook)
 {
 	const xxx_p xxxp = node->private;
 	struct ng_mesg *resp = NULL;
 	int error = 0;
+	struct ng_mesg *msg;
 
+	NGI_GET_MSG(item, msg);
 	/* Deal with message according to cookie and command */
 	switch (msg->header.typecookie) {
 	case NGM_XXX_COOKIE: 
@@ -298,18 +294,17 @@ ng_xxx_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
 	}
 
 	/* Take care of synchronous response, if any */
-	if (rptr)
-		*rptr = resp;
-	else if (resp)
-		FREE(resp, M_NETGRAPH);
-
+	NG_RESPOND_MSG(error, node, item, resp);
 	/* Free the message and return */
-	FREE(msg, M_NETGRAPH);
+	NG_FREE_MSG(msg);
 	return(error);
 }
 
 /*
  * Receive data, and do something with it.
+ * Actually we receive a queue item which holds the data.
+ * If we free the item it wil also froo the data and metadata unless
+ * we have previously disassociated them using the NGI_GET_xxx() macros.
  * Possibly send it out on another link after processing.
  * Possibly do something different if it comes from different
  * hooks. the caller will never free m or meta, so
@@ -321,14 +316,17 @@ ng_xxx_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
  * in the connect() method. 
  */
 static int
-ng_xxx_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
+ng_xxx_rcvdata(hook_p hook, item_p item )
 {
 	const xxx_p xxxp = hook->node->private;
 	int chan = -2;
 	int dlci = -2;
 	int error;
+	struct mbuf *m;
+	meta_p meta;
 
+
+	NGI_GET_M(item, m);
 	if (hook->private) {
 		dlci = ((struct XXX_hookinfo *) hook->private)->dlci;
 		chan = ((struct XXX_hookinfo *) hook->private)->channel;
@@ -339,8 +337,8 @@ ng_xxx_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 			 * the front here */
 			/* M_PREPEND(....)	; */
 			/* mtod(m, xxxxxx)->dlci = dlci; */
-			NG_SEND_DATA(error, xxxp->downstream_hook.hook,
-			    m, meta);
+			NG_FWD_NEW_DATA(error, item, 
+				xxxp->downstream_hook.hook, m);
 			xxxp->packets_out++;
 		} else {
 			/* data came from the multiplexed link */
@@ -350,7 +348,8 @@ ng_xxx_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 				if (xxxp->channel[chan].dlci == dlci)
 					break;
 			if (chan == XXX_NUM_DLCIS) {
-				NG_FREE_DATA(m, meta);
+				NG_FREE_ITEM(item);
+				NG_FREE_M(m);
 				return (ENETUNREACH);
 			}
 			/* If we were called at splnet, use the following:
@@ -364,13 +363,15 @@ ng_xxx_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 			 * the processing of the data can continue. after
 			 * these are run 'm' and 'meta' should be considered
 			 * as invalid and NG_SEND_DATA actually zaps them. */
-			NG_SEND_DATA(error, xxxp->channel[chan].hook, m, meta);
+			NG_FWD_NEW_DATA(error, item,
+				xxxp->channel[chan].hook, m);
 			xxxp->packets_in++;
 		}
 	} else {
 		/* It's the debug hook, throw it away.. */
 		if (hook == xxxp->downstream_hook.hook)
-			NG_FREE_DATA(m, meta);
+			NG_FREE_ITEM(item);
+			NG_FREE_M(m);
 	}
 	return 0;
 }
@@ -398,24 +399,46 @@ devintr()
 
 /*
  * Do local shutdown processing..
+ * All our links and the name have already been removed.
  * If we are a persistant device, we might refuse to go away, and
- * we'd only remove our links and reset ourself.
+ * we'd create a new node immediatly.
  */
 static int
-ng_xxx_rmnode(node_p node)
+ng_xxx_shutdown(node_p node)
 {
 	const xxx_p privdata = node->private;
+	int error;
 
 	node->flags |= NG_INVALID;
-	ng_cutlinks(node);
-#ifndef PERSISTANT_NODE
-	ng_unname(node);
 	node->private = NULL;
 	ng_unref(privdata->node);
+#ifndef PERSISTANT_NODE
 	FREE(privdata, M_NETGRAPH);
 #else
+	/* 
+	 * Create a new node. This is basically what a device 
+	 * driver would do in the attach routine.
+	 */
+	error = ng_make_node_common(&typestruct, &node);
+	if (node == NULL) {
+		printf ("node recreation failed:");
+		return (error);
+	}
+	if ( ng_name_node(node, "name")) {	/* whatever name is needed */
+		printf("something informative");
+		ng_unref(node);			/* drop it again */
+		return (0);
+	}
 	privdata->packets_in = 0;		/* reset stats */
 	privdata->packets_out = 0;
+	for (i = 0; i < XXX_NUM_DLCIS; i++) {
+		privdata->channel[i].dlci = -2;
+		privdata->channel[i].channel = i;
+	}
+
+	/* Link structs together; this counts as our one reference to node */
+	privdata->node = node;
+	node->private = privdata;
 	node->flags &= ~NG_INVALID;		/* reset invalid flag */
 #endif /* PERSISTANT_NODE */
 	return (0);
@@ -470,8 +493,9 @@ ng_xxx_disconnect(hook_p hook)
 {
 	if (hook->private)
 		((struct XXX_hookinfo *) (hook->private))->hook = NULL;
-	if (hook->node->numhooks == 0)
-		ng_rmnode(hook->node);
+	if ((hook->node->numhooks == 0)
+	&& ((hook->node->flags & NG_INVALID) == 0)) /* already shutting down? */
+		ng_rmnode_self(hook->node);
 	return (0);
 }
 
diff --git a/sys/netgraph/ng_socket.c b/sys/netgraph/ng_socket.c
index 19ea5e0..953793c 100644
--- a/sys/netgraph/ng_socket.c
+++ b/sys/netgraph/ng_socket.c
@@ -98,7 +98,7 @@
 /* Netgraph node methods */
 static ng_constructor_t	ngs_constructor;
 static ng_rcvmsg_t	ngs_rcvmsg;
-static ng_shutdown_t	ngs_rmnode;
+static ng_shutdown_t	ngs_shutdown;
 static ng_newhook_t	ngs_newhook;
 static ng_rcvdata_t	ngs_rcvdata;
 static ng_disconnect_t	ngs_disconnect;
@@ -111,7 +111,6 @@ static void	ng_detach_common(struct ngpcb *pcbp, int type);
 /*static int	ng_internalize(struct mbuf *m, struct proc *p); */
 
 static int	ng_connect_data(struct sockaddr *nam, struct ngpcb *pcbp);
-static int	ng_connect_cntl(struct sockaddr *nam, struct ngpcb *pcbp);
 static int	ng_bind(struct sockaddr *nam, struct ngpcb *pcbp);
 
 static int	ngs_mod_event(module_t mod, int event, void *data);
@@ -125,7 +124,7 @@ static struct ng_type typestruct = {
 	ngs_mod_event,
 	ngs_constructor,
 	ngs_rcvmsg,
-	ngs_rmnode,
+	ngs_shutdown,
 	ngs_newhook,
 	NULL,
 	NULL,
@@ -182,9 +181,9 @@ ngc_send(struct socket *so, int flags, struct mbuf *m, struct sockaddr *addr,
 {
 	struct ngpcb *const pcbp = sotongpcb(so);
 	struct sockaddr_ng *const sap = (struct sockaddr_ng *) addr;
-	struct ng_mesg *resp;
+	struct ng_mesg *msg;
 	struct mbuf *m0;
-	char *msg, *path = NULL;
+	char *path = NULL;
 	int len, error = 0;
 
 	if (pcbp == NULL) {
@@ -229,21 +228,15 @@ ngc_send(struct socket *so, int flags, struct mbuf *m, struct sockaddr *addr,
 
 	/* Move the data into a linear buffer as well. Messages are not
 	 * delivered in mbufs. */
-	MALLOC(msg, char *, len + 1, M_NETGRAPH, M_WAITOK);
+	MALLOC(msg, struct ng_mesg *, len + 1, M_NETGRAPH_MSG, M_WAITOK);
 	if (msg == NULL) {
 		error = ENOMEM;
 		goto release;
 	}
-	m_copydata(m, 0, len, msg);
+	m_copydata(m, 0, len, (char *)msg);
 
-	/* The callee will free the msg when done. The addr is our business. */
-	error = ng_send_msg(pcbp->sockdata->node,
-			    (struct ng_mesg *) msg, path, NULL, NULL, &resp);
-
-	/* If the callee responded with a synchronous response, then put it
-	 * back on the receive side of the socket; sap is source address. */
-	if (error == 0 && resp != NULL)
-		error = ship_msg(pcbp, resp, sap);
+	/* The callee will free the msg when done. The path is our business. */
+	NG_SEND_MSG_PATH(error, pcbp->sockdata->node, msg, path, NULL);
 
 release:
 	if (path != NULL)
@@ -268,11 +261,11 @@ ngc_bind(struct socket *so, struct sockaddr *nam, struct proc *p)
 static int
 ngc_connect(struct socket *so, struct sockaddr *nam, struct proc *p)
 {
-	struct ngpcb *const pcbp = sotongpcb(so);
-
-	if (pcbp == 0)
-		return (EINVAL);
-	return (ng_connect_cntl(nam, pcbp));
+	/*
+	 * At this time refuse to do this.. it used to 
+	 * do something but it was undocumented and not used.
+	 */
+	return (EINVAL);
 }
 
 /***************************************************************
@@ -517,7 +510,7 @@ ng_detach_common(struct ngpcb *pcbp, int which)
 			panic(__FUNCTION__);
 		}
 		if ((--sockdata->refs == 0) && (sockdata->node != NULL))
-			ng_rmnode(sockdata->node);
+			ng_rmnode_self(sockdata->node);
 	}
 	pcbp->ng_socket->so_pcb = NULL;
 	pcbp->ng_socket = NULL;
@@ -598,62 +591,54 @@ ng_connect_data(struct sockaddr *nam, struct ngpcb *pcbp)
 	node_p farnode;
 	struct ngsock *sockdata;
 	int error;
+	item_p item;
 
 	/* If we are already connected, don't do it again */
 	if (pcbp->sockdata != NULL)
 		return (EISCONN);
 
 	/* Find the target (victim) and check it doesn't already have a data
-	 * socket. Also check it is a 'socket' type node. */
+	 * socket. Also check it is a 'socket' type node.
+	 * Use ng_package_data() and address_path() to do this.
+	 */
+
 	sap = (struct sockaddr_ng *) nam;
-	if ((error = ng_path2node(NULL, sap->sg_data, &farnode, NULL)))
-		return (error);
+	/* The item will hold the node reference */
+	item = ng_package_data(NULL, NULL);
+	if (item == NULL) {
+		return (ENOMEM);
+	}
+	if ((error = ng_address_path(NULL, item,  sap->sg_data, NULL)))
+		return (error); /* item is freed on failure */
 
-	if (strcmp(farnode->type->name, NG_SOCKET_NODE_TYPE) != 0)
+	/*
+	 * Extract node from item and free item. Remember we now have 
+	 * a reference on the node. The item holds it for us.
+	 * when we free the item we release the reference.
+	 */
+	farnode = item->el_dest; /* shortcut */
+	if (strcmp(farnode->type->name, NG_SOCKET_NODE_TYPE) != 0) {
+		NG_FREE_ITEM(item); /* drop the reference to the node */
 		return (EINVAL);
+	}
 	sockdata = farnode->private;
-	if (sockdata->datasock != NULL)
+	if (sockdata->datasock != NULL) {
+		NG_FREE_ITEM(item);	/* drop the reference to the node */
 		return (EADDRINUSE);
+	}
 
-	/* Link the PCB and the private data struct. and note the extra
-	 * reference */
+	/*
+	 * Link the PCB and the private data struct. and note the extra
+	 * reference. Drop the extra reference on the node.
+	 */
 	sockdata->datasock = pcbp;
 	pcbp->sockdata = sockdata;
-	sockdata->refs++;
+	sockdata->refs++; /* XXX possible race if it's being freed */
+	NG_FREE_ITEM(item);	/* drop the reference to the node */
 	return (0);
 }
 
 /*
- * Connect the existing control socket node to a named node:hook.
- * The hook we use on this end is the same name as the remote node name.
- */
-static int
-ng_connect_cntl(struct sockaddr *nam, struct ngpcb *pcbp)
-{
-	struct ngsock *const sockdata = pcbp->sockdata;
-	struct sockaddr_ng *sap;
-	char *node, *hook;
-	node_p farnode;
-	int rtn, error;
-
-	sap = (struct sockaddr_ng *) nam;
-	rtn = ng_path_parse(sap->sg_data, &node, NULL, &hook);
-	if (rtn < 0 || node == NULL || hook == NULL) {
-		TRAP_ERROR;
-		return (EINVAL);
-	}
-	farnode = ng_findname(sockdata->node, node);
-	if (farnode == NULL) {
-		TRAP_ERROR;
-		return (EADDRNOTAVAIL);
-	}
-
-	/* Connect, using a hook name the same as the far node name. */
-	error = ng_con_nodes(sockdata->node, node, farnode, hook);
-	return error;
-}
-
-/*
  * Binding a socket means giving the corresponding node a name
  */
 static int
@@ -693,7 +678,7 @@ ship_msg(struct ngpcb *pcbp, struct ng_mesg *msg, struct sockaddr_ng *addr)
 
 	/* Here we free the message, as we are the end of the line.
 	 * We need to do that regardless of whether we got mbufs. */
-	FREE(msg, M_NETGRAPH);
+	NG_FREE_MSG(msg);
 
 	if (mdata == NULL) {
 		TRAP_ERROR;
@@ -715,7 +700,7 @@ ship_msg(struct ngpcb *pcbp, struct ng_mesg *msg, struct sockaddr_ng *addr)
  * You can only create new nodes from the socket end of things.
  */
 static int
-ngs_constructor(node_p *nodep)
+ngs_constructor(node_p nodep)
 {
 	return (EINVAL);
 }
@@ -736,14 +721,19 @@ ngs_newhook(node_p node, hook_p hook, const char *name)
  * Unless they are for us specifically (socket_type)
  */
 static int
-ngs_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
-	   struct ng_mesg **resp, hook_p lasthook)
+ngs_rcvmsg(node_p node, item_p item, hook_p lasthook)
 {
 	struct ngsock *const sockdata = node->private;
 	struct ngpcb *const pcbp = sockdata->ctlsock;
 	struct sockaddr_ng *addr;
 	int addrlen;
 	int error = 0;
+	struct	ng_mesg *msg;
+	ng_ID_t	retaddr = NGI_RETADDR(item);
+	char	retabuf[32];
+
+	NGI_GET_MSG(item, msg);
+	NG_FREE_ITEM(item); /* we have all we need */
 
 	/* Only allow mesgs to be passed if we have the control socket.
 	 * Data sockets can only support the generic messages. */
@@ -764,14 +754,13 @@ ngs_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
 			error = EINVAL;		/* unknown command */
 		}
 		/* Free the message and return */
-		FREE(msg, M_NETGRAPH);
+		NG_FREE_MSG(msg);
 		return(error);
 
 	}
 	/* Get the return address into a sockaddr */
-	if ((retaddr == NULL) || (*retaddr == '\0'))
-		retaddr = "";
-	addrlen = strlen(retaddr);
+	sprintf(retabuf,"[%x]:", retaddr);
+	addrlen = strlen(retabuf);
 	MALLOC(addr, struct sockaddr_ng *, addrlen + 4, M_NETGRAPH, M_NOWAIT);
 	if (addr == NULL) {
 		TRAP_ERROR;
@@ -779,7 +768,7 @@ ngs_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
 	}
 	addr->sg_len = addrlen + 3;
 	addr->sg_family = AF_NETGRAPH;
-	bcopy(retaddr, addr->sg_data, addrlen);
+	bcopy(retabuf, addr->sg_data, addrlen);
 	addr->sg_data[addrlen] = '\0';
 
 	/* Send it up */
@@ -792,8 +781,7 @@ ngs_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
  * Receive data on a hook
  */
 static int
-ngs_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
+ngs_rcvdata(hook_p hook, item_p item)
 {
 	struct ngsock *const sockdata = hook->node->private;
 	struct ngpcb *const pcbp = sockdata->datasock;
@@ -801,10 +789,13 @@ ngs_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 	struct sockaddr_ng *addr;
 	char *addrbuf[NG_HOOKLEN + 1 + 4];
 	int addrlen;
+	struct mbuf *m;
 
+	NGI_GET_M(item, m);
+	NG_FREE_ITEM(item);
 	/* If there is no data socket, black-hole it */
 	if (pcbp == NULL) {
-		NG_FREE_DATA(m, meta);
+		NG_FREE_M(m);
 		return (0);
 	}
 	so = pcbp->ng_socket;
@@ -817,9 +808,6 @@ ngs_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 	bcopy(hook->name, addr->sg_data, addrlen);
 	addr->sg_data[addrlen] = '\0';
 
-	/* We have no use for the meta data, free/clear it now. */
-	NG_FREE_META(meta);
-
 	/* Try to tell the socket which hook it came in on */
 	if (sbappendaddr(&so->so_rcv, (struct sockaddr *) addr, m, NULL) == 0) {
 		m_freem(m);
@@ -842,8 +830,9 @@ ngs_disconnect(hook_p hook)
 	struct ngsock *const sockdata = hook->node->private;
 
 	if ((sockdata->flags & NGS_FLAG_NOLINGER )
-	&& (hook->node->numhooks == 0)) {
-		ng_rmnode(hook->node);
+	&& (hook->node->numhooks == 0)
+	&& ((hook->node->flags & NG_INVALID) == 0)) {
+		ng_rmnode_self(hook->node);
 	}
 	return (0);
 }
@@ -854,15 +843,12 @@ ngs_disconnect(hook_p hook)
  * knows we should be shutting down.
  */
 static int
-ngs_rmnode(node_p node)
+ngs_shutdown(node_p node)
 {
 	struct ngsock *const sockdata = node->private;
 	struct ngpcb *const dpcbp = sockdata->datasock;
 	struct ngpcb *const pcbp = sockdata->ctlsock;
 
-	ng_cutlinks(node);
-	ng_unname(node);
-
 	if (dpcbp != NULL) {
 		soisdisconnected(dpcbp->ng_socket);
 		dpcbp->sockdata = NULL;
@@ -939,39 +925,41 @@ extern struct domain ngdomain;		/* stop compiler warnings */
 
 static struct protosw ngsw[] = {
 	{
-		SOCK_DGRAM,
-		&ngdomain,
+		SOCK_DGRAM,		/* protocol type */
+		&ngdomain,		/* backpointer to domain */
 		NG_CONTROL,
-		PR_ATOMIC | PR_ADDR /* | PR_RIGHTS */,
-		0, 0, 0, 0,
-		NULL,
-		0, 0, 0, 0,
-		&ngc_usrreqs
+		PR_ATOMIC | PR_ADDR /* | PR_RIGHTS */,	/* flags */
+		0, 0, 0, 0,		/* input, output, ctlinput, ctloutput */
+		NULL,			/* ousrreq */
+		0, 0, 0, 0,		/* init, fasttimeo, slowtimo, drain */
+		&ngc_usrreqs,		/* usrreq table (above) */
+		/*{NULL}*/		/* pffh (protocol filter head?) */
 	},
 	{
-		SOCK_DGRAM,
-		&ngdomain,
+		SOCK_DGRAM,		/* protocol type */
+		&ngdomain,		/* backpointer to domain */
 		NG_DATA,
-		PR_ATOMIC | PR_ADDR,
-		0, 0, 0, 0,
-		NULL,
-		0, 0, 0, 0,
-		&ngd_usrreqs
+		PR_ATOMIC | PR_ADDR,	/* flags */
+		0, 0, 0, 0,		/* input, output, ctlinput, ctloutput */
+		NULL,			/* ousrreq() */
+		0, 0, 0, 0,		/* init, fasttimeo, slowtimo, drain */
+		&ngd_usrreqs,		/* usrreq table (above) */
+		/*{NULL}*/		/* pffh (protocol filter head?) */
 	}
 };
 
 struct domain ngdomain = {
 	AF_NETGRAPH,
 	"netgraph",
-	0,
-	NULL,
-	NULL,
-	ngsw,
-	&ngsw[sizeof(ngsw) / sizeof(ngsw[0])],
-	0,
-	NULL,
-	0,
-	0
+	NULL,					/* init() */
+	NULL,					/* externalise() */
+	NULL,					/* dispose() */
+	ngsw,					/* protosw entry */
+	&ngsw[sizeof(ngsw) / sizeof(ngsw[0])], 	/* Number of protosw entries */
+	NULL,					/* next domain in list */
+	NULL,					/* rtattach() */
+	0,					/* arg to rtattach in bits */
+	0					/* maxrtkey */
 };
 
 /*
@@ -996,12 +984,13 @@ ngs_mod_event(module_t mod, int event, void *data)
 		}
 
 #ifdef NOTYET
+		if ((LIST_EMPTY(&ngsocklist)) && (typestruct.refs == 0)) {
 		/* Unregister protocol domain XXX can't do this yet.. */
-		if ((error = net_rm_domain(&ngdomain)) != 0)
-			break;
-#else
-		error = EBUSY;
+			if ((error = net_rm_domain(&ngdomain)) != 0)
+				break;
+		} else
 #endif
+			error = EBUSY;
 		break;
 	default:
 		error = EOPNOTSUPP;
diff --git a/sys/netgraph/ng_tee.c b/sys/netgraph/ng_tee.c
index 89ae12e..02ae393 100644
--- a/sys/netgraph/ng_tee.c
+++ b/sys/netgraph/ng_tee.c
@@ -79,7 +79,7 @@ typedef struct privdata *sc_p;
 /* Netgraph methods */
 static ng_constructor_t	ngt_constructor;
 static ng_rcvmsg_t	ngt_rcvmsg;
-static ng_shutdown_t	ngt_rmnode;
+static ng_shutdown_t	ngt_shutdown;
 static ng_newhook_t	ngt_newhook;
 static ng_rcvdata_t	ngt_rcvdata;
 static ng_disconnect_t	ngt_disconnect;
@@ -133,7 +133,7 @@ static struct ng_type ng_tee_typestruct = {
 	NULL,
 	ngt_constructor,
 	ngt_rcvmsg,
-	ngt_rmnode,
+	ngt_shutdown,
 	ngt_newhook,
 	NULL,
 	NULL,
@@ -147,21 +147,16 @@ NETGRAPH_INIT(tee, &ng_tee_typestruct);
  * Node constructor
  */
 static int
-ngt_constructor(node_p *nodep)
+ngt_constructor(node_p node)
 {
 	sc_p privdata;
-	int error = 0;
 
 	MALLOC(privdata, sc_p, sizeof(*privdata), M_NETGRAPH, M_NOWAIT|M_ZERO);
 	if (privdata == NULL)
 		return (ENOMEM);
 
-	if ((error = ng_make_node_common(&ng_tee_typestruct, nodep))) {
-		FREE(privdata, M_NETGRAPH);
-		return (error);
-	}
-	(*nodep)->private = privdata;
-	privdata->node = *nodep;
+	node->private = privdata;
+	privdata->node = node;
 	return (0);
 }
 
@@ -198,13 +193,14 @@ ngt_newhook(node_p node, hook_p hook, const char *name)
  * Receive a control message
  */
 static int
-ngt_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
-	   struct ng_mesg **rptr, hook_p lasthook)
+ngt_rcvmsg(node_p node, item_p item, hook_p lasthook)
 {
 	const sc_p sc = node->private;
 	struct ng_mesg *resp = NULL;
 	int error = 0;
+	struct ng_mesg *msg;
 
+	NGI_GET_MSG(item, msg);
 	switch (msg->header.typecookie) {
 	case NGM_TEE_COOKIE:
 		switch (msg->header.cmd) {
@@ -248,17 +244,32 @@ ngt_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
 			break;
 		}
 		break;
+	case NGM_FLOW_COOKIE:
+		if (lasthook)  {
+			if (lasthook == sc->left.hook) {
+				if (sc->right.hook) {
+					NGI_MSG(item) = msg;
+					NG_FWD_MSG_HOOK(error, node, item,
+						sc->right.hook, 0);
+					return (error);
+				}
+			} else {
+				if (sc->left.hook) {
+					NGI_MSG(item) = msg;
+					NG_FWD_MSG_HOOK(error, node, item, 
+						sc->left.hook, 0);
+					return (error);
+				}
+			}
+		}
+		break;
 	default:
 		error = EINVAL;
 		break;
 	}
-	if (rptr)
-		*rptr = resp;
-	else if (resp)
-		FREE(resp, M_NETGRAPH);
-
 done:
-	FREE(msg, M_NETGRAPH);
+	NG_RESPOND_MSG(error, node, item, resp);
+	NG_FREE_MSG(msg);
 	return (error);
 }
 
@@ -273,15 +284,18 @@ done:
  * from the other side.
  */
 static int
-ngt_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
+ngt_rcvdata(hook_p hook, item_p item)
 {
 	const sc_p sc = hook->node->private;
 	struct hookinfo *const hinfo = (struct hookinfo *) hook->private;
 	struct hookinfo *dest;
 	struct hookinfo *dup;
 	int error = 0;
+	struct mbuf *m;
+	meta_p meta;
 
+	m = NGI_M(item);
+	meta = NGI_META(item); /* leave these owned by the item */
 	/* Which hook? */
 	if (hinfo == &sc->left) {
 		dup = &sc->left2right;
@@ -302,42 +316,43 @@ ngt_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 	hinfo->stats.inOctets += m->m_pkthdr.len;
 	hinfo->stats.inFrames++;
 
+	/*
+	 * Don't make a copy if only the dup hook exists.
+	 */
+	if ((dup && dup->hook) && (dest->hook == NULL)) {
+		dest = dup;
+		dup = NULL;
+	}
+
 	/* Duplicate packet and meta info if requried */
 	if (dup != NULL) {
 		struct mbuf *m2;
 		meta_p meta2;
 
-		/* Copy packet */
+		/* Copy packet (failure will not stop the original)*/
 		m2 = m_dup(m, M_NOWAIT);
-		if (m2 == NULL) {
-			NG_FREE_DATA(m, meta);
-			return (ENOBUFS);
+		if (m2) {
+
+			/* Copy meta info */
+			/* If we can't get a copy, tough.. */
+			if (meta != NULL) {
+				meta2 = ng_copy_meta(meta);
+			} else
+				meta2 = NULL;
+
+			/* Deliver duplicate */
+			dup->stats.outOctets += m->m_pkthdr.len;
+			dup->stats.outFrames++;
+			NG_SEND_DATA(error, dup->hook, m2, meta2);
 		}
-
-		/* Copy meta info */
-		if (meta != NULL) {
-			MALLOC(meta2, meta_p,
-			    meta->used_len, M_NETGRAPH, M_NOWAIT);
-			if (meta2 == NULL) {
-				m_freem(m2);
-				NG_FREE_DATA(m, meta);
-				return (ENOMEM);
-			}
-			bcopy(meta, meta2, meta->used_len);
-			meta2->allocated_len = meta->used_len;
-		} else
-			meta2 = NULL;
-
-		/* Deliver duplicate */
-		dup->stats.outOctets += m->m_pkthdr.len;
-		dup->stats.outFrames++;
-		NG_SEND_DATA(error, dup->hook, m2, meta2);
 	}
-
 	/* Deliver frame out destination hook */
 	dest->stats.outOctets += m->m_pkthdr.len;
 	dest->stats.outFrames++;
-	NG_SEND_DATA(error, dest->hook, m, meta);
+	if (dest->hook)
+		NG_FWD_DATA(error, item, dest->hook);
+	else
+		NG_FREE_ITEM(item);
 	return (0);
 }
 
@@ -353,15 +368,15 @@ ngt_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
  * from two links is in ng_base.c.
  */
 static int
-ngt_rmnode(node_p node)
+ngt_shutdown(node_p node)
 {
 	const sc_p privdata = node->private;
 
 	node->flags |= NG_INVALID;
+#if 0 /* can never happen as cutlinks is already called */
 	if (privdata->left.hook && privdata->right.hook)
 		ng_bypass(privdata->left.hook, privdata->right.hook);
-	ng_cutlinks(node);
-	ng_unname(node);
+#endif
 	node->private = NULL;
 	ng_unref(privdata->node);
 	FREE(privdata, M_NETGRAPH);
@@ -378,8 +393,9 @@ ngt_disconnect(hook_p hook)
 
 	KASSERT(hinfo != NULL, ("%s: null info", __FUNCTION__));
 	hinfo->hook = NULL;
-	if (hook->node->numhooks == 0)
-		ng_rmnode(hook->node);
+	if ((hook->node->numhooks == 0)
+	&& ((hook->node->flags & NG_INVALID) == 0))
+		ng_rmnode_self(hook->node);
 	return (0);
 }
 
diff --git a/sys/netgraph/ng_tty.c b/sys/netgraph/ng_tty.c
index 54ad7d5..2f95e85 100644
--- a/sys/netgraph/ng_tty.c
+++ b/sys/netgraph/ng_tty.c
@@ -229,19 +229,19 @@ ngt_open(dev_t dev, struct tty *tp)
 	}
 	snprintf(name, sizeof(name), "%s%d", typestruct.name, ngt_unit++);
 
-	/* Set back pointers */
-	sc->node->private = sc;
-	tp->t_sc = (caddr_t) sc;
-
 	/* Assign node its name */
 	if ((error = ng_name_node(sc->node, name))) {
 		log(LOG_ERR, "%s: node name exists?\n", name);
 		ngt_nodeop_ok = 1;
-		ng_rmnode(sc->node);
+		ng_unref(sc->node);
 		ngt_nodeop_ok = 0;
 		goto done;
 	}
 
+	/* Set back pointers */
+	sc->node->private = sc;
+	tp->t_sc = (caddr_t) sc;
+
 	/*
 	 * Pre-allocate cblocks to the an appropriate amount.
 	 * I'm not sure what is appropriate.
@@ -279,7 +279,7 @@ ngt_close(struct tty *tp, int flag)
 			sc->flags &= ~FLG_TIMEOUT;
 		}
 		ngt_nodeop_ok = 1;
-		ng_rmnode(sc->node);
+		ng_rmnode_self(sc->node);
 		ngt_nodeop_ok = 0;
 		tp->t_sc = NULL;
 	}
@@ -493,11 +493,9 @@ ngt_timeout(void *arg)
  * the line discipline on a tty, so always return an error if not.
  */
 static int
-ngt_constructor(node_p *nodep)
+ngt_constructor(node_p node)
 {
-	if (!ngt_nodeop_ok)
-		return (EOPNOTSUPP);
-	return (ng_make_node_common(&typestruct, nodep));
+	return (EOPNOTSUPP);
 }
 
 /*
@@ -521,12 +519,13 @@ done:
 }
 
 /*
- * set the hooks into queueing mode (for outgoing packets)
+ * Set the hooks into queueing mode (for outgoing packets)
+ * Force single client at a time.
  */
 static int
 ngt_connect(hook_p hook)
 {
-	hook->peer->flags |= HK_QUEUE;
+	hook->peer->flags |= HK_QUEUE|HK_FORCE_WRITER;
 	return (0);
 }
 
@@ -560,8 +559,6 @@ ngt_shutdown(node_p node)
 
 	if (!ngt_nodeop_ok)
 		return (EOPNOTSUPP);
-	ng_unname(node);
-	ng_cutlinks(node);
 	node->private = NULL;
 	ng_unref(sc->node);
 	m_freem(sc->qhead);
@@ -576,15 +573,17 @@ ngt_shutdown(node_p node)
  * output queue and start output if necessary.
  */
 static int
-ngt_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
+ngt_rcvdata(hook_p hook, item_p item)
 {
 	const sc_p sc = hook->node->private;
 	int s, error = 0;
+	struct mbuf *m;
 
 	if (hook != sc->hook)
 		panic(__FUNCTION__);
-	NG_FREE_META(meta);
+
+	NGI_GET_M(item, m);
+	NG_FREE_ITEM(item);
 	s = spltty();
 	if (sc->qlen >= MAX_MBUFQ)
 		ERROUT(ENOBUFS);
@@ -607,13 +606,14 @@ done:
  * Receive control message
  */
 static int
-ngt_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
-	   struct ng_mesg **rptr, hook_p lasthook)
+ngt_rcvmsg(node_p node, item_p item, hook_p lasthook)
 {
 	const sc_p sc = (sc_p) node->private;
 	struct ng_mesg *resp = NULL;
 	int error = 0;
+	struct ng_mesg *msg;
 
+	NGI_GET_MSG(item, msg);
 	switch (msg->header.typecookie) {
 	case NGM_TTY_COOKIE:
 		switch (msg->header.cmd) {
@@ -643,13 +643,9 @@ ngt_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
 	default:
 		ERROUT(EINVAL);
 	}
-	if (rptr)
-		*rptr = resp;
-	else if (resp)
-		FREE(resp, M_NETGRAPH);
-
 done:
-	FREE(msg, M_NETGRAPH);
+	NG_RESPOND_MSG(error, node, item, resp);
+	NG_FREE_MSG(msg);
 	return (error);
 }
 
diff --git a/sys/netgraph/ng_vjc.c b/sys/netgraph/ng_vjc.c
index 60da6b2..a9c9515 100644
--- a/sys/netgraph/ng_vjc.c
+++ b/sys/netgraph/ng_vjc.c
@@ -89,7 +89,7 @@ typedef struct ng_vjc_private *priv_p;
 /* Netgraph node methods */
 static ng_constructor_t	ng_vjc_constructor;
 static ng_rcvmsg_t	ng_vjc_rcvmsg;
-static ng_shutdown_t	ng_vjc_rmnode;
+static ng_shutdown_t	ng_vjc_shutdown;
 static ng_newhook_t	ng_vjc_newhook;
 static ng_rcvdata_t	ng_vjc_rcvdata;
 static ng_disconnect_t	ng_vjc_disconnect;
@@ -227,7 +227,7 @@ static struct ng_type ng_vjc_typestruct = {
 	NULL,
 	ng_vjc_constructor,
 	ng_vjc_rcvmsg,
-	ng_vjc_rmnode,
+	ng_vjc_shutdown,
 	ng_vjc_newhook,
 	NULL,
 	NULL,
@@ -245,22 +245,16 @@ NETGRAPH_INIT(vjc, &ng_vjc_typestruct);
  * Create a new node
  */
 static int
-ng_vjc_constructor(node_p *nodep)
+ng_vjc_constructor(node_p node)
 {
 	priv_p priv;
-	int error;
 
 	/* Allocate private structure */
 	MALLOC(priv, priv_p, sizeof(*priv), M_NETGRAPH, M_NOWAIT | M_ZERO);
 	if (priv == NULL)
 		return (ENOMEM);
 
-	/* Call generic node constructor */
-	if ((error = ng_make_node_common(&ng_vjc_typestruct, nodep))) {
-		FREE(priv, M_NETGRAPH);
-		return (error);
-	}
-	(*nodep)->private = priv;
+	node->private = priv;
 
 	/* Done */
 	return (0);
@@ -300,13 +294,14 @@ ng_vjc_newhook(node_p node, hook_p hook, const char *name)
  * Receive a control message
  */
 static int
-ng_vjc_rcvmsg(node_p node, struct ng_mesg *msg,
-	      const char *raddr, struct ng_mesg **rptr, hook_p lasthook)
+ng_vjc_rcvmsg(node_p node, item_p item, hook_p lasthook)
 {
 	const priv_p priv = (priv_p) node->private;
 	struct ng_mesg *resp = NULL;
 	int error = 0;
+	struct ng_mesg *msg;
 
+	NGI_GET_MSG(item, msg);
 	/* Check type cookie */
 	switch (msg->header.typecookie) {
 	case NGM_VJC_COOKIE:
@@ -396,13 +391,9 @@ ng_vjc_rcvmsg(node_p node, struct ng_mesg *msg,
 		error = EINVAL;
 		break;
 	}
-	if (rptr)
-		*rptr = resp;
-	else if (resp)
-		FREE(resp, M_NETGRAPH);
-
 done:
-	FREE(msg, M_NETGRAPH);
+	NG_RESPOND_MSG(error, node, item, resp);
+	NG_FREE_MSG(msg);
 	return (error);
 }
 
@@ -410,13 +401,14 @@ done:
  * Receive data
  */
 static int
-ng_vjc_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
+ng_vjc_rcvdata(hook_p hook, item_p item)
 {
 	const node_p node = hook->node;
 	const priv_p priv = (priv_p) node->private;
 	int error = 0;
+	struct mbuf *m;
 
+	NGI_GET_M(item, m);
 	if (hook == priv->ip) {			/* outgoing packet */
 		u_int type = TYPE_IP;
 
@@ -425,7 +417,7 @@ ng_vjc_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 			struct ip *ip;
 
 			if ((m = ng_vjc_pulluphdrs(m, 0)) == NULL) {
-				NG_FREE_META(meta);
+				NG_FREE_ITEM(item);
 				return (ENOBUFS);
 			}
 			ip = mtod(m, struct ip *);
@@ -460,7 +452,8 @@ ng_vjc_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 
 		/* Are we decompressing? */
 		if (!priv->conf.enableDecomp) {
-			NG_FREE_DATA(m, meta);
+			NG_FREE_M(m);
+			NG_FREE_ITEM(item);
 			return (ENXIO);
 		}
 
@@ -471,7 +464,7 @@ ng_vjc_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 		if (m->m_len < need2pullup
 		    && (m = m_pullup(m, need2pullup)) == NULL) {
 			priv->slc.sls_errorin++;
-			NG_FREE_META(meta);
+			NG_FREE_ITEM(item);
 			return (ENOBUFS);
 		}
 
@@ -480,7 +473,8 @@ ng_vjc_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 		    m->m_len, m->m_pkthdr.len, TYPE_COMPRESSED_TCP,
 		    &priv->slc, &hdr, &hlen);
 		if (vjlen <= 0) {
-			NG_FREE_DATA(m, meta);
+			NG_FREE_M(m);
+			NG_FREE_ITEM(item);
 			return (EINVAL);
 		}
 		m_adj(m, vjlen);
@@ -489,7 +483,8 @@ ng_vjc_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 		MGETHDR(hm, M_DONTWAIT, MT_DATA);
 		if (hm == NULL) {
 			priv->slc.sls_errorin++;
-			NG_FREE_DATA(m, meta);
+			NG_FREE_M(m);
+			NG_FREE_ITEM(item);
 			return (ENOBUFS);
 		}
 		hm->m_len = 0;
@@ -499,7 +494,8 @@ ng_vjc_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 			if ((hm->m_flags & M_EXT) == 0) {
 				m_freem(hm);
 				priv->slc.sls_errorin++;
-				NG_FREE_DATA(m, meta);
+				NG_FREE_M(m);
+				NG_FREE_ITEM(item);
 				return (ENOBUFS);
 			}
 		}
@@ -517,13 +513,14 @@ ng_vjc_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 
 		/* Are we decompressing? */
 		if (!priv->conf.enableDecomp) {
-			NG_FREE_DATA(m, meta);
+			NG_FREE_M(m);
+			NG_FREE_ITEM(item);
 			return (ENXIO);
 		}
 
 		/* Pull up IP+TCP headers */
 		if ((m = ng_vjc_pulluphdrs(m, 1)) == NULL) {
-			NG_FREE_META(meta);
+			NG_FREE_ITEM(item);
 			return (ENOBUFS);
 		}
 
@@ -531,7 +528,8 @@ ng_vjc_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 		if (sl_uncompress_tcp_core(mtod(m, u_char *),
 		    m->m_len, m->m_pkthdr.len, TYPE_UNCOMPRESSED_TCP,
 		    &priv->slc, &hdr, &hlen) < 0) {
-			NG_FREE_DATA(m, meta);
+			NG_FREE_M(m);
+			NG_FREE_ITEM(item);
 			return (EINVAL);
 		}
 		hook = priv->ip;
@@ -541,7 +539,7 @@ ng_vjc_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 		panic("%s: unknown hook", __FUNCTION__);
 
 	/* Send result back out */
-	NG_SEND_DATA(error, hook, m, meta);
+	NG_FWD_NEW_DATA(error, item, hook, m);
 	return (error);
 }
 
@@ -549,13 +547,11 @@ ng_vjc_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
  * Shutdown node
  */
 static int
-ng_vjc_rmnode(node_p node)
+ng_vjc_shutdown(node_p node)
 {
 	const priv_p priv = (priv_p) node->private;
 
 	node->flags |= NG_INVALID;
-	ng_cutlinks(node);
-	ng_unname(node);
 	bzero(priv, sizeof(*priv));
 	FREE(priv, M_NETGRAPH);
 	node->private = NULL;
@@ -585,8 +581,9 @@ ng_vjc_disconnect(hook_p hook)
 		panic("%s: unknown hook", __FUNCTION__);
 
 	/* Go away if no hooks left */
-	if (node->numhooks == 0)
-		ng_rmnode(node);
+	if ((node->numhooks == 0)
+	&& ((node->flags & NG_INVALID) == 0))
+		ng_rmnode_self(node);
 	return (0);
 }
 
diff --git a/sys/pci/if_mn.c b/sys/pci/if_mn.c
index e3af171..679fd30 100644
--- a/sys/pci/if_mn.c
+++ b/sys/pci/if_mn.c
@@ -267,7 +267,7 @@ struct softc {
 };
 
 static int
-ngmn_constructor(node_p *nodep)
+ngmn_constructor(node_p node)
 {
 
 	return (EINVAL);
@@ -281,29 +281,29 @@ ngmn_shutdown(node_p nodep)
 }
 
 static int
-ngmn_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
-								struct ng_mesg **rptr, hook_p lasthook)
+ngmn_rcvmsg(node_p node, item_p item, hook_p lasthook)
 {
 	struct softc *sc;
 	struct ng_mesg *resp = NULL;
 	struct schan *sch;
 	char *arg;
 	int pos, i;
+	struct ng_mesg *msg;
 
+	NGI_GET_MSG(item, msg);
 	sc = node->private;
 
 	if (msg->header.typecookie != NGM_GENERIC_COOKIE ||
-		rptr == NULL ||  /* temporary */
 	    msg->header.cmd != NGM_TEXT_STATUS) {
-		if (rptr)
-			*rptr = NULL;
-		FREE(msg, M_NETGRAPH);
+		NG_FREE_ITEM(item);
+		NG_FREE_MSG(msg);
 		return (EINVAL);
 	}
 	NG_MKRESPONSE(resp, msg, sizeof(struct ng_mesg) + NG_TEXTRESPONSE,
 	    M_NOWAIT);
 	if (resp == NULL) {
-		FREE(msg, M_NETGRAPH);
+		NG_FREE_ITEM(item);
+		NG_FREE_MSG(msg);
 		return (ENOMEM);
 	}
 	arg = (char *)resp->data;
@@ -377,12 +377,8 @@ ngmn_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
 	resp->header.arglen = pos + 1;
 
 	/* Take care of synchronous response, if any */
-	if (rptr)
-		*rptr = resp;
-	else if (resp)
-		FREE(resp, M_NETGRAPH);	/* Will eventually send the hard way */
-
-	FREE(msg, M_NETGRAPH);
+	NG_RESPOND_MSG(i, node, item, resp);
+	NG_FREE_MSG(msg);
 	return (0);
 }
 
@@ -505,28 +501,30 @@ mn_fmt_ts(char *p, u_int32_t ts)
  */
 
 static int
-ngmn_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
+ngmn_rcvdata(hook_p hook, item_p item)
 {
 	struct mbuf  *m2;
 	struct trxd *dp, *dp2;
 	struct schan *sch;
 	struct softc *sc;
 	int chan, pitch, len;
+	struct mbuf *m;
 
 	sch = hook->private;
 	sc = sch->sc;
 	chan = sch->chan;
 
 	if (sch->state != UP) {
-		NG_FREE_DATA(m, meta);
+		NG_FREE_ITEM(item);
 		return (0);
 	}
+	NGI_GET_M(item, m);
 	if (sch->tx_pending + m->m_pkthdr.len > sch->tx_limit * mn_maxlatency) {
-		NG_FREE_DATA(m, meta);
+		NG_FREE_M(m);
+		NG_FREE_ITEM(item);
 		return (0);
 	}
-	NG_FREE_META(meta);
+	NG_FREE_ITEM(item);
 	pitch = 0;
 	m2 = m;
 	dp2 = sc->ch[chan]->xl;
@@ -1350,7 +1348,6 @@ mn_attach (device_t self)
 	sc->node->private = sc;
 	sprintf(sc->nodename, "%s%d", NG_MN_NODE_TYPE, sc->unit);
 	if (ng_name_node(sc->node, sc->nodename)) {
-		ng_rmnode(sc->node);
 		ng_unref(sc->node);
 		return (0);
 	}