Reviewed by: Archie@freebsd.org

This clears out my outstanding netgraph changes.
There is a netgraph change of design in the offing and this is to some
extent a superset of soem of the new functionality and some of the old
functionality that may be removed.

This code works as before, but allows some new features that I want to
work with and evaluate. It is the basis for a version of netgraph
with integral locking for SMP use.

This is running on my test machine with no new problems :-)

36 files changed, 493 insertions, 238 deletions

diff --git a/sys/dev/ar/if_ar.c b/sys/dev/ar/if_ar.c
index 9acc402..ad5aac9 100644
--- a/sys/dev/ar/if_ar.c
+++ b/sys/dev/ar/if_ar.c
@@ -285,7 +285,6 @@ static struct ng_type typestruct = {
 	NULL,
 	ngar_connect,
 	ngar_rcvdata,
-	ngar_rcvdata,
 	ngar_disconnect,
 	NULL
 };
@@ -1732,6 +1731,9 @@ ar_get_packets(struct ar_softc *sc)
 	int i;
 	int len;
 	u_char rxstat;
+#ifdef NETGRAPH
+	int error;
+#endif
 
 	while(ar_packet_avail(sc, &len, &rxstat)) {
 		TRC(printf("apa: len %d, rxstat %x\n", len, rxstat));
@@ -1765,7 +1767,7 @@ ar_get_packets(struct ar_softc *sc)
 			sppp_input(&sc->ifsppp.pp_if, m);
 			sc->ifsppp.pp_if.if_ipackets++;
 #else	/* NETGRAPH */
-			ng_queue_data(sc->hook, m, NULL);
+			NG_SEND_DATA_ONLY(error, sc->hook, m);
 			sc->ipackets++;
 #endif	/* NETGRAPH */
 
@@ -2283,7 +2285,7 @@ ngar_rcvmsg(node_p node, struct ng_mesg *msg,
  */
 static	int
 ngar_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-			struct mbuf **ret_m, meta_p *ret_meta)
+		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
 {
 	int s;
 	int error = 0;
@@ -2350,6 +2352,8 @@ ngar_rmnode(node_p node)
 static	int
 ngar_connect(hook_p hook)
 {
+	/* probably not at splnet, force outward queueing */
+	hook->peer->flags |= HK_QUEUE;
 	/* be really amiable and just say "YUP that's OK by me! " */
 	return (0);
 }
diff --git a/sys/dev/ar/if_ar_isa.c b/sys/dev/ar/if_ar_isa.c
index 9acc402..ad5aac9 100644
--- a/sys/dev/ar/if_ar_isa.c
+++ b/sys/dev/ar/if_ar_isa.c
@@ -285,7 +285,6 @@ static struct ng_type typestruct = {
 	NULL,
 	ngar_connect,
 	ngar_rcvdata,
-	ngar_rcvdata,
 	ngar_disconnect,
 	NULL
 };
@@ -1732,6 +1731,9 @@ ar_get_packets(struct ar_softc *sc)
 	int i;
 	int len;
 	u_char rxstat;
+#ifdef NETGRAPH
+	int error;
+#endif
 
 	while(ar_packet_avail(sc, &len, &rxstat)) {
 		TRC(printf("apa: len %d, rxstat %x\n", len, rxstat));
@@ -1765,7 +1767,7 @@ ar_get_packets(struct ar_softc *sc)
 			sppp_input(&sc->ifsppp.pp_if, m);
 			sc->ifsppp.pp_if.if_ipackets++;
 #else	/* NETGRAPH */
-			ng_queue_data(sc->hook, m, NULL);
+			NG_SEND_DATA_ONLY(error, sc->hook, m);
 			sc->ipackets++;
 #endif	/* NETGRAPH */
 
@@ -2283,7 +2285,7 @@ ngar_rcvmsg(node_p node, struct ng_mesg *msg,
  */
 static	int
 ngar_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-			struct mbuf **ret_m, meta_p *ret_meta)
+		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
 {
 	int s;
 	int error = 0;
@@ -2350,6 +2352,8 @@ ngar_rmnode(node_p node)
 static	int
 ngar_connect(hook_p hook)
 {
+	/* probably not at splnet, force outward queueing */
+	hook->peer->flags |= HK_QUEUE;
 	/* be really amiable and just say "YUP that's OK by me! " */
 	return (0);
 }
diff --git a/sys/dev/lmc/if_lmc.c b/sys/dev/lmc/if_lmc.c
index cf220f7..3bec1fa 100644
--- a/sys/dev/lmc/if_lmc.c
+++ b/sys/dev/lmc/if_lmc.c
@@ -184,7 +184,6 @@ static struct ng_type typestruct = {
         NULL,
         ng_lmc_connect,
         ng_lmc_rcvdata,
-        ng_lmc_rcvdata,
         ng_lmc_disconnect,
         ng_lmc_cmdlist
 };
@@ -623,9 +622,11 @@ lmc_rx_intr(lmc_softc_t * const sc)
 				}
 			}
 			if (accept) {
+				int error;
+
 				ms->m_pkthdr.len = total_len;
 				ms->m_pkthdr.rcvif = NULL;
-				ng_queue_data(sc->lmc_hook, ms, NULL);
+				NG_SEND_DATA_ONLY(error, sc->lmc_hook, ms);
 			}
 			ms = m0;
 		}
@@ -1396,7 +1397,7 @@ ng_lmc_rcvmsg(node_p node, struct ng_mesg *msg,
  */
 static  int
 ng_lmc_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-	struct mbuf **ret_m, meta_p *ret_meta)
+	struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
 {
         int s;
         int error = 0;
@@ -1462,6 +1463,8 @@ ng_lmc_rmnode(node_p node)
 static  int
 ng_lmc_connect(hook_p hook)
 {
+	/* We are probably not at splnet.. force outward queueing */
+	hook->peer->flags |= HK_QUEUE;
         /* be really amiable and just say "YUP that's OK by me! " */
         return (0);
 }
diff --git a/sys/dev/sr/if_sr.c b/sys/dev/sr/if_sr.c
index b27d2f5..86efdb7 100644
--- a/sys/dev/sr/if_sr.c
+++ b/sys/dev/sr/if_sr.c
@@ -2408,6 +2408,8 @@ sr_get_packets(struct sr_softc *sc)
 	sca_descriptor *rxdesc;	/* descriptor in memory */
 #ifndef NETGRAPH
 	struct ifnet *ifp;	/* network intf ctl table */
+#else
+	int error;
 #endif /* NETGRAPH */
 	struct mbuf *m = NULL;	/* message buffer */
 
@@ -2533,7 +2535,7 @@ sr_get_packets(struct sr_softc *sc)
 				       bp[9], bp[10], bp[11]);
 			}
 #endif
-			ng_queue_data(sc->hook, m, NULL);
+			NG_SEND_DATA_ONLY(error, sc->hook, m);
 			sc->ipackets++;
 #endif /* NETGRAPH */
 			/*
@@ -3239,7 +3241,7 @@ ngsr_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
  */
 static	int
 ngsr_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-			struct mbuf **ret_m, meta_p *ret_meta)
+		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
 {
 	int s;
 	int error = 0;
@@ -3306,6 +3308,8 @@ ngsr_rmnode(node_p node)
 static	int
 ngsr_connect(hook_p hook)
 {
+	/* probably not at splnet, force outward queueing */
+	hook->peer->flags |= HK_QUEUE;
 	/* be really amiable and just say "YUP that's OK by me! " */
 	return (0);
 }
diff --git a/sys/dev/sr/if_sr_isa.c b/sys/dev/sr/if_sr_isa.c
index b27d2f5..86efdb7 100644
--- a/sys/dev/sr/if_sr_isa.c
+++ b/sys/dev/sr/if_sr_isa.c
@@ -2408,6 +2408,8 @@ sr_get_packets(struct sr_softc *sc)
 	sca_descriptor *rxdesc;	/* descriptor in memory */
 #ifndef NETGRAPH
 	struct ifnet *ifp;	/* network intf ctl table */
+#else
+	int error;
 #endif /* NETGRAPH */
 	struct mbuf *m = NULL;	/* message buffer */
 
@@ -2533,7 +2535,7 @@ sr_get_packets(struct sr_softc *sc)
 				       bp[9], bp[10], bp[11]);
 			}
 #endif
-			ng_queue_data(sc->hook, m, NULL);
+			NG_SEND_DATA_ONLY(error, sc->hook, m);
 			sc->ipackets++;
 #endif /* NETGRAPH */
 			/*
@@ -3239,7 +3241,7 @@ ngsr_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
  */
 static	int
 ngsr_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-			struct mbuf **ret_m, meta_p *ret_meta)
+		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
 {
 	int s;
 	int error = 0;
@@ -3306,6 +3308,8 @@ ngsr_rmnode(node_p node)
 static	int
 ngsr_connect(hook_p hook)
 {
+	/* probably not at splnet, force outward queueing */
+	hook->peer->flags |= HK_QUEUE;
 	/* be really amiable and just say "YUP that's OK by me! " */
 	return (0);
 }
diff --git a/sys/dev/usb/udbp.c b/sys/dev/usb/udbp.c
index d75e1f2..2cfbf0b 100644
--- a/sys/dev/usb/udbp.c
+++ b/sys/dev/usb/udbp.c
@@ -195,7 +195,6 @@ Static struct ng_type ng_udbp_typestruct = {
 	NULL,
 	ng_udbp_connect,
 	ng_udbp_rcvdata,
-	ng_udbp_rcvdata,
 	ng_udbp_disconnect,
 	ng_udbp_cmdlist
 };
@@ -485,7 +484,6 @@ udbp_in_transfer_cb(usbd_xfer_handle xfer, usbd_private_handle priv,
 	udbp_p 		sc = priv;		/* XXX see priv above */
 	int		s;
 	int		len;
-	meta_p		meta = NULL;
 	struct		mbuf *m;
 
 	if (err) {
@@ -506,7 +504,7 @@ udbp_in_transfer_cb(usbd_xfer_handle xfer, usbd_private_handle priv,
 		if (sc->hook) {
 			/* get packet from device and send on */
 			m = m_devget(sc->sc_bulkin_buffer, len, 0, NULL, NULL);
-	    		NG_SEND_DATAQ(err, sc->hook, m, meta);
+	    		NG_SEND_DATA_ONLY(err, sc->hook, m);
 		}
 		splx(s);
 	
@@ -723,7 +721,7 @@ ng_udbp_rcvmsg(node_p node,
  */
 Static int
 ng_udbp_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta)
+		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
 {
 	const udbp_p sc = hook->node->private;
 	int error;
@@ -793,6 +791,8 @@ ng_udbp_rmnode(node_p node)
 Static int
 ng_udbp_connect(hook_p hook)
 {
+	/* probably not at splnet, force outward queueing */
+	hook->peer->flags |= HK_QUEUE;
 	/* be really amiable and just say "YUP that's OK by me! " */
 	return (0);
 }
diff --git a/sys/i386/isa/if_ar.c b/sys/i386/isa/if_ar.c
index 9acc402..ad5aac9 100644
--- a/sys/i386/isa/if_ar.c
+++ b/sys/i386/isa/if_ar.c
@@ -285,7 +285,6 @@ static struct ng_type typestruct = {
 	NULL,
 	ngar_connect,
 	ngar_rcvdata,
-	ngar_rcvdata,
 	ngar_disconnect,
 	NULL
 };
@@ -1732,6 +1731,9 @@ ar_get_packets(struct ar_softc *sc)
 	int i;
 	int len;
 	u_char rxstat;
+#ifdef NETGRAPH
+	int error;
+#endif
 
 	while(ar_packet_avail(sc, &len, &rxstat)) {
 		TRC(printf("apa: len %d, rxstat %x\n", len, rxstat));
@@ -1765,7 +1767,7 @@ ar_get_packets(struct ar_softc *sc)
 			sppp_input(&sc->ifsppp.pp_if, m);
 			sc->ifsppp.pp_if.if_ipackets++;
 #else	/* NETGRAPH */
-			ng_queue_data(sc->hook, m, NULL);
+			NG_SEND_DATA_ONLY(error, sc->hook, m);
 			sc->ipackets++;
 #endif	/* NETGRAPH */
 
@@ -2283,7 +2285,7 @@ ngar_rcvmsg(node_p node, struct ng_mesg *msg,
  */
 static	int
 ngar_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-			struct mbuf **ret_m, meta_p *ret_meta)
+		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
 {
 	int s;
 	int error = 0;
@@ -2350,6 +2352,8 @@ ngar_rmnode(node_p node)
 static	int
 ngar_connect(hook_p hook)
 {
+	/* probably not at splnet, force outward queueing */
+	hook->peer->flags |= HK_QUEUE;
 	/* be really amiable and just say "YUP that's OK by me! " */
 	return (0);
 }
diff --git a/sys/i386/isa/if_sr.c b/sys/i386/isa/if_sr.c
index b27d2f5..86efdb7 100644
--- a/sys/i386/isa/if_sr.c
+++ b/sys/i386/isa/if_sr.c
@@ -2408,6 +2408,8 @@ sr_get_packets(struct sr_softc *sc)
 	sca_descriptor *rxdesc;	/* descriptor in memory */
 #ifndef NETGRAPH
 	struct ifnet *ifp;	/* network intf ctl table */
+#else
+	int error;
 #endif /* NETGRAPH */
 	struct mbuf *m = NULL;	/* message buffer */
 
@@ -2533,7 +2535,7 @@ sr_get_packets(struct sr_softc *sc)
 				       bp[9], bp[10], bp[11]);
 			}
 #endif
-			ng_queue_data(sc->hook, m, NULL);
+			NG_SEND_DATA_ONLY(error, sc->hook, m);
 			sc->ipackets++;
 #endif /* NETGRAPH */
 			/*
@@ -3239,7 +3241,7 @@ ngsr_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
  */
 static	int
 ngsr_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-			struct mbuf **ret_m, meta_p *ret_meta)
+		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
 {
 	int s;
 	int error = 0;
@@ -3306,6 +3308,8 @@ ngsr_rmnode(node_p node)
 static	int
 ngsr_connect(hook_p hook)
 {
+	/* probably not at splnet, force outward queueing */
+	hook->peer->flags |= HK_QUEUE;
 	/* be really amiable and just say "YUP that's OK by me! " */
 	return (0);
 }
diff --git a/sys/i4b/driver/i4b_ing.c b/sys/i4b/driver/i4b_ing.c
index 50e1c4b..5510f50 100644
--- a/sys/i4b/driver/i4b_ing.c
+++ b/sys/i4b/driver/i4b_ing.c
@@ -464,6 +464,7 @@ ing_rx_data_rdy(int unit)
 {
 	register struct ing_softc *sc = &ing_softc[unit];
 	register struct mbuf *m;
+	int error;
 	
 	if((m = *isdn_linktab[unit]->rx_mbuf) == NULL)
 		return;
@@ -476,7 +477,7 @@ ing_rx_data_rdy(int unit)
 
 	sc->sc_inpkt++;
 	
-	ng_queue_data(sc->hook, m, NULL);
+	NG_SEND_DATA_ONLY(error, sc->hook, m);
 }
 
 /*---------------------------------------------------------------------------*
@@ -744,7 +745,7 @@ ng_ing_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
 #if defined(__FreeBSD_version) && __FreeBSD_version >= 500000
 static int
 ng_ing_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-                struct mbuf **ret_m, meta_p *ret_meta)
+                struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
 #else
 static int
 ng_ing_rcvdata(hook_p hook, struct mbuf *m, meta_p meta)
@@ -829,6 +830,8 @@ ng_ing_rmnode(node_p node)
 static int
 ng_ing_connect(hook_p hook)
 {
+	/* probably not at splnet, force outward queueing */
+	hook->peer->flags |= HK_QUEUE;
 	return (0);
 }
 
diff --git a/sys/netgraph/netgraph.h b/sys/netgraph/netgraph.h
index a74fb20..48fb4f5 100644
--- a/sys/netgraph/netgraph.h
+++ b/sys/netgraph/netgraph.h
@@ -67,6 +67,7 @@ typedef struct ng_hook *hook_p;
 
 /* Flags for a hook */
 #define HK_INVALID		0x0001	/* don't trust it! */
+#define HK_QUEUE		0x0002	/* queue for later delivery */
 
 /*
  * Structure of a node
@@ -139,7 +140,7 @@ typedef	int	ng_newhook_t(node_p node, hook_p hook, const char *name);
 typedef	hook_p	ng_findhook_t(node_p node, const char *name);
 typedef	int	ng_connect_t(hook_p hook);
 typedef	int	ng_rcvdata_t(hook_p hook, struct mbuf *m, meta_p meta,
-			struct mbuf **ret_m, meta_p *ret_meta);
+		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp);
 typedef	int	ng_disconnect_t(hook_p hook);
 
 /*
@@ -158,6 +159,16 @@ struct ng_cmdlist {
 
 /*
  * Structure of a node type
+ * If data is sent to the "rcvdata()" entrypoint then the system
+ * may decide to defer it until later by queing it with the normal netgraph
+ * input queuing system.  This is decidde by the HK_QUEUE flag being set in
+ * the flags word of the peer (receiving) hook. The dequeuing mechanism will
+ * ensure it is not requeued again.
+ * Note the input queueing system is to allow modules
+ * to 'release the stack' or to pass data across spl layers.
+ * The data will be redelivered as soon as the NETISR code runs
+ * which may be almost immediatly.  A node may also do it's own queueing
+ * for other reasons (e.g. device output queuing).
  */
 struct ng_type {
 
@@ -170,8 +181,7 @@ struct ng_type {
 	ng_newhook_t	*newhook;	/* first notification of new hook */
 	ng_findhook_t	*findhook;	/* only if you have lots of hooks */
 	ng_connect_t	*connect;	/* final notification of new hook */
-	ng_rcvdata_t	*rcvdata;	/* date comes here */
-	ng_rcvdata_t	*rcvdataq;	/* or here if being queued */
+	ng_rcvdata_t	*rcvdata;	/* data comes here */
 	ng_disconnect_t	*disconnect;	/* notify on disconnect */
 
 	const struct	ng_cmdlist *cmdlist;	/* commands we can convert */
@@ -182,47 +192,64 @@ struct ng_type {
 };
 
 /* Send data packet with meta-data */
-#define NG_SEND_DATA(error, hook, m, a)					\
+#define NG_SEND_DATA(err, hook, m, meta)				\
 	do {								\
-		(error) = ng_send_data((hook), (m), (a), NULL, NULL);	\
+		(err) = ng_send_data((hook), (m), (meta),	\
+						NULL, NULL, NULL);	\
 		(m) = NULL;						\
-		(a) = NULL;						\
+		(meta) = NULL;						\
 	} while (0)
 
-/* Send  queued data packet with meta-data */
-#define NG_SEND_DATAQ(error, hook, m, a)				\
+/* Send data packet with no meta-data */
+#define NG_SEND_DATA_ONLY(err, hook, m)					\
 	do {								\
-		(error) = ng_send_dataq((hook), (m), (a), NULL, NULL);	\
+		(err) = ng_send_data((hook), (m), NULL,	\
+						NULL, NULL, NULL);	\
 		(m) = NULL;						\
-		(a) = NULL;						\
 	} while (0)
 
-#define NG_SEND_DATA_RET(error, hook, m, a)				\
+/* Send data packet including a possible sync response pointer */
+#define NG_SEND_DATA_RESP(err, hook, m, meta, rmsg)			\
+	do {								\
+		(err) = ng_send_data((hook), (m), (meta),	\
+					NULL, NULL, rmsg);		\
+		(m) = NULL;						\
+		(meta) = NULL;						\
+	} while (0)
+
+/*
+ * Send data packet including a possible sync response pointer
+ * Allow the callee to replace the data and pass it back
+ * or to simply 'reject it' or 'keep it'
+ */
+#define NG_SEND_DATA_RET(err, hook, m, meta, rmsg)			\
 	do {								\
 		struct mbuf *rm = NULL;					\
-		meta_p ra = NULL;					\
-		(error) = ng_send_data((hook), (m), (a), &rm, &ra);	\
+		meta_p rmeta = NULL;					\
+		(err) = ng_send_data((hook), (m), (meta),	\
+					&rm, &rmeta, (rmsg));		\
 		(m) = rm;						\
-		(a) = ra;						\
+		(meta) = rmeta;						\
 	} while (0)
 
+
 /* Free metadata */
-#define NG_FREE_META(a)							\
+#define NG_FREE_META(meta)						\
 	do {								\
-		if ((a)) {						\
-			FREE((a), M_NETGRAPH);				\
-			a = NULL;					\
+		if ((meta)) {						\
+			FREE((meta), M_NETGRAPH);			\
+			meta = NULL;					\
 		}							\
 	} while (0)
 
 /* Free any data packet and/or meta-data */
-#define NG_FREE_DATA(m, a)						\
+#define NG_FREE_DATA(m, meta)						\
 	do {								\
 		if ((m)) {						\
 			m_freem((m));					\
 			m = NULL;					\
 		}							\
-		NG_FREE_META((a));					\
+		NG_FREE_META((meta));					\
 	} while (0)
 
 /*
@@ -268,19 +295,18 @@ int	ng_mod_event(module_t mod, int what, void *arg);
 int	ng_name_node(node_p node, const char *name);
 int	ng_newtype(struct ng_type *tp);
 ng_ID_t ng_node2ID(node_p node);
-int	ng_path2node(node_p here, const char *path, node_p *dest, char **rtnp,
-			hook_p *lasthook);
+int	ng_path2node(node_p here, const char *path,
+		node_p *dest, hook_p *lasthook);
 int	ng_path_parse(char *addr, char **node, char **path, char **hook);
 int	ng_queue_data(hook_p hook, struct mbuf *m, meta_p meta);
-int	ng_queue_msg(node_p here, struct ng_mesg *msg, const char *address);
+int	ng_queue_msg(node_p here, struct ng_mesg *msg, const char *address,
+		hook_p hook, char *retaddr);
 void	ng_release_node(node_p node);
 void	ng_rmnode(node_p node);
 int	ng_send_data(hook_p hook, struct mbuf *m, meta_p meta,
-			struct mbuf **ret_m, meta_p *ret_meta);
-int	ng_send_dataq(hook_p hook, struct mbuf *m, meta_p meta,
-			struct mbuf **ret_m, meta_p *ret_meta);
-int	ng_send_msg(node_p here, struct ng_mesg *msg,
-	    const char *address, struct ng_mesg **resp);
+		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp);
+int	ng_send_msg(node_p here, struct ng_mesg *msg, const char *address,
+		hook_p hook, char *retaddr, struct ng_mesg **resp);
 void	ng_unname(node_p node);
 void	ng_unref(node_p node);
 int	ng_wait_node(node_p node, char *msg);
diff --git a/sys/netgraph/ng_UI.c b/sys/netgraph/ng_UI.c
index b410a51..9d490b9 100644
--- a/sys/netgraph/ng_UI.c
+++ b/sys/netgraph/ng_UI.c
@@ -87,7 +87,6 @@ static struct ng_type typestruct = {
 	NULL,
 	NULL,
 	ng_UI_rcvdata,
-	ng_UI_rcvdata,
 	ng_UI_disconnect,
 	NULL
 };
@@ -151,6 +150,22 @@ static int
 ng_UI_rcvmsg(node_p node, struct ng_mesg *msg,
 	     const char *raddr, struct ng_mesg **rp, hook_p lasthook)
 {
+	const priv_p priv = node->private;
+
+	if ((msg->header.typecookie == NGM_FLOW_COOKIE) && lasthook)  {
+		if (lasthook == priv->downlink) {
+			if (priv->uplink) {
+				return (ng_send_msg(node, msg, NULL,
+					priv->uplink, raddr, rp));
+			}
+		} else {
+			if (priv->downlink) {
+				return (ng_send_msg(node, msg, NULL,
+					priv->downlink, raddr, rp));
+			}
+		}
+	}
+		
 	FREE(msg, M_NETGRAPH);
 	return (EINVAL);
 }
@@ -163,7 +178,7 @@ ng_UI_rcvmsg(node_p node, struct ng_mesg *msg,
  */
 static int
 ng_UI_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta)
+		struct mbuf **ret_m, meta_p *ret_meta,  struct ng_mesg **resp)
 {
 	const node_p node = hook->node;
 	const priv_p priv = node->private;
diff --git a/sys/netgraph/ng_async.c b/sys/netgraph/ng_async.c
index 43781fe..8dc951a 100644
--- a/sys/netgraph/ng_async.c
+++ b/sys/netgraph/ng_async.c
@@ -158,7 +158,6 @@ static struct ng_type typestruct = {
 	NULL,
 	NULL,
 	nga_rcvdata,
-	nga_rcvdata,
 	nga_disconnect,
 	nga_cmdlist
 };
@@ -232,7 +231,7 @@ nga_newhook(node_p node, hook_p hook, const char *name)
  */
 static int
 nga_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta)
+		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
 {
 	const sc_p sc = hook->node->private;
 
diff --git a/sys/netgraph/ng_base.c b/sys/netgraph/ng_base.c
index 84336d5..335518f 100644
--- a/sys/netgraph/ng_base.c
+++ b/sys/netgraph/ng_base.c
@@ -85,6 +85,9 @@ static int	ng_generic_msg(node_p here, struct ng_mesg *msg,
 			hook_p hook);
 static ng_ID_t	ng_decodeidname(const char *name);
 static int	ngb_mod_event(module_t mod, int event, void *data);
+static int	ng_send_data_dont_queue(hook_p hook, struct mbuf *m,
+		meta_p meta, struct mbuf **ret_m, meta_p *ret_meta,
+		struct ng_mesg **resp);
 static void	ngintr(void);
 
 /* Our own netgraph malloc type */
@@ -321,7 +324,7 @@ ng_make_node(const char *typename, node_p *nodepp)
 		int error;
 
 		/* Not found, try to load it as a loadable module */
-		snprintf(filename, sizeof(filename), "ng_%s", typename);
+		snprintf(filename, sizeof(filename), "/boot/kernel/ng_%s", typename);
 		error = linker_load_file(filename, &lf);
 		if (error != 0)
 			return (error);
@@ -1028,10 +1031,8 @@ ng_path_parse(char *addr, char **nodep, char **pathp, char **hookp)
  * return the destination node. Compute the "return address" if desired.
  */
 int
-ng_path2node(node_p here, const char *address, node_p *destp, char **rtnp,
-	hook_p *lasthook)
+ng_path2node(node_p here, const char *address, node_p *destp, hook_p *lasthook)
 {
-	const	node_p start = here;
 	char    fullpath[NG_PATHLEN + 1];
 	char   *nodename, *path, pbuf[2];
 	node_p  node;
@@ -1039,8 +1040,6 @@ ng_path2node(node_p here, const char *address, node_p *destp, char **rtnp,
 	hook_p hook = NULL;
 
 	/* Initialize */
-	if (rtnp)
-		*rtnp = NULL;
 	if (destp == NULL)
 		return EINVAL;
 	*destp = NULL;
@@ -1111,19 +1110,6 @@ ng_path2node(node_p here, const char *address, node_p *destp, char **rtnp,
 		return (ENXIO);
 	}
 
-	/* Now compute return address, i.e., the path to the sender */
-	if (rtnp != NULL) {
-		MALLOC(*rtnp, char *, NG_NODELEN + 2, M_NETGRAPH, M_NOWAIT);
-		if (*rtnp == NULL) {
-			TRAP_ERROR;
-			return (ENOMEM);
-		}
-		if (start->name != NULL)
-			sprintf(*rtnp, "%s:", start->name);
-		else
-			sprintf(*rtnp, "[%x]:", ng_node2ID(start));
-	}
-
 	/* Done */
 	*destp = node;
 	if (lasthook != NULL)
@@ -1160,22 +1146,49 @@ do {									\
 
 
 /*
- * Send a control message to a node
+ * Send a control message to a node.
+ * If hook is supplied, use it in preference to the address.
+ * If the return address is not supplied it will be set to this node.
  */
 int
 ng_send_msg(node_p here, struct ng_mesg *msg, const char *address,
-	    struct ng_mesg **rptr)
+		hook_p hook, char *retaddr, struct ng_mesg **rptr)
 {
 	node_p  dest = NULL;
-	char   *retaddr = NULL;
 	int     error;
 	hook_p	lasthook;
 
-	/* Find the target node */
-	error = ng_path2node(here, address, &dest, &retaddr, &lasthook);
-	if (error) {
-		FREE(msg, M_NETGRAPH);
-		return error;
+	/*
+	 * Find the target node.
+	 * If there is a HOOK argument, then use that in preference
+	 * to the address.
+	 */
+	if (hook) {
+		lasthook = hook->peer;
+		dest = lasthook->node;
+	} else {
+		error = ng_path2node(here, address, &dest, &lasthook);
+		if (error) {
+			FREE(msg, M_NETGRAPH);
+			return (error);
+		}
+	}
+
+	/* If the user didn't supply a return addres, assume it's "here". */
+	if (retaddr == NULL) {
+		/*
+		 * Now fill out the return address,
+		 * i.e. the name/ID of the sender. (If we didn't get one)
+		 */
+		MALLOC(retaddr, char *, NG_NODELEN + 2, M_NETGRAPH, M_NOWAIT);
+		if (retaddr == NULL) {
+			TRAP_ERROR;
+			return (ENOMEM);
+		}
+		if (here->name != NULL)
+			sprintf(retaddr, "%s:", here->name);
+		else
+			sprintf(retaddr, "[%x]:", ng_node2ID(here));
 	}
 
 	/* Make sure the resp field is null before we start */
@@ -1242,7 +1255,7 @@ ng_generic_msg(node_p here, struct ng_mesg *msg, const char *retaddr,
 		con->path[sizeof(con->path) - 1] = '\0';
 		con->ourhook[sizeof(con->ourhook) - 1] = '\0';
 		con->peerhook[sizeof(con->peerhook) - 1] = '\0';
-		error = ng_path2node(here, con->path, &node2, NULL, NULL);
+		error = ng_path2node(here, con->path, &node2, NULL);
 		if (error)
 			break;
 		error = ng_con_nodes(here, con->ourhook, node2, con->peerhook);
@@ -1632,65 +1645,54 @@ ng_generic_msg(node_p here, struct ng_mesg *msg, const char *retaddr,
 /*
  * Send a data packet to a node. If the recipient has no
  * 'receive data' method, then silently discard the packet.
+ * The receiving node may elect to put the data onto the netgraph
+ * NETISR queue for later delivery. It may do this because it knows there
+ * is some recursion and wishes to unwind the stack, or because it has
+ * some suspicion that it is being called at (say) splimp instead of
+ * splnet.
  */
 int 
 ng_send_data(hook_p hook, struct mbuf *m, meta_p meta,
-	struct mbuf **ret_m, meta_p *ret_meta)
+	struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
 {
 	ng_rcvdata_t *rcvdata;
-	int error;
 
 	CHECK_DATA_MBUF(m);
-	if (hook && (hook->flags & HK_INVALID) == 0) {
-		rcvdata = hook->peer->node->type->rcvdata;
-		if (rcvdata != NULL)
-			error = (*rcvdata)(hook->peer, m, meta,
-					ret_m, ret_meta);
-		else {
-			error = 0;
-			NG_FREE_DATA(m, meta);
-		}
-	} else {
+	if ((hook == NULL)
+	|| ((hook->flags & HK_INVALID) != 0)
+	|| ((rcvdata = hook->peer->node->type->rcvdata) == NULL)) {
 		TRAP_ERROR;
-		error = ENOTCONN;
 		NG_FREE_DATA(m, meta);
+		return (ENOTCONN);
 	}
-	return (error);
+	if (hook->peer->flags & HK_QUEUE) {
+		return (ng_queue_data(hook, m, meta));
+	}
+	return ( (*rcvdata)(hook->peer, m, meta, ret_m, ret_meta, resp));
 }
 
 /*
- * Send a queued data packet to a node. If the recipient has no
- * 'receive queued data' method, then try the 'receive data' method above.
+ * Send a queued data packet to a node. 
+ *
+ * This is meant for data that is being dequeued and should therefore NOT
+ * be queued again. It ignores the queue flag and should NOT be called
+ * outside of this file. (thus it is static)
  */
-int 
-ng_send_dataq(hook_p hook, struct mbuf *m, meta_p meta,
-	struct mbuf **ret_m, meta_p *ret_meta)
+static int 
+ng_send_data_dont_queue(hook_p hook, struct mbuf *m, meta_p meta,
+	struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
 {
 	ng_rcvdata_t *rcvdata;
-	int error;
 
 	CHECK_DATA_MBUF(m);
-	if (hook && (hook->flags & HK_INVALID) == 0) {
-		rcvdata = hook->peer->node->type->rcvdataq;
-		if (rcvdata != NULL)
-			error = (*rcvdata)(hook->peer, m, meta,
-					ret_m, ret_meta);
-		else {
-			rcvdata = hook->peer->node->type->rcvdata;
-			if (rcvdata != NULL) {
-				error = (*rcvdata)(hook->peer, m, meta,
-					ret_m, ret_meta);
-			} else {
-				error = 0;
-				NG_FREE_DATA(m, meta);
-			}
-		}
-	} else {
+	if ((hook == NULL)
+	|| ((hook->flags & HK_INVALID) != 0)
+	|| ((rcvdata = hook->peer->node->type->rcvdata) == NULL)) {
 		TRAP_ERROR;
-		error = ENOTCONN;
 		NG_FREE_DATA(m, meta);
-	}
-	return (error);
+		return (ENOTCONN);
+	} 
+	return ((*rcvdata)(hook->peer, m, meta, ret_m, ret_meta, resp));
 }
 
 /*
@@ -1821,8 +1823,8 @@ struct ng_queue_entry {
 		struct {
 			struct ng_mesg	*msg_msg;
 			node_p		msg_node;
-			void		*msg_retaddr;
 			hook_p		msg_lasthook;
+			char		*msg_retaddr;
 		} msg;
 	} body;
 };
@@ -1928,23 +1930,50 @@ ng_queue_data(hook_p hook, struct mbuf *m, meta_p meta)
 /*
  * Running at a raised (but we don't know which) processor priority level,
  * put the msg onto a queue to be picked up by another PPL (probably splnet)
+ * Either specify an address, or a hook to traverse.
+ * The return address can be specified, or it will be pointed at this node.
  */
 int
-ng_queue_msg(node_p here, struct ng_mesg *msg, const char *address)
+ng_queue_msg(node_p here, struct ng_mesg *msg, const char *address, hook_p hook,char *retaddr)
 {
 	register struct ng_queue_entry *q;
 	int     s;
 	node_p  dest = NULL;
-	char   *retaddr = NULL;
 	int     error;
 	hook_p	lasthook = NULL;
 
-	/* Find the target node. */
-	error = ng_path2node(here, address, &dest, &retaddr, &lasthook);
-	if (error) {
-		FREE(msg, M_NETGRAPH);
-		return (error);
+	/*
+	 * Find the target node.
+	 * If there is a HOOK argument, then use that in preference
+	 * to the address.
+	 */
+	if (hook) {
+		lasthook = hook->peer;
+		dest = lasthook->node;
+	} else {
+		error = ng_path2node(here, address, &dest, &lasthook);
+		if (error) {
+			FREE(msg, M_NETGRAPH);
+			return (error);
+		}
 	}
+
+	if (retaddr == NULL) {
+		/*
+		 * Now fill out the return address,
+		 * i.e. the name/ID of the sender. (If we didn't get one)
+		 */
+		MALLOC(retaddr, char *, NG_NODELEN + 2, M_NETGRAPH, M_NOWAIT);
+		if (retaddr == NULL) {
+			TRAP_ERROR;
+			return (ENOMEM);
+		}
+		if (here->name != NULL)
+			sprintf(retaddr, "%s:", here->name);
+		else
+			sprintf(retaddr, "[%x]:", ng_node2ID(here));
+	}
+
 	if ((q = ng_getqblk()) == NULL) {
 		FREE(msg, M_NETGRAPH);
 		if (retaddr)
@@ -1957,8 +1986,8 @@ ng_queue_msg(node_p here, struct ng_mesg *msg, const char *address)
 	q->next = NULL;
 	q->body.msg.msg_node = dest;
 	q->body.msg.msg_msg = msg;
-	q->body.msg.msg_retaddr = retaddr;
-	q->body.msg.msg_lasthook = lasthook;
+	q->body.msg.msg_retaddr = retaddr; /* XXX malloc'd, give it away */
+	q->body.msg.msg_lasthook = lasthook; /* XXX needs reference */
 	s = splhigh();		/* protect refs and queue */
 	dest->refs++;		/* don't let it go away while on the queue */
 	if (lasthook)
@@ -2011,7 +2040,10 @@ ngintr(void)
 			m = ngq->body.data.da_m;
 			meta = ngq->body.data.da_meta;
 			RETURN_QBLK(ngq);
-			NG_SEND_DATAQ(error, hook, m, meta);
+			ng_send_data_dont_queue(hook, m, meta,
+				NULL, NULL, NULL);
+			m = NULL;
+			meta = NULL;
 			ng_unref_hook(hook);
 			break;
 		case NGQF_MESG:
diff --git a/sys/netgraph/ng_bpf.c b/sys/netgraph/ng_bpf.c
index 8b09b6a..49cc3e7 100644
--- a/sys/netgraph/ng_bpf.c
+++ b/sys/netgraph/ng_bpf.c
@@ -196,7 +196,6 @@ static struct ng_type typestruct = {
 	NULL,
 	NULL,
 	ng_bpf_rcvdata,
-	ng_bpf_rcvdata,
 	ng_bpf_disconnect,
 	ng_bpf_cmdlist
 };
@@ -375,7 +374,7 @@ done:
  */
 static int
 ng_bpf_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta)
+		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
 {
 	const hinfo_p hip = hook->private;
 	int totlen = m->m_pkthdr.len;
diff --git a/sys/netgraph/ng_bridge.c b/sys/netgraph/ng_bridge.c
index e33d743..68a4c0c 100644
--- a/sys/netgraph/ng_bridge.c
+++ b/sys/netgraph/ng_bridge.c
@@ -276,7 +276,6 @@ static struct ng_type ng_bridge_typestruct = {
 	NULL,
 	NULL,
 	ng_bridge_rcvdata,
-	ng_bridge_rcvdata,
 	ng_bridge_disconnect,
 	ng_bridge_cmdlist,
 };
@@ -511,7 +510,7 @@ ng_bridge_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
  */
 static int
 ng_bridge_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta)
+		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
 {
 	const node_p node = hook->node;
 	const priv_p priv = node->private;
diff --git a/sys/netgraph/ng_cisco.c b/sys/netgraph/ng_cisco.c
index 8355ee2..76316c4 100644
--- a/sys/netgraph/ng_cisco.c
+++ b/sys/netgraph/ng_cisco.c
@@ -181,7 +181,6 @@ static struct ng_type typestruct = {
 	NULL,
 	NULL,
 	cisco_rcvdata,
-	cisco_rcvdata,
 	cisco_disconnect,
 	ng_cisco_cmdlist
 };
@@ -351,7 +350,7 @@ cisco_rcvmsg(node_p node, struct ng_mesg *msg,
  */
 static int
 cisco_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta)
+		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
 {
 	const sc_p sc = hook->node->private;
 	struct protoent *pep;
@@ -502,8 +501,8 @@ cisco_input(sc_p sc, struct mbuf *m, meta_p meta)
 				    NGM_CISCO_GET_IPADDR, 0, M_NOWAIT);
 				if (msg == NULL)
 					goto nomsg;
-				ng_send_msg(sc->node, msg,
-				    NG_CISCO_HOOK_INET, &resp);
+				ng_send_msg(sc->node, msg, NULL,
+					sc->inet.hook, NULL, &resp);
 				if (resp != NULL)
 					cisco_rcvmsg(sc->node, resp, ".",
 								NULL, NULL);
diff --git a/sys/netgraph/ng_echo.c b/sys/netgraph/ng_echo.c
index 69d1604..8ef95a4 100644
--- a/sys/netgraph/ng_echo.c
+++ b/sys/netgraph/ng_echo.c
@@ -71,7 +71,6 @@ static struct ng_type typestruct = {
 	NULL,
 	NULL,
 	nge_rcvdata,
-	nge_rcvdata,
 	nge_disconnect,
 	NULL
 };
@@ -98,7 +97,7 @@ nge_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
  */
 static int
 nge_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta)
+		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
 {
 	int error = 0;
 
diff --git a/sys/netgraph/ng_ether.c b/sys/netgraph/ng_ether.c
index 775c213..806798a 100644
--- a/sys/netgraph/ng_ether.c
+++ b/sys/netgraph/ng_ether.c
@@ -100,6 +100,7 @@ static ng_constructor_t	ng_ether_constructor;
 static ng_rcvmsg_t	ng_ether_rcvmsg;
 static ng_shutdown_t	ng_ether_rmnode;
 static ng_newhook_t	ng_ether_newhook;
+static ng_connect_t	ng_ether_connect;
 static ng_rcvdata_t	ng_ether_rcvdata;
 static ng_disconnect_t	ng_ether_disconnect;
 static int		ng_ether_mod_event(module_t mod, int event, void *data);
@@ -187,8 +188,7 @@ static struct ng_type ng_ether_typestruct = {
 	ng_ether_rmnode,
 	ng_ether_newhook,
 	NULL,
-	NULL,
-	ng_ether_rcvdata,
+	ng_ether_connect,
 	ng_ether_rcvdata,
 	ng_ether_disconnect,
 	ng_ether_cmdlist,
@@ -242,7 +242,7 @@ ng_ether_input_orphan(struct ifnet *ifp,
 }
 
 /*
- * Handle a packet that has come in on an interface.
+ * Handle a packet that has come in on an ethernet interface.
  * The Ethernet header has already been detached from the mbuf,
  * so we have to put it back.
  *
@@ -252,7 +252,6 @@ static void
 ng_ether_input2(node_p node, struct mbuf **mp, struct ether_header *eh)
 {
 	const priv_p priv = node->private;
-	meta_p meta = NULL;
 	int error;
 
 	/* Glue Ethernet header back on */
@@ -260,7 +259,7 @@ ng_ether_input2(node_p node, struct mbuf **mp, struct ether_header *eh)
 		return;
 
 	/* Send out lower/orphan hook */
-	(void)ng_queue_data(priv->lower, *mp, meta);
+	NG_SEND_DATA_ONLY(error, priv->lower, *mp);
 	*mp = NULL;
 }
 
@@ -281,7 +280,7 @@ ng_ether_output(struct ifnet *ifp, struct mbuf **mp)
 		return (0);
 
 	/* Send it out "upper" hook */
-	NG_SEND_DATA_RET(error, priv->upper, *mp, meta);
+	NG_SEND_DATA_RET(error, priv->upper, *mp, meta, NULL);
 
 	/* If we got a reflected packet back, handle it */
 	if (error == 0 && *mp != NULL) {
@@ -481,6 +480,18 @@ ng_ether_newhook(node_p node, hook_p hook, const char *name)
 }
 
 /*
+ * Hooks are attached, adjust to force queueing.
+ * We don't really care which hook it is.
+ * they should all be queuing for outgoing data.
+ */
+static	int
+ng_ether_connect(hook_p hook)
+{
+	hook->peer->flags |= HK_QUEUE;
+	return (0);
+}
+
+/*
  * Receive an incoming control message.
  */
 static int
@@ -591,7 +602,7 @@ ng_ether_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
  */
 static int
 ng_ether_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta)
+		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
 {
 	const node_p node = hook->node;
 	const priv_p priv = node->private;
diff --git a/sys/netgraph/ng_frame_relay.c b/sys/netgraph/ng_frame_relay.c
index e58eb80..830800b 100644
--- a/sys/netgraph/ng_frame_relay.c
+++ b/sys/netgraph/ng_frame_relay.c
@@ -147,7 +147,6 @@ static struct ng_type typestruct = {
 	NULL,
 	NULL,
 	ngfrm_rcvdata,
-	ngfrm_rcvdata,
 	ngfrm_disconnect,
 	NULL
 };
@@ -337,7 +336,7 @@ ngfrm_addrlen(char *hdr)
  */
 static int
 ngfrm_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta)
+		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
 {
 	struct	ctxinfo *const ctxp = hook->private;
 	int     error = 0;
diff --git a/sys/netgraph/ng_hole.c b/sys/netgraph/ng_hole.c
index 3c538f9..43ed1f2 100644
--- a/sys/netgraph/ng_hole.c
+++ b/sys/netgraph/ng_hole.c
@@ -67,7 +67,6 @@ static struct ng_type typestruct = {
 	NULL,
 	NULL,
 	ngh_rcvdata,
-	ngh_rcvdata,
 	ngh_disconnect,
 	NULL
 };
@@ -78,7 +77,7 @@ NETGRAPH_INIT(hole, &typestruct);
  */
 static int
 ngh_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta)
+		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
 {
 	NG_FREE_DATA(m, meta);
 	return 0;
diff --git a/sys/netgraph/ng_iface.c b/sys/netgraph/ng_iface.c
index 146e4b6..992d9b9 100644
--- a/sys/netgraph/ng_iface.c
+++ b/sys/netgraph/ng_iface.c
@@ -191,7 +191,6 @@ static struct ng_type typestruct = {
 	NULL,
 	NULL,
 	ng_iface_rcvdata,
-	ng_iface_rcvdata,
 	ng_iface_disconnect,
 	ng_iface_cmds
 };
@@ -721,7 +720,7 @@ ng_iface_rcvmsg(node_p node, struct ng_mesg *msg,
  */
 static int
 ng_iface_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta)
+		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
 {
 	const priv_p priv = hook->node->private;
 	const iffam_p iffam = get_iffam_from_hook(priv, hook);
diff --git a/sys/netgraph/ng_ksocket.c b/sys/netgraph/ng_ksocket.c
index 8b2e54e..c01f679 100644
--- a/sys/netgraph/ng_ksocket.c
+++ b/sys/netgraph/ng_ksocket.c
@@ -469,7 +469,6 @@ static struct ng_type ng_ksocket_typestruct = {
 	NULL,
 	NULL,
 	ng_ksocket_rcvdata,
-	ng_ksocket_rcvdata,
 	ng_ksocket_disconnect,
 	ng_ksocket_cmds
 };
@@ -782,7 +781,7 @@ done:
  */
 static int
 ng_ksocket_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta)
+		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
 {
 	struct proc *p = curproc ? curproc : &proc0;	/* XXX broken */
 	const node_p node = hook->node;
diff --git a/sys/netgraph/ng_lmi.c b/sys/netgraph/ng_lmi.c
index c2eba39..7019464 100644
--- a/sys/netgraph/ng_lmi.c
+++ b/sys/netgraph/ng_lmi.c
@@ -108,7 +108,6 @@ static struct ng_type typestruct = {
 	NULL,
 	NULL,
 	nglmi_rcvdata,
-	nglmi_rcvdata,
 	nglmi_disconnect,
 	NULL
 };
@@ -559,7 +558,7 @@ nglmi_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
  */
 static int
 nglmi_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta)
+		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
 {
 	sc_p    sc = hook->node->private;
 	u_char *data;
diff --git a/sys/netgraph/ng_message.h b/sys/netgraph/ng_message.h
index 6b1b139..3eb6096 100644
--- a/sys/netgraph/ng_message.h
+++ b/sys/netgraph/ng_message.h
@@ -83,7 +83,7 @@ struct ng_mesg {
 }
 
 /* Negraph type binary compatibility field */
-#define NG_VERSION	3
+#define NG_VERSION	4
 
 /* Flags field flags */
 #define NGF_ORIG	0x0000		/* the msg is the original request */
@@ -117,6 +117,34 @@ struct ng_mesg {
 #define	NGM_ASCII2BINARY	13	/* convert ascii to struct ng_mesg */
 #define	NGM_TEXT_CONFIG		14	/* (optional) get/set text config */
 
+/*
+ * Flow control and intra node control messages.
+ * These are routed between nodes to allow flow control and to allow
+ * events to be passed around the graph. 
+ * There will be some form of default handling for these but I 
+ * do not yet know what it is..
+ */
+
+/* Generic message type cookie */
+#define NGM_FLOW_COOKIE	851672669 /* temp for debugging */
+
+/* Upstream messages */
+#define NGM_LINK_IS_UP		32	/* e.g. carrier found - no data */
+#define NGM_LINK_IS_DOWN	33	/* carrier lost, includes queue state */
+#define NGM_HIGH_WATER_PASSED	34	/* includes queue state */
+#define NGM_LOW_WATER_PASSED	35	/* includes queue state */
+#define NGM_SYNC_QUEUE_STATE	36	/* sync response from sending packet */
+
+/* Downstream messages */
+#define NGM_DROP_LINK		41	/* drop DTR, etc. - stay in the graph */
+#define NGM_RAISE LINK		42	/* if you previously dropped it */
+#define NGM_FLUSH_QUEUE		43	/* no data */
+#define NGM_GET_BANDWIDTH	44	/* either real or measured */
+#define NGM_SET_XMIT_Q_LIMITS	45	/* includes queue state */
+#define NGM_GET_XMIT_Q_LIMITS	46	/* returns queue state */
+#define NGM_MICROMANAGE		47	/* We want sync. queue state reply 
+					   for each packet sent down */
+#define NGM_SET_FLOW_MANAGER	48	/* send flow control here */ 
 /* Structure used for NGM_MKPEER */
 struct ngm_mkpeer {
 	char	type[NG_TYPELEN + 1];			/* peer type */
@@ -271,6 +299,69 @@ struct typelist {
 	}							\
 }
 
+struct ngm_bandwidth {
+	u_int64_t	nominal_in;
+	u_int64_t	seen_in;
+	u_int64_t	nominal_out;
+	u_int64_t	seen_out;
+};
+
+/* Keep this in sync with the above structure definition */
+#define NG_GENERIC_BANDWIDTH_INFO()	{			\
+	{							\
+	  { "nominal_in",	&ng_parse_uint64_type	},	\
+	  { "seen_in",		&ng_parse_uint64_type	},	\
+	  { "nominal_out",	&ng_parse_uint64_type	},	\
+	  { "seen_out",		&ng_parse_uint64_type	},	\
+	  { NULL },						\
+	}							\
+}
+
+/*
+ * Information about a node's 'output' queue.
+ * This is NOT the netgraph input queueing mechanism,
+ * but rather any queue the node may implement internally
+ * This has to consider ALTQ if we are to work with it.
+ * As far as I can see, ALTQ counts PACKETS, not bytes.
+ * If ALTQ has several queues and one has passed a watermark
+ * we should have the priority of that queue be real (and not -1)
+ * XXX ALTQ stuff is just an idea.....
+ */
+struct ngm_queue_state {
+	u_int queue_priority; /* maybe only low-pri is full. -1 = all*/
+	u_int	max_queuelen_bytes;
+	u_int	max_queuelen_packets;
+	u_int	low_watermark;
+	u_int	high_watermark;
+	u_int	current;
+};
+
+/* Keep this in sync with the above structure definition */
+#define NG_GENERIC_QUEUE_INFO()	{				\
+	{							\
+	  { "max_queuelen_bytes", &ng_parse_uint_type	},	\
+	  { "max_queuelen_packets", &ng_parse_uint_type	},	\
+	  { "high_watermark",	&ng_parse_uint_type	},	\
+	  { "low_watermark",	&ng_parse_uint_type	},	\
+	  { "current",		&ng_parse_uint_type	},	\
+	  { NULL },						\
+	}							\
+}
+
+/* Tell a node who to send async flow control info to. */
+struct flow_manager {
+	ng_ID_t		id;			/* unique identifier */
+};
+
+/* Keep this in sync with the above structure definition */
+#define NG_GENERIC_FLOW_MANAGER_INFO()	{			\
+	{							\
+	  { "id",		&ng_parse_hint32_type	},	\
+	  { NULL },						\
+	}							\
+}
+
+
 /*
  * For netgraph nodes that are somehow associated with file descriptors
  * (e.g., a device that has a /dev entry and is also a netgraph node),
diff --git a/sys/netgraph/ng_mppc.c b/sys/netgraph/ng_mppc.c
index ddd48b7..6c84cda 100644
--- a/sys/netgraph/ng_mppc.c
+++ b/sys/netgraph/ng_mppc.c
@@ -153,7 +153,6 @@ static struct ng_type ng_mppc_typestruct = {
 	NULL,
 	NULL,
 	ng_mppc_rcvdata,
-	ng_mppc_rcvdata,
 	ng_mppc_disconnect,
 	NULL
 };
@@ -347,7 +346,7 @@ done:
  */
 static int
 ng_mppc_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta)
+		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
 {
 	const node_p node = hook->node;
 	const priv_p priv = node->private;
@@ -385,7 +384,9 @@ ng_mppc_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 				    NGM_MPPC_RESETREQ, 0, M_NOWAIT);
 				if (msg == NULL)
 					return (error);
-				ng_send_msg(node, msg, priv->ctrlpath, NULL);
+				/* XXX can we use a hook instead of ctrlpath? */
+				ng_send_msg(node, msg, priv->ctrlpath,
+					NULL, NULL, NULL); 
 			}
 			return (error);
 		}
diff --git a/sys/netgraph/ng_one2many.c b/sys/netgraph/ng_one2many.c
index 3189b95..96f93c9 100644
--- a/sys/netgraph/ng_one2many.c
+++ b/sys/netgraph/ng_one2many.c
@@ -172,7 +172,6 @@ static struct ng_type ng_one2many_typestruct = {
 	NULL,
 	NULL,
 	ng_one2many_rcvdata,
-	ng_one2many_rcvdata,
 	ng_one2many_disconnect,
 	ng_one2many_cmdlist,
 };
@@ -383,7 +382,7 @@ ng_one2many_rcvmsg(node_p node, struct ng_mesg *msg,
  */
 static int
 ng_one2many_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-	struct mbuf **ret_m, meta_p *ret_meta)
+	struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
 {
 	const node_p node = hook->node;
 	const priv_p priv = node->private;
diff --git a/sys/netgraph/ng_ppp.c b/sys/netgraph/ng_ppp.c
index ba16fad..d5afd9c 100644
--- a/sys/netgraph/ng_ppp.c
+++ b/sys/netgraph/ng_ppp.c
@@ -352,7 +352,6 @@ static struct ng_type ng_ppp_typestruct = {
 	NULL,
 	NULL,
 	ng_ppp_rcvdata,
-	ng_ppp_rcvdata,
 	ng_ppp_disconnect,
 	ng_ppp_cmds
 };
@@ -559,12 +558,15 @@ ng_ppp_rcvmsg(node_p node, struct ng_mesg *msg,
 		char path[NG_PATHLEN + 1];
 		node_p origNode;
 
-		if ((error = ng_path2node(node,
-		    raddr, &origNode, NULL, NULL)) != 0)
+		if ((error = ng_path2node(node, raddr, &origNode, NULL)) != 0)
 			ERROUT(error);
 		snprintf(path, sizeof(path), "[%lx]:%s",
 		    (long)node, NG_PPP_HOOK_VJC_IP);
-		return ng_send_msg(origNode, msg, path, rptr);
+		return ng_send_msg(origNode, msg, path, NULL, NULL, rptr);
+/* XXX Archie, looks like you are using the wrong value for the ID here..
+ you can't use a node address as a node-ID any more.. 
+But it may be that you can use the new 'hook' arg for ng_send_msg()
+to achieve a more efficient resuld than an ID anyhow. */
 	    }
 	default:
 		error = EINVAL;
@@ -585,7 +587,7 @@ done:
  */
 static int
 ng_ppp_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta)
+		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
 {
 	const node_p node = hook->node;
 	const priv_p priv = node->private;
@@ -781,9 +783,9 @@ ng_ppp_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 	}
 
 	/* Send packet out hook */
-	NG_SEND_DATA_RET(error, outHook, m, meta);
+	NG_SEND_DATA_RET(error, outHook, m, meta, resp);
 	if (m != NULL || meta != NULL)
-		return ng_ppp_rcvdata(outHook, m, meta, NULL, NULL);
+		return ng_ppp_rcvdata(outHook, m, meta, NULL, NULL, resp);
 	return (error);
 }
 
diff --git a/sys/netgraph/ng_pppoe.c b/sys/netgraph/ng_pppoe.c
index 193c99a..e41c80e 100644
--- a/sys/netgraph/ng_pppoe.c
+++ b/sys/netgraph/ng_pppoe.c
@@ -117,6 +117,13 @@ static const struct ng_cmdlist ng_pppoe_cmds[] = {
 	},
 	{
 	  NGM_PPPOE_COOKIE,
+	  NGM_PPPOE_SERVICE,
+	  "pppoe_service",
+	  &ngpppoe_init_data_state_type,
+	  NULL
+	},
+	{
+	  NGM_PPPOE_COOKIE,
 	  NGM_PPPOE_SUCCESS,
 	  "pppoe_success",
 	  &ng_pppoe_sts_state_type,
@@ -151,7 +158,6 @@ static struct ng_type typestruct = {
 	NULL,
 	ng_pppoe_connect,
 	ng_pppoe_rcvdata,
-	ng_pppoe_rcvdata,
 	ng_pppoe_disconnect,
 	ng_pppoe_cmds
 };
@@ -524,10 +530,6 @@ AAA
  * with a single reference for us.. we transfer it to the
  * private structure.. when we free the private struct we must
  * unref the node so it gets freed too.
- *
- * If this were a device node than this work would be done in the attach()
- * routine and the constructor would return EINVAL as you should not be able
- * to creatednodes that depend on hardware (unless you can add the hardware :)
  */
 static int
 ng_pppoe_constructor(node_p *nodep)
@@ -561,6 +563,7 @@ AAA
  * The following hook names are special:
  *  Ethernet:  the hook that should be connected to a NIC.
  *  debug:	copies of data sent out here  (when I write the code).
+ * All other hook names need only be unique. (the framework checks this).
  */
 static int
 ng_pppoe_newhook(node_p node, hook_p hook, const char *name)
@@ -618,6 +621,7 @@ AAA
 		case NGM_PPPOE_CONNECT:
 		case NGM_PPPOE_LISTEN: 
 		case NGM_PPPOE_OFFER: 
+		case NGM_PPPOE_SERVICE: 
 			ourmsg = (struct ngpppoe_init_data *)msg->data;
 			if (msg->header.arglen < sizeof(*ourmsg)) {
 				printf("pppoe: init data too small\n");
@@ -653,6 +657,14 @@ AAA
 				LEAVE(EINVAL);
 			}
 			sp = hook->private;
+
+			/*
+			 * PPPOE_SERVICE advertisments are set up
+			 * on sessions that are in PRIMED state.
+			 */
+			if (msg->header.cmd == NGM_PPPOE_SERVICE) {
+				break;
+			}
 			if (sp->state |= PPPOE_SNONE) {
 				printf("pppoe: Session already active\n");
 				LEAVE(EISCONN);
@@ -735,7 +747,6 @@ AAA
 			 * Store the originator of this message so we can send
 			 * a success of fail message to them later.
 			 * Move the hook to 'LISTENING'
-
 			 */
 			neg->service.hdr.tag_type = PTT_SRV_NAME;
 			neg->service.hdr.tag_len =
@@ -771,6 +782,25 @@ AAA
 			 */
 			sp->state = PPPOE_PRIMED;
 			break;
+		case NGM_PPPOE_SERVICE: 
+			/* 
+			 * Check the session is primed.
+			 * for now just allow ONE service to be advertised.
+			 * If you do it twice you just overwrite.
+			 */
+			if (sp->state |= PPPOE_PRIMED) {
+				printf("pppoe: Session not primed\n");
+				LEAVE(EISCONN);
+			}
+			neg->service.hdr.tag_type = PTT_SRV_NAME;
+			neg->service.hdr.tag_len =
+			    htons((u_int16_t)ourmsg->data_len);
+
+			if (ourmsg->data_len)
+				bcopy(ourmsg->data, neg->service.data,
+				    ourmsg->data_len);
+			neg->service_len = ourmsg->data_len;
+			break;
 		default:
 			LEAVE(EINVAL);
 		}
@@ -828,7 +858,7 @@ AAA
  */
 static int
 ng_pppoe_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta)
+		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
 {
 	node_p			node = hook->node;
 	const priv_p		privp = node->private;
@@ -1268,10 +1298,20 @@ AAA
 			insert_tag(sp, &neg->ac_name.hdr); /* AC_NAME */
 			if ((tag = get_tag(ph, PTT_SRV_NAME)))
 				insert_tag(sp, tag);	  /* return service */
+			/*
+			 * If we have a NULL service request
+			 * and have an extra service defined in this hook,
+			 * then also add a tag for the extra service.
+			 * XXX this is a hack. eventually we should be able
+			 * to support advertising many services, not just one 
+			 */
+			if (((tag == NULL) || (tag->tag_len == 0))
+			&& (neg->service.hdr.tag_len != 0)) {
+				insert_tag(sp, &neg->service.hdr); /* SERVICE */
+			}
 			if ((tag = get_tag(ph, PTT_HOST_UNIQ)))
 				insert_tag(sp, tag); /* returned hostunique */
 			insert_tag(sp, &uniqtag.hdr);
-			/* XXX maybe put the tag in the session store */
 			scan_tags(sp, ph);
 			make_packet(sp);
 			sendpacket(sp);
@@ -1583,8 +1623,10 @@ pppoe_send_event(sessp sp, enum cmd cmdid)
 AAA
 	NG_MKMESSAGE(msg, NGM_PPPOE_COOKIE, cmdid,
 			sizeof(struct ngpppoe_sts), M_NOWAIT);
+	if (msg == NULL)
+		return (ENOMEM);
 	sts = (struct ngpppoe_sts *)msg->data;
 	strncpy(sts->hook, sp->hook->name, NG_HOOKLEN + 1);
-	error = ng_send_msg(sp->hook->node, msg, sp->creator, NULL);
+	error = ng_send_msg(sp->hook->node, msg, sp->creator, NULL, NULL, NULL);
 	return (error);
 }
diff --git a/sys/netgraph/ng_pptpgre.c b/sys/netgraph/ng_pptpgre.c
index 60892ba..7d4fedf 100644
--- a/sys/netgraph/ng_pptpgre.c
+++ b/sys/netgraph/ng_pptpgre.c
@@ -255,7 +255,6 @@ static struct ng_type ng_pptpgre_typestruct = {
 	NULL,
 	NULL,
 	ng_pptpgre_rcvdata,
-	ng_pptpgre_rcvdata,
 	ng_pptpgre_disconnect,
 	ng_pptpgre_cmdlist
 };
@@ -394,7 +393,7 @@ done:
  */
 static int
 ng_pptpgre_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta)
+		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
 {
 	const node_p node = hook->node;
 	const priv_p priv = node->private;
diff --git a/sys/netgraph/ng_rfc1490.c b/sys/netgraph/ng_rfc1490.c
index 02b1c3b..0eb090f 100644
--- a/sys/netgraph/ng_rfc1490.c
+++ b/sys/netgraph/ng_rfc1490.c
@@ -106,7 +106,6 @@ static struct ng_type typestruct = {
 	NULL,
 	NULL,
 	ng_rfc1490_rcvdata,
-	ng_rfc1490_rcvdata,
 	ng_rfc1490_disconnect,
 	NULL
 };
@@ -215,7 +214,7 @@ ng_rfc1490_rcvmsg(node_p node, struct ng_mesg *msg,
 
 static int
 ng_rfc1490_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta)
+		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
 {
 	const node_p node = hook->node;
 	const priv_p priv = node->private;
diff --git a/sys/netgraph/ng_sample.c b/sys/netgraph/ng_sample.c
index d9ec259..4d4bfba 100644
--- a/sys/netgraph/ng_sample.c
+++ b/sys/netgraph/ng_sample.c
@@ -65,7 +65,6 @@ static ng_shutdown_t	ng_xxx_rmnode;
 static ng_newhook_t	ng_xxx_newhook;
 static ng_connect_t	ng_xxx_connect;
 static ng_rcvdata_t	ng_xxx_rcvdata;	 /* note these are both ng_rcvdata_t */
-static ng_rcvdata_t	ng_xxx_rcvdataq; /* note these are both ng_rcvdata_t */
 static ng_disconnect_t	ng_xxx_disconnect;
 
 /* Parse type for struct ngxxxstat */
@@ -107,7 +106,6 @@ static struct ng_type typestruct = {
 	NULL,
 	ng_xxx_connect,
 	ng_xxx_rcvdata,
-	ng_xxx_rcvdataq,
 	ng_xxx_disconnect,
 	ng_xxx_cmdlist
 };
@@ -318,34 +316,13 @@ ng_xxx_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
  * if we use up this data or abort we must free BOTH of these.
  *
  * If we want, we may decide to force this data to be queued and reprocessed
- * at the netgraph NETISR time. (at which time it will be entered using ng_xxx_rcvdataq().
+ * at the netgraph NETISR time.
+ * We would do that by setting the HK_QUEUE flag on our hook. We would do that
+ * in the connect() method. 
  */
 static int
 ng_xxx_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta)
-{
-	int dlci = -2;
-
-	if (hook->private) {
-		/* 
-		 * If it's dlci 1023, requeue it so that it's handled
-		 * at a lower priority. This is how a node decides to
-		 * defer a data message.
-		 */
-		dlci = ((struct XXX_hookinfo *) hook->private)->dlci;
-		if (dlci == 1023) {
-			return(ng_queue_data(hook->peer, m, meta));
-		}
-	}
-	return(ng_xxx_rcvdataq(hook, m, meta));
-}
-
-/*
- * Always accept the data. This version of rcvdata is called from the dequeueing routine.
- */
-static int
-ng_xxx_rcvdataq(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta)
+		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
 {
 	const xxx_p xxxp = hook->node->private;
 	int chan = -2;
@@ -362,7 +339,7 @@ ng_xxx_rcvdataq(hook_p hook, struct mbuf *m, meta_p meta,
 			 * the front here */
 			/* M_PREPEND(....)	; */
 			/* mtod(m, xxxxxx)->dlci = dlci; */
-			error = ng_send_data(xxxp->downstream_hook.hook,
+			NG_SEND_DATA(error, xxxp->downstream_hook.hook,
 			    m, meta);
 			xxxp->packets_out++;
 		} else {
@@ -406,13 +383,15 @@ ng_xxx_rcvdataq(hook_p hook, struct mbuf *m, meta_p meta,
  */
 devintr()
 {
-	meta_p  meta = NULL;	/* whatever metadata we might imagine goes
+	int error;
 				 * here */
 
 	/* get packet from device and send on */
 	m = MGET(blah blah)
-	    error = ng_queueit(upstream, m, meta);	/* see note above in
-							 * xxx_rcvdata() */
+	
+	NG_SEND_DATA_ONLY(error, xxxp->upstream_hook.hook, m);
+				/* see note above in xxx_rcvdata() */
+				/* and ng_xxx_connect() */
 }
 
 #endif				/* 0 */
@@ -449,7 +428,35 @@ ng_xxx_rmnode(node_p node)
 static int
 ng_xxx_connect(hook_p hook)
 {
-	/* be really amiable and just say "YUP that's OK by me! " */
+#if 0
+	/*
+	 * If we were a driver running at other than splnet then
+	 * we should set the QUEUE bit on the edge so that we
+	 * will deliver by queing.
+	 */
+	if /*it is the upstream hook */
+	hook->peer->flags |= HK_QUEUE;
+#endif
+#if 0
+	/*
+	 * If for some reason we want incoming date to be queued
+	 * by the NETISR system and delivered later we can set the same bit on
+	 * OUR hook. (maybe to allow unwinding of the stack)
+	 */
+
+	if (hook->private) {
+		int dlci;
+		/* 
+		 * If it's dlci 1023, requeue it so that it's handled
+		 * at a lower priority. This is how a node decides to
+		 * defer a data message.
+		 */
+		dlci = ((struct XXX_hookinfo *) hook->private)->dlci;
+		if (dlci == 1023) {
+			hook->flags |= HK_QUEUE;
+		}
+#endif
+	/* otherwise be really amiable and just say "YUP that's OK by me! " */
 	return (0);
 }
 
diff --git a/sys/netgraph/ng_socket.c b/sys/netgraph/ng_socket.c
index 4e31b30..7608aba 100644
--- a/sys/netgraph/ng_socket.c
+++ b/sys/netgraph/ng_socket.c
@@ -130,7 +130,6 @@ static struct ng_type typestruct = {
 	NULL,
 	NULL,
 	ngs_rcvdata,
-	ngs_rcvdata,
 	ngs_disconnect,
 	NULL
 };
@@ -239,7 +238,7 @@ ngc_send(struct socket *so, int flags, struct mbuf *m, struct sockaddr *addr,
 
 	/* The callee will free the msg when done. The addr is our business. */
 	error = ng_send_msg(pcbp->sockdata->node,
-			    (struct ng_mesg *) msg, path, &resp);
+			    (struct ng_mesg *) msg, path, NULL, NULL, &resp);
 
 	/* If the callee responded with a synchronous response, then put it
 	 * back on the receive side of the socket; sap is source address. */
@@ -307,7 +306,6 @@ ngd_send(struct socket *so, int flags, struct mbuf *m, struct sockaddr *addr,
 {
 	struct ngpcb *const pcbp = sotongpcb(so);
 	struct sockaddr_ng *const sap = (struct sockaddr_ng *) addr;
-	meta_p  mp = NULL;
 	int     len, error;
 	hook_p  hook = NULL;
 	char	hookname[NG_HOOKLEN + 1];
@@ -359,7 +357,7 @@ ngd_send(struct socket *so, int flags, struct mbuf *m, struct sockaddr *addr,
 	}
 
 	/* Send data (OK if hook is NULL) */
-	NG_SEND_DATA(error, hook, m, mp);	/* makes m NULL */
+	NG_SEND_DATA_ONLY(error, hook, m);	/* makes m NULL */
 
 release:
 	if (control != NULL)
@@ -394,7 +392,7 @@ ng_setsockaddr(struct socket *so, struct sockaddr **addr)
 
 	s = splnet();
 	pcbp = sotongpcb(so);
-	if (pcbp == 0) {
+	if ((pcbp == NULL) || (pcbp->sockdata == NULL)) {
 		splx(s);
 		return (EINVAL);
 	}
@@ -608,7 +606,7 @@ ng_connect_data(struct sockaddr *nam, struct ngpcb *pcbp)
 	/* Find the target (victim) and check it doesn't already have a data
 	 * socket. Also check it is a 'socket' type node. */
 	sap = (struct sockaddr_ng *) nam;
-	if ((error = ng_path2node(NULL, sap->sg_data, &farnode, NULL, NULL)))
+	if ((error = ng_path2node(NULL, sap->sg_data, &farnode, NULL)))
 		return (error);
 
 	if (strcmp(farnode->type->name, NG_SOCKET_NODE_TYPE) != 0)
@@ -668,7 +666,10 @@ ng_bind(struct sockaddr *nam, struct ngpcb *pcbp)
 		TRAP_ERROR;
 		return (EINVAL);
 	}
-	if (sap->sg_len < 3 || sap->sg_data[sap->sg_len - 3] != '\0') {
+	if ((sap->sg_len < 4)
+	||  (sap->sg_len > (NG_NODELEN + 3))
+	||  (sap->sg_data[0] == '\0')
+	||  (sap->sg_data[sap->sg_len - 3] != '\0')) {
 		TRAP_ERROR;
 		return (EINVAL);
 	}
@@ -792,7 +793,7 @@ ngs_rcvmsg(node_p node, struct ng_mesg *msg, const char *retaddr,
  */
 static int
 ngs_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta)
+		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
 {
 	struct ngsock *const sockdata = hook->node->private;
 	struct ngpcb *const pcbp = sockdata->datasock;
diff --git a/sys/netgraph/ng_tee.c b/sys/netgraph/ng_tee.c
index b27d435..be6111a 100644
--- a/sys/netgraph/ng_tee.c
+++ b/sys/netgraph/ng_tee.c
@@ -138,7 +138,6 @@ static struct ng_type ng_tee_typestruct = {
 	NULL,
 	NULL,
 	ngt_rcvdata,
-	ngt_rcvdata,
 	ngt_disconnect,
 	ng_tee_cmds
 };
@@ -275,7 +274,7 @@ done:
  */
 static int
 ngt_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta)
+		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
 {
 	const sc_p sc = hook->node->private;
 	struct hookinfo *const hinfo = (struct hookinfo *) hook->private;
diff --git a/sys/netgraph/ng_tty.c b/sys/netgraph/ng_tty.c
index 74e4d18..6236d6e 100644
--- a/sys/netgraph/ng_tty.c
+++ b/sys/netgraph/ng_tty.c
@@ -134,6 +134,7 @@ static ng_constructor_t	ngt_constructor;
 static ng_rcvmsg_t	ngt_rcvmsg;
 static ng_shutdown_t	ngt_shutdown;
 static ng_newhook_t	ngt_newhook;
+static ng_connect_t	ngt_connect;
 static ng_rcvdata_t	ngt_rcvdata;
 static ng_disconnect_t	ngt_disconnect;
 static int	ngt_mod_event(module_t mod, int event, void *data);
@@ -166,8 +167,7 @@ static struct ng_type typestruct = {
 	ngt_shutdown,
 	ngt_newhook,
 	NULL,
-	NULL,
-	ngt_rcvdata,
+	ngt_connect,
 	ngt_rcvdata,
 	ngt_disconnect,
 	NULL
@@ -394,7 +394,7 @@ ngt_input(int c, struct tty *tp)
 	/* Ship off mbuf if it's time */
 	if (sc->hotchar == -1 || c == sc->hotchar || m->m_len >= MHLEN) {
 		m->m_data = m->m_pktdat;
-		error = ng_queue_data(sc->hook, m, NULL);
+		NG_SEND_DATA_ONLY(error, sc->hook, m);
 		sc->m = NULL;
 	}
 done:
@@ -521,6 +521,16 @@ done:
 }
 
 /*
+ * set the hooks into queueing mode (for outgoing packets)
+ */
+static int
+ngt_connect(hook_p hook)
+{
+	hook->peer->flags |= HK_QUEUE;
+	return (0);
+}
+
+/*
  * Disconnect the hook
  */
 static int
@@ -567,7 +577,7 @@ ngt_shutdown(node_p node)
  */
 static int
 ngt_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta)
+		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
 {
 	const sc_p sc = hook->node->private;
 	int s, error = 0;
diff --git a/sys/netgraph/ng_vjc.c b/sys/netgraph/ng_vjc.c
index 6faf6c4..41a16d6 100644
--- a/sys/netgraph/ng_vjc.c
+++ b/sys/netgraph/ng_vjc.c
@@ -232,7 +232,6 @@ static struct ng_type ng_vjc_typestruct = {
 	NULL,
 	NULL,
 	ng_vjc_rcvdata,
-	ng_vjc_rcvdata,
 	ng_vjc_disconnect,
 	ng_vjc_cmds
 };
@@ -412,7 +411,7 @@ done:
  */
 static int
 ng_vjc_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-		struct mbuf **ret_m, meta_p *ret_meta)
+		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
 {
 	const node_p node = hook->node;
 	const priv_p priv = (priv_p) node->private;
diff --git a/sys/pci/if_mn.c b/sys/pci/if_mn.c
index 8c222ba..170284c 100644
--- a/sys/pci/if_mn.c
+++ b/sys/pci/if_mn.c
@@ -497,7 +497,7 @@ mn_fmt_ts(char *p, u_int32_t ts)
 
 static int
 ngmn_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
-			struct mbuf **ret_m, meta_p *ret_meta)
+		struct mbuf **ret_m, meta_p *ret_meta, struct ng_mesg **resp)
 {
 	struct mbuf  *m2;
 	struct trxd *dp, *dp2;
@@ -653,6 +653,8 @@ ngmn_connect(hook_p hook)
 	if (!(u & 1))
 		printf("%s: init chan %d stat %08x\n", sc->name, chan, u);
 	sc->m32x->stat = 1; 
+	/* probably not at splnet, force outward queueing */
+	hook->peer->flags |= HK_QUEUE;
 
 	return (0);
 }
@@ -1030,9 +1032,9 @@ mn_rx_intr(struct softc *sc, u_int32_t vector)
 		m->m_pkthdr.len = m->m_len = (dp->status >> 16) & 0x1fff;
 		err = (dp->status >> 8) & 0xff;
 		if (!err) {
-			ng_queue_data(sch->hook, m, NULL);
+			int error;
+			NG_SEND_DATA_ONLY(error, sch->hook, m);
 			sch->last_recv = time_second;
-			m = 0;
 			/* we could be down by now... */
 			if (sch->state != UP) 
 				return;