Fix broken multi-link fragment reassembly algorithm.

Add hook for IPv6. Misc cleanups.

PR:		kern/16335

1 files changed, 666 insertions, 230 deletions

diff --git a/sys/netgraph/ng_ppp.c b/sys/netgraph/ng_ppp.c
index db4dcbc..ba1dacf 100644
--- a/sys/netgraph/ng_ppp.c
+++ b/sys/netgraph/ng_ppp.c
@@ -2,7 +2,7 @@
 /*
  * ng_ppp.c
  *
- * Copyright (c) 1996-1999 Whistle Communications, Inc.
+ * Copyright (c) 1996-2000 Whistle Communications, Inc.
  * All rights reserved.
  * 
  * Subject to the following obligations and disclaimer of warranty, use and
@@ -47,12 +47,15 @@
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
+#include <sys/time.h>
 #include <sys/mbuf.h>
 #include <sys/malloc.h>
 #include <sys/errno.h>
 #include <sys/syslog.h>
 #include <sys/ctype.h>
 
+#include <machine/limits.h>
+
 #include <netgraph/ng_message.h>
 #include <netgraph/netgraph.h>
 #include <netgraph/ng_parse.h>
@@ -67,6 +70,7 @@
 #define PROT_COMPD		0x00fd
 #define PROT_CRYPTD		0x0053
 #define PROT_IP			0x0021
+#define PROT_IPV6		0x0057
 #define PROT_IPX		0x002b
 #define PROT_LCP		0xc021
 #define PROT_MP			0x003d
@@ -78,10 +82,6 @@
 #define MP_INITIAL_SEQ		0		/* per RFC 1990 */
 #define MP_MIN_LINK_MRU		32
 
-#define MP_MAX_SEQ_LINGER	64		/* max frags we will hold */
-#define MP_INSANE_SEQ_JUMP	128		/* a sequence # jump too far */
-#define MP_MIN_FRAG_LEN		6		/* don't frag smaller frames */
-
 #define MP_SHORT_SEQ_MASK	0x00000fff	/* short seq # mask */
 #define MP_SHORT_SEQ_HIBIT	0x00000800	/* short seq # high bit */
 #define MP_SHORT_FIRST_FLAG	0x00008000	/* first fragment in frame */
@@ -92,6 +92,8 @@
 #define MP_LONG_FIRST_FLAG	0x80000000	/* first fragment in frame */
 #define MP_LONG_LAST_FLAG	0x40000000	/* last fragment in frame */
 
+#define MP_NOSEQ		INT_MAX		/* impossible sequence number */
+
 #define MP_SEQ_MASK(priv)	((priv)->conf.recvShortSeq ? \
 				    MP_SHORT_SEQ_MASK : MP_LONG_SEQ_MASK)
 
@@ -109,20 +111,27 @@
 				    MP_SHORT_SEQ_DIFF((x), (y)) : \
 				    MP_LONG_SEQ_DIFF((x), (y)))
 
+#define MP_NEXT_SEQ(priv,seq)	(((seq) + 1) & MP_SEQ_MASK(priv))
+#define MP_PREV_SEQ(priv,seq)	(((seq) - 1) & MP_SEQ_MASK(priv))
+
+/* Don't fragment transmitted packets smaller than this */
+#define MP_MIN_FRAG_LEN		6
+
+/* Maximum fragment reasssembly queue length */
+#define MP_MAX_QUEUE_LEN	128
+
+/* Fragment queue scanner period */
+#define MP_FRAGTIMER_INTERVAL	(hz/2)
+
 /* We store incoming fragments this way */
 struct ng_ppp_frag {
-	int				seq;
-	u_char				first;
-	u_char				last;
-	struct mbuf			*data;
-	meta_p				meta;
-	CIRCLEQ_ENTRY(ng_ppp_frag)	f_qent;
-};
-
-/* We keep track of link queue status this way */
-struct ng_ppp_link_qstat {
-	struct timeval		lastWrite;	/* time of last write */
-	int			bytesInQueue;	/* bytes in the queue then */
+	int				seq;		/* fragment seq# */
+	u_char				first;		/* First in packet? */
+	u_char				last;		/* Last in packet? */
+	struct timeval			timestamp;	/* time of reception */
+	struct mbuf			*data;		/* Fragment data */
+	meta_p				meta;		/* Fragment meta */
+	CIRCLEQ_ENTRY(ng_ppp_frag)	f_qent;		/* Fragment queue */
 };
 
 /* We use integer indicies to refer to the non-link hooks */
@@ -151,8 +160,10 @@ static const char *const ng_ppp_hook_names[] = {
 #define HOOK_INDEX_VJC_UNCOMP		10
 	NG_PPP_HOOK_VJC_VJIP,
 #define HOOK_INDEX_VJC_VJIP		11
+	NG_PPP_HOOK_IPV6,
+#define HOOK_INDEX_IPV6			12
 	NULL
-#define HOOK_INDEX_MAX			12
+#define HOOK_INDEX_MAX			13
 };
 
 /* We store index numbers in the hook private pointer. The HOOK_INDEX()
@@ -160,22 +171,34 @@ static const char *const ng_ppp_hook_names[] = {
    complement of the link number for link hooks. */
 #define HOOK_INDEX(hook)	(*((int16_t *) &(hook)->private))
 
-/* Node private data */
+/* Per-link private information */
+struct ng_ppp_link {
+	struct ng_ppp_link_conf	conf;		/* link configuration */
+	hook_p			hook;		/* connection to link data */
+	int			seq;		/* highest rec'd seq# - MSEQ */
+	struct timeval		lastWrite;	/* time of last write */
+	int			bytesInQueue;	/* bytes in the output queue */
+	struct ng_ppp_link_stat	stats;		/* Link stats */
+};
+
+/* Total per-node private information */
 struct ng_ppp_private {
-	struct ng_ppp_node_config	conf;
-	struct ng_ppp_link_stat		bundleStats;
-	struct ng_ppp_link_stat		linkStats[NG_PPP_MAX_LINKS];
-	hook_p				links[NG_PPP_MAX_LINKS];
-	hook_p				hooks[HOOK_INDEX_MAX];
-	u_char				vjCompHooked;
-	u_char				allLinksEqual;
-	u_short				activeLinks[NG_PPP_MAX_LINKS];
-	u_int				numActiveLinks;
-	u_int				lastLink;	/* for round robin */
-	struct ng_ppp_link_qstat	qstat[NG_PPP_MAX_LINKS];
-	CIRCLEQ_HEAD(ng_ppp_fraglist, ng_ppp_frag)
-					frags;		/* incoming fragments */
-	int				mpSeqOut;	/* next out MP seq # */
+	struct ng_ppp_bund_conf	conf;			/* bundle config */
+	struct ng_ppp_link_stat	bundleStats;		/* bundle stats */
+	struct ng_ppp_link	links[NG_PPP_MAX_LINKS];/* per-link info */
+	int			xseq;			/* next out MP seq # */
+	int			mseq;			/* min links[i].seq */
+	u_char			vjCompHooked;		/* VJ comp hooked up? */
+	u_char			allLinksEqual;		/* all xmit the same? */
+	u_char			timerActive;		/* frag timer active? */
+	u_int			numActiveLinks;		/* how many links up */
+	int			activeLinks[NG_PPP_MAX_LINKS];	/* indicies */
+	u_int			lastLink;		/* for round robin */
+	hook_p			hooks[HOOK_INDEX_MAX];	/* non-link hooks */
+	CIRCLEQ_HEAD(ng_ppp_fraglist, ng_ppp_frag)	/* fragment queue */
+				frags;
+	int			qlen;			/* fraq queue length */
+	struct callout_handle	fragTimer;		/* fraq queue check */
 };
 typedef struct ng_ppp_private *priv_p;
 
@@ -194,17 +217,25 @@ static int	ng_ppp_output(node_p node, int bypass, int proto,
 			int linkNum, struct mbuf *m, meta_p meta);
 static int	ng_ppp_mp_input(node_p node, int linkNum,
 			struct mbuf *m, meta_p meta);
+static int	ng_ppp_check_packet(node_p node);
+static void	ng_ppp_get_packet(node_p node, struct mbuf **mp, meta_p *metap);
+static int	ng_ppp_frag_process(node_p node);
+static int	ng_ppp_frag_trim(node_p node);
+static void	ng_ppp_frag_timeout(void *arg);
+static void	ng_ppp_frag_checkstale(node_p node);
+static void	ng_ppp_frag_reset(node_p node);
 static int	ng_ppp_mp_output(node_p node, struct mbuf *m, meta_p meta);
 static void	ng_ppp_mp_strategy(node_p node, int len, int *distrib);
 static int	ng_ppp_intcmp(const void *v1, const void *v2);
 static struct	mbuf *ng_ppp_addproto(struct mbuf *m, int proto, int compOK);
 static struct	mbuf *ng_ppp_prepend(struct mbuf *m, const void *buf, int len);
 static int	ng_ppp_config_valid(node_p node,
-			const struct ng_ppp_node_config *newConf);
+			const struct ng_ppp_node_conf *newConf);
 static void	ng_ppp_update(node_p node, int newConf);
-static void	ng_ppp_free_frags(node_p node);
+static void	ng_ppp_start_frag_timer(node_p node);
+static void	ng_ppp_stop_frag_timer(node_p node);
 
-/* Parse type for struct ng_ppp_link_config */
+/* Parse type for struct ng_ppp_link_conf */
 static const struct ng_parse_struct_info
 	ng_ppp_link_type_info = NG_PPP_LINK_TYPE_INFO;
 static const struct ng_parse_type ng_ppp_link_type = {
@@ -212,7 +243,15 @@ static const struct ng_parse_type ng_ppp_link_type = {
 	&ng_ppp_link_type_info,
 };
 
-/* Parse type for struct ng_ppp_node_config */
+/* Parse type for struct ng_ppp_bund_conf */
+static const struct ng_parse_struct_info
+	ng_ppp_bund_type_info = NG_PPP_BUND_TYPE_INFO;
+static const struct ng_parse_type ng_ppp_bund_type = {
+	&ng_parse_struct_type,
+	&ng_ppp_bund_type_info,
+};
+
+/* Parse type for struct ng_ppp_node_conf */
 struct ng_parse_fixedarray_info ng_ppp_array_info = {
 	&ng_ppp_link_type,
 	NG_PPP_MAX_LINKS
@@ -221,11 +260,11 @@ static const struct ng_parse_type ng_ppp_link_array_type = {
 	&ng_parse_fixedarray_type,
 	&ng_ppp_array_info,
 };
-static const struct ng_parse_struct_info ng_ppp_config_type_info
-	= NG_PPP_CONFIG_TYPE_INFO(&ng_ppp_link_array_type);
-static const struct ng_parse_type ng_ppp_config_type = {
+static const struct ng_parse_struct_info ng_ppp_conf_type_info
+	= NG_PPP_CONFIG_TYPE_INFO(&ng_ppp_bund_type, &ng_ppp_link_array_type);
+static const struct ng_parse_type ng_ppp_conf_type = {
 	&ng_parse_struct_type,
-	&ng_ppp_config_type_info
+	&ng_ppp_conf_type_info
 };
 
 /* Parse type for struct ng_ppp_link_stat */
@@ -242,7 +281,7 @@ static const struct ng_cmdlist ng_ppp_cmds[] = {
 	  NGM_PPP_COOKIE,
 	  NGM_PPP_SET_CONFIG,
 	  "setconfig",
-	  &ng_ppp_config_type,
+	  &ng_ppp_conf_type,
 	  NULL
 	},
 	{
@@ -250,7 +289,7 @@ static const struct ng_cmdlist ng_ppp_cmds[] = {
 	  NGM_PPP_GET_CONFIG,
 	  "getconfig",
 	  NULL,
-	  &ng_ppp_config_type
+	  &ng_ppp_conf_type
 	},
 	{
 	  NGM_PPP_COOKIE,
@@ -294,10 +333,13 @@ static struct ng_type ng_ppp_typestruct = {
 };
 NETGRAPH_INIT(ppp, &ng_ppp_typestruct);
 
-static int	*compareLatencies;		/* hack for ng_ppp_intcmp() */
+static int *compareLatencies;			/* hack for ng_ppp_intcmp() */
 
 /* Address and control field header */
-static const u_char	ng_ppp_acf[2] = { 0xff, 0x03 };
+static const u_char ng_ppp_acf[2] = { 0xff, 0x03 };
+
+/* Maximum time we'll let a complete incoming packet sit in the queue */
+static const struct timeval ng_ppp_max_staleness = { 2, 0 };	/* 2 seconds */
 
 #define ERROUT(x)	do { error = (x); goto done; } while (0)
 
@@ -312,7 +354,7 @@ static int
 ng_ppp_constructor(node_p *nodep)
 {
 	priv_p priv;
-	int error;
+	int i, error;
 
 	/* Allocate private structure */
 	MALLOC(priv, priv_p, sizeof(*priv), M_NETGRAPH, M_WAITOK);
@@ -329,6 +371,9 @@ ng_ppp_constructor(node_p *nodep)
 
 	/* Initialize state */
 	CIRCLEQ_INIT(&priv->frags);
+	for (i = 0; i < NG_PPP_MAX_LINKS; i++)
+		priv->links[i].seq = MP_NOSEQ;
+	callout_handle_init(&priv->fragTimer);
 
 	/* Done */
 	return (0);
@@ -357,7 +402,7 @@ ng_ppp_newhook(node_p node, hook_p hook, const char *name)
 		linkNum = (int)strtoul(cp, &eptr, 10);
 		if (*eptr != '\0' || linkNum < 0 || linkNum >= NG_PPP_MAX_LINKS)
 			return (EINVAL);
-		hookPtr = &priv->links[linkNum];
+		hookPtr = &priv->links[linkNum].hook;
 		hookIndex = ~linkNum;
 	} else {				/* must be a non-link hook */
 		int i;
@@ -378,7 +423,7 @@ ng_ppp_newhook(node_p node, hook_p hook, const char *name)
 		return (EISCONN);
 
 	/* Disallow more than one link unless multilink is enabled */
-	if (linkNum != -1 && priv->conf.links[linkNum].enableLink
+	if (linkNum != -1 && priv->links[linkNum].conf.enableLink
 	    && !priv->conf.enableMultilink && priv->numActiveLinks >= 1)
 		return (ENODEV);
 
@@ -405,24 +450,37 @@ ng_ppp_rcvmsg(node_p node, struct ng_mesg *msg,
 		switch (msg->header.cmd) {
 		case NGM_PPP_SET_CONFIG:
 		    {
-			struct ng_ppp_node_config *const newConf =
-			    (struct ng_ppp_node_config *) msg->data;
+			struct ng_ppp_node_conf *const conf =
+			    (struct ng_ppp_node_conf *)msg->data;
+			int i;
 
 			/* Check for invalid or illegal config */
-			if (msg->header.arglen != sizeof(*newConf))
+			if (msg->header.arglen != sizeof(*conf))
 				ERROUT(EINVAL);
-			if (!ng_ppp_config_valid(node, newConf))
+			if (!ng_ppp_config_valid(node, conf))
 				ERROUT(EINVAL);
-			priv->conf = *newConf;
+
+			/* Copy config */
+			priv->conf = conf->bund;
+			for (i = 0; i < NG_PPP_MAX_LINKS; i++)
+				priv->links[i].conf = conf->links[i];
 			ng_ppp_update(node, 1);
 			break;
 		    }
 		case NGM_PPP_GET_CONFIG:
-			NG_MKRESPONSE(resp, msg, sizeof(priv->conf), M_NOWAIT);
+		    {
+			struct ng_ppp_node_conf *conf;
+			int i;
+
+			NG_MKRESPONSE(resp, msg, sizeof(*conf), M_NOWAIT);
 			if (resp == NULL)
 				ERROUT(ENOMEM);
-			bcopy(&priv->conf, resp->data, sizeof(priv->conf));
+			conf = (struct ng_ppp_node_conf *)resp->data;
+			conf->bund = priv->conf;
+			for (i = 0; i < NG_PPP_MAX_LINKS; i++)
+				conf->links[i] = priv->links[i].conf;
 			break;
+		    }
 		case NGM_PPP_GET_LINK_STATS:
 		case NGM_PPP_CLR_LINK_STATS:
 		case NGM_PPP_GETCLR_LINK_STATS:
@@ -437,7 +495,7 @@ ng_ppp_rcvmsg(node_p node, struct ng_mesg *msg,
 			    && linkNum != NG_PPP_BUNDLE_LINKNUM)
 				ERROUT(EINVAL);
 			stats = (linkNum == NG_PPP_BUNDLE_LINKNUM) ?
-				&priv->bundleStats : &priv->linkStats[linkNum];
+			    &priv->bundleStats : &priv->links[linkNum].stats;
 			if (msg->header.cmd != NGM_PPP_CLR_LINK_STATS) {
 				NG_MKRESPONSE(resp, msg,
 				    sizeof(struct ng_ppp_link_stat), M_NOWAIT);
@@ -455,19 +513,17 @@ ng_ppp_rcvmsg(node_p node, struct ng_mesg *msg,
 		}
 		break;
 	case NGM_VJC_COOKIE:
-	   {
-	       char path[NG_PATHLEN + 1];
-	       node_p origNode;
-	       hook_p lasthook;
-
-	       if ((error = ng_path2node(node, raddr, &origNode,
-									NULL, &lasthook)) != 0)
-		       ERROUT(error);
-	       snprintf(path, sizeof(path), "[%lx]:%s",
-		   (long) node, NG_PPP_HOOK_VJC_IP);
-	       return ng_send_msg(origNode, msg, path, rptr);
-	       break;
-	   }
+	    {
+		char path[NG_PATHLEN + 1];
+		node_p origNode;
+
+		if ((error = ng_path2node(node,
+		    raddr, &origNode, NULL, NULL)) != 0)
+			ERROUT(error);
+		snprintf(path, sizeof(path), "[%lx]:%s",
+		    (long)node, NG_PPP_HOOK_VJC_IP);
+		return ng_send_msg(origNode, msg, path, rptr);
+	    }
 	default:
 		error = EINVAL;
 		break;
@@ -498,15 +554,17 @@ ng_ppp_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 
 	/* Did it come from a link hook? */
 	if (index < 0) {
+		struct ng_ppp_link *link;
 
 		/* Convert index into a link number */
 		linkNum = (u_int16_t)~index;
 		KASSERT(linkNum < NG_PPP_MAX_LINKS,
 		    ("%s: bogus index 0x%x", __FUNCTION__, index));
+		link = &priv->links[linkNum];
 
 		/* Stats */
-		priv->linkStats[linkNum].recvFrames++;
-		priv->linkStats[linkNum].recvOctets += m->m_pkthdr.len;
+		link->stats.recvFrames++;
+		link->stats.recvOctets += m->m_pkthdr.len;
 
 		/* Strip address and control fields, if present */
 		if (m->m_pkthdr.len >= 2) {
@@ -520,7 +578,7 @@ ng_ppp_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 
 		/* Dispatch incoming frame (if not enabled, to bypass) */
 		return ng_ppp_input(node,
-		    !priv->conf.links[linkNum].enableLink, linkNum, m, meta);
+		    !link->conf.enableLink, linkNum, m, meta);
 	}
 
 	/* Get protocol & check if data allowed from this hook */
@@ -541,6 +599,13 @@ ng_ppp_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 		}
 		proto = PROT_IPX;
 		break;
+	case HOOK_INDEX_IPV6:
+		if (!priv->conf.enableIPv6) {
+			NG_FREE_DATA(m, meta);
+			return (ENXIO);
+		}
+		proto = PROT_IPV6;
+		break;
 	case HOOK_INDEX_INET:
 	case HOOK_INDEX_VJC_VJIP:
 		if (!priv->conf.enableIP) {
@@ -630,6 +695,7 @@ ng_ppp_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 		}
 		/* FALLTHROUGH */
 	case HOOK_INDEX_ATALK:
+	case HOOK_INDEX_IPV6:
 	case HOOK_INDEX_IPX:
 	case HOOK_INDEX_VJC_COMP:
 	case HOOK_INDEX_VJC_UNCOMP:
@@ -673,7 +739,9 @@ ng_ppp_rcvdata(hook_p hook, struct mbuf *m, meta_p meta,
 	}
 
 	/* Send packet out hook */
-	NG_SEND_DATA(error, outHook, m, meta);
+	NG_SEND_DATA_RET(error, outHook, m, meta);
+	if (m != NULL || meta != NULL)
+		return ng_ppp_rcvdata(outHook, m, meta, NULL, NULL);
 	return (error);
 }
 
@@ -685,11 +753,14 @@ ng_ppp_rmnode(node_p node)
 {
 	const priv_p priv = node->private;
 
+	/* Stop fragment queue timer */
+	ng_ppp_stop_frag_timer(node);
+
 	/* Take down netgraph node */
 	node->flags |= NG_INVALID;
 	ng_cutlinks(node);
 	ng_unname(node);
-	ng_ppp_free_frags(node);
+	ng_ppp_frag_reset(node);
 	bzero(priv, sizeof(*priv));
 	FREE(priv, M_NETGRAPH);
 	node->private = NULL;
@@ -709,7 +780,7 @@ ng_ppp_disconnect(hook_p hook)
 
 	/* Zero out hook pointer */
 	if (index < 0)
-		priv->links[~index] = NULL;
+		priv->links[~index].hook = NULL;
 	else
 		priv->hooks[index] = NULL;
 
@@ -717,7 +788,7 @@ ng_ppp_disconnect(hook_p hook)
 	if (node->numhooks > 0)
 		ng_ppp_update(node, 0);
 	else
-		ng_rmnode(hook->node);
+		ng_rmnode(node);
 	return (0);
 }
 
@@ -749,7 +820,7 @@ ng_ppp_input(node_p node, int bypass, int linkNum, struct mbuf *m, meta_p meta)
 		if (linkNum == NG_PPP_BUNDLE_LINKNUM)
 			priv->bundleStats.badProtos++;
 		else
-			priv->linkStats[linkNum].badProtos++;
+			priv->links[linkNum].stats.badProtos++;
 		NG_FREE_DATA(m, meta);
 		return (EINVAL);
 	}
@@ -777,7 +848,8 @@ ng_ppp_input(node_p node, int bypass, int linkNum, struct mbuf *m, meta_p meta)
 			outHook = priv->hooks[HOOK_INDEX_VJC_UNCOMP];
 		break;
 	case PROT_MP:
-		if (priv->conf.enableMultilink)
+		if (priv->conf.enableMultilink
+		    && linkNum != NG_PPP_BUNDLE_LINKNUM)
 			return ng_ppp_mp_input(node, linkNum, m, meta);
 		break;
 	case PROT_APPLETALK:
@@ -792,6 +864,10 @@ ng_ppp_input(node_p node, int bypass, int linkNum, struct mbuf *m, meta_p meta)
 		if (priv->conf.enableIP)
 			outHook = priv->hooks[HOOK_INDEX_INET];
 		break;
+	case PROT_IPV6:
+		if (priv->conf.enableIPv6)
+			outHook = priv->hooks[HOOK_INDEX_IPV6];
+		break;
 	}
 
 bypass:
@@ -822,19 +898,24 @@ ng_ppp_output(node_p node, int bypass,
 	int proto, int linkNum, struct mbuf *m, meta_p meta)
 {
 	const priv_p priv = node->private;
+	struct ng_ppp_link *link;
 	int len, error;
 
 	/* If not doing MP, map bundle virtual link to (the only) link */
 	if (linkNum == NG_PPP_BUNDLE_LINKNUM && !priv->conf.enableMultilink)
 		linkNum = priv->activeLinks[0];
 
+	/* Get link pointer (optimization) */
+	link = (linkNum != NG_PPP_BUNDLE_LINKNUM) ?
+	    &priv->links[linkNum] : NULL;
+
 	/* Check link status (if real) */
 	if (linkNum != NG_PPP_BUNDLE_LINKNUM) {
-		if (!bypass && !priv->conf.links[linkNum].enableLink) {
+		if (!bypass && !link->conf.enableLink) {
 			NG_FREE_DATA(m, meta);
 			return (ENXIO);
 		}
-		if (priv->links[linkNum] == NULL) {
+		if (link->hook == NULL) {
 			NG_FREE_DATA(m, meta);
 			return (ENETDOWN);
 		}
@@ -843,7 +924,7 @@ ng_ppp_output(node_p node, int bypass,
 	/* Prepend protocol number, possibly compressed */
 	if ((m = ng_ppp_addproto(m, proto,
 	    linkNum == NG_PPP_BUNDLE_LINKNUM
-	      || priv->conf.links[linkNum].enableProtoComp)) == NULL) {
+	      || link->conf.enableProtoComp)) == NULL) {
 		NG_FREE_META(meta);
 		return (ENOBUFS);
 	}
@@ -853,7 +934,7 @@ ng_ppp_output(node_p node, int bypass,
 		return ng_ppp_mp_output(node, m, meta);
 
 	/* Prepend address and control field (unless compressed) */
-	if (proto == PROT_LCP || !priv->conf.links[linkNum].enableACFComp) {
+	if (proto == PROT_LCP || !link->conf.enableACFComp) {
 		if ((m = ng_ppp_prepend(m, &ng_ppp_acf, 2)) == NULL) {
 			NG_FREE_META(meta);
 			return (ENOBUFS);
@@ -862,36 +943,85 @@ ng_ppp_output(node_p node, int bypass,
 
 	/* Deliver frame */
 	len = m->m_pkthdr.len;
-	NG_SEND_DATA(error, priv->links[linkNum], m, meta);
+	NG_SEND_DATA(error, link->hook, m, meta);
 
 	/* Update stats and 'bytes in queue' counter */
 	if (error == 0) {
-		priv->linkStats[linkNum].xmitFrames++;
-		priv->linkStats[linkNum].xmitOctets += len;
-		priv->qstat[linkNum].bytesInQueue += len;
-		microtime(&priv->qstat[linkNum].lastWrite);
+		link->stats.xmitFrames++;
+		link->stats.xmitOctets += len;
+		link->bytesInQueue += len;
+		getmicrouptime(&link->lastWrite);
 	}
 	return error;
 }
 
 /*
  * Handle an incoming multi-link fragment
+ *
+ * The fragment reassembly algorithm is somewhat complex. This is mainly
+ * because we are required not to reorder the reconstructed packets, yet
+ * fragments are only guaranteed to arrive in order on a per-link basis.
+ * In other words, when we have a complete packet ready, but the previous
+ * packet is still incomplete, we have to decide between delivering the
+ * complete packet and throwing away the incomplete one, or waiting to
+ * see if the remainder of the incomplete one arrives, at which time we
+ * can deliver both packets, in order.
+ *
+ * This problem is exacerbated by "sequence number slew", which is when
+ * the sequence numbers coming in from different links are far apart from
+ * each other. In particular, certain unnamed equipment (*cough* Ascend)
+ * has been seen to generate sequence number slew of up to 10 on an ISDN
+ * 2B-channel MP link. There is nothing invalid about sequence number slew
+ * but it makes the reasssembly process have to work harder.
+ *
+ * However, the peer is required to transmit fragments in order on each
+ * link. That means if we define MSEQ as the minimum over all links of
+ * the highest sequence number received on that link, then we can always
+ * give up any hope of receiving a fragment with sequence number < MSEQ in
+ * the future (all of this using 'wraparound' sequence number space).
+ * Therefore we can always immediately throw away incomplete packets
+ * missing fragments with sequence numbers < MSEQ.
+ *
+ * Here is an overview of our algorithm:
+ *
+ *    o Received fragments are inserted into a queue, for which we
+ *	maintain these invariants between calls to this function:
+ *
+ *	- Fragments are ordered in the queue by sequence number
+ *	- If a complete packet is at the head of the queue, then
+ *	  the first fragment in the packet has seq# > MSEQ + 1
+ *	  (otherwise, we could deliver it immediately)
+ *	- If any fragments have seq# < MSEQ, then they are necessarily
+ *	  part of a packet whose missing seq#'s are all > MSEQ (otherwise,
+ *	  we can throw them away because they'll never be completed)
+ *	- The queue contains at most MP_MAX_QUEUE_LEN fragments
+ *
+ *    o We have a periodic timer that checks the queue for the first
+ *	complete packet that has been sitting in the queue "too long".
+ *	When one is detected, all previous (incomplete) fragments are
+ *	discarded, their missing fragments are declared lost and MSEQ
+ *	is increased.
+ *
+ *    o If we recieve a fragment with seq# < MSEQ, we throw it away
+ *	because we've already delcared it lost.
+ *
+ * This assumes linkNum != NG_PPP_BUNDLE_LINKNUM.
  */
 static int
 ng_ppp_mp_input(node_p node, int linkNum, struct mbuf *m, meta_p meta)
 {
 	const priv_p priv = node->private;
+	struct ng_ppp_link *const link = &priv->links[linkNum];
 	struct ng_ppp_frag frag0, *frag = &frag0;
-	struct ng_ppp_frag *qent, *qnext;
-	struct ng_ppp_frag *first, *last;
-	int diff, highSeq, nextSeq, inserted;
-	struct mbuf *tail;
+	struct ng_ppp_frag *qent;
+	int i, diff, inserted;
 
 	/* Extract fragment information from MP header */
 	if (priv->conf.recvShortSeq) {
 		u_int16_t shdr;
 
 		if (m->m_pkthdr.len < 2) {
+			link->stats.runts++;
 			NG_FREE_DATA(m, meta);
 			return (EINVAL);
 		}
@@ -903,14 +1033,13 @@ ng_ppp_mp_input(node_p node, int linkNum, struct mbuf *m, meta_p meta)
 		frag->seq = shdr & MP_SHORT_SEQ_MASK;
 		frag->first = (shdr & MP_SHORT_FIRST_FLAG) != 0;
 		frag->last = (shdr & MP_SHORT_LAST_FLAG) != 0;
-		highSeq = CIRCLEQ_EMPTY(&priv->frags) ?
-		    frag->seq : CIRCLEQ_FIRST(&priv->frags)->seq;
-		diff = MP_SHORT_SEQ_DIFF(frag->seq, highSeq);
+		diff = MP_SHORT_SEQ_DIFF(frag->seq, priv->mseq);
 		m_adj(m, 2);
 	} else {
 		u_int32_t lhdr;
 
 		if (m->m_pkthdr.len < 4) {
+			link->stats.runts++;
 			NG_FREE_DATA(m, meta);
 			return (EINVAL);
 		}
@@ -922,91 +1051,119 @@ ng_ppp_mp_input(node_p node, int linkNum, struct mbuf *m, meta_p meta)
 		frag->seq = lhdr & MP_LONG_SEQ_MASK;
 		frag->first = (lhdr & MP_LONG_FIRST_FLAG) != 0;
 		frag->last = (lhdr & MP_LONG_LAST_FLAG) != 0;
-		highSeq = CIRCLEQ_EMPTY(&priv->frags) ?
-		    frag->seq : CIRCLEQ_FIRST(&priv->frags)->seq;
-		diff = MP_LONG_SEQ_DIFF(frag->seq, highSeq);
+		diff = MP_LONG_SEQ_DIFF(frag->seq, priv->mseq);
 		m_adj(m, 4);
 	}
 	frag->data = m;
 	frag->meta = meta;
+	getmicrouptime(&frag->timestamp);
+
+	/* If sequence number is < MSEQ, we've already declared this
+	   fragment as lost, so we have no choice now but to drop it */
+	if (diff < 0) {
+		link->stats.dropFragments++;
+		NG_FREE_DATA(m, meta);
+		return (0);
+	}
 
-	/* If the sequence number makes a large jump, empty the queue */
-	if (diff <= -MP_INSANE_SEQ_JUMP || diff >= MP_INSANE_SEQ_JUMP)
-		ng_ppp_free_frags(node);
+	/* Update highest received sequence number on this link and MSEQ */
+	priv->mseq = link->seq = frag->seq;
+	for (i = 0; i < priv->numActiveLinks; i++) {
+		struct ng_ppp_link *const alink =
+		    &priv->links[priv->activeLinks[i]];
 
-	/* Optimization: handle a frame that's all in one fragment */
-	if (frag->first && frag->last)
-		return ng_ppp_input(node, 0, NG_PPP_BUNDLE_LINKNUM, m, meta);
+		if (MP_SEQ_DIFF(priv, alink->seq, priv->mseq) < 0)
+			priv->mseq = alink->seq;
+	}
 
 	/* Allocate a new frag struct for the queue */
 	MALLOC(frag, struct ng_ppp_frag *, sizeof(*frag), M_NETGRAPH, M_NOWAIT);
 	if (frag == NULL) {
 		NG_FREE_DATA(m, meta);
+		ng_ppp_frag_process(node);
 		return (ENOMEM);
 	}
 	*frag = frag0;
-	meta = NULL;
-	m = NULL;
 
-	/* Add fragment to queue, which is reverse sorted by sequence number */
+	/* Add fragment to queue, which is sorted by sequence number */
 	inserted = 0;
-	CIRCLEQ_FOREACH(qent, &priv->frags, f_qent) {
+	CIRCLEQ_FOREACH_REVERSE(qent, &priv->frags, f_qent) {
 		diff = MP_SEQ_DIFF(priv, frag->seq, qent->seq);
 		if (diff > 0) {
-			CIRCLEQ_INSERT_BEFORE(&priv->frags, qent, frag, f_qent);
+			CIRCLEQ_INSERT_AFTER(&priv->frags, qent, frag, f_qent);
 			inserted = 1;
 			break;
 		} else if (diff == 0) {	     /* should never happen! */
-			log(LOG_ERR, "%s: rec'd dup MP fragment\n", node->name);
-			if (linkNum == NG_PPP_BUNDLE_LINKNUM)
-				priv->linkStats[linkNum].dupFragments++;
-			else
-				priv->bundleStats.dupFragments++;
+			link->stats.dupFragments++;
 			NG_FREE_DATA(frag->data, frag->meta);
 			FREE(frag, M_NETGRAPH);
 			return (EINVAL);
 		}
 	}
 	if (!inserted)
-		CIRCLEQ_INSERT_TAIL(&priv->frags, frag, f_qent);
-
-	/* Find the last fragment in the possibly newly completed frame */
-	last = NULL;
-	qent = frag;
-	nextSeq = frag->seq;
-	while (qent != (void *)&priv->frags && qent->seq == nextSeq) {
-		if (qent->last) {
-			last = qent;
-			break;
-		}
-		qent = CIRCLEQ_PREV(qent, f_qent);
-		nextSeq = (nextSeq + 1) & MP_SEQ_MASK(priv);
-	}
-	if (last == NULL)
-		goto incomplete;
-
-	/* Find the first fragment in the possibly newly completed frame */
-	first = NULL;
-	qent = frag;
-	nextSeq = frag->seq;
-	while (qent != (void *)&priv->frags && qent->seq == nextSeq) {
-		if (qent->first) {
-			first = qent;
-			break;
-		}
-		qent = CIRCLEQ_NEXT(qent, f_qent);
-		nextSeq = (nextSeq - 1) & MP_SEQ_MASK(priv);
+		CIRCLEQ_INSERT_HEAD(&priv->frags, frag, f_qent);
+	priv->qlen++;
+
+	/* Process the queue */
+	return ng_ppp_frag_process(node);
+}
+
+/*
+ * Examine our list of fragments, and determine if there is a
+ * complete and deliverable packet at the head of the list.
+ * Return 1 if so, zero otherwise.
+ */
+static int
+ng_ppp_check_packet(node_p node)
+{
+	const priv_p priv = node->private;
+	struct ng_ppp_frag *qent, *qnext;
+
+	/* Check for empty queue */
+	if (CIRCLEQ_EMPTY(&priv->frags))
+		return (0);
+
+	/* Check first fragment is the start of a deliverable packet */
+	qent = CIRCLEQ_FIRST(&priv->frags);
+	if (!qent->first || MP_SEQ_DIFF(priv, qent->seq, priv->mseq) > 1)
+		return (0);
+
+	/* Check that all the fragments are there */
+	while (!qent->last) {
+		qnext = CIRCLEQ_NEXT(qent, f_qent);
+		if (qnext == (void *)&priv->frags)	/* end of queue */
+			return (0);
+		if (qnext->seq != MP_NEXT_SEQ(priv, qent->seq))
+			return (0);
+		qent = qnext;
 	}
-	if (first == NULL)
-		goto incomplete;
 
-	/* We have a complete frame, extract it from the queue */
-	for (tail = NULL, qent = first; qent != NULL; qent = qnext) {
-		qnext = CIRCLEQ_PREV(qent, f_qent);
+	/* Got one */
+	return (1);
+}
+
+/*
+ * Pull a completed packet off the head of the incoming fragment queue.
+ * This assumes there is a completed packet there to pull off.
+ */
+static void
+ng_ppp_get_packet(node_p node, struct mbuf **mp, meta_p *metap)
+{
+	const priv_p priv = node->private;
+	struct ng_ppp_frag *qent, *qnext;
+	struct mbuf *m = NULL, *tail;
+
+	qent = CIRCLEQ_FIRST(&priv->frags);
+	KASSERT(!CIRCLEQ_EMPTY(&priv->frags) && qent->first,
+	    ("%s: no packet", __FUNCTION__));
+	for (tail = NULL; qent != NULL; qent = qnext) {
+		qnext = CIRCLEQ_NEXT(qent, f_qent);
+		KASSERT(!CIRCLEQ_EMPTY(&priv->frags),
+		    ("%s: empty q", __FUNCTION__));
 		CIRCLEQ_REMOVE(&priv->frags, qent, f_qent);
 		if (tail == NULL) {
 			tail = m = qent->data;
-			meta = qent->meta;	/* inherit first frag's meta */
+			*metap = qent->meta;	/* inherit first frag's meta */
 		} else {
 			m->m_pkthdr.len += qent->data->m_pkthdr.len;
 			tail->m_next = qent->data;
@@ -1014,25 +1171,249 @@ ng_ppp_mp_input(node_p node, int linkNum, struct mbuf *m, meta_p meta)
 		}
 		while (tail->m_next != NULL)
 			tail = tail->m_next;
-		if (qent == last)
+		if (qent->last)
 			qnext = NULL;
 		FREE(qent, M_NETGRAPH);
+		priv->qlen--;
 	}
+	*mp = m;
+}
 
-incomplete:
-	/* Prune out stale entries in the queue */
-	for (qent = CIRCLEQ_LAST(&priv->frags); 
-	    qent != (void *)&priv->frags; qent = qnext) {
-		if (MP_SEQ_DIFF(priv, highSeq, qent->seq) <= MP_MAX_SEQ_LINGER)
+/*
+ * Trim fragments from the queue whose packets can never be completed.
+ * This assumes a complete packet is NOT at the beginning of the queue.
+ * Returns 1 if fragments were removed, zero otherwise.
+ */
+static int
+ng_ppp_frag_trim(node_p node)
+{
+	const priv_p priv = node->private;
+	struct ng_ppp_frag *qent, *qnext = NULL;
+	int removed = 0;
+
+	/* Scan for "dead" fragments and remove them */
+	while (1) {
+		int dead = 0;
+
+		/* If queue is empty, we're done */
+		if (CIRCLEQ_EMPTY(&priv->frags))
+			break;
+
+		/* Determine whether first fragment can ever be completed */
+		CIRCLEQ_FOREACH(qent, &priv->frags, f_qent) {
+			if (MP_SEQ_DIFF(priv, qent->seq, priv->mseq) >= 0)
+				break;
+			qnext = CIRCLEQ_NEXT(qent, f_qent);
+			KASSERT(qnext != (void*)&priv->frags,
+			    ("%s: last frag < MSEQ?", __FUNCTION__));
+			if (qnext->seq != MP_NEXT_SEQ(priv, qent->seq)
+			    || qent->last || qnext->first) {
+				dead = 1;
+				break;
+			}
+		}
+		if (!dead)
 			break;
-		qnext = CIRCLEQ_PREV(qent, f_qent);
+
+		/* Remove fragment and all others in the same packet */
+		while ((qent = CIRCLEQ_FIRST(&priv->frags)) != qnext) {
+			KASSERT(!CIRCLEQ_EMPTY(&priv->frags),
+			    ("%s: empty q", __FUNCTION__));
+			priv->bundleStats.dropFragments++;
+			CIRCLEQ_REMOVE(&priv->frags, qent, f_qent);
+			NG_FREE_DATA(qent->data, qent->meta);
+			FREE(qent, M_NETGRAPH);
+			priv->qlen--;
+			removed = 1;
+		}
+	}
+	return (removed);
+}
+
+/*
+ * Run the queue, restoring the queue invariants
+ */
+static int
+ng_ppp_frag_process(node_p node)
+{
+	const priv_p priv = node->private;
+	struct mbuf *m;
+	meta_p meta;
+
+	/* Deliver any deliverable packets */
+	while (ng_ppp_check_packet(node)) {
+		ng_ppp_get_packet(node, &m, &meta);
+		ng_ppp_input(node, 0, NG_PPP_BUNDLE_LINKNUM, m, meta);
+	}
+
+	/* Delete dead fragments and try again */
+	if (ng_ppp_frag_trim(node)) {
+		while (ng_ppp_check_packet(node)) {
+			ng_ppp_get_packet(node, &m, &meta);
+			ng_ppp_input(node, 0, NG_PPP_BUNDLE_LINKNUM, m, meta);
+		}
+	}
+
+	/* Check for stale fragments while we're here */
+	ng_ppp_frag_checkstale(node);
+
+	/* Check queue length */
+	if (priv->qlen > MP_MAX_QUEUE_LEN) {
+		struct ng_ppp_frag *qent;
+		int i;
+
+		/* Get oldest fragment */
+		KASSERT(!CIRCLEQ_EMPTY(&priv->frags),
+		    ("%s: empty q", __FUNCTION__));
+		qent = CIRCLEQ_FIRST(&priv->frags);
+
+		/* Bump MSEQ if necessary */
+		if (MP_SEQ_DIFF(priv, priv->mseq, qent->seq) < 0) {
+			priv->mseq = qent->seq;
+			for (i = 0; i < priv->numActiveLinks; i++) {
+				struct ng_ppp_link *const alink =
+				    &priv->links[priv->activeLinks[i]];
+
+				if (MP_SEQ_DIFF(priv,
+				    alink->seq, priv->mseq) < 0)
+					alink->seq = priv->mseq;
+			}
+		}
+
+		/* Drop it */
+		priv->bundleStats.dropFragments++;
 		CIRCLEQ_REMOVE(&priv->frags, qent, f_qent);
 		NG_FREE_DATA(qent->data, qent->meta);
 		FREE(qent, M_NETGRAPH);
+		priv->qlen--;
+
+		/* Process queue again */
+		return ng_ppp_frag_process(node);
+	}
+
+	/* Done */
+	return (0);
+}
+
+/*
+ * Check for 'stale' completed packets that need to be delivered
+ *
+ * If a link goes down or has a temporary failure, MSEQ can get
+ * "stuck", because no new incoming fragments appear on that link.
+ * This can cause completed packets to never get delivered if
+ * their sequence numbers are all > MSEQ + 1.
+ *
+ * This routine checks how long all of the completed packets have
+ * been sitting in the queue, and if too long, removes fragments
+ * from the queue and increments MSEQ to allow them to be delivered.
+ */
+static void
+ng_ppp_frag_checkstale(node_p node)
+{
+	const priv_p priv = node->private;
+	struct ng_ppp_frag *qent, *beg, *end;
+	struct timeval now, age;
+	struct mbuf *m;
+	meta_p meta;
+	int i, seq;
+
+	now.tv_sec = 0;			/* uninitialized state */
+	while (1) {
+
+		/* If queue is empty, we're done */
+		if (CIRCLEQ_EMPTY(&priv->frags))
+			break;
+
+		/* Find the first complete packet in the queue */
+		beg = end = NULL;
+		seq = CIRCLEQ_FIRST(&priv->frags)->seq;
+		CIRCLEQ_FOREACH(qent, &priv->frags, f_qent) {
+			if (qent->first)
+				beg = qent;
+			else if (qent->seq != seq)
+				beg = NULL;
+			if (beg != NULL && qent->last) {
+				end = qent;
+				break;
+			}
+			seq = MP_NEXT_SEQ(priv, seq);
+		}
+
+		/* If none found, exit */
+		if (end == NULL)
+			break;
+
+		/* Get current time (we assume we've been up for >= 1 second) */
+		if (now.tv_sec == 0)
+			getmicrouptime(&now);
+
+		/* Check if packet has been queued too long */
+		age = now;
+		timevalsub(&age, &beg->timestamp);
+		if (timevalcmp(&age, &ng_ppp_max_staleness, < ))
+			break;
+
+		/* Throw away junk fragments in front of the completed packet */
+		while ((qent = CIRCLEQ_FIRST(&priv->frags)) != beg) {
+			KASSERT(!CIRCLEQ_EMPTY(&priv->frags),
+			    ("%s: empty q", __FUNCTION__));
+			priv->bundleStats.dropFragments++;
+			CIRCLEQ_REMOVE(&priv->frags, qent, f_qent);
+			NG_FREE_DATA(qent->data, qent->meta);
+			FREE(qent, M_NETGRAPH);
+			priv->qlen--;
+		}
+
+		/* Extract completed packet */
+		ng_ppp_get_packet(node, &m, &meta);
+
+		/* Bump MSEQ if necessary */
+		if (MP_SEQ_DIFF(priv, priv->mseq, end->seq) < 0) {
+			priv->mseq = end->seq;
+			for (i = 0; i < priv->numActiveLinks; i++) {
+				struct ng_ppp_link *const alink =
+				    &priv->links[priv->activeLinks[i]];
+
+				if (MP_SEQ_DIFF(priv,
+				    alink->seq, priv->mseq) < 0)
+					alink->seq = priv->mseq;
+			}
+		}
+
+		/* Deliver packet */
+		ng_ppp_input(node, 0, NG_PPP_BUNDLE_LINKNUM, m, meta);
+	}
+}
+
+/*
+ * Periodically call ng_ppp_frag_checkstale()
+ */
+static void
+ng_ppp_frag_timeout(void *arg)
+{
+	const node_p node = arg;
+	const priv_p priv = node->private;
+	int s = splnet();
+
+	/* Handle the race where shutdown happens just before splnet() above */
+	if ((node->flags & NG_INVALID) != 0) {
+		ng_unref(node);
+		splx(s);
+		return;
 	}
 
-	/* Deliver newly completed frame, if any */
-	return m ? ng_ppp_input(node, 0, NG_PPP_BUNDLE_LINKNUM, m, meta) : 0;
+	/* Reset timer state after timeout */
+	KASSERT(priv->timerActive, ("%s: !timerActive", __FUNCTION__));
+	priv->timerActive = 0;
+	KASSERT(node->refs > 1, ("%s: refs=%d", __FUNCTION__, node->refs));
+	ng_unref(node);
+
+	/* Start timer again */
+	ng_ppp_start_frag_timer(node);
+
+	/* Scan the fragment queue */
+	ng_ppp_frag_checkstale(node);
+	splx(s);
 }
 
 /*
@@ -1089,6 +1470,7 @@ deliver:
 	for (firstFragment = 1, activeLinkNum = priv->numActiveLinks - 1;
 	    activeLinkNum >= 0; activeLinkNum--) {
 		const int linkNum = priv->activeLinks[activeLinkNum];
+		struct ng_ppp_link *const link = &priv->links[linkNum];
 
 		/* Deliver fragment(s) out the next link */
 		for ( ; distrib[activeLinkNum] > 0; firstFragment = 0) {
@@ -1098,8 +1480,8 @@ deliver:
 
 			/* Calculate fragment length; don't exceed link MTU */
 			len = distrib[activeLinkNum];
-			if (len > priv->conf.links[linkNum].mru)
-				len = priv->conf.links[linkNum].mru;
+			if (len > link->conf.mru)
+				len = link->conf.mru;
 			distrib[activeLinkNum] -= len;
 			lastFragment = (len == m->m_pkthdr.len);
 
@@ -1119,9 +1501,9 @@ deliver:
 			if (priv->conf.xmitShortSeq) {
 				u_int16_t shdr;
 
-				shdr = priv->mpSeqOut;
-				priv->mpSeqOut =
-				    (priv->mpSeqOut + 1) % MP_SHORT_SEQ_MASK;
+				shdr = priv->xseq;
+				priv->xseq =
+				    (priv->xseq + 1) % MP_SHORT_SEQ_MASK;
 				if (firstFragment)
 					shdr |= MP_SHORT_FIRST_FLAG;
 				if (lastFragment)
@@ -1131,9 +1513,9 @@ deliver:
 			} else {
 				u_int32_t lhdr;
 
-				lhdr = priv->mpSeqOut;
-				priv->mpSeqOut =
-				    (priv->mpSeqOut + 1) % MP_LONG_SEQ_MASK;
+				lhdr = priv->xseq;
+				priv->xseq =
+				    (priv->xseq + 1) % MP_LONG_SEQ_MASK;
 				if (firstFragment)
 					lhdr |= MP_LONG_FIRST_FLAG;
 				if (lastFragment)
@@ -1149,18 +1531,7 @@ deliver:
 			}
 
 			/* Copy the meta information, if any */
-			if (meta != NULL && !lastFragment) {
-				MALLOC(meta2, meta_p,
-				    meta->used_len, M_NETGRAPH, M_NOWAIT);
-				if (meta2 == NULL) {
-					m_freem(m2);
-					NG_FREE_DATA(m, meta);
-					return (ENOMEM);
-				}
-				meta2->allocated_len = meta->used_len;
-				bcopy(meta, meta2, meta->used_len);
-			} else
-				meta2 = meta;
+			meta2 = lastFragment ? meta : ng_copy_meta(meta);
 
 			/* Send fragment */
 			error = ng_ppp_output(node, 0,
@@ -1265,7 +1636,7 @@ ng_ppp_mp_strategy(node_p node, int len, int *distrib)
 	const priv_p priv = node->private;
 	int latency[NG_PPP_MAX_LINKS];
 	int sortByLatency[NG_PPP_MAX_LINKS];
-	int activeLinkNum, linkNum;
+	int activeLinkNum;
 	int t0, total, topSum, botSum;
 	struct timeval now;
 	int i, numFragments;
@@ -1277,45 +1648,44 @@ ng_ppp_mp_strategy(node_p node, int len, int *distrib)
 	}
 
 	/* Get current time */
-	microtime(&now);
+	getmicrouptime(&now);
 
 	/* Compute latencies for each link at this point in time */
 	for (activeLinkNum = 0;
 	    activeLinkNum < priv->numActiveLinks; activeLinkNum++) {
+		struct ng_ppp_link *alink;
 		struct timeval diff;
 		int xmitBytes;
 
 		/* Start with base latency value */
-		linkNum = priv->activeLinks[activeLinkNum];
-		latency[activeLinkNum] = priv->conf.links[linkNum].latency;
+		alink = &priv->links[priv->activeLinks[activeLinkNum]];
+		latency[activeLinkNum] = alink->conf.latency;
 		sortByLatency[activeLinkNum] = activeLinkNum;	/* see below */
 
 		/* Any additional latency? */
-		if (priv->qstat[activeLinkNum].bytesInQueue == 0)
+		if (alink->bytesInQueue == 0)
 			continue;
 
 		/* Compute time delta since last write */
 		diff = now;
-		timevalsub(&diff, &priv->qstat[activeLinkNum].lastWrite);
+		timevalsub(&diff, &alink->lastWrite);
 		if (now.tv_sec < 0 || diff.tv_sec >= 10) {	/* sanity */
-			priv->qstat[activeLinkNum].bytesInQueue = 0;
+			alink->bytesInQueue = 0;
 			continue;
 		}
 
 		/* How many bytes could have transmitted since last write? */
-		xmitBytes = priv->conf.links[linkNum].bandwidth * diff.tv_sec
-		    + (priv->conf.links[linkNum].bandwidth
-			* (diff.tv_usec / 1000)) / 100;
-		priv->qstat[activeLinkNum].bytesInQueue -= xmitBytes;
-		if (priv->qstat[activeLinkNum].bytesInQueue < 0)
-			priv->qstat[activeLinkNum].bytesInQueue = 0;
+		xmitBytes = (alink->conf.bandwidth * diff.tv_sec)
+		    + (alink->conf.bandwidth * (diff.tv_usec / 1000)) / 100;
+		alink->bytesInQueue -= xmitBytes;
+		if (alink->bytesInQueue < 0)
+			alink->bytesInQueue = 0;
 		else
 			latency[activeLinkNum] +=
-			    (100 * priv->qstat[activeLinkNum].bytesInQueue)
-				/ priv->conf.links[linkNum].bandwidth;
+			    (100 * alink->bytesInQueue) / alink->conf.bandwidth;
 	}
 
-	/* Sort links by latency */
+	/* Sort active links by latency */
 	compareLatencies = latency;
 	qsort(sortByLatency,
 	    priv->numActiveLinks, sizeof(*sortByLatency), ng_ppp_intcmp);
@@ -1329,8 +1699,8 @@ ng_ppp_mp_strategy(node_p node, int len, int *distrib)
 
 			flowTime = latency[sortByLatency[numFragments]]
 			    - latency[sortByLatency[i]];
-			total += ((flowTime * priv->conf.links[
-			    priv->activeLinks[sortByLatency[i]]].bandwidth)
+			total += ((flowTime * priv->links[
+			    priv->activeLinks[sortByLatency[i]]].conf.bandwidth)
 			    	+ 99) / 100;
 		}
 		if (total >= len)
@@ -1339,8 +1709,8 @@ ng_ppp_mp_strategy(node_p node, int len, int *distrib)
 
 	/* Solve for t_0 in that interval */
 	for (topSum = botSum = i = 0; i < numFragments; i++) {
-		int bw = priv->conf.links[
-		    priv->activeLinks[sortByLatency[i]]].bandwidth;
+		int bw = priv->links[
+		    priv->activeLinks[sortByLatency[i]]].conf.bandwidth;
 
 		topSum += latency[sortByLatency[i]] * bw;	/* / 100 */
 		botSum += bw;					/* / 100 */
@@ -1350,8 +1720,8 @@ ng_ppp_mp_strategy(node_p node, int len, int *distrib)
 	/* Compute f_i(t_0) all i */
 	bzero(distrib, priv->numActiveLinks * sizeof(*distrib));
 	for (total = i = 0; i < numFragments; i++) {
-		int bw = priv->conf.links[
-		    priv->activeLinks[sortByLatency[i]]].bandwidth;
+		int bw = priv->links[
+		    priv->activeLinks[sortByLatency[i]]].conf.bandwidth;
 
 		distrib[sortByLatency[i]] =
 		    (bw * (t0 - latency[sortByLatency[i]]) + 50) / 100;
@@ -1360,29 +1730,38 @@ ng_ppp_mp_strategy(node_p node, int len, int *distrib)
 
 	/* Deal with any rounding error */
 	if (total < len) {
+		struct ng_ppp_link *fastLink =
+		    &priv->links[priv->activeLinks[sortByLatency[0]]];
 		int fast = 0;
 
 		/* Find the fastest link */
 		for (i = 1; i < numFragments; i++) {
-			if (priv->conf.links[
-			      priv->activeLinks[sortByLatency[i]]].bandwidth >
-			    priv->conf.links[
-			      priv->activeLinks[sortByLatency[fast]]].bandwidth)
+			struct ng_ppp_link *const link =
+			    &priv->links[priv->activeLinks[sortByLatency[i]]];
+
+			if (link->conf.bandwidth > fastLink->conf.bandwidth) {
 				fast = i;
+				fastLink = link;
+			}
 		}
 		distrib[sortByLatency[fast]] += len - total;
 	} else while (total > len) {
+		struct ng_ppp_link *slowLink =
+		    &priv->links[priv->activeLinks[sortByLatency[0]]];
 		int delta, slow = 0;
 
 		/* Find the slowest link that still has bytes to remove */
 		for (i = 1; i < numFragments; i++) {
+			struct ng_ppp_link *const link =
+			    &priv->links[priv->activeLinks[sortByLatency[i]]];
+
 			if (distrib[sortByLatency[slow]] == 0
 			  || (distrib[sortByLatency[i]] > 0
-			    && priv->conf.links[priv->activeLinks[
-					sortByLatency[i]]].bandwidth <
-			      priv->conf.links[priv->activeLinks[
-					sortByLatency[slow]]].bandwidth))
+			    && link->conf.bandwidth <
+			      slowLink->conf.bandwidth)) {
 				slow = i;
+				slowLink = link;
+			}
 		}
 		delta = total - len;
 		if (delta > distrib[sortByLatency[slow]])
@@ -1454,11 +1833,11 @@ ng_ppp_update(node_p node, int newConf)
 		for (i = 0; i < NG_PPP_MAX_LINKS; i++) {
 			int hdrBytes;
 
-			hdrBytes = (priv->conf.links[i].enableACFComp ? 0 : 2)
-			    + (priv->conf.links[i].enableProtoComp ? 1 : 2)
+			hdrBytes = (priv->links[i].conf.enableACFComp ? 0 : 2)
+			    + (priv->links[i].conf.enableProtoComp ? 1 : 2)
 			    + (priv->conf.xmitShortSeq ? 2 : 4);
-			priv->conf.links[i].latency +=
-			    ((hdrBytes * priv->conf.links[i].bandwidth) + 50)
+			priv->links[i].conf.latency +=
+			    ((hdrBytes * priv->links[i].conf.bandwidth) + 50)
 				/ 100;
 		}
 	}
@@ -1468,23 +1847,45 @@ ng_ppp_update(node_p node, int newConf)
 	priv->numActiveLinks = 0;
 	priv->allLinksEqual = 1;
 	for (i = 0; i < NG_PPP_MAX_LINKS; i++) {
-		struct ng_ppp_link_config *const lc = &priv->conf.links[i];
+		struct ng_ppp_link *const link = &priv->links[i];
+
+		/* Is link active? */
+		if (link->conf.enableLink && link->hook != NULL) {
+			struct ng_ppp_link *link0;
 
-		if (lc->enableLink && priv->links[i] != NULL) {
+			/* Add link to list of active links */
 			priv->activeLinks[priv->numActiveLinks++] = i;
-			if (lc->latency !=
-			      priv->conf.links[priv->activeLinks[0]].latency
-			  || lc->bandwidth !=
-			      priv->conf.links[priv->activeLinks[0]].bandwidth)
+			link0 = &priv->links[priv->activeLinks[0]];
+
+			/* Determine if all links are still equal */
+			if (link->conf.latency != link0->conf.latency
+			  || link->conf.bandwidth != link0->conf.bandwidth)
 				priv->allLinksEqual = 0;
-		}
+
+			/* Initialize rec'd sequence number */
+			if (link->seq == MP_NOSEQ) {
+				link->seq = (link == link0) ?
+				    MP_INITIAL_SEQ : link0->seq;
+			}
+		} else
+			link->seq = MP_NOSEQ;
 	}
 
-	/* Reset MP state if multi-link is no longer active */
-	if (!priv->conf.enableMultilink || priv->numActiveLinks == 0) {
-		ng_ppp_free_frags(node);
-		priv->mpSeqOut = MP_INITIAL_SEQ;
-		bzero(&priv->qstat, sizeof(priv->qstat));
+	/* Update MP state as multi-link is active or not */
+	if (priv->conf.enableMultilink && priv->numActiveLinks > 0)
+		ng_ppp_start_frag_timer(node);
+	else {
+		ng_ppp_stop_frag_timer(node);
+		ng_ppp_frag_reset(node);
+		priv->xseq = MP_INITIAL_SEQ;
+		priv->mseq = MP_INITIAL_SEQ;
+		for (i = 0; i < NG_PPP_MAX_LINKS; i++) {
+			struct ng_ppp_link *const link = &priv->links[i];
+
+			bzero(&link->lastWrite, sizeof(link->lastWrite));
+			link->bytesInQueue = 0;
+			link->seq = MP_NOSEQ;
+		}
 	}
 }
 
@@ -1493,14 +1894,14 @@ ng_ppp_update(node_p node, int newConf)
  * from the current configuration and link activity status.
  */
 static int
-ng_ppp_config_valid(node_p node, const struct ng_ppp_node_config *newConf)
+ng_ppp_config_valid(node_p node, const struct ng_ppp_node_conf *newConf)
 {
 	const priv_p priv = node->private;
 	int i, newNumLinksActive;
 
 	/* Check per-link config and count how many links would be active */
 	for (newNumLinksActive = i = 0; i < NG_PPP_MAX_LINKS; i++) {
-		if (newConf->links[i].enableLink && priv->links[i] != NULL)
+		if (newConf->links[i].enableLink && priv->links[i].hook != NULL)
 			newNumLinksActive++;
 		if (!newConf->links[i].enableLink)
 			continue;
@@ -1515,19 +1916,20 @@ ng_ppp_config_valid(node_p node, const struct ng_ppp_node_config *newConf)
 	}
 
 	/* Check bundle parameters */
-	if (newConf->enableMultilink && newConf->mrru < MP_MIN_MRRU)
+	if (newConf->bund.enableMultilink && newConf->bund.mrru < MP_MIN_MRRU)
 		return (0);
 
 	/* Disallow changes to multi-link configuration while MP is active */
 	if (priv->numActiveLinks > 0 && newNumLinksActive > 0) {
-		if (!priv->conf.enableMultilink != !newConf->enableMultilink
-		    || !priv->conf.xmitShortSeq != !newConf->xmitShortSeq
-		    || !priv->conf.recvShortSeq != !newConf->recvShortSeq)
+		if (!priv->conf.enableMultilink
+				!= !newConf->bund.enableMultilink
+		    || !priv->conf.xmitShortSeq != !newConf->bund.xmitShortSeq
+		    || !priv->conf.recvShortSeq != !newConf->bund.recvShortSeq)
 			return (0);
 	}
 
 	/* At most one link can be active unless multi-link is enabled */
-	if (!newConf->enableMultilink && newNumLinksActive > 1)
+	if (!newConf->bund.enableMultilink && newNumLinksActive > 1)
 		return (0);
 
 	/* Configuration change would be valid */
@@ -1538,7 +1940,7 @@ ng_ppp_config_valid(node_p node, const struct ng_ppp_node_config *newConf)
  * Free all entries in the fragment queue
  */
 static void
-ng_ppp_free_frags(node_p node)
+ng_ppp_frag_reset(node_p node)
 {
 	const priv_p priv = node->private;
 	struct ng_ppp_frag *qent, *qnext;
@@ -1550,5 +1952,39 @@ ng_ppp_free_frags(node_p node)
 		FREE(qent, M_NETGRAPH);
 	}
 	CIRCLEQ_INIT(&priv->frags);
+	priv->qlen = 0;
+}
+
+/*
+ * Start fragment queue timer
+ */
+static void
+ng_ppp_start_frag_timer(node_p node)
+{
+	const priv_p priv = node->private;
+
+	if (!priv->timerActive) {
+		priv->fragTimer = timeout(ng_ppp_frag_timeout,
+		    node, MP_FRAGTIMER_INTERVAL);
+		priv->timerActive = 1;
+		node->refs++;
+	}
+}
+
+/*
+ * Stop fragment queue timer
+ */
+static void
+ng_ppp_stop_frag_timer(node_p node)
+{
+	const priv_p priv = node->private;
+
+	if (priv->timerActive) {
+		untimeout(ng_ppp_frag_timeout, node, priv->fragTimer);
+		priv->timerActive = 0;
+		KASSERT(node->refs > 1,
+		    ("%s: refs=%d", __FUNCTION__, node->refs));
+		ng_unref(node);
+	}
 }