FreeBSD-ify ALTQ:

 - add locking
 - disable ALTQ3_COMPAT by default (do not remove the code to keep the diff
   towards KAME small)
 - put some more code under ALTQ3 conditional compilation as it should be
 - account for if_xname
 - some more minor compile fixes

As people started wondering:
The strange path layout "altq/altq" is there to avoid "-Isys/contrib" and
make it "-Isys/contrib/altq" instead, as we will need at least <altq/altq.h>
and <altq/if_altq.h> for kernel compilation.

The "freebsd4_..." in the privious commit is just the best tag name in the
KAME tree I could find to classify this in order to track its history. It
does *not* mean that this will go to 4-STABLE or anything of that kind.

9 files changed, 164 insertions, 21 deletions

diff --git a/sys/contrib/altq/altq/altq.h b/sys/contrib/altq/altq/altq.h
index 64ff22e..c740ed3 100644
--- a/sys/contrib/altq/altq/altq.h
+++ b/sys/contrib/altq/altq/altq.h
@@ -1,3 +1,4 @@
+/*	$FreeBSD$	*/
 /*	$KAME: altq.h,v 1.10 2003/07/10 12:07:47 kjc Exp $	*/
 
 /*
@@ -28,7 +29,7 @@
 #ifndef _ALTQ_ALTQ_H_
 #define	_ALTQ_ALTQ_H_
 
-#if 1
+#if 0
 /*
  * allow altq-3 (altqd(8) and /dev/altq) to coexist with the new pf-based altq.
  * altq3 is mainly for research experiments. pf-based altq is for daily use.
diff --git a/sys/contrib/altq/altq/altq_cbq.c b/sys/contrib/altq/altq/altq_cbq.c
index c12828d..0080895 100644
--- a/sys/contrib/altq/altq/altq_cbq.c
+++ b/sys/contrib/altq/altq/altq_cbq.c
@@ -1,3 +1,4 @@
+/*	$FreeBSD$	*/
 /*	$KAME: altq_cbq.c,v 1.19 2003/09/17 14:23:25 kjc Exp $	*/
 
 /*
@@ -198,6 +199,8 @@ cbq_request(struct ifaltq *ifq, int req, void *arg)
 {
 	cbq_state_t	*cbqp = (cbq_state_t *)ifq->altq_disc;
 
+	IFQ_LOCK_ASSERT(ifq);
+
 	switch (req) {
 	case ALTRQ_PURGE:
 		cbq_purge(cbqp);
@@ -500,10 +503,13 @@ cbq_enqueue(struct ifaltq *ifq, struct mbuf *m, struct altq_pktattr *pktattr)
 	struct m_tag	*t;
 	int		 len;
 
+	IFQ_LOCK_ASSERT(ifq);
+
 	/* grab class set by classifier */
 	if ((m->m_flags & M_PKTHDR) == 0) {
 		/* should not happen */
-#if defined(__NetBSD__) || defined(__OpenBSD__)
+#if defined(__NetBSD__) || defined(__OpenBSD__)\
+    || (defined(__FreeBSD__) && __FreeBSD_version >= 501113)
 		printf("altq: packet for %s does not have pkthdr\n",
 		    ifq->altq_ifp->if_xname);
 #else
@@ -552,6 +558,8 @@ cbq_dequeue(struct ifaltq *ifq, int op)
 	cbq_state_t	*cbqp = (cbq_state_t *)ifq->altq_disc;
 	struct mbuf	*m;
 
+	IFQ_LOCK_ASSERT(ifq);
+
 	m = rmc_dequeue_next(&cbqp->ifnp, op);
 
 	if (m && op == ALTDQ_REMOVE) {
@@ -578,6 +586,8 @@ cbqrestart(struct ifaltq *ifq)
 	cbq_state_t	*cbqp;
 	struct ifnet	*ifp;
 
+	IFQ_LOCK_ASSERT(ifq);
+
 	if (!ALTQ_IS_ENABLED(ifq))
 		/* cbq must have been detached */
 		return;
@@ -588,8 +598,11 @@ cbqrestart(struct ifaltq *ifq)
 
 	ifp = ifq->altq_ifp;
 	if (ifp->if_start &&
-	    cbqp->cbq_qlen > 0 && (ifp->if_flags & IFF_OACTIVE) == 0)
+	    cbqp->cbq_qlen > 0 && (ifp->if_flags & IFF_OACTIVE) == 0) {
+	    	IFQ_UNLOCK(ifq);
 		(*ifp->if_start)(ifp);
+		IFQ_LOCK(ifq);
+	}
 }
 
 static void cbq_purge(cbq_state_t *cbqp)
@@ -1014,7 +1027,8 @@ cbqclose(dev, flag, fmt, p)
 
 	while (cbq_list) {
 		ifp = cbq_list->ifnp.ifq_->altq_ifp;
-#if defined(__NetBSD__) || defined(__OpenBSD__)
+#if defined(__NetBSD__) || defined(__OpenBSD__)\
+    || (defined(__FreeBSD__) && __FreeBSD_version >= 501113)
 		sprintf(iface.cbq_ifacename, "%s", ifp->if_xname);
 #else
 		sprintf(iface.cbq_ifacename,
diff --git a/sys/contrib/altq/altq/altq_cdnr.c b/sys/contrib/altq/altq/altq_cdnr.c
index ba61e7d..d0a1313 100644
--- a/sys/contrib/altq/altq/altq_cdnr.c
+++ b/sys/contrib/altq/altq/altq_cdnr.c
@@ -1,3 +1,4 @@
+/*	$FreeBSD$	*/
 /*	$KAME: altq_cdnr.c,v 1.14 2003/09/05 22:40:36 itojun Exp $	*/
 
 /*
@@ -57,7 +58,9 @@
 #endif
 
 #include <altq/altq.h>
+#ifdef ALTQ3_COMPAT
 #include <altq/altq_conf.h>
+#endif
 #include <altq/altq_cdnr.h>
 
 #ifdef ALTQ3_COMPAT
diff --git a/sys/contrib/altq/altq/altq_hfsc.c b/sys/contrib/altq/altq/altq_hfsc.c
index 4ac035e..7eea5c7 100644
--- a/sys/contrib/altq/altq/altq_hfsc.c
+++ b/sys/contrib/altq/altq/altq_hfsc.c
@@ -1,3 +1,4 @@
+/*	$FreeBSD$	*/
 /*	$KAME: altq_hfsc.c,v 1.24 2003/12/05 05:40:46 kjc Exp $	*/
 
 /*
@@ -360,6 +361,8 @@ hfsc_request(struct ifaltq *ifq, int req, void *arg)
 {
 	struct hfsc_if	*hif = (struct hfsc_if *)ifq->altq_disc;
 
+	IFQ_LOCK_ASSERT(ifq);
+
 	switch (req) {
 	case ALTRQ_PURGE:
 		hfsc_purge(hif);
@@ -503,6 +506,7 @@ hfsc_class_create(struct hfsc_if *hif, struct service_curve *rsc,
 #else
 	s = splimp();
 #endif
+	IFQ_LOCK(hif->hif_ifq);
 	hif->hif_classes++;
 
 	/*
@@ -520,6 +524,7 @@ hfsc_class_create(struct hfsc_if *hif, struct service_curve *rsc,
 				break;
 			}
 		if (i == HFSC_MAX_CLASSES) {
+			IFQ_UNLOCK(hif->hif_ifq);
 			splx(s);
 			goto err_ret;
 		}
@@ -541,6 +546,7 @@ hfsc_class_create(struct hfsc_if *hif, struct service_curve *rsc,
 			p->cl_siblings = cl;
 		}
 	}
+	IFQ_UNLOCK(hif->hif_ifq);
 	splx(s);
 
 	return (cl);
@@ -586,6 +592,7 @@ hfsc_class_destroy(struct hfsc_class *cl)
 #else
 	s = splimp();
 #endif
+	IFQ_LOCK(cl->cl_hif->hif_ifq);
 
 #ifdef ALTQ3_COMPAT
 	/* delete filters referencing to this class */
@@ -618,6 +625,7 @@ hfsc_class_destroy(struct hfsc_class *cl)
 		}
 
 	cl->cl_hif->hif_classes--;
+	IFQ_UNLOCK(cl->cl_hif->hif_ifq);
 	splx(s);
 
 	actlist_destroy(cl->cl_actc);
@@ -633,10 +641,12 @@ hfsc_class_destroy(struct hfsc_class *cl)
 #endif
 	}
 
+	IFQ_LOCK(cl->cl_hif->hif_ifq);
 	if (cl == cl->cl_hif->hif_rootclass)
 		cl->cl_hif->hif_rootclass = NULL;
 	if (cl == cl->cl_hif->hif_defaultclass)
 		cl->cl_hif->hif_defaultclass = NULL;
+	IFQ_UNLOCK(cl->cl_hif->hif_ifq);
 
 	if (cl->cl_usc != NULL)
 		FREE(cl->cl_usc, M_DEVBUF);
@@ -686,10 +696,13 @@ hfsc_enqueue(struct ifaltq *ifq, struct mbuf *m, struct altq_pktattr *pktattr)
 	struct m_tag *t;
 	int len;
 
+	IFQ_LOCK_ASSERT(ifq);
+
 	/* grab class set by classifier */
 	if ((m->m_flags & M_PKTHDR) == 0) {
 		/* should not happen */
-#if defined(__NetBSD__) || defined(__OpenBSD__)
+#if defined(__NetBSD__) || defined(__OpenBSD__)\
+    || (defined(__FreeBSD__) && __FreeBSD_version >= 501113)
 		printf("altq: packet for %s does not have pkthdr\n",
 		    ifq->altq_ifp->if_xname);
 #else
@@ -754,6 +767,8 @@ hfsc_dequeue(struct ifaltq *ifq, int op)
 	int realtime = 0;
 	u_int64_t cur_time;
 
+	IFQ_LOCK_ASSERT(ifq);
+
 	if (hif->hif_packets == 0)
 		/* no packet in the tree */
 		return (NULL);
@@ -1815,6 +1830,7 @@ hfsc_class_modify(cl, rsc, fsc, usc)
 #else
 	s = splimp();
 #endif
+	IFQ_LOCK(cl->cl_hif->hif_ifq);
 
 	if (rsc != NULL) {
 		if (rsc->m1 == 0 && rsc->m2 == 0) {
@@ -1879,6 +1895,7 @@ hfsc_class_modify(cl, rsc, fsc, usc)
 		/* is this enough? */
 	}
 
+	IFQ_UNLOCK(cl->cl_hif->hif_ifq);
 	splx(s);
 
 	return (0);
diff --git a/sys/contrib/altq/altq/altq_priq.c b/sys/contrib/altq/altq/altq_priq.c
index 7211277..e4e84e6 100644
--- a/sys/contrib/altq/altq/altq_priq.c
+++ b/sys/contrib/altq/altq/altq_priq.c
@@ -1,3 +1,4 @@
+/*	$FreeBSD$	*/
 /*	$KAME: altq_priq.c,v 1.11 2003/09/17 14:23:25 kjc Exp $	*/
 /*
  * Copyright (C) 2000-2003
@@ -56,7 +57,9 @@
 
 #include <net/pfvar.h>
 #include <altq/altq.h>
+#ifdef ALTQ3_COMPAT
 #include <altq/altq_conf.h>
+#endif
 #include <altq/altq_priq.h>
 
 /*
@@ -256,6 +259,8 @@ priq_request(struct ifaltq *ifq, int req, void *arg)
 {
 	struct priq_if	*pif = (struct priq_if *)ifq->altq_disc;
 
+	IFQ_LOCK_ASSERT(ifq);
+
 	switch (req) {
 	case ALTRQ_PURGE:
 		priq_purge(pif);
@@ -301,8 +306,10 @@ priq_class_create(struct priq_if *pif, int pri, int qlimit, int flags, int qid)
 #else
 		s = splimp();
 #endif
+		IFQ_LOCK(cl->cl_pif->pif_ifq);
 		if (!qempty(cl->cl_q))
 			priq_purgeq(cl);
+		IFQ_UNLOCK(cl->cl_pif->pif_ifq);
 		splx(s);
 #ifdef ALTQ_RIO
 		if (q_is_rio(cl->cl_q))
@@ -406,6 +413,7 @@ priq_class_destroy(struct priq_class *cl)
 #else
 	s = splimp();
 #endif
+	IFQ_LOCK(cl->cl_pif->pif_ifq);
 
 #ifdef ALTQ3_CLFIER_COMPAT
 	/* delete filters referencing to this class */
@@ -426,6 +434,7 @@ priq_class_destroy(struct priq_class *cl)
 		if (pri < 0)
 			pif->pif_maxpri = -1;
 	}
+	IFQ_UNLOCK(cl->cl_pif->pif_ifq);
 	splx(s);
 
 	if (cl->cl_red != NULL) {
@@ -455,10 +464,13 @@ priq_enqueue(struct ifaltq *ifq, struct mbuf *m, struct altq_pktattr *pktattr)
 	struct m_tag *t;
 	int len;
 
+	IFQ_LOCK_ASSERT(ifq);
+
 	/* grab class set by classifier */
 	if ((m->m_flags & M_PKTHDR) == 0) {
 		/* should not happen */
-#if defined(__NetBSD__) || defined(__OpenBSD__)
+#if defined(__NetBSD__) || defined(__OpenBSD__)\
+    || (defined(__FreeBSD__) && __FreeBSD_version >= 501113)
 		printf("altq: packet for %s does not have pkthdr\n",
 		    ifq->altq_ifp->if_xname);
 #else
@@ -517,6 +529,8 @@ priq_dequeue(struct ifaltq *ifq, int op)
 	struct mbuf *m;
 	int pri;
 
+	IFQ_LOCK_ASSERT(ifq);
+
 	if (IFQ_IS_EMPTY(ifq))
 		/* no packet in the queue */
 		return (NULL);
diff --git a/sys/contrib/altq/altq/altq_red.c b/sys/contrib/altq/altq/altq_red.c
index b4aa9d3..ff6d832 100644
--- a/sys/contrib/altq/altq/altq_red.c
+++ b/sys/contrib/altq/altq/altq_red.c
@@ -1,3 +1,4 @@
+/*	$FreeBSD$	*/
 /*	$KAME: altq_red.c,v 1.18 2003/09/05 22:40:36 itojun Exp $	*/
 
 /*
@@ -210,7 +211,9 @@ static __inline struct fve *flowlist_reclaim(struct flowvalve *,
 					     struct altq_pktattr *);
 static __inline void flowlist_move_to_head(struct flowvalve *, struct fve *);
 static __inline int fv_p2f(struct flowvalve *, int);
+#if 0 /* XXX: make the compiler happy (fv_alloc unused) */
 static struct flowvalve *fv_alloc(struct red *);
+#endif
 static void fv_destroy(struct flowvalve *);
 static int fv_checkflow(struct flowvalve *, struct altq_pktattr *,
 			struct fve **);
@@ -1034,6 +1037,8 @@ red_enqueue(ifq, m, pktattr)
 {
 	red_queue_t *rqp = (red_queue_t *)ifq->altq_disc;
 
+	IFQ_LOCK_ASSERT(ifq);
+
 	if (red_addq(rqp->rq_red, rqp->rq_q, m, pktattr) < 0)
 		return ENOBUFS;
 	ifq->ifq_len++;
@@ -1056,6 +1061,8 @@ red_dequeue(ifq, op)
 	red_queue_t *rqp = (red_queue_t *)ifq->altq_disc;
 	struct mbuf *m;
 
+	IFQ_LOCK_ASSERT(ifq);
+
 	if (op == ALTDQ_POLL)
 		return qhead(rqp->rq_q);
 
@@ -1074,6 +1081,8 @@ red_request(ifq, req, arg)
 {
 	red_queue_t *rqp = (red_queue_t *)ifq->altq_disc;
 
+	IFQ_LOCK_ASSERT(ifq);
+
 	switch (req) {
 	case ALTRQ_PURGE:
 		red_purgeq(rqp);
@@ -1277,6 +1286,7 @@ flowlist_move_to_head(fv, fve)
 	}
 }
 
+#if 0 /* XXX: make the compiler happy (fv_alloc unused) */
 /*
  * allocate flowvalve structure
  */
@@ -1335,6 +1345,7 @@ fv_alloc(rp)
 
 	return (fv);
 }
+#endif
 
 static void fv_destroy(fv)
 	struct flowvalve *fv;
diff --git a/sys/contrib/altq/altq/altq_rio.c b/sys/contrib/altq/altq/altq_rio.c
index 56c8ee8..6bc4a95 100644
--- a/sys/contrib/altq/altq/altq_rio.c
+++ b/sys/contrib/altq/altq/altq_rio.c
@@ -1,3 +1,4 @@
+/*	$FreeBSD$	*/
 /*	$KAME: altq_rio.c,v 1.17 2003/07/10 12:07:49 kjc Exp $	*/
 
 /*
@@ -771,6 +772,8 @@ rio_request(ifq, req, arg)
 {
 	rio_queue_t *rqp = (rio_queue_t *)ifq->altq_disc;
 
+	IFQ_LOCK_ASSERT(ifq);
+
 	switch (req) {
 	case ALTRQ_PURGE:
 		_flushq(rqp->rq_q);
@@ -796,6 +799,8 @@ rio_enqueue(ifq, m, pktattr)
 	rio_queue_t *rqp = (rio_queue_t *)ifq->altq_disc;
 	int error = 0;
 
+	IFQ_LOCK_ASSERT(ifq);
+
 	if (rio_addq(rqp->rq_rio, rqp->rq_q, m, pktattr) == 0)
 		ifq->ifq_len++;
 	else
@@ -819,6 +824,8 @@ rio_dequeue(ifq, op)
 	rio_queue_t *rqp = (rio_queue_t *)ifq->altq_disc;
 	struct mbuf *m = NULL;
 
+	IFQ_LOCK_ASSERT(ifq);
+
 	if (op == ALTDQ_POLL)
 		return qhead(rqp->rq_q);
 
diff --git a/sys/contrib/altq/altq/altq_rmclass.c b/sys/contrib/altq/altq/altq_rmclass.c
index 70f7926..63194ee 100644
--- a/sys/contrib/altq/altq/altq_rmclass.c
+++ b/sys/contrib/altq/altq/altq_rmclass.c
@@ -1,3 +1,4 @@
+/*	$FreeBSD$	*/
 /*	$KAME: altq_rmclass.c,v 1.18 2003/11/06 06:32:53 kjc Exp $	*/
 
 /*
@@ -315,6 +316,7 @@ rmc_newclass(int pri, struct rm_ifdat *ifd, u_int nsecPerByte,
 #else
 	s = splimp();
 #endif
+	IFQ_LOCK(ifd->ifq_);
 	if ((peer = ifd->active_[pri]) != NULL) {
 		/* find the last class at this pri */
 		cl->peer_ = peer;
@@ -346,6 +348,7 @@ rmc_newclass(int pri, struct rm_ifdat *ifd, u_int nsecPerByte,
 		ifd->alloc_[pri] += cl->allotment_;
 		rmc_wrr_set_weights(ifd);
 	}
+	IFQ_UNLOCK(ifd->ifq_);
 	splx(s);
 	return (cl);
 }
@@ -366,6 +369,7 @@ rmc_modclass(struct rm_class *cl, u_int nsecPerByte, int maxq, u_int maxidle,
 #else
 	s = splimp();
 #endif
+	IFQ_LOCK(ifd->ifq_);
 	cl->allotment_ = RM_NS_PER_SEC / nsecPerByte; /* Bytes per sec */
 	cl->qthresh_ = 0;
 	cl->ns_per_byte_ = nsecPerByte;
@@ -399,6 +403,7 @@ rmc_modclass(struct rm_class *cl, u_int nsecPerByte, int maxq, u_int maxidle,
 		ifd->alloc_[cl->pri_] += cl->allotment_ - old_allotment;
 		rmc_wrr_set_weights(ifd);
 	}
+	IFQ_UNLOCK(ifd->ifq_);
 	splx(s);
 	return (0);
 }
@@ -564,6 +569,7 @@ rmc_delete_class(struct rm_ifdat *ifd, struct rm_class *cl)
 #else
 	s = splimp();
 #endif
+	IFQ_LOCK(ifd->ifq_);
 	/*
 	 * Free packets in the packet queue.
 	 * XXX - this may not be a desired behavior.  Packets should be
@@ -636,6 +642,7 @@ rmc_delete_class(struct rm_ifdat *ifd, struct rm_class *cl)
 	rmc_depth_recompute(ifd->root_);
 #endif
 
+	IFQ_UNLOCK(ifd->ifq_);
 	splx(s);
 
 	/*
@@ -1571,6 +1578,7 @@ rmc_restart(struct rm_class *cl)
 #else
 	s = splimp();
 #endif
+	IFQ_LOCK(ifd->ifq_);
 	if (cl->sleeping_) {
 		cl->sleeping_ = 0;
 		cl->undertime_.tv_sec = 0;
@@ -1580,6 +1588,7 @@ rmc_restart(struct rm_class *cl)
 			(ifd->restart)(ifd->ifq_);
 		}
 	}
+	IFQ_UNLOCK(ifd->ifq_);
 	splx(s);
 }
 
diff --git a/sys/contrib/altq/altq/altq_subr.c b/sys/contrib/altq/altq/altq_subr.c
index 09482ce..13af544 100644
--- a/sys/contrib/altq/altq/altq_subr.c
+++ b/sys/contrib/altq/altq/altq_subr.c
@@ -1,3 +1,4 @@
+/*	$FreeBSD$	*/
 /*	$KAME: altq_subr.c,v 1.21 2003/11/06 06:32:53 kjc Exp $	*/
 
 /*
@@ -131,6 +132,7 @@ altq_lookup(name, type)
 	struct ifnet *ifp;
 
 	if ((ifp = ifunit(name)) != NULL) {
+		/* read if_snd unlocked */
 		if (type != ALTQT_NONE && ifp->if_snd.altq_type == type)
 			return (ifp->if_snd.altq_disc);
 	}
@@ -149,8 +151,11 @@ altq_attach(ifq, type, discipline, enqueue, dequeue, request, clfier, classify)
 	void *clfier;
 	void *(*classify)(void *, struct mbuf *, int);
 {
-	if (!ALTQ_IS_READY(ifq))
+	IFQ_LOCK(ifq);
+	if (!ALTQ_IS_READY(ifq)) {
+		IFQ_UNLOCK(ifq);
 		return ENXIO;
+	}
 
 #ifdef ALTQ3_COMPAT
 	/*
@@ -158,10 +163,14 @@ altq_attach(ifq, type, discipline, enqueue, dequeue, request, clfier, classify)
 	 * check these if clfier is not NULL (which implies altq3).
 	 */
 	if (clfier != NULL) {
-		if (ALTQ_IS_ENABLED(ifq))
+		if (ALTQ_IS_ENABLED(ifq)) {
+			IFQ_UNLOCK(ifq);
 			return EBUSY;
-		if (ALTQ_IS_ATTACHED(ifq))
+		}
+		if (ALTQ_IS_ATTACHED(ifq)) {
+			IFQ_UNLOCK(ifq);
 			return EEXIST;
+		}
 	}
 #endif
 	ifq->altq_type     = type;
@@ -177,6 +186,7 @@ altq_attach(ifq, type, discipline, enqueue, dequeue, request, clfier, classify)
 	altq_module_incref(type);
 #endif
 #endif
+	IFQ_UNLOCK(ifq);
 	return 0;
 }
 
@@ -184,12 +194,20 @@ int
 altq_detach(ifq)
 	struct ifaltq *ifq;
 {
-	if (!ALTQ_IS_READY(ifq))
+	IFQ_LOCK(ifq);
+
+	if (!ALTQ_IS_READY(ifq)) {
+		IFQ_UNLOCK(ifq);
 		return ENXIO;
-	if (ALTQ_IS_ENABLED(ifq))
+	}
+	if (ALTQ_IS_ENABLED(ifq)) {
+		IFQ_UNLOCK(ifq);
 		return EBUSY;
-	if (!ALTQ_IS_ATTACHED(ifq))
+	}
+	if (!ALTQ_IS_ATTACHED(ifq)) {
+		IFQ_UNLOCK(ifq);
 		return (0);
+	}
 #ifdef ALTQ3_COMPAT
 #ifdef ALTQ_KLD
 	altq_module_declref(ifq->altq_type);
@@ -204,6 +222,8 @@ altq_detach(ifq)
 	ifq->altq_clfier   = NULL;
 	ifq->altq_classify = NULL;
 	ifq->altq_flags &= ALTQF_CANTCHANGE;
+
+	IFQ_UNLOCK(ifq);
 	return 0;
 }
 
@@ -213,23 +233,30 @@ altq_enable(ifq)
 {
 	int s;
 
-	if (!ALTQ_IS_READY(ifq))
+	IFQ_LOCK(ifq);
+
+	if (!ALTQ_IS_READY(ifq)) {
+		IFQ_UNLOCK(ifq);
 		return ENXIO;
-	if (ALTQ_IS_ENABLED(ifq))
+	}
+	if (ALTQ_IS_ENABLED(ifq)) {
+		IFQ_UNLOCK(ifq);
 		return 0;
+	}
 
 #ifdef __NetBSD__
 	s = splnet();
 #else
 	s = splimp();
 #endif
-	IFQ_PURGE(ifq);
+	IFQ_PURGE_NOLOCK(ifq);
 	ASSERT(ifq->ifq_len == 0);
 	ifq->altq_flags |= ALTQF_ENABLED;
 	if (ifq->altq_clfier != NULL)
 		ifq->altq_flags |= ALTQF_CLASSIFY;
 	splx(s);
 
+	IFQ_UNLOCK(ifq);
 	return 0;
 }
 
@@ -239,18 +266,23 @@ altq_disable(ifq)
 {
 	int s;
 
-	if (!ALTQ_IS_ENABLED(ifq))
+	IFQ_LOCK(ifq);
+	if (!ALTQ_IS_ENABLED(ifq)) {
+		IFQ_UNLOCK(ifq);
 		return 0;
+	}
 
 #ifdef __NetBSD__
 	s = splnet();
 #else
 	s = splimp();
 #endif
-	IFQ_PURGE(ifq);
+	IFQ_PURGE_NOLOCK(ifq);
 	ASSERT(ifq->ifq_len == 0);
 	ifq->altq_flags &= ~(ALTQF_ENABLED|ALTQF_CLASSIFY);
 	splx(s);
+	
+	IFQ_UNLOCK(ifq);
 	return 0;
 }
 
@@ -288,6 +320,7 @@ tbr_dequeue(ifq, op)
 	int64_t interval;
 	u_int64_t now;
 
+	IFQ_LOCK_ASSERT(ifq);
 	tbr = ifq->altq_tbr;
 	if (op == ALTDQ_REMOVE && tbr->tbr_lastop == ALTDQ_POLL) {
 		/* if this is a remove after poll, bypass tbr check */
@@ -314,9 +347,9 @@ tbr_dequeue(ifq, op)
 		m = (*ifq->altq_dequeue)(ifq, op);
 	else {
 		if (op == ALTDQ_POLL)
-			IF_POLL(ifq, m);
+			_IF_POLL(ifq, m);
 		else
-			IF_DEQUEUE(ifq, m);
+			_IF_DEQUEUE(ifq, m);
 	}
 
 	if (m != NULL && op == ALTDQ_REMOVE)
@@ -343,19 +376,26 @@ tbr_set(ifq, profile)
 		return (ENXIO);
 	}
 
+	IFQ_LOCK(ifq);
 	if (profile->rate == 0) {
 		/* delete this tbr */
-		if ((tbr = ifq->altq_tbr) == NULL)
+		if ((tbr = ifq->altq_tbr) == NULL) {
+			IFQ_UNLOCK(ifq);
 			return (ENOENT);
+		}
 		ifq->altq_tbr = NULL;
 		FREE(tbr, M_DEVBUF);
+		IFQ_UNLOCK(ifq);
 		return (0);
 	}
 
+	IFQ_UNLOCK(ifq);
 	MALLOC(tbr, struct tb_regulator *, sizeof(struct tb_regulator),
 	       M_DEVBUF, M_WAITOK);
-	if (tbr == NULL)
+	if (tbr == NULL) {		/* can not happen */
+		IFQ_UNLOCK(ifq);
 		return (ENOMEM);
+	}
 	bzero(tbr, sizeof(struct tb_regulator));
 
 	tbr->tbr_rate = TBR_SCALE(profile->rate / 8) / machclk_freq;
@@ -368,6 +408,7 @@ tbr_set(ifq, profile)
 	tbr->tbr_last = read_machclk();
 	tbr->tbr_lastop = ALTDQ_REMOVE;
 
+	IFQ_LOCK(ifq);
 	otbr = ifq->altq_tbr;
 	ifq->altq_tbr = tbr;	/* set the new tbr */
 
@@ -379,12 +420,15 @@ tbr_set(ifq, profile)
 			tbr_timer = 1;
 		}
 	}
+	IFQ_UNLOCK(ifq);
 	return (0);
 }
 
 /*
  * tbr_timeout goes through the interface list, and kicks the drivers
  * if necessary.
+ *
+ * MPSAFE
  */
 static void
 tbr_timeout(arg)
@@ -399,13 +443,20 @@ tbr_timeout(arg)
 #else
 	s = splimp();
 #endif
+#if defined(__FreeBSD__) && (__FreeBSD_version >= 500000)
+	IFNET_RLOCK();
+#endif
 	for (ifp = TAILQ_FIRST(&ifnet); ifp; ifp = TAILQ_NEXT(ifp, if_list)) {
+		/* read from if_snd unlocked */
 		if (!TBR_IS_ENABLED(&ifp->if_snd))
 			continue;
 		active++;
 		if (!IFQ_IS_EMPTY(&ifp->if_snd) && ifp->if_start != NULL)
 			(*ifp->if_start)(ifp);
 	}
+#if defined(__FreeBSD__) && (__FreeBSD_version >= 500000)
+	IFNET_RUNLOCK();
+#endif
 	splx(s);
 	if (active > 0)
 		CALLOUT_RESET(&tbr_callout, 1, tbr_timeout, (void *)0);
@@ -437,6 +488,7 @@ tbr_get(ifq, profile)
 {
 	struct tb_regulator *tbr;
 
+	IFQ_LOCK(ifq);
 	if ((tbr = ifq->altq_tbr) == NULL) {
 		profile->rate = 0;
 		profile->depth = 0;
@@ -445,12 +497,15 @@ tbr_get(ifq, profile)
 		    (u_int)TBR_UNSCALE(tbr->tbr_rate * 8 * machclk_freq);
 		profile->depth = (u_int)TBR_UNSCALE(tbr->tbr_depth);
 	}
+	IFQ_UNLOCK(ifq);
 	return (0);
 }
 
 /*
  * attach a discipline to the interface.  if one already exists, it is
  * overridden.
+ * Locking is done in the discipline specific attach functions. Basically
+ * they call back to altq_attach which takes care of the attach and locking.
  */
 int
 altq_pfattach(struct pf_altq *a)
@@ -497,6 +552,7 @@ altq_pfdetach(struct pf_altq *a)
 		return (EINVAL);
 
 	/* if this discipline is no longer referenced, just return */
+	/* read unlocked from if_snd */
 	if (a->altq_disc == NULL || a->altq_disc != ifp->if_snd.altq_disc)
 		return (0);
 
@@ -505,6 +561,7 @@ altq_pfdetach(struct pf_altq *a)
 #else
 	s = splimp();
 #endif
+	/* read unlocked from if_snd, _disable and _detach take care */
 	if (ALTQ_IS_ENABLED(&ifp->if_snd))
 		error = altq_disable(&ifp->if_snd);
 	if (error == 0)
@@ -516,6 +573,8 @@ altq_pfdetach(struct pf_altq *a)
 
 /*
  * add a discipline or a queue
+ * Locking is done in the discipline specific functions with regards to
+ * malloc with WAITOK, also it is not yet clear which lock to use.
  */
 int
 altq_add(struct pf_altq *a)
@@ -555,6 +614,8 @@ altq_add(struct pf_altq *a)
 
 /*
  * remove a discipline or a queue
+ * It is yet unclear what lock to use to protect this operation, the
+ * discipline specific functions will determine and grab it
  */
 int
 altq_remove(struct pf_altq *a)
@@ -589,6 +650,8 @@ altq_remove(struct pf_altq *a)
 
 /*
  * add a queue to the discipline
+ * It is yet unclear what lock to use to protect this operation, the
+ * discipline specific functions will determine and grab it
  */
 int
 altq_add_queue(struct pf_altq *a)
@@ -620,6 +683,8 @@ altq_add_queue(struct pf_altq *a)
 
 /*
  * remove a queue from the discipline
+ * It is yet unclear what lock to use to protect this operation, the
+ * discipline specific functions will determine and grab it
  */
 int
 altq_remove_queue(struct pf_altq *a)
@@ -651,6 +716,8 @@ altq_remove_queue(struct pf_altq *a)
 
 /*
  * get queue statistics
+ * Locking is done in the discipline specific functions with regards to
+ * copyout operations, also it is not yet clear which lock to use.
  */
 int
 altq_getqstats(struct pf_altq *a, void *ubuf, int *nbytes)