Implement the check I was talking about in the previous message already.

Introduce domain_init_status to keep track of the init status of the domains
list (surprise). 0 = uninitialized, 1 = initialized/unpopulated, 2 =
initialized/done. Higher values can be used to support late addition of
domains which right now "works", but is potential dangerous. I choose to
only give a warning when doing so.

Use domain_init_status with if_attachdomain[1]() to ensure that we have a
complete domains list when we init the if_afdata array. Store the current
value of domain_init_status in if_afdata_initialized. This way we can update
if_afdata after a new protocol has been added (once that is allowed).

Submitted by:	se (with changes)
Reviewed by:	julian, glebius, se
PR:		kern/73321	(partly)

3 files changed, 42 insertions, 4 deletions

diff --git a/sys/kern/uipc_domain.c b/sys/kern/uipc_domain.c
index 1be391c..5dc5430 100644
--- a/sys/kern/uipc_domain.c
+++ b/sys/kern/uipc_domain.c
@@ -59,6 +59,10 @@ __FBSDID("$FreeBSD$");
 static void domaininit(void *);
 SYSINIT(domain, SI_SUB_PROTO_DOMAIN, SI_ORDER_FIRST, domaininit, NULL)
 
+static void domainfinalize(void *);
+SYSINIT(domainfin, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_FIRST, domainfinalize,
+    NULL)
+
 static struct callout pffast_callout;
 static struct callout pfslow_callout;
 
@@ -66,6 +70,7 @@ static void	pffasttimo(void *);
 static void	pfslowtimo(void *);
 
 struct domain *domains;		/* registered protocol domains */
+int domain_init_status = 0;
 struct mtx dom_mtx;		/* domain list lock */
 MTX_SYSINIT(domain, &dom_mtx, "domain list", MTX_DEF);
 
@@ -160,6 +165,24 @@ net_add_domain(void *data)
 	mtx_lock(&dom_mtx);
 	dp->dom_next = domains;
 	domains = dp;
+
+	KASSERT(domain_init_status >= 1,
+	    ("attempt to net_add_domain(%s) before domaininit()",
+	    dp->dom_name));
+#ifndef INVARIANTS
+	if (domain_init_status == 0)
+		printf("WARNING: attempt to net_add_domain(%s) before "
+		    "domaininit()\n", dp->dom_name);
+#endif
+#ifdef notyet
+	KASSERT(domain_init_status < 2,
+	    ("attempt to net_add_domain(%s) after domainfinalize()",
+	    dp->dom_name));
+#else
+	if (domain_init_status != 0)
+		printf("WARNING: attempt to net_add_domain(%s) after "
+		    "domainfinalize()\n", dp->dom_name);
+#endif
 	mtx_unlock(&dom_mtx);
 	net_init_domain(dp);
 }
@@ -188,10 +211,24 @@ domaininit(void *dummy)
 		callout_init(&pfslow_callout, 0);
 	}
 
+	mtx_lock(&dom_mtx);
+	KASSERT(domain_init_status == 0, ("domaininit called too late!"));
+	domain_init_status = 1;
+	mtx_unlock(&dom_mtx);
+
 	callout_reset(&pffast_callout, 1, pffasttimo, NULL);
 	callout_reset(&pfslow_callout, 1, pfslowtimo, NULL);
 }
 
+/* ARGSUSED*/
+static void
+domainfinalize(void *dummy)
+{
+	mtx_lock(&dom_mtx);
+	KASSERT(domain_init_status == 1, ("domainfinalize called too late!"));
+	domain_init_status = 2;
+	mtx_unlock(&dom_mtx);	
+}
 
 struct protosw *
 pffindtype(family, type)
diff --git a/sys/net/if.c b/sys/net/if.c
index f6728df..b21701d 100644
--- a/sys/net/if.c
+++ b/sys/net/if.c
@@ -451,7 +451,7 @@ if_attach(struct ifnet *ifp)
 	ifp->if_snd.altq_tbr  = NULL;
 	ifp->if_snd.altq_ifp  = ifp;
 
-	if (domains)
+	if (domain_init_status >= 2)
 		if_attachdomain1(ifp);
 
 	EVENTHANDLER_INVOKE(ifnet_arrival_event, ifp);
@@ -471,7 +471,7 @@ if_attachdomain(void *dummy)
 		if_attachdomain1(ifp);
 	splx(s);
 }
-SYSINIT(domainifattach, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_FIRST,
+SYSINIT(domainifattach, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_SECOND,
     if_attachdomain, NULL);
 
 static void
@@ -490,14 +490,14 @@ if_attachdomain1(struct ifnet *ifp)
 		splx(s);
 		return;
 	}
-	if (ifp->if_afdata_initialized) {
+	if (ifp->if_afdata_initialized >= domain_init_status) {
 		IF_AFDATA_UNLOCK(ifp);
 		splx(s);
 		printf("if_attachdomain called more than once on %s\n",
 		    ifp->if_xname);
 		return;
 	}
-	ifp->if_afdata_initialized = 1;
+	ifp->if_afdata_initialized = domain_init_status;
 	IF_AFDATA_UNLOCK(ifp);
 
 	/* address family dependent data region */
diff --git a/sys/sys/domain.h b/sys/sys/domain.h
index a789e1b..b410650 100644
--- a/sys/sys/domain.h
+++ b/sys/sys/domain.h
@@ -64,6 +64,7 @@ struct domain {
 };
 
 #ifdef _KERNEL
+extern int	domain_init_status;
 extern struct	domain *domains;
 extern struct	domain localdomain;
 extern void	net_add_domain(void *);