Allow carp(4) to be loaded as a kernel module. Follow precedent set by

bridge(4), lagg(4) etc. and make use of function pointers and
pf_proto_register() to hook carp into the network stack.

Currently, because of the uncertainty about whether the unload path is free
of race condition panics, unloads are disallowed by default.  Compiling with
CARPMOD_CAN_UNLOAD in CFLAGS removes this anti foot shooting measure.

This commit requires IP6PROTOSPACER, introduced in r211115.

Reviewed by:	bz, simon
Approved by:	ken (mentor)
MFC after:	2 weeks

13 files changed, 214 insertions, 149 deletions

diff --git a/sys/modules/Makefile b/sys/modules/Makefile
index d033a34..028c4cc 100644
--- a/sys/modules/Makefile
+++ b/sys/modules/Makefile
@@ -112,6 +112,7 @@ SUBDIR=	${_3dfx} \
 	${_ida} \
 	${_ie} \
 	if_bridge \
+	if_carp \
 	if_disc \
 	if_edsc \
 	if_ef \
diff --git a/sys/modules/if_carp/Makefile b/sys/modules/if_carp/Makefile
new file mode 100644
index 0000000..f315a80
--- /dev/null
+++ b/sys/modules/if_carp/Makefile
@@ -0,0 +1,21 @@
+# $FreeBSD$
+
+.PATH: ${.CURDIR}/../../netinet
+
+.include <bsd.own.mk>
+
+KMOD=	if_carp
+SRCS=	ip_carp.c
+SRCS+=	opt_carp.h opt_bpf.h opt_inet.h opt_inet6.h vnode_if.h
+
+.if !defined(KERNBUILDDIR)
+opt_inet.h:
+	@echo "#define INET 1" > ${.TARGET}
+
+.if ${MK_INET6_SUPPORT} != "no"
+opt_inet6.h:
+	@echo "#define INET6 1" > ${.TARGET}
+.endif
+.endif
+
+.include <bsd.kmod.mk>
diff --git a/sys/net/if.c b/sys/net/if.c
index da323cc..3c4ad16 100644
--- a/sys/net/if.c
+++ b/sys/net/if.c
@@ -33,7 +33,6 @@
 #include "opt_compat.h"
 #include "opt_inet6.h"
 #include "opt_inet.h"
-#include "opt_carp.h"
 #include "opt_ddb.h"
 
 #include <sys/param.h>
@@ -89,11 +88,6 @@
 #ifdef INET
 #include <netinet/if_ether.h>
 #endif
-#if defined(INET) || defined(INET6)
-#ifdef DEV_CARP
-#include <netinet/ip_carp.h>
-#endif
-#endif
 
 #include <security/mac/mac_framework.h>
 
@@ -130,6 +124,7 @@ SX_SYSINIT(ifdescr_sx, &ifdescr_sx, "ifnet descr");
 void	(*bstp_linkstate_p)(struct ifnet *ifp, int state);
 void	(*ng_ether_link_state_p)(struct ifnet *ifp, int state);
 void	(*lagg_linkstate_p)(struct ifnet *ifp, int state);
+void	(*carp_linkstate_p)(struct ifnet *ifp);
 
 struct mbuf *(*tbr_dequeue_ptr)(struct ifaltq *, int) = NULL;
 
@@ -1813,12 +1808,8 @@ if_unroute(struct ifnet *ifp, int flag, int fam)
 			pfctlinput(PRC_IFDOWN, ifa->ifa_addr);
 	ifp->if_qflush(ifp);
 
-#if defined(INET) || defined(INET6)
-#ifdef DEV_CARP
 	if (ifp->if_carp)
-		carp_carpdev_state(ifp->if_carp);
-#endif
-#endif
+		(*carp_linkstate_p)(ifp);
 	rt_ifmsg(ifp);
 }
 
@@ -1839,12 +1830,8 @@ if_route(struct ifnet *ifp, int flag, int fam)
 	TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link)
 		if (fam == PF_UNSPEC || (fam == ifa->ifa_addr->sa_family))
 			pfctlinput(PRC_IFUP, ifa->ifa_addr);
-#if defined(INET) || defined(INET6)
-#ifdef DEV_CARP
 	if (ifp->if_carp)
-		carp_carpdev_state(ifp->if_carp);
-#endif
-#endif
+		(*carp_linkstate_p)(ifp);
 	rt_ifmsg(ifp);
 #ifdef INET6
 	in6_if_up(ifp);
@@ -1887,12 +1874,8 @@ do_link_state_change(void *arg, int pending)
 	if ((ifp->if_type == IFT_ETHER || ifp->if_type == IFT_L2VLAN) &&
 	    IFP2AC(ifp)->ac_netgraph != NULL)
 		(*ng_ether_link_state_p)(ifp, link_state);
-#if defined(INET) || defined(INET6)
-#ifdef DEV_CARP
 	if (ifp->if_carp)
-		carp_carpdev_state(ifp->if_carp);
-#endif
-#endif
+		(*carp_linkstate_p)(ifp);
 	if (ifp->if_bridge) {
 		KASSERT(bstp_linkstate_p != NULL,("if_bridge bstp not loaded!"));
 		(*bstp_linkstate_p)(ifp, link_state);
diff --git a/sys/net/if_bridge.c b/sys/net/if_bridge.c
index 5f33dd5..fb04731 100644
--- a/sys/net/if_bridge.c
+++ b/sys/net/if_bridge.c
@@ -79,7 +79,6 @@ __FBSDID("$FreeBSD$");
 
 #include "opt_inet.h"
 #include "opt_inet6.h"
-#include "opt_carp.h"
 
 #include <sys/param.h>
 #include <sys/mbuf.h>
@@ -121,10 +120,8 @@ __FBSDID("$FreeBSD$");
 #include <netinet6/ip6_var.h>
 #endif
 #if defined(INET) || defined(INET6)
-#ifdef DEV_CARP
 #include <netinet/ip_carp.h>
 #endif
-#endif
 #include <machine/in_cksum.h>
 #include <netinet/if_ether.h> /* for struct arpcom */
 #include <net/bridgestp.h>
@@ -2144,6 +2141,10 @@ drop:
 	m_freem(m);
 }
 
+#if defined(INET) || defined(INET6)
+int (*carp_forus_p)(struct carp_if *, u_char *);
+#endif
+
 /*
  * bridge_input:
  *
@@ -2252,13 +2253,13 @@ bridge_input(struct ifnet *ifp, struct mbuf *m)
 		return (m);
 	}
 
-#if (defined(INET) || defined(INET6)) && defined(DEV_CARP)
+#if (defined(INET) || defined(INET6))
 #   define OR_CARP_CHECK_WE_ARE_DST(iface) \
 	|| ((iface)->if_carp \
-	    && carp_forus((iface)->if_carp, eh->ether_dhost))
+	    && (*carp_forus_p)((iface)->if_carp, eh->ether_dhost))
 #   define OR_CARP_CHECK_WE_ARE_SRC(iface) \
 	|| ((iface)->if_carp \
-	    && carp_forus((iface)->if_carp, eh->ether_shost))
+	    && (*carp_forus_p)((iface)->if_carp, eh->ether_shost))
 #else
 #   define OR_CARP_CHECK_WE_ARE_DST(iface)
 #   define OR_CARP_CHECK_WE_ARE_SRC(iface)
diff --git a/sys/net/if_ethersubr.c b/sys/net/if_ethersubr.c
index bbf9753..984d0bb 100644
--- a/sys/net/if_ethersubr.c
+++ b/sys/net/if_ethersubr.c
@@ -35,7 +35,6 @@
 #include "opt_inet6.h"
 #include "opt_ipx.h"
 #include "opt_netgraph.h"
-#include "opt_carp.h"
 #include "opt_mbuf_profiling.h"
 
 #include <sys/param.h>
@@ -70,6 +69,7 @@
 #include <netinet/in.h>
 #include <netinet/in_var.h>
 #include <netinet/if_ether.h>
+#include <netinet/ip_carp.h>
 #include <netinet/ip_var.h>
 #include <netinet/ip_fw.h>
 #include <netinet/ipfw/ip_fw_private.h>
@@ -78,12 +78,6 @@
 #include <netinet6/nd6.h>
 #endif
 
-#if defined(INET) || defined(INET6)
-#ifdef DEV_CARP
-#include <netinet/ip_carp.h>
-#endif
-#endif
-
 #ifdef IPX
 #include <netipx/ipx.h>
 #include <netipx/ipx_if.h>
@@ -120,6 +114,11 @@ void	(*ng_ether_attach_p)(struct ifnet *ifp);
 void	(*ng_ether_detach_p)(struct ifnet *ifp);
 
 void	(*vlan_input_p)(struct ifnet *, struct mbuf *);
+#if defined(INET) || defined(INET6)
+int (*carp_forus_p)(struct ifnet *, u_char *);
+int (*carp_output_p)(struct ifnet *, struct mbuf *, struct sockaddr *,
+    struct rtentry *);
+#endif
 
 /* if_bridge(4) support */
 struct mbuf *(*bridge_input_p)(struct ifnet *, struct mbuf *); 
@@ -399,12 +398,10 @@ ether_output(struct ifnet *ifp, struct mbuf *m,
 	}
 
 #if defined(INET) || defined(INET6)
-#ifdef DEV_CARP
 	if (ifp->if_carp &&
-	    (error = carp_output(ifp, m, dst, NULL)))
+	    (error = (*carp_output_p)(ifp, m, dst, NULL)))
 		goto bad;
 #endif
-#endif
 
 	/* Handle ng_ether(4) processing, if any */
 	if (IFP2AC(ifp)->ac_netgraph != NULL) {
@@ -724,7 +721,6 @@ ether_input(struct ifnet *ifp, struct mbuf *m)
 	}
 
 #if defined(INET) || defined(INET6)
-#ifdef DEV_CARP
 	/*
 	 * Clear M_PROMISC on frame so that carp(4) will see it when the
 	 * mbuf flows up to Layer 3.
@@ -735,11 +731,10 @@ ether_input(struct ifnet *ifp, struct mbuf *m)
 	 * TODO: Maintain a hash table of ethernet addresses other than
 	 * ether_dhost which may be active on this ifp.
 	 */
-	if (ifp->if_carp && carp_forus(ifp->if_carp, eh->ether_dhost)) {
+	if (ifp->if_carp && (*carp_forus_p)(ifp, eh->ether_dhost)) {
 		m->m_flags &= ~M_PROMISC;
 	} else
 #endif
-#endif
 	{
 		/*
 		 * If the frame received was not for our MAC address, set the
diff --git a/sys/netinet/if_ether.c b/sys/netinet/if_ether.c
index ec03268..bdc981a 100644
--- a/sys/netinet/if_ether.c
+++ b/sys/netinet/if_ether.c
@@ -39,7 +39,6 @@
 __FBSDID("$FreeBSD$");
 
 #include "opt_inet.h"
-#include "opt_carp.h"
 
 #include <sys/param.h>
 #include <sys/kernel.h>
@@ -69,10 +68,6 @@ __FBSDID("$FreeBSD$");
 #include <net/if_arc.h>
 #include <net/iso88025.h>
 
-#ifdef DEV_CARP
-#include <netinet/ip_carp.h>
-#endif
-
 #include <security/mac/mac_framework.h>
 
 #define SIN(s) ((struct sockaddr_in *)s)
@@ -123,6 +118,10 @@ static void	arptimer(void *);
 #ifdef INET
 static void	in_arpinput(struct mbuf *);
 #endif
+#if defined(INET) || defined(INET6)
+int		(*carp_iamatch_p)(struct ifnet *, struct in_ifaddr *, struct in_addr *,
+		    u_int8_t **);
+#endif
 
 static const struct netisr_handler arp_nh = {
 	.nh_name = "arp",
@@ -494,9 +493,7 @@ in_arpinput(struct mbuf *m)
 	int op, flags;
 	int req_len;
 	int bridged = 0, is_bridge = 0;
-#ifdef DEV_CARP
 	int carp_match = 0;
-#endif
 	struct sockaddr_in sin;
 	sin.sin_len = sizeof(struct sockaddr_in);
 	sin.sin_family = AF_INET;
@@ -539,16 +536,14 @@ in_arpinput(struct mbuf *m)
 			IN_IFADDR_RUNLOCK();
 			goto match;
 		}
-#ifdef DEV_CARP
 		if (ifp->if_carp != NULL &&
-		    carp_iamatch(ifp->if_carp, ia, &isaddr, &enaddr) &&
+		    (*carp_iamatch_p)(ifp, ia, &isaddr, &enaddr) &&
 		    itaddr.s_addr == ia->ia_addr.sin_addr.s_addr) {
 			carp_match = 1;
 			ifa_ref(&ia->ia_ifa);
 			IN_IFADDR_RUNLOCK();
 			goto match;
 		}
-#endif
 	}
 	LIST_FOREACH(ia, INADDR_HASH(isaddr.s_addr), ia_hash)
 		if (((bridged && ia->ia_ifp->if_bridge != NULL) ||
@@ -648,11 +643,7 @@ match:
 	IF_AFDATA_UNLOCK(ifp);
 	if (la != NULL) {
 		/* the following is not an error when doing bridging */
-		if (!bridged && la->lle_tbl->llt_ifp != ifp
-#ifdef DEV_CARP
-		    && (ifp->if_type != IFT_CARP || !carp_match)
-#endif
-			) {
+		if (!bridged && la->lle_tbl->llt_ifp != ifp && !carp_match) {
 			if (log_arp_wrong_iface)
 				log(LOG_ERR, "arp: %s is on %s "
 				    "but got reply from %*D on %s\n",
diff --git a/sys/netinet/in.c b/sys/netinet/in.c
index db7c8fe..b5873d8 100644
--- a/sys/netinet/in.c
+++ b/sys/netinet/in.c
@@ -33,7 +33,6 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
-#include "opt_carp.h"
 #include "opt_mpath.h"
 
 #include <sys/param.h>
@@ -891,13 +890,11 @@ in_ifinit(struct ifnet *ifp, struct in_ifaddr *ia, struct sockaddr_in *sin,
 	ia->ia_net = i & ia->ia_netmask;
 	ia->ia_subnet = i & ia->ia_subnetmask;
 	in_socktrim(&ia->ia_sockmask);
-#ifdef DEV_CARP
 	/*
 	 * XXX: carp(4) does not have interface route
 	 */
 	if (ifp->if_type == IFT_CARP)
 		return (0);
-#endif
 	/*
 	 * Add route for the network.
 	 */
@@ -1152,12 +1149,12 @@ in_scrubprefix(struct in_ifaddr *target)
 		 * the route itself to it.  Make sure that routing daemons
 		 * get a heads-up.
 		 *
-		 * XXX: a special case for carp(4) interface
+		 * XXX: a special case for carp(4) interface - this should
+		 *      be more generally specified as an interface that
+		 *      doesn't support such action.
 		 */
 		if ((ia->ia_flags & IFA_ROUTE) == 0
-#ifdef DEV_CARP
 		    && (ia->ia_ifp->if_type != IFT_CARP)
-#endif
 							) {
 			IN_IFADDR_RUNLOCK();
 			rtinit(&(target->ia_ifa), (int)RTM_DELETE,
diff --git a/sys/netinet/in_proto.c b/sys/netinet/in_proto.c
index 6d72fbe..110301a 100644
--- a/sys/netinet/in_proto.c
+++ b/sys/netinet/in_proto.c
@@ -37,7 +37,6 @@ __FBSDID("$FreeBSD$");
 #include "opt_ipsec.h"
 #include "opt_inet6.h"
 #include "opt_pf.h"
-#include "opt_carp.h"
 #include "opt_sctp.h"
 #include "opt_mpath.h"
 
@@ -94,10 +93,6 @@ static struct pr_usrreqs nousrreqs;
 #include <net/if_pfsync.h>
 #endif
 
-#ifdef DEV_CARP
-#include <netinet/ip_carp.h>
-#endif
-
 extern	struct domain inetdomain;
 
 /* Spacer for loadable protocols. */
@@ -330,18 +325,6 @@ struct protosw inetsw[] = {
 	.pr_usrreqs =		&rip_usrreqs
 },
 #endif	/* DEV_PFSYNC */
-#ifdef DEV_CARP
-{
-	.pr_type =		SOCK_RAW,
-	.pr_domain =		&inetdomain,
-	.pr_protocol =		IPPROTO_CARP,
-	.pr_flags =		PR_ATOMIC|PR_ADDR,
-	.pr_input =		carp_input,
-	.pr_output =		(pr_output_t*)rip_output,
-	.pr_ctloutput =		rip_ctloutput,
-	.pr_usrreqs =		&rip_usrreqs
-},
-#endif /* DEV_CARP */
 /* Spacer n-times for loadable protocols. */
 IPPROTOSPACER,
 IPPROTOSPACER,
@@ -413,6 +396,3 @@ SYSCTL_NODE(_net_inet, IPPROTO_RAW,	raw,	CTLFLAG_RW, 0,	"RAW");
 #ifdef DEV_PFSYNC
 SYSCTL_NODE(_net_inet, IPPROTO_PFSYNC,	pfsync,	CTLFLAG_RW, 0,	"PFSYNC");
 #endif
-#ifdef DEV_CARP
-SYSCTL_NODE(_net_inet, IPPROTO_CARP,	carp,	CTLFLAG_RW, 0,	"CARP");
-#endif
diff --git a/sys/netinet/ip_carp.c b/sys/netinet/ip_carp.c
index 743a901..1820ae0 100644
--- a/sys/netinet/ip_carp.c
+++ b/sys/netinet/ip_carp.c
@@ -27,7 +27,6 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
-#include "opt_carp.h"
 #include "opt_bpf.h"
 #include "opt_inet.h"
 #include "opt_inet6.h"
@@ -44,6 +43,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/time.h>
 #include <sys/priv.h>
 #include <sys/proc.h>
+#include <sys/protosw.h>
 #include <sys/sysctl.h>
 #include <sys/syslog.h>
 #include <sys/signalvar.h>
@@ -79,6 +79,7 @@ __FBSDID("$FreeBSD$");
 #ifdef INET6
 #include <netinet/icmp6.h>
 #include <netinet/ip6.h>
+#include <netinet6/ip6protosw.h>
 #include <netinet6/ip6_var.h>
 #include <netinet6/scope6_var.h>
 #include <netinet6/nd6.h>
@@ -134,8 +135,29 @@ struct carp_softc {
 };
 #define	SC2IFP(sc)	((sc)->sc_ifp)
 
+/* These are external networking stack hooks for CARP */
+/* net/if.c */
+extern void (*carp_linkstate_p)(struct ifnet *);
+/* net/if_bridge.c net/if_ethersubr.c */
+extern struct ifnet *(*carp_forus_p)(struct ifnet *, u_char *);
+/* net/if_ethersubr.c */
+extern int (*carp_output_p)(struct ifnet *, struct mbuf *,
+    struct sockaddr *, struct rtentry *);
+#ifdef INET
+/* netinet/if_ether.c */
+extern int (*carp_iamatch_p)(struct ifnet *, struct in_ifaddr *,
+    struct in_addr *, u_int8_t **);
+#endif
+#ifdef INET6
+/* netinet6/nd6_nbr.c */
+extern struct ifaddr *(*carp_iamatch6_p)(struct ifnet *, struct in6_addr *);
+extern caddr_t (*carp_macmatch6_p)(struct ifnet *, struct mbuf *,
+    const struct in6_addr *);
+#endif
+
 int carp_suppress_preempt = 0;
 int carp_opts[CARPCTL_MAXID] = { 0, 1, 0, 1, 0, 0 };	/* XXX for now */
+SYSCTL_NODE(_net_inet, IPPROTO_CARP,	carp,	CTLFLAG_RW, 0,	"CARP");
 SYSCTL_INT(_net_inet_carp, CARPCTL_ALLOW, allow, CTLFLAG_RW,
     &carp_opts[CARPCTL_ALLOW], 0, "Accept incoming CARP packets");
 SYSCTL_INT(_net_inet_carp, CARPCTL_PREEMPT, preempt, CTLFLAG_RW,
@@ -160,6 +182,10 @@ struct carp_if {
 	struct mtx	 vhif_mtx;
 };
 
+#define	CARP_INET	0
+#define	CARP_INET6	1
+static int proto_reg[] = {-1, -1};
+
 /* Get carp_if from softc. Valid after carp_set_addr{,6}. */
 #define	SC2CIF(sc)		((struct carp_if *)(sc)->sc_carpdev->if_carp)
 
@@ -1146,14 +1172,15 @@ carp_addrcount(struct carp_if *cif, struct in_ifaddr *ia, int type)
 }
 
 int
-carp_iamatch(void *v, struct in_ifaddr *ia,
+carp_iamatch(struct ifnet *ifp, struct in_ifaddr *ia,
     struct in_addr *isaddr, u_int8_t **enaddr)
 {
-	struct carp_if *cif = v;
+	struct carp_if *cif;
 	struct carp_softc *vh;
 	int index, count = 0;
 	struct ifaddr *ifa;
 
+	cif = ifp->if_carp;
 	CARP_LOCK(cif);
 
 	if (carp_opts[CARPCTL_ARPBALANCE]) {
@@ -1222,12 +1249,13 @@ carp_iamatch(void *v, struct in_ifaddr *ia,
 
 #ifdef INET6
 struct ifaddr *
-carp_iamatch6(void *v, struct in6_addr *taddr)
+carp_iamatch6(struct ifnet *ifp, struct in6_addr *taddr)
 {
-	struct carp_if *cif = v;
+	struct carp_if *cif;
 	struct carp_softc *vh;
 	struct ifaddr *ifa;
 
+	cif = ifp->if_carp;
 	CARP_LOCK(cif);
 	TAILQ_FOREACH(vh, &cif->vhif_vrs, sc_list) {
 		IF_ADDR_LOCK(SC2IFP(vh));
@@ -1250,14 +1278,15 @@ carp_iamatch6(void *v, struct in6_addr *taddr)
 	return (NULL);
 }
 
-void *
-carp_macmatch6(void *v, struct mbuf *m, const struct in6_addr *taddr)
+caddr_t
+carp_macmatch6(struct ifnet *ifp, struct mbuf *m, const struct in6_addr *taddr)
 {
 	struct m_tag *mtag;
-	struct carp_if *cif = v;
+	struct carp_if *cif;
 	struct carp_softc *sc;
 	struct ifaddr *ifa;
 
+	cif = ifp->if_carp;
 	CARP_LOCK(cif);
 	TAILQ_FOREACH(sc, &cif->vhif_vrs, sc_list) {
 		IF_ADDR_LOCK(SC2IFP(sc));
@@ -1293,15 +1322,16 @@ carp_macmatch6(void *v, struct mbuf *m, const struct in6_addr *taddr)
 #endif
 
 struct ifnet *
-carp_forus(void *v, void *dhost)
+carp_forus(struct ifnet *ifp, u_char *dhost)
 {
-	struct carp_if *cif = v;
+	struct carp_if *cif;
 	struct carp_softc *vh;
 	u_int8_t *ena = dhost;
 
 	if (ena[0] || ena[1] || ena[2] != 0x5e || ena[3] || ena[4] != 1)
 		return (NULL);
 
+	cif = ifp->if_carp;
 	CARP_LOCK(cif);
 	TAILQ_FOREACH(vh, &cif->vhif_vrs, sc_list)
 		if ((SC2IFP(vh)->if_flags & IFF_UP) &&
@@ -2210,10 +2240,11 @@ carp_set_state(struct carp_softc *sc, int state)
 }
 
 void
-carp_carpdev_state(void *v)
+carp_carpdev_state(struct ifnet *ifp)
 {
-	struct carp_if *cif = v;
+	struct carp_if *cif;
 
+	cif = ifp->if_carp;
 	CARP_LOCK(cif);
 	carp_carpdev_state_locked(cif);
 	CARP_UNLOCK(cif);
@@ -2264,24 +2295,121 @@ carp_sc_state_locked(struct carp_softc *sc)
 	return;
 }
 
+#ifdef INET
+extern  struct domain inetdomain;
+static struct protosw in_carp_protosw = {
+	.pr_type =		SOCK_RAW,
+	.pr_domain =		&inetdomain,
+	.pr_protocol =		IPPROTO_CARP,
+	.pr_flags =		PR_ATOMIC|PR_ADDR,
+	.pr_input =		carp_input,
+	.pr_output =		(pr_output_t *)rip_output,
+	.pr_ctloutput =		rip_ctloutput,
+	.pr_usrreqs =		&rip_usrreqs
+};
+#endif
+
+#ifdef INET6
+extern	struct domain inet6domain;
+static struct ip6protosw in6_carp_protosw = {
+	.pr_type =		SOCK_RAW,
+	.pr_domain =		&inet6domain,
+	.pr_protocol =		IPPROTO_CARP,
+	.pr_flags =		PR_ATOMIC|PR_ADDR,
+	.pr_input =		carp6_input,
+	.pr_output =		rip6_output,
+	.pr_ctloutput =		rip6_ctloutput,
+	.pr_usrreqs =		&rip6_usrreqs
+};
+#endif
+
+static void
+carp_mod_cleanup(void)
+{
+
+	if (if_detach_event_tag == NULL)
+		return;
+	EVENTHANDLER_DEREGISTER(ifnet_departure_event, if_detach_event_tag);
+	if_clone_detach(&carp_cloner);
+#ifdef INET
+	if (proto_reg[CARP_INET] == 0) {
+		pf_proto_unregister(PF_INET, IPPROTO_CARP, SOCK_RAW);
+		proto_reg[CARP_INET] = -1;
+	}
+	carp_iamatch_p = NULL;
+#endif
+#ifdef INET6
+	if (proto_reg[CARP_INET6] == 0) {
+		pf_proto_unregister(PF_INET6, IPPROTO_CARP, SOCK_RAW);
+		proto_reg[CARP_INET6] = -1;
+	}
+	carp_iamatch6_p = NULL;
+	carp_macmatch6_p = NULL;
+#endif
+	carp_linkstate_p = NULL;
+	carp_forus_p = NULL;
+	carp_output_p = NULL;
+	mtx_destroy(&carp_mtx);
+}
+
+static int
+carp_mod_load(void)
+{
+
+	if_detach_event_tag = EVENTHANDLER_REGISTER(ifnet_departure_event,
+		carp_ifdetach, NULL, EVENTHANDLER_PRI_ANY);
+	if (if_detach_event_tag == NULL)
+		return (ENOMEM);
+	mtx_init(&carp_mtx, "carp_mtx", NULL, MTX_DEF);
+	LIST_INIT(&carpif_list);
+	if_clone_attach(&carp_cloner);
+	carp_linkstate_p = carp_carpdev_state;
+	carp_forus_p = carp_forus;
+	carp_output_p = carp_output;
+#ifdef INET6
+	carp_iamatch6_p = carp_iamatch6;
+	carp_macmatch6_p = carp_macmatch6;
+	proto_reg[CARP_INET6] = pf_proto_register(PF_INET6,
+	    (struct protosw *)&in6_carp_protosw);
+	if (proto_reg[CARP_INET6] != 0) {
+		printf("carp: error %d attaching to PF_INET6\n",
+		    proto_reg[CARP_INET6]);
+		carp_mod_cleanup();
+		return (EINVAL);
+	}
+#endif
+#ifdef INET
+	carp_iamatch_p = carp_iamatch;
+	proto_reg[CARP_INET] = pf_proto_register(PF_INET, &in_carp_protosw);
+	if (proto_reg[CARP_INET] != 0) {
+		printf("carp: error %d attaching to PF_INET\n",
+		    proto_reg[CARP_INET]);
+		carp_mod_cleanup();
+		return (EINVAL);
+	}
+#endif
+	return 0;
+}
+
 static int
 carp_modevent(module_t mod, int type, void *data)
 {
 	switch (type) {
 	case MOD_LOAD:
-		if_detach_event_tag = EVENTHANDLER_REGISTER(ifnet_departure_event,
-		    carp_ifdetach, NULL, EVENTHANDLER_PRI_ANY);
-		if (if_detach_event_tag == NULL)
-			return (ENOMEM);
-		mtx_init(&carp_mtx, "carp_mtx", NULL, MTX_DEF);
-		LIST_INIT(&carpif_list);
-		if_clone_attach(&carp_cloner);
-		break;
-
+		return carp_mod_load();
+		/* NOTREACHED */
 	case MOD_UNLOAD:
-		EVENTHANDLER_DEREGISTER(ifnet_departure_event, if_detach_event_tag);
-		if_clone_detach(&carp_cloner);
-		mtx_destroy(&carp_mtx);
+		/*
+		 * XXX: For now, disallow module unloading by default due to
+		 * a race condition where a thread may dereference one of the
+		 * function pointer hooks after the module has been
+		 * unloaded, during processing of a packet, causing a panic.
+		 */
+#ifdef CARPMOD_CAN_UNLOAD
+		carp_mod_cleanup();
+#else
+		return (EBUSY);
+#endif
 		break;
 
 	default:
diff --git a/sys/netinet/ip_carp.h b/sys/netinet/ip_carp.h
index 47bb494..1e06806 100644
--- a/sys/netinet/ip_carp.h
+++ b/sys/netinet/ip_carp.h
@@ -157,15 +157,15 @@ struct carpreq {
 }
 
 #ifdef _KERNEL
-void		 carp_carpdev_state(void *);
+void		 carp_carpdev_state(struct ifnet *);
 void		 carp_input (struct mbuf *, int);
 int		 carp6_input (struct mbuf **, int *, int);
 int		 carp_output (struct ifnet *, struct mbuf *, struct sockaddr *,
 		     struct rtentry *);
-int		 carp_iamatch (void *, struct in_ifaddr *, struct in_addr *,
+int		 carp_iamatch (struct ifnet *, struct in_ifaddr *, struct in_addr *,
 		     u_int8_t **);
-struct ifaddr	*carp_iamatch6(void *, struct in6_addr *);
-void		*carp_macmatch6(void *, struct mbuf *, const struct in6_addr *);
-struct	ifnet	*carp_forus (void *, void *);
+struct ifaddr	*carp_iamatch6(struct ifnet *, struct in6_addr *);
+caddr_t		carp_macmatch6(struct ifnet *, struct mbuf *, const struct in6_addr *);
+struct	ifnet	*carp_forus (struct ifnet *, u_char *);
 #endif
 #endif /* _IP_CARP_H */
diff --git a/sys/netinet/ip_input.c b/sys/netinet/ip_input.c
index a17907c..40b494a 100644
--- a/sys/netinet/ip_input.c
+++ b/sys/netinet/ip_input.c
@@ -37,7 +37,6 @@ __FBSDID("$FreeBSD$");
 #include "opt_ipstealth.h"
 #include "opt_ipsec.h"
 #include "opt_route.h"
-#include "opt_carp.h"
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -74,9 +73,7 @@ __FBSDID("$FreeBSD$");
 #include <netinet/ip_icmp.h>
 #include <netinet/ip_options.h>
 #include <machine/in_cksum.h>
-#ifdef DEV_CARP
 #include <netinet/ip_carp.h>
-#endif
 #ifdef IPSEC
 #include <netinet/ip_ipsec.h>
 #endif /* IPSEC */
@@ -606,10 +603,7 @@ passin:
 	 */
 	checkif = V_ip_checkinterface && (V_ipforwarding == 0) && 
 	    ifp != NULL && ((ifp->if_flags & IFF_LOOPBACK) == 0) &&
-#ifdef DEV_CARP
-	    !ifp->if_carp &&
-#endif
-	    (dchg == 0);
+	    ifp->if_carp == NULL && (dchg == 0);
 
 	/*
 	 * Check for exact addresses in the hash bucket.
diff --git a/sys/netinet6/in6_proto.c b/sys/netinet6/in6_proto.c
index 4aec9cc..3a8a0e6 100644
--- a/sys/netinet6/in6_proto.c
+++ b/sys/netinet6/in6_proto.c
@@ -67,7 +67,6 @@ __FBSDID("$FreeBSD$");
 #include "opt_inet6.h"
 #include "opt_ipsec.h"
 #include "opt_ipstealth.h"
-#include "opt_carp.h"
 #include "opt_sctp.h"
 #include "opt_mpath.h"
 #include "opt_route.h"
@@ -112,10 +111,6 @@ __FBSDID("$FreeBSD$");
 #include <netinet6/pim6_var.h>
 #include <netinet6/nd6.h>
 
-#ifdef DEV_CARP
-#include <netinet/ip_carp.h>
-#endif
-
 #ifdef SCTP
 #include <netinet/in_pcb.h>
 #include <netinet/sctp_pcb.h>
@@ -336,18 +331,6 @@ struct ip6protosw inet6sw[] = {
 	.pr_ctloutput =		rip6_ctloutput,
 	.pr_usrreqs =		&rip6_usrreqs
 },
-#ifdef DEV_CARP
-{
-	.pr_type =		SOCK_RAW,
-	.pr_domain =		&inet6domain,
-	.pr_protocol =		IPPROTO_CARP,
-	.pr_flags =		PR_ATOMIC|PR_ADDR,
-	.pr_input =		carp6_input,
-	.pr_output =		rip6_output,
-	.pr_ctloutput =		rip6_ctloutput,
-	.pr_usrreqs =		&rip6_usrreqs
-},
-#endif /* DEV_CARP */
 /* Spacer n-times for loadable protocols. */
 IP6PROTOSPACER,
 IP6PROTOSPACER,
diff --git a/sys/netinet6/nd6_nbr.c b/sys/netinet6/nd6_nbr.c
index a50925b..eca85eb 100644
--- a/sys/netinet6/nd6_nbr.c
+++ b/sys/netinet6/nd6_nbr.c
@@ -35,7 +35,6 @@ __FBSDID("$FreeBSD$");
 #include "opt_inet.h"
 #include "opt_inet6.h"
 #include "opt_ipsec.h"
-#include "opt_carp.h"
 #include "opt_mpath.h"
 
 #include <sys/param.h>
@@ -74,10 +73,6 @@ __FBSDID("$FreeBSD$");
 #include <netinet6/nd6.h>
 #include <netinet/icmp6.h>
 
-#ifdef DEV_CARP
-#include <netinet/ip_carp.h>
-#endif
-
 #define SDL(s) ((struct sockaddr_dl *)s)
 
 struct dadq;
@@ -89,6 +84,10 @@ static void nd6_dad_ns_output(struct dadq *, struct ifaddr *);
 static void nd6_dad_ns_input(struct ifaddr *);
 static void nd6_dad_na_input(struct ifaddr *);
 
+struct ifaddr *(*carp_iamatch6_p)(struct ifnet *, struct in6_addr *);
+caddr_t (*carp_macmatch6_p)(struct ifnet *, struct mbuf *,
+    const struct in6_addr *);
+
 VNET_DEFINE(int, dad_ignore_ns) = 0;	/* ignore NS in DAD - specwise incorrect*/
 VNET_DEFINE(int, dad_maxtry) = 15;	/* max # of *tries* to transmit DAD packet */
 #define	V_dad_ignore_ns			VNET(dad_ignore_ns)
@@ -222,14 +221,10 @@ nd6_ns_input(struct mbuf *m, int off, int icmp6len)
 	 * (3) "tentative" address on which DAD is being performed.
 	 */
 	/* (1) and (3) check. */
-#ifdef DEV_CARP
 	if (ifp->if_carp)
-		ifa = carp_iamatch6(ifp->if_carp, &taddr6);
+		ifa = (*carp_iamatch6_p)(ifp, &taddr6);
 	if (ifa == NULL)
 		ifa = (struct ifaddr *)in6ifa_ifpwithaddr(ifp, &taddr6);
-#else
-	ifa = (struct ifaddr *)in6ifa_ifpwithaddr(ifp, &taddr6);
-#endif
 
 	/* (2) check. */
 	if (ifa == NULL) {
@@ -1029,14 +1024,10 @@ nd6_na_output(struct ifnet *ifp, const struct in6_addr *daddr6_0,
 		 * my address) use lladdr configured for the interface.
 		 */
 		if (sdl0 == NULL) {
-#ifdef DEV_CARP
 			if (ifp->if_carp)
-				mac = carp_macmatch6(ifp->if_carp, m, taddr6);
+				mac = (*carp_macmatch6_p)(ifp, m, taddr6);
 			if (mac == NULL)
 				mac = nd6_ifptomac(ifp);
-#else
-			mac = nd6_ifptomac(ifp);
-#endif
 		} else if (sdl0->sa_family == AF_LINK) {
 			struct sockaddr_dl *sdl;
 			sdl = (struct sockaddr_dl *)sdl0;