Restore a feature that was present in 5.x and 6.x, and was cleared in

7.x, 8.x and 9.x with pf(4) imports: pfsync(4) should suppress CARP
preemption, while it is running its bulk update.

However, reimplement the feature in more elegant manner, that is
partially inspired by newer OpenBSD:

- Rename term "suppression" to "demotion", to match with OpenBSD.
- Keep a global demotion factor, that can be raised by several
  conditions, for now these are:
  - interface goes down
  - carp(4) has problems with ip_output() or ip6_output()
  - pfsync performs bulk update
- Unlike in OpenBSD the demotion factor isn't a counter, but
  is actual value added to advskew. The adjustment values for
  particular error conditions are also configurable, and their
  defaults are maximum advskew value, so a single failure bumps
  demotion to maximum. This is for POLA compatibility, and should
  satisfy most users.
- Demotion factor is a writable sysctl, so user can do
  foot shooting, if he desires to.

1 files changed, 27 insertions, 39 deletions

diff --git a/sys/contrib/pf/net/if_pfsync.c b/sys/contrib/pf/net/if_pfsync.c
index a8e5fa9..8614c00 100644
--- a/sys/contrib/pf/net/if_pfsync.c
+++ b/sys/contrib/pf/net/if_pfsync.c
@@ -62,12 +62,6 @@ __FBSDID("$FreeBSD$");
 #else
 #define	NPFSYNC		0
 #endif
-
-#ifdef DEV_CARP
-#define	NCARP		DEV_CARP
-#else
-#define	NCARP		0
-#endif
 #endif /* __FreeBSD__ */
 
 #include <sys/param.h>
@@ -127,12 +121,14 @@ __FBSDID("$FreeBSD$");
 #include <netinet6/nd6.h>
 #endif /* INET6 */
 
-#ifndef __FreeBSD__
+#ifdef __FreeBSD__
+#include <netinet/ip_carp.h>
+#else
 #include "carp.h"
-#endif
 #if NCARP > 0
 #include <netinet/ip_carp.h>
 #endif
+#endif
 
 #include <net/pfvar.h>
 #include <net/if_pfsync.h>
@@ -308,11 +304,15 @@ static VNET_DEFINE(struct pfsync_softc	*, pfsyncif) = NULL;
 
 static VNET_DEFINE(struct pfsyncstats, pfsyncstats);
 #define	V_pfsyncstats		VNET(pfsyncstats)
+static VNET_DEFINE(int, pfsync_carp_adj) = CARP_MAXSKEW;
+#define	V_pfsync_carp_adj	VNET(pfsync_carp_adj)
 
 SYSCTL_NODE(_net, OID_AUTO, pfsync, CTLFLAG_RW, 0, "PFSYNC");
 SYSCTL_VNET_STRUCT(_net_pfsync, OID_AUTO, stats, CTLFLAG_RW,
     &VNET_NAME(pfsyncstats), pfsyncstats,
     "PFSYNC statistics (struct pfsyncstats, net/if_pfsync.h)");
+SYSCTL_INT(_net_pfsync, OID_AUTO, carp_demotion_factor, CTLFLAG_RW,
+    &VNET_NAME(pfsync_carp_adj), 0, "pfsync's CARP demotion factor adjustment");
 #else
 struct pfsync_softc	*pfsyncif = NULL;
 struct pfsyncstats	 pfsyncstats;
@@ -505,11 +505,11 @@ pfsync_clone_create(struct if_clone *ifc, int unit)
 	if_attach(ifp);
 #ifndef __FreeBSD__
 	if_alloc_sadl(ifp);
-#endif
 
 #if NCARP > 0
 	if_addgroup(ifp, "carp");
 #endif
+#endif
 
 #if NBPFILTER > 0
 #ifdef __FreeBSD__
@@ -545,14 +545,11 @@ pfsync_clone_destroy(struct ifnet *ifp)
 	timeout_del(&sc->sc_tmo);
 #ifdef __FreeBSD__
 	PF_UNLOCK();
-#endif
-#if NCARP > 0
-#ifdef notyet
-#ifdef __FreeBSD__
-	if (!sc->pfsync_sync_ok)
+	if (!sc->pfsync_sync_ok && carp_demote_adj_p)
+		(*carp_demote_adj_p)(-V_pfsync_carp_adj, "pfsync destroy");
 #else
+#if NCARP > 0
 	if (!pfsync_sync_ok)
-#endif
 		carp_group_demote_adj(&sc->sc_if, -1);
 #endif
 #endif
@@ -1636,19 +1633,16 @@ pfsync_in_bus(struct pfsync_pkt *pkt, struct mbuf *m, int offset, int count)
 			sc->sc_ureq_sent = 0;
 			sc->sc_bulk_tries = 0;
 			timeout_del(&sc->sc_bulkfail_tmo);
-#if NCARP > 0
-#ifdef notyet
 #ifdef __FreeBSD__
-			if (!sc->pfsync_sync_ok)
+			if (!sc->pfsync_sync_ok && carp_demote_adj_p)
+				(*carp_demote_adj_p)(-V_pfsync_carp_adj,
+				    "pfsync bulk done");
+			sc->pfsync_sync_ok = 1;
 #else
+#if NCARP > 0
 			if (!pfsync_sync_ok)
-#endif
 				carp_group_demote_adj(&sc->sc_if, -1);
 #endif
-#endif
-#ifdef __FreeBSD__
-			sc->pfsync_sync_ok = 1;
-#else
 			pfsync_sync_ok = 1;
 #endif
 #ifdef __FreeBSD__
@@ -1988,19 +1982,16 @@ pfsyncioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 		if (sc->sc_sync_if) {
 			/* Request a full state table update. */
 			sc->sc_ureq_sent = time_uptime;
-#if NCARP > 0
-#ifdef notyet
 #ifdef __FreeBSD__
-			if (sc->pfsync_sync_ok)
+			if (sc->pfsync_sync_ok && carp_demote_adj_p)
+				(*carp_demote_adj_p)(V_pfsync_carp_adj,
+				    "pfsync bulk start");
+			sc->pfsync_sync_ok = 0;
 #else
+#if NCARP > 0
 			if (pfsync_sync_ok)
-#endif
 				carp_group_demote_adj(&sc->sc_if, 1);
 #endif
-#endif
-#ifdef __FreeBSD__
-			sc->pfsync_sync_ok = 0;
-#else
 			pfsync_sync_ok = 0;
 #endif
 #ifdef __FreeBSD__
@@ -3159,19 +3150,16 @@ pfsync_bulk_fail(void *arg)
 		/* Pretend like the transfer was ok */
 		sc->sc_ureq_sent = 0;
 		sc->sc_bulk_tries = 0;
-#if NCARP > 0
-#ifdef notyet
 #ifdef __FreeBSD__
-		if (!sc->pfsync_sync_ok)
+		if (!sc->pfsync_sync_ok && carp_demote_adj_p)
+			(*carp_demote_adj_p)(-V_pfsync_carp_adj,
+			    "pfsync bulk fail");
+		sc->pfsync_sync_ok = 1;
 #else
+#if NCARP > 0
 		if (!pfsync_sync_ok)
-#endif
 			carp_group_demote_adj(&sc->sc_if, -1);
 #endif
-#endif
-#ifdef __FreeBSD__
-		sc->pfsync_sync_ok = 1;
-#else
 		pfsync_sync_ok = 1;
 #endif
 #ifdef __FreeBSD__