Make PFIL use per-VNET lock instead of per-AF lock. Since most used packet

filters (ipfw and PF) use the same ruleset with the same lock for both
AF_INET and AF_INET6 there is no need in more fine-grade locking.
However, it is possible to request personal lock by specifying
PFIL_FLAG_PRIVATE_LOCK flag in pfil_head structure (see pfil.9 for
more details).

Export PFIL lock via rw_lock(9)/rm_lock(9)-like API permitting pfil consumers
to use this lock instead of own lock. This help reducing locks on main
traffic path.

pfil_assert() is currently not implemented due to absense of rm_assert().
Waiting for some kind of r234648 to be merged in HEAD.

This change is part of bigger patch reducing routing locking.

Sponsored by:	Yandex LLC
Reviewed by:	glebius, ae
OK'd by:	silence on net@
MFC after:	3 weeks

3 files changed, 147 insertions, 12 deletions

diff --git a/share/man/man9/pfil.9 b/share/man/man9/pfil.9
index b6d3a03..9425f5d 100644
--- a/share/man/man9/pfil.9
+++ b/share/man/man9/pfil.9
@@ -28,7 +28,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd October 6, 2012
+.Dd October 22, 2012
 .Dt PFIL 9
 .Os
 .Sh NAME
@@ -39,7 +39,11 @@
 .Nm pfil_hook_get ,
 .Nm pfil_add_hook ,
 .Nm pfil_remove_hook ,
-.Nm pfil_run_hooks
+.Nm pfil_run_hooks ,
+.Nm pfil_rlock ,
+.Nm pfil_runlock ,
+.Nm pfil_wlock ,
+.Nm pfil_wunlock
 .Nd packet filter interface
 .Sh SYNOPSIS
 .In sys/param.h
@@ -62,6 +66,14 @@
 .Fn (*func) "void *arg" "struct mbuf **mp" "struct ifnet *" "int dir" "struct inpcb *"
 .Ft int
 .Fn pfil_run_hooks "struct pfil_head *head" "struct mbuf **mp" "struct ifnet *" "int dir" "struct inpcb *"
+.Ft void
+.Fn pfil_rlock "struct pfil_head *" "struct rm_priotracker *"
+.Ft void
+.Fn pfil_runlock "struct pfil_head *" "struct rm_priotracker *"
+.Ft void
+.Fn pfil_wlock "struct pfil_head *"
+.Ft void
+.Fn pfil_wunlock "struct pfil_head *"
 .Sh DESCRIPTION
 The
 .Nm
@@ -86,6 +98,16 @@ The data link type is a
 .Xr bpf 4
 DLT constant indicating what kind of header is present on the packet
 at the filtering point.
+Each filtering point uses common per-VNET rmlock by default.
+This can be changed by specifying
+.Vt PFIL_FLAG_PRIVATE_LOCK
+as
+.Vt "flags"
+field in the
+.Vt pfil_head
+structure.
+Note that specifying private lock can break filters sharing the same
+ruleset and/or state between different data link types.
 Filtering points may be unregistered with the
 .Fn pfil_head_unregister
 function.
@@ -122,6 +144,31 @@ The filter returns an error (errno) if the packet processing is to stop, or 0
 if the processing is to continue.
 If the packet processing is to stop, it is the responsibility of the
 filter to free the packet.
+.Pp
+Every filter hook is called with
+.Nm
+read lock held.
+All heads uses the same lock within the same VNET instance.
+Packet filter can use this lock instead of own locking model to
+improve performance.
+Since
+.Nm
+uses
+.Xr rmlock 9
+.Fn pfil_rlock
+and
+.Fn pfil_runlock
+require
+.Va struct rm_priotracker
+to be passed as argument.
+Filter can acquire and release writer lock via
+.Fn pfil_wlock
+and
+.Fn pfil_wunlock
+functions.
+See
+.Xr rmlock 9
+for more details.
 .Sh FILTERING POINTS
 Currently, filtering points are implemented for the following link types:
 .Pp
@@ -157,6 +204,7 @@ might sleep!
 .Sh SEE ALSO
 .Xr bpf 4 ,
 .Xr if_bridge 4
+.Xr rmlock 4
 .Sh HISTORY
 The
 .Nm
@@ -192,6 +240,9 @@ as well as be less IP-centric.
 .Pp
 Fine-grained locking was added in
 .Fx 5.2 .
+.Nm
+lock export was added in
+.Fx 10.0 .
 .Sh BUGS
 The
 .Fn pfil_hook_get
diff --git a/sys/net/pfil.c b/sys/net/pfil.c
index a11950f..06da0be 100644
--- a/sys/net/pfil.c
+++ b/sys/net/pfil.c
@@ -61,6 +61,8 @@ static int pfil_list_remove(pfil_list_t *,
 LIST_HEAD(pfilheadhead, pfil_head);
 VNET_DEFINE(struct pfilheadhead, pfil_head_list);
 #define	V_pfil_head_list	VNET(pfil_head_list)
+VNET_DEFINE(struct rmlock, pfil_lock);
+#define	V_pfil_lock	VNET(pfil_lock)
 
 /*
  * pfil_run_hooks() runs the specified packet filter hooks.
@@ -91,6 +93,60 @@ pfil_run_hooks(struct pfil_head *ph, struct mbuf **mp, struct ifnet *ifp,
 }
 
 /*
+ * pfil_try_rlock() acquires rm reader lock for specified head
+ * if this is immediately possible,
+ */
+int
+pfil_try_rlock(struct pfil_head *ph, struct rm_priotracker *tracker)
+{
+	return PFIL_TRY_RLOCK(ph, tracker);
+}
+
+/*
+ * pfil_rlock() acquires rm reader lock for specified head.
+ */
+void
+pfil_rlock(struct pfil_head *ph, struct rm_priotracker *tracker)
+{
+	PFIL_RLOCK(ph, tracker);
+}
+
+/*
+ * pfil_runlock() releases reader lock for specified head.
+ */
+void
+pfil_runlock(struct pfil_head *ph, struct rm_priotracker *tracker)
+{
+	PFIL_RUNLOCK(ph, tracker);
+}
+
+/*
+ * pfil_wlock() acquires writer lock for specified head.
+ */
+void
+pfil_wlock(struct pfil_head *ph)
+{
+	PFIL_WLOCK(ph);
+}
+
+/*
+ * pfil_wunlock() releases writer lock for specified head.
+ */
+void
+pfil_wunlock(struct pfil_head *ph)
+{
+	PFIL_WUNLOCK(ph);
+}
+
+/*
+ * pfil_wowned() releases writer lock for specified head.
+ */
+int
+pfil_wowned(struct pfil_head *ph)
+{
+	return PFIL_WOWNED(ph);
+}
+/*
  * pfil_head_register() registers a pfil_head with the packet filter hook
  * mechanism.
  */
@@ -295,6 +351,7 @@ vnet_pfil_init(const void *unused)
 {
 
 	LIST_INIT(&V_pfil_head_list);
+	PFIL_LOCK_INIT_REAL(&V_pfil_lock, "shared");
 	return (0);
 }
 
@@ -306,6 +363,7 @@ vnet_pfil_uninit(const void *unused)
 {
 
 	/*  XXX should panic if list is not empty */
+	PFIL_LOCK_DESTROY_REAL(&V_pfil_lock);
 	return (0);
 }
 
diff --git a/sys/net/pfil.h b/sys/net/pfil.h
index a8738f1..fabfe9a 100644
--- a/sys/net/pfil.h
+++ b/sys/net/pfil.h
@@ -64,6 +64,8 @@ typedef	TAILQ_HEAD(pfil_list, packet_filter_hook) pfil_list_t;
 #define	PFIL_TYPE_AF		1	/* key is AF_* type */
 #define	PFIL_TYPE_IFNET		2	/* key is ifnet pointer */
 
+#define	PFIL_FLAG_PRIVATE_LOCK	0x01	/* Personal lock instead of global */
+
 struct pfil_head {
 	pfil_list_t	ph_in;
 	pfil_list_t	ph_out;
@@ -72,7 +74,9 @@ struct pfil_head {
 #if defined( __linux__ ) || defined( _WIN32 )
 	rwlock_t	ph_mtx;
 #else
-	struct rmlock	ph_lock;
+	struct rmlock	*ph_plock;	/* Pointer to the used lock */
+	struct rmlock	ph_lock;	/* Private lock storage */
+	int		flags;
 #endif
 	union {
 		u_long		phu_val;
@@ -90,21 +94,43 @@ int	pfil_remove_hook(int (*func)(void *, struct mbuf **, struct ifnet *,
 int	pfil_run_hooks(struct pfil_head *, struct mbuf **, struct ifnet *,
 	    int, struct inpcb *inp);
 
+struct rm_priotracker;	/* Do not require including rmlock header */
+int pfil_try_rlock(struct pfil_head *, struct rm_priotracker *);
+void pfil_rlock(struct pfil_head *, struct rm_priotracker *);
+void pfil_runlock(struct pfil_head *, struct rm_priotracker *);
+void pfil_wlock(struct pfil_head *);
+void pfil_wunlock(struct pfil_head *);
+int pfil_wowned(struct pfil_head *ph);
+
 int	pfil_head_register(struct pfil_head *);
 int	pfil_head_unregister(struct pfil_head *);
 
 struct pfil_head *pfil_head_get(int, u_long);
 
 #define	PFIL_HOOKED(p) ((p)->ph_nhooks > 0)
-#define	PFIL_LOCK_INIT(p) \
-    rm_init_flags(&(p)->ph_lock, "PFil hook read/write mutex", RM_RECURSE)
-#define	PFIL_LOCK_DESTROY(p) rm_destroy(&(p)->ph_lock)
-#define PFIL_RLOCK(p, t) rm_rlock(&(p)->ph_lock, (t))
-#define PFIL_WLOCK(p) rm_wlock(&(p)->ph_lock)
-#define PFIL_RUNLOCK(p, t) rm_runlock(&(p)->ph_lock, (t))
-#define PFIL_WUNLOCK(p) rm_wunlock(&(p)->ph_lock)
-#define PFIL_LIST_LOCK() mtx_lock(&pfil_global_lock)
-#define PFIL_LIST_UNLOCK() mtx_unlock(&pfil_global_lock)
+#define	PFIL_LOCK_INIT_REAL(l, t)	\
+	rm_init_flags(l, "PFil " t " rmlock", RM_RECURSE)
+#define	PFIL_LOCK_DESTROY_REAL(l)	\
+	rm_destroy(l)
+#define	PFIL_LOCK_INIT(p)	do {			\
+	if ((p)->flags & PFIL_FLAG_PRIVATE_LOCK) {	\
+		PFIL_LOCK_INIT_REAL(&(p)->ph_lock, "private");	\
+		(p)->ph_plock = &(p)->ph_lock;		\
+	} else						\
+		(p)->ph_plock = &V_pfil_lock;		\
+} while (0)
+#define	PFIL_LOCK_DESTROY(p)	do {			\
+	if ((p)->flags & PFIL_FLAG_PRIVATE_LOCK)	\
+		PFIL_LOCK_DESTROY_REAL((p)->ph_plock);	\
+} while (0)
+#define PFIL_TRY_RLOCK(p, t)	rm_try_rlock((p)->ph_plock, (t))
+#define PFIL_RLOCK(p, t)	rm_rlock((p)->ph_plock, (t))
+#define PFIL_WLOCK(p)		rm_wlock((p)->ph_plock)
+#define PFIL_RUNLOCK(p, t)	rm_runlock((p)->ph_plock, (t))
+#define PFIL_WUNLOCK(p)		rm_wunlock((p)->ph_plock)
+#define PFIL_WOWNED(p)		rm_wowned((p)->ph_plock)
+#define PFIL_LIST_LOCK()	mtx_lock(&pfil_global_lock)
+#define PFIL_LIST_UNLOCK()	mtx_unlock(&pfil_global_lock)
 
 static __inline struct packet_filter_hook *
 pfil_hook_get(int dir, struct pfil_head *ph)