Speed up rctl operation with large rulesets, by holding the lock

during iteration instead of relocking it for each traversed rule.

Reviewed by:	mjg@
MFC after:	1 month
Sponsored by:	The FreeBSD Foundation
Differential Revision:	https://reviews.freebsd.org/D4110

5 files changed, 72 insertions, 20 deletions

diff --git a/sys/kern/kern_jail.c b/sys/kern/kern_jail.c
index 05a5632..c8b541f 100644
--- a/sys/kern/kern_jail.c
+++ b/sys/kern/kern_jail.c
@@ -4480,15 +4480,20 @@ SYSCTL_JAIL_PARAM(_allow_mount, zfs, CTLTYPE_INT | CTLFLAG_RW,
 #ifdef RACCT
 void
 prison_racct_foreach(void (*callback)(struct racct *racct,
-    void *arg2, void *arg3), void *arg2, void *arg3)
+    void *arg2, void *arg3), void (*pre)(void), void (*post)(void),
+    void *arg2, void *arg3)
 {
 	struct prison_racct *prr;
 
 	ASSERT_RACCT_ENABLED();
 
 	sx_slock(&allprison_lock);
+	if (pre != NULL)
+		(pre)();
 	LIST_FOREACH(prr, &allprison_racct, prr_next)
 		(callback)(prr->prr_racct, arg2, arg3);
+	if (post != NULL)
+		(post)();
 	sx_sunlock(&allprison_lock);
 }
 
diff --git a/sys/kern/kern_loginclass.c b/sys/kern/kern_loginclass.c
index 62e6421..07ef910 100644
--- a/sys/kern/kern_loginclass.c
+++ b/sys/kern/kern_loginclass.c
@@ -234,12 +234,17 @@ sys_setloginclass(struct thread *td, struct setloginclass_args *uap)
 
 void
 loginclass_racct_foreach(void (*callback)(struct racct *racct,
-    void *arg2, void *arg3), void *arg2, void *arg3)
+    void *arg2, void *arg3), void (*pre)(void), void (*post)(void),
+    void *arg2, void *arg3)
 {
 	struct loginclass *lc;
 
 	rw_rlock(&loginclasses_lock);
+	if (pre != NULL)
+		(pre)();
 	LIST_FOREACH(lc, &loginclasses, lc_next)
 		(callback)(lc->lc_racct, arg2, arg3);
+	if (post != NULL)
+		(post)();
 	rw_runlock(&loginclasses_lock);
 }
diff --git a/sys/kern/kern_racct.c b/sys/kern/kern_racct.c
index dde0baf..941da42 100644
--- a/sys/kern/kern_racct.c
+++ b/sys/kern/kern_racct.c
@@ -1156,6 +1156,7 @@ racct_decay_resource(struct racct *racct, void * res, void* dummy)
 	int64_t r_old, r_new;
 
 	ASSERT_RACCT_ENABLED();
+	mtx_assert(&racct_lock, MA_OWNED);
 
 	resource = *(int *)res;
 	r_old = racct->r_resources[resource];
@@ -1164,9 +1165,21 @@ racct_decay_resource(struct racct *racct, void * res, void* dummy)
 	if (r_old <= 0)
 		return;
 
-	mtx_lock(&racct_lock);
 	r_new = r_old * RACCT_DECAY_FACTOR / FSCALE;
 	racct->r_resources[resource] = r_new;
+}
+
+static void
+racct_decay_pre(void)
+{
+
+	mtx_lock(&racct_lock);
+}
+
+static void
+racct_decay_post(void)
+{
+
 	mtx_unlock(&racct_lock);
 }
 
@@ -1176,9 +1189,12 @@ racct_decay(int resource)
 
 	ASSERT_RACCT_ENABLED();
 
-	ui_racct_foreach(racct_decay_resource, &resource, NULL);
-	loginclass_racct_foreach(racct_decay_resource, &resource, NULL);
-	prison_racct_foreach(racct_decay_resource, &resource, NULL);
+	ui_racct_foreach(racct_decay_resource, racct_decay_pre,
+	    racct_decay_post, &resource, NULL);
+	loginclass_racct_foreach(racct_decay_resource, racct_decay_pre,
+	    racct_decay_post, &resource, NULL);
+	prison_racct_foreach(racct_decay_resource, racct_decay_pre,
+	    racct_decay_post, &resource, NULL);
 }
 
 static void
diff --git a/sys/kern/kern_rctl.c b/sys/kern/kern_rctl.c
index 18e4cc2..c04c58b 100644
--- a/sys/kern/kern_rctl.c
+++ b/sys/kern/kern_rctl.c
@@ -1150,16 +1150,29 @@ rctl_rule_add(struct rctl_rule *rule)
 }
 
 static void
+rctl_rule_pre_callback(void)
+{
+
+	rw_wlock(&rctl_lock);
+}
+
+static void
+rctl_rule_post_callback(void)
+{
+
+	rw_wunlock(&rctl_lock);
+}
+
+static void
 rctl_rule_remove_callback(struct racct *racct, void *arg2, void *arg3)
 {
 	struct rctl_rule *filter = (struct rctl_rule *)arg2;
 	int found = 0;
 
 	ASSERT_RACCT_ENABLED();
+	rw_assert(&rctl_lock, RA_WLOCKED);
 
-	rw_wlock(&rctl_lock);
 	found += rctl_racct_remove_rules(racct, filter);
-	rw_wunlock(&rctl_lock);
 
 	*((int *)arg3) += found;
 }
@@ -1186,12 +1199,15 @@ rctl_rule_remove(struct rctl_rule *filter)
 		return (ESRCH);
 	}
 
-	loginclass_racct_foreach(rctl_rule_remove_callback, filter,
-	    (void *)&found);
-	ui_racct_foreach(rctl_rule_remove_callback, filter,
-	    (void *)&found);
-	prison_racct_foreach(rctl_rule_remove_callback, filter,
-	    (void *)&found);
+	loginclass_racct_foreach(rctl_rule_remove_callback,
+	    rctl_rule_pre_callback, rctl_rule_post_callback,
+	    filter, (void *)&found);
+	ui_racct_foreach(rctl_rule_remove_callback,
+	    rctl_rule_pre_callback, rctl_rule_post_callback,
+	    filter, (void *)&found);
+	prison_racct_foreach(rctl_rule_remove_callback,
+	    rctl_rule_pre_callback, rctl_rule_post_callback,
+	    filter, (void *)&found);
 
 	sx_assert(&allproc_lock, SA_LOCKED);
 	rw_wlock(&rctl_lock);
@@ -1425,15 +1441,14 @@ rctl_get_rules_callback(struct racct *racct, void *arg2, void *arg3)
 	struct sbuf *sb = (struct sbuf *)arg3;
 
 	ASSERT_RACCT_ENABLED();
+	rw_assert(&rctl_lock, RA_LOCKED);
 
-	rw_rlock(&rctl_lock);
 	LIST_FOREACH(link, &racct->r_rule_links, rrl_next) {
 		if (!rctl_rule_matches(link->rrl_rule, filter))
 			continue;
 		rctl_rule_to_sbuf(sb, link->rrl_rule);
 		sbuf_printf(sb, ",");
 	}
-	rw_runlock(&rctl_lock);
 }
 
 int
@@ -1494,9 +1509,15 @@ sys_rctl_get_rules(struct thread *td, struct rctl_get_rules_args *uap)
 		rw_runlock(&rctl_lock);
 	}
 
-	loginclass_racct_foreach(rctl_get_rules_callback, filter, sb);
-	ui_racct_foreach(rctl_get_rules_callback, filter, sb);
-	prison_racct_foreach(rctl_get_rules_callback, filter, sb);
+	loginclass_racct_foreach(rctl_get_rules_callback,
+	    rctl_rule_pre_callback, rctl_rule_post_callback,
+	    filter, sb);
+	ui_racct_foreach(rctl_get_rules_callback,
+	    rctl_rule_pre_callback, rctl_rule_post_callback,
+	    filter, sb);
+	prison_racct_foreach(rctl_get_rules_callback,
+	    rctl_rule_pre_callback, rctl_rule_post_callback,
+	    filter, sb);
 	if (sbuf_error(sb) == ENOMEM) {
 		error = ERANGE;
 		goto out;
diff --git a/sys/kern/kern_resource.c b/sys/kern/kern_resource.c
index b6dbab9..4edbcba 100644
--- a/sys/kern/kern_resource.c
+++ b/sys/kern/kern_resource.c
@@ -1356,17 +1356,22 @@ uifree(struct uidinfo *uip)
 #ifdef RACCT
 void
 ui_racct_foreach(void (*callback)(struct racct *racct,
-    void *arg2, void *arg3), void *arg2, void *arg3)
+    void *arg2, void *arg3), void (*pre)(void), void (*post)(void),
+    void *arg2, void *arg3)
 {
 	struct uidinfo *uip;
 	struct uihashhead *uih;
 
 	rw_rlock(&uihashtbl_lock);
+	if (pre != NULL)
+		(pre)();
 	for (uih = &uihashtbl[uihash]; uih >= uihashtbl; uih--) {
 		LIST_FOREACH(uip, uih, ui_hash) {
 			(callback)(uip->ui_racct, arg2, arg3);
 		}
 	}
+	if (post != NULL)
+		(post)();
 	rw_runlock(&uihashtbl_lock);
 }
 #endif