Add user priority loaning code to support priority propagation for

1:1 threading's POSIX priority mutexes, the code is no-op unless
priority-aware umtx code is committed.

6 files changed, 184 insertions, 10 deletions

diff --git a/sys/kern/kern_umtx.c b/sys/kern/kern_umtx.c
index cb248a8..c6db80c 100644
--- a/sys/kern/kern_umtx.c
+++ b/sys/kern/kern_umtx.c
@@ -601,6 +601,11 @@ kern_umtx_wake(struct thread *td, void *uaddr, int n_wake)
 	return (0);
 }
 
+void
+umtx_pi_adjust(struct thread *td __unused, u_char oldpri __unused)
+{
+}
+
 int
 _umtx_lock(struct thread *td, struct _umtx_lock_args *uap)
     /* struct umtx *umtx */
diff --git a/sys/kern/sched_4bsd.c b/sys/kern/sched_4bsd.c
index 1e34e80..ce7582f 100644
--- a/sys/kern/sched_4bsd.c
+++ b/sys/kern/sched_4bsd.c
@@ -53,6 +53,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/sysctl.h>
 #include <sys/sx.h>
 #include <sys/turnstile.h>
+#include <sys/umtx.h>
 #include <machine/pcb.h>
 #include <machine/smp.h>
 
@@ -586,7 +587,7 @@ resetpriority(struct ksegrp *kg)
 		    NICE_WEIGHT * (kg->kg_proc->p_nice - PRIO_MIN);
 		newpriority = min(max(newpriority, PRI_MIN_TIMESHARE),
 		    PRI_MAX_TIMESHARE);
-		kg->kg_user_pri = newpriority;
+		sched_user_prio(kg, newpriority);
 	}
 }
 
@@ -864,6 +865,60 @@ sched_prio(struct thread *td, u_char prio)
 }
 
 void
+sched_user_prio(struct ksegrp *kg, u_char prio)
+{
+	struct thread *td;
+	u_char oldprio;
+
+	kg->kg_base_user_pri = prio;
+
+	/* XXXKSE only for 1:1 */
+
+	td = TAILQ_FIRST(&kg->kg_threads);
+	if (td == NULL) {
+		kg->kg_user_pri = prio;
+		return;
+	}
+
+	if (td->td_flags & TDF_UBORROWING && kg->kg_user_pri <= prio)
+		return;
+
+	oldprio = kg->kg_user_pri;
+	kg->kg_user_pri = prio;
+
+	if (TD_ON_UPILOCK(td) && oldprio != prio)
+		umtx_pi_adjust(td, oldprio);
+}
+
+void
+sched_lend_user_prio(struct thread *td, u_char prio)
+{
+	u_char oldprio;
+
+	td->td_flags |= TDF_UBORROWING;
+
+	oldprio = td->td_ksegrp->kg_user_pri;
+	td->td_ksegrp->kg_user_pri = prio;
+
+	if (TD_ON_UPILOCK(td) && oldprio != prio)
+		umtx_pi_adjust(td, oldprio);
+}
+
+void
+sched_unlend_user_prio(struct thread *td, u_char prio)
+{
+	struct ksegrp *kg = td->td_ksegrp;
+	u_char base_pri;
+
+	base_pri = kg->kg_base_user_pri;
+	if (prio >= base_pri) {
+		td->td_flags &= ~TDF_UBORROWING;
+		sched_user_prio(kg, base_pri);
+	} else
+		sched_lend_user_prio(td, prio);
+}
+
+void
 sched_sleep(struct thread *td)
 {
 
diff --git a/sys/kern/sched_core.c b/sys/kern/sched_core.c
index f0c4fc2..b2de346 100644
--- a/sys/kern/sched_core.c
+++ b/sys/kern/sched_core.c
@@ -49,6 +49,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/sysctl.h>
 #include <sys/sysproto.h>
 #include <sys/turnstile.h>
+#include <sys/umtx.h>
 #include <sys/unistd.h>
 #include <sys/vmmeter.h>
 #ifdef KTRACE
@@ -633,7 +634,7 @@ sched_calc_pri(struct ksegrp *kg)
 			pri = PUSER_MAX;
 		return (pri);
 	}
-	return (kg->kg_user_pri);
+	return (kg->kg_base_user_pri);
 }
 
 static int
@@ -646,7 +647,7 @@ sched_recalc_pri(struct kse *ke, uint64_t now)
 	kg = ke->ke_ksegrp;
 	delta = now - ke->ke_timestamp;
 	if (__predict_false(!sched_is_timeshare(kg)))
-		return (kg->kg_user_pri);
+		return (kg->kg_base_user_pri);
 
 	if (delta > NS_MAX_SLEEP_TIME)
 		sleep_time = NS_MAX_SLEEP_TIME;
@@ -948,6 +949,60 @@ sched_prio(struct thread *td, u_char prio)
 }
 
 void
+sched_user_prio(struct ksegrp *kg, u_char prio)
+{
+	struct thread *td;
+	u_char oldprio;
+
+	kg->kg_base_user_pri = prio;
+
+	/* XXXKSE only for 1:1 */
+
+	td = TAILQ_FIRST(&kg->kg_threads);
+	if (td == NULL) {
+		kg->kg_user_pri = prio;
+		return;
+	}
+
+	if (td->td_flags & TDF_UBORROWING && kg->kg_user_pri <= prio)
+		return;
+
+	oldprio = kg->kg_user_pri;
+	kg->kg_user_pri = prio;
+
+	if (TD_ON_UPILOCK(td) && oldprio != prio)
+		umtx_pi_adjust(td, oldprio);
+}
+
+void
+sched_lend_user_prio(struct thread *td, u_char prio)
+{
+	u_char oldprio;
+
+	td->td_flags |= TDF_UBORROWING;
+
+	oldprio = td->td_ksegrp->kg_user_pri;
+	td->td_ksegrp->kg_user_pri = prio;
+
+	if (TD_ON_UPILOCK(td) && oldprio != prio)
+		umtx_pi_adjust(td, oldprio);
+}
+
+void
+sched_unlend_user_prio(struct thread *td, u_char prio)
+{
+	struct ksegrp *kg = td->td_ksegrp;
+	u_char base_pri;
+
+	base_pri = kg->kg_base_user_pri;
+	if (prio >= base_pri) {
+		td->td_flags &= ~TDF_UBORROWING;
+		sched_user_prio(kg, base_pri);
+	} else
+		sched_lend_user_prio(td, prio);
+}
+
+void
 sched_switch(struct thread *td, struct thread *newtd, int flags)
 {
 	struct kseq *ksq;
@@ -1038,7 +1093,7 @@ sched_nice(struct proc *p, int nice)
 	p->p_nice = nice;
 	FOREACH_KSEGRP_IN_PROC(p, kg) {
 		if (kg->kg_pri_class == PRI_TIMESHARE) {
-			kg->kg_user_pri = sched_calc_pri(kg);
+			sched_user_prio(kg, sched_calc_pri(kg));
 			FOREACH_THREAD_IN_GROUP(kg, td)
 				td->td_flags |= TDF_NEEDRESCHED;
 		}
@@ -1082,7 +1137,7 @@ sched_wakeup(struct thread *td)
 				now = now - mykseq->ksq_last_timestamp +
 				    kseq->ksq_last_timestamp;
 #endif
-			kg->kg_user_pri = sched_recalc_pri(ke, now);
+			sched_user_prio(kg, sched_recalc_pri(ke, now));
 		}
 	}
 	setrunqueue(td, SRQ_BORING);
@@ -1109,7 +1164,7 @@ sched_fork_ksegrp(struct thread *td, struct ksegrp *child)
 	mtx_assert(&sched_lock, MA_OWNED);
 	child->kg_slptime = kg->kg_slptime * CHILD_WEIGHT / 100;
 	if (child->kg_pri_class == PRI_TIMESHARE)
-		child->kg_user_pri = sched_calc_pri(child);
+		sched_user_prio(child, sched_calc_pri(child));
 	kg->kg_slptime = kg->kg_slptime * PARENT_WEIGHT / 100;
 }
 
@@ -1275,7 +1330,7 @@ sched_tick(void)
 		td->td_flags |= TDF_NEEDRESCHED;
 		sched_update_runtime(ke, now);
 		sched_commit_runtime(ke);
-		kg->kg_user_pri = sched_calc_pri(kg);
+		sched_user_prio(kg, sched_calc_pri(kg));
 		ke->ke_slice = sched_timeslice(ke);
 		ke->ke_flags &= ~KEF_FIRST_SLICE;
 		if (ke->ke_flags & KEF_BOUND || td->td_pinned) {
diff --git a/sys/kern/sched_ule.c b/sys/kern/sched_ule.c
index a352445..0c77299 100644
--- a/sys/kern/sched_ule.c
+++ b/sys/kern/sched_ule.c
@@ -48,6 +48,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/sysctl.h>
 #include <sys/sysproto.h>
 #include <sys/turnstile.h>
+#include <sys/umtx.h>
 #include <sys/vmmeter.h>
 #ifdef KTRACE
 #include <sys/uio.h>
@@ -1060,7 +1061,7 @@ sched_priority(struct ksegrp *kg)
 	else if (pri < PRI_MIN_TIMESHARE)
 		pri = PRI_MIN_TIMESHARE;
 
-	kg->kg_user_pri = pri;
+	sched_user_prio(kg, pri);
 
 	return;
 }
@@ -1344,6 +1345,60 @@ sched_prio(struct thread *td, u_char prio)
 }
 
 void
+sched_user_prio(struct ksegrp *kg, u_char prio)
+{
+	struct thread *td;
+	u_char oldprio;
+
+	kg->kg_base_user_pri = prio;
+
+	/* XXXKSE only for 1:1 */
+
+	td = TAILQ_FIRST(&kg->kg_threads);
+	if (td == NULL) {
+		kg->kg_user_pri = prio;
+		return;
+	}
+
+	if (td->td_flags & TDF_UBORROWING && kg->kg_user_pri <= prio)
+		return;
+
+	oldprio = kg->kg_user_pri;
+	kg->kg_user_pri = prio;
+
+	if (TD_ON_UPILOCK(td) && oldprio != prio)
+		umtx_pi_adjust(td, oldprio);
+}
+
+void
+sched_lend_user_prio(struct thread *td, u_char prio)
+{
+	u_char oldprio;
+
+	td->td_flags |= TDF_UBORROWING;
+
+	oldprio = td->td_ksegrp->kg_user_pri;
+	td->td_ksegrp->kg_user_pri = prio;
+
+	if (TD_ON_UPILOCK(td) && oldprio != prio)
+		umtx_pi_adjust(td, oldprio);
+}
+
+void
+sched_unlend_user_prio(struct thread *td, u_char prio)
+{
+	struct ksegrp *kg = td->td_ksegrp;
+	u_char base_pri;
+
+	base_pri = kg->kg_base_user_pri;
+	if (prio >= base_pri) {
+		td->td_flags &= ~TDF_UBORROWING;
+		sched_user_prio(kg, base_pri);
+	} else
+		sched_lend_user_prio(td, prio);
+}
+
+void
 sched_switch(struct thread *td, struct thread *newtd, int flags)
 {
 	struct kseq *ksq;
@@ -1519,6 +1574,7 @@ sched_fork_ksegrp(struct thread *td, struct ksegrp *child)
 	child->kg_slptime = kg->kg_slptime;
 	child->kg_runtime = kg->kg_runtime;
 	child->kg_user_pri = kg->kg_user_pri;
+	child->kg_base_user_pri = kg->kg_base_user_pri;
 	sched_interact_fork(child);
 	kg->kg_runtime += tickincr;
 	sched_interact_update(kg);
diff --git a/sys/sys/sched.h b/sys/sys/sched.h
index b782c1f..ccb335d 100644
--- a/sys/sys/sched.h
+++ b/sys/sys/sched.h
@@ -63,12 +63,15 @@ void	sched_nice(struct proc *p, int nice);
  */
 void	sched_exit_thread(struct thread *td, struct thread *child);
 void	sched_fork_thread(struct thread *td, struct thread *child);
+void	sched_lend_prio(struct thread *td, u_char prio);
+void	sched_lend_user_prio(struct thread *td, u_char pri);
 fixpt_t	sched_pctcpu(struct thread *td);
 void	sched_prio(struct thread *td, u_char prio);
-void	sched_lend_prio(struct thread *td, u_char prio);
 void	sched_sleep(struct thread *td);
 void	sched_switch(struct thread *td, struct thread *newtd, int flags);
 void	sched_unlend_prio(struct thread *td, u_char prio);
+void	sched_unlend_user_prio(struct thread *td, u_char pri);
+void	sched_user_prio(struct ksegrp *kg, u_char prio);
 void	sched_userret(struct thread *td);
 void	sched_wakeup(struct thread *td);
 
diff --git a/sys/sys/umtx.h b/sys/sys/umtx.h
index 57389fd..796ee47 100644
--- a/sys/sys/umtx.h
+++ b/sys/sys/umtx.h
@@ -142,6 +142,6 @@ struct umtx_q *umtxq_alloc(void);
 void umtxq_free(struct umtx_q *);
 struct thread;
 int kern_umtx_wake(struct thread *td, void *uaddr, int n_wake);
-
+void umtx_pi_adjust(struct thread *td, u_char oldpri);
 #endif /* !_KERNEL */
 #endif /* !_SYS_UMTX_H_ */