Modify the way we account for CPU time spent (step 1)

Keep track of time spent by the cpu in various contexts in units of
"cputicks" and scale to real-world microsec^H^H^H^H^H^H^H^Hclock_t
only when somebody wants to inspect the numbers.

For now "cputicks" are still derived from the current timecounter
and therefore things should by definition remain sensible also on
SMP machines.  (The main reason for this first milestone commit is
to verify that hypothesis.)

On slower machines, the avoided multiplications to normalize timestams
at every context switch, comes out as a 5-7% better score on the
unixbench/context1 microbenchmark.  On more modern hardware no change
in performance is seen.

7 files changed, 44 insertions, 25 deletions

diff --git a/sys/kern/init_main.c b/sys/kern/init_main.c
index b63c4e8..e2ec53f 100644
--- a/sys/kern/init_main.c
+++ b/sys/kern/init_main.c
@@ -459,11 +459,10 @@ proc0_post(void *dummy __unused)
 	sx_slock(&allproc_lock);
 	LIST_FOREACH(p, &allproc, p_list) {
 		microuptime(&p->p_stats->p_start);
-		p->p_rux.rux_runtime.sec = 0;
-		p->p_rux.rux_runtime.frac = 0;
+		p->p_rux.rux_runtime = 0;
 	}
 	sx_sunlock(&allproc_lock);
-	binuptime(PCPU_PTR(switchtime));
+	PCPU_SET(switchtime, cpu_ticks());
 	PCPU_SET(switchticks, ticks);
 
 	/*
diff --git a/sys/kern/kern_exit.c b/sys/kern/kern_exit.c
index b907981..8803ac9 100644
--- a/sys/kern/kern_exit.c
+++ b/sys/kern/kern_exit.c
@@ -110,7 +110,7 @@ sys_exit(struct thread *td, struct sys_exit_args *uap)
 void
 exit1(struct thread *td, int rv)
 {
-	struct bintime new_switchtime;
+	uint64_t new_switchtime;
 	struct proc *p, *nq, *q;
 	struct tty *tp;
 	struct vnode *ttyvp;
@@ -543,9 +543,8 @@ retry:
 	ruadd(p->p_ru, &p->p_rux, &p->p_stats->p_cru, &p->p_crux);
 
 	/* Do the same timestamp bookkeeping that mi_switch() would do. */
-	binuptime(&new_switchtime);
-	bintime_add(&p->p_rux.rux_runtime, &new_switchtime);
-	bintime_sub(&p->p_rux.rux_runtime, PCPU_PTR(switchtime));
+	new_switchtime = cpu_ticks();
+	p->p_rux.rux_runtime += (new_switchtime - PCPU_GET(switchtime));
 	PCPU_SET(switchtime, new_switchtime);
 	PCPU_SET(switchticks, ticks);
 	cnt.v_swtch++;
diff --git a/sys/kern/kern_proc.c b/sys/kern/kern_proc.c
index 59d257b..e02b947 100644
--- a/sys/kern/kern_proc.c
+++ b/sys/kern/kern_proc.c
@@ -624,7 +624,6 @@ fill_kinfo_proc_only(struct proc *p, struct kinfo_proc *kp)
 	struct thread *td0;
 	struct tty *tp;
 	struct session *sp;
-	struct timeval tv;
 	struct ucred *cred;
 	struct sigacts *ps;
 
@@ -695,8 +694,7 @@ fill_kinfo_proc_only(struct proc *p, struct kinfo_proc *kp)
 	kp->ki_swtime = p->p_swtime;
 	kp->ki_pid = p->p_pid;
 	kp->ki_nice = p->p_nice;
-	bintime2timeval(&p->p_rux.rux_runtime, &tv);
-	kp->ki_runtime = tv.tv_sec * (u_int64_t)1000000 + tv.tv_usec;
+	kp->ki_runtime = p->p_rux.rux_runtime * 1000000 / cpu_tickrate();
 	mtx_unlock_spin(&sched_lock);
 	if ((p->p_sflag & PS_INMEM) && p->p_stats != NULL) {
 		kp->ki_start = p->p_stats->p_start;
diff --git a/sys/kern/kern_resource.c b/sys/kern/kern_resource.c
index 397a7c5..8ecac2e 100644
--- a/sys/kern/kern_resource.c
+++ b/sys/kern/kern_resource.c
@@ -704,7 +704,7 @@ calcru(p, up, sp)
 	struct timeval *up;
 	struct timeval *sp;
 {
-	struct bintime bt;
+	uint64_t bt;
 	struct rusage_ext rux;
 	struct thread *td;
 	int bt_valid;
@@ -712,6 +712,7 @@ calcru(p, up, sp)
 	PROC_LOCK_ASSERT(p, MA_OWNED);
 	mtx_assert(&sched_lock, MA_NOTOWNED);
 	bt_valid = 0;
+	bt = 0;
 	mtx_lock_spin(&sched_lock);
 	rux = p->p_rux;
 	FOREACH_THREAD_IN_PROC(p, td) {
@@ -725,12 +726,16 @@ calcru(p, up, sp)
 			KASSERT(td->td_oncpu != NOCPU,
 			    ("%s: running thread has no CPU", __func__));
 			if (!bt_valid) {
-				binuptime(&bt);
+				bt = cpu_ticks();
 				bt_valid = 1;
 			}
-			bintime_add(&rux.rux_runtime, &bt);
-			bintime_sub(&rux.rux_runtime,
-			    &pcpu_find(td->td_oncpu)->pc_switchtime);
+			/*
+			 * XXX: Doesn't this mean that this quantum will
+			 * XXX: get counted twice if calcru() is called
+			 * XXX: from SIGINFO ?
+			 */
+			rux.rux_runtime +=
+			    (bt - pcpu_find(td->td_oncpu)->pc_switchtime);
 		}
 	}
 	mtx_unlock_spin(&sched_lock);
@@ -758,7 +763,6 @@ calcru1(p, ruxp, up, sp)
 	struct timeval *up;
 	struct timeval *sp;
 {
-	struct timeval tv;
 	/* {user, system, interrupt, total} {ticks, usec}; previous tu: */
 	u_int64_t ut, uu, st, su, it, iu, tt, tu, ptu;
 
@@ -770,8 +774,7 @@ calcru1(p, ruxp, up, sp)
 		st = 1;
 		tt = 1;
 	}
-	bintime2timeval(&ruxp->rux_runtime, &tv);
-	tu = (u_int64_t)tv.tv_sec * 1000000 + tv.tv_usec;
+	tu = (ruxp->rux_runtime * 1000000LL) / cpu_tickrate();
 	ptu = ruxp->rux_uu + ruxp->rux_su + ruxp->rux_iu;
 	if (tu < ptu) {
 		printf(
@@ -884,7 +887,7 @@ ruadd(ru, rux, ru2, rux2)
 	register long *ip, *ip2;
 	register int i;
 
-	bintime_add(&rux->rux_runtime, &rux2->rux_runtime);
+	rux->rux_runtime += rux2->rux_runtime;
 	rux->rux_uticks += rux2->rux_uticks;
 	rux->rux_sticks += rux2->rux_sticks;
 	rux->rux_iticks += rux2->rux_iticks;
diff --git a/sys/kern/kern_synch.c b/sys/kern/kern_synch.c
index 6764e35..3ccbc15 100644
--- a/sys/kern/kern_synch.c
+++ b/sys/kern/kern_synch.c
@@ -355,7 +355,7 @@ wakeup_one(ident)
 void
 mi_switch(int flags, struct thread *newtd)
 {
-	struct bintime new_switchtime;
+	uint64_t new_switchtime;
 	struct thread *td;
 	struct proc *p;
 
@@ -384,9 +384,8 @@ mi_switch(int flags, struct thread *newtd)
 	 * Compute the amount of time during which the current
 	 * process was running, and add that to its total so far.
 	 */
-	binuptime(&new_switchtime);
-	bintime_add(&p->p_rux.rux_runtime, &new_switchtime);
-	bintime_sub(&p->p_rux.rux_runtime, PCPU_PTR(switchtime));
+	new_switchtime = cpu_ticks();
+	p->p_rux.rux_runtime += (new_switchtime - PCPU_GET(switchtime));
 
 	td->td_generation++;	/* bump preempt-detect counter */
 
@@ -405,7 +404,7 @@ mi_switch(int flags, struct thread *newtd)
 	 * it reaches the max, arrange to kill the process in ast().
 	 */
 	if (p->p_cpulimit != RLIM_INFINITY &&
-	    p->p_rux.rux_runtime.sec >= p->p_cpulimit) {
+	    p->p_rux.rux_runtime >= p->p_cpulimit * cpu_tickrate()) {
 		p->p_sflag |= PS_XCPU;
 		td->td_flags |= TDF_ASTPENDING;
 	}
diff --git a/sys/kern/kern_tc.c b/sys/kern/kern_tc.c
index 929e570..397fd2f 100644
--- a/sys/kern/kern_tc.c
+++ b/sys/kern/kern_tc.c
@@ -131,6 +131,7 @@ sysctl_kern_boottime(SYSCTL_HANDLER_ARGS)
 #endif
 		return SYSCTL_OUT(req, &boottime, sizeof(boottime));
 }
+
 /*
  * Return the difference between the timehands' counter value now and what
  * was when we copied it to the timehands' offset_count.
@@ -782,3 +783,23 @@ inittimecounter(void *dummy)
 }
 
 SYSINIT(timecounter, SI_SUB_CLOCKS, SI_ORDER_SECOND, inittimecounter, NULL)
+
+static
+uint64_t
+tc_cpu_ticks(void)
+{
+	static uint64_t base;
+	static unsigned last;
+	uint64_t u;
+	struct timecounter *tc;
+
+	tc = timehands->th_counter;
+	u = tc->tc_get_timecount(tc) & tc->tc_counter_mask;
+	if (u < last)
+		base += tc->tc_counter_mask + 1;
+	last = u;
+	return (u + base);
+}
+
+uint64_t (*cpu_ticks)(void) = tc_cpu_ticks;
+uint64_t (*cpu_tickrate)(void) = tc_getfrequency;
diff --git a/sys/kern/subr_trap.c b/sys/kern/subr_trap.c
index 7b6c5b8..f4e0a53 100644
--- a/sys/kern/subr_trap.c
+++ b/sys/kern/subr_trap.c
@@ -238,7 +238,7 @@ ast(struct trapframe *framep)
 		PROC_LOCK(p);
 		lim_rlimit(p, RLIMIT_CPU, &rlim);
 		mtx_lock_spin(&sched_lock);
-		if (p->p_rux.rux_runtime.sec >= rlim.rlim_max) {
+		if (p->p_rux.rux_runtime >= rlim.rlim_max * cpu_tickrate()) {
 			mtx_unlock_spin(&sched_lock);
 			killproc(p, "exceeded maximum CPU limit");
 		} else {