Modify the way we account for CPU time spent (step 1)

Keep track of time spent by the cpu in various contexts in units of
"cputicks" and scale to real-world microsec^H^H^H^H^H^H^H^Hclock_t
only when somebody wants to inspect the numbers.

For now "cputicks" are still derived from the current timecounter
and therefore things should by definition remain sensible also on
SMP machines.  (The main reason for this first milestone commit is
to verify that hypothesis.)

On slower machines, the avoided multiplications to normalize timestams
at every context switch, comes out as a 5-7% better score on the
unixbench/context1 microbenchmark.  On more modern hardware no change
in performance is seen.

16 files changed, 71 insertions, 32 deletions

diff --git a/sys/alpha/alpha/mp_machdep.c b/sys/alpha/alpha/mp_machdep.c
index e548d4f..9d6cdc5 100644
--- a/sys/alpha/alpha/mp_machdep.c
+++ b/sys/alpha/alpha/mp_machdep.c
@@ -225,7 +225,7 @@ smp_init_secondary(void)
 	spinlock_exit();
 	KASSERT(curthread->td_md.md_spinlock_count == 1, ("invalid count"));
 
-	binuptime(PCPU_PTR(switchtime));
+	PCPU_SET(switchtime, cpu_ticks());
 	PCPU_SET(switchticks, ticks);
 
 	cpu_throw(NULL, choosethread());	/* doesn't return */
diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c
index c05b05f..a152bc1 100644
--- a/sys/amd64/amd64/mp_machdep.c
+++ b/sys/amd64/amd64/mp_machdep.c
@@ -582,7 +582,7 @@ init_secondary(void)
 	spinlock_exit();
 	KASSERT(curthread->td_md.md_spinlock_count == 1, ("invalid count"));
 
-	binuptime(PCPU_PTR(switchtime));
+	PCPU_SET(switchtime, cpu_ticks());
 	PCPU_SET(switchticks, ticks);
 
 	cpu_throw(NULL, choosethread());	/* doesn't return */
diff --git a/sys/i386/i386/mp_machdep.c b/sys/i386/i386/mp_machdep.c
index 95928a1..bd8dfdf 100644
--- a/sys/i386/i386/mp_machdep.c
+++ b/sys/i386/i386/mp_machdep.c
@@ -631,7 +631,7 @@ init_secondary(void)
 	spinlock_exit();
 	KASSERT(curthread->td_md.md_spinlock_count == 1, ("invalid count"));
 
-	binuptime(PCPU_PTR(switchtime));
+	PCPU_SET(switchtime, cpu_ticks());
 	PCPU_SET(switchticks, ticks);
 
 	cpu_throw(NULL, choosethread());	/* doesn't return */
diff --git a/sys/ia64/ia64/mp_machdep.c b/sys/ia64/ia64/mp_machdep.c
index dedc395..0b9e7dd 100644
--- a/sys/ia64/ia64/mp_machdep.c
+++ b/sys/ia64/ia64/mp_machdep.c
@@ -138,7 +138,7 @@ ia64_ap_startup(void)
 	spinlock_exit();
 	KASSERT(curthread->td_md.md_spinlock_count == 1, ("invalid count"));
 
-	binuptime(PCPU_PTR(switchtime));
+	PCPU_SET(switchtime, cpu_ticks());
 	PCPU_SET(switchticks, ticks);
 
 	ia64_set_tpr(0);
diff --git a/sys/kern/init_main.c b/sys/kern/init_main.c
index b63c4e8..e2ec53f 100644
--- a/sys/kern/init_main.c
+++ b/sys/kern/init_main.c
@@ -459,11 +459,10 @@ proc0_post(void *dummy __unused)
 	sx_slock(&allproc_lock);
 	LIST_FOREACH(p, &allproc, p_list) {
 		microuptime(&p->p_stats->p_start);
-		p->p_rux.rux_runtime.sec = 0;
-		p->p_rux.rux_runtime.frac = 0;
+		p->p_rux.rux_runtime = 0;
 	}
 	sx_sunlock(&allproc_lock);
-	binuptime(PCPU_PTR(switchtime));
+	PCPU_SET(switchtime, cpu_ticks());
 	PCPU_SET(switchticks, ticks);
 
 	/*
diff --git a/sys/kern/kern_exit.c b/sys/kern/kern_exit.c
index b907981..8803ac9 100644
--- a/sys/kern/kern_exit.c
+++ b/sys/kern/kern_exit.c
@@ -110,7 +110,7 @@ sys_exit(struct thread *td, struct sys_exit_args *uap)
 void
 exit1(struct thread *td, int rv)
 {
-	struct bintime new_switchtime;
+	uint64_t new_switchtime;
 	struct proc *p, *nq, *q;
 	struct tty *tp;
 	struct vnode *ttyvp;
@@ -543,9 +543,8 @@ retry:
 	ruadd(p->p_ru, &p->p_rux, &p->p_stats->p_cru, &p->p_crux);
 
 	/* Do the same timestamp bookkeeping that mi_switch() would do. */
-	binuptime(&new_switchtime);
-	bintime_add(&p->p_rux.rux_runtime, &new_switchtime);
-	bintime_sub(&p->p_rux.rux_runtime, PCPU_PTR(switchtime));
+	new_switchtime = cpu_ticks();
+	p->p_rux.rux_runtime += (new_switchtime - PCPU_GET(switchtime));
 	PCPU_SET(switchtime, new_switchtime);
 	PCPU_SET(switchticks, ticks);
 	cnt.v_swtch++;
diff --git a/sys/kern/kern_proc.c b/sys/kern/kern_proc.c
index 59d257b..e02b947 100644
--- a/sys/kern/kern_proc.c
+++ b/sys/kern/kern_proc.c
@@ -624,7 +624,6 @@ fill_kinfo_proc_only(struct proc *p, struct kinfo_proc *kp)
 	struct thread *td0;
 	struct tty *tp;
 	struct session *sp;
-	struct timeval tv;
 	struct ucred *cred;
 	struct sigacts *ps;
 
@@ -695,8 +694,7 @@ fill_kinfo_proc_only(struct proc *p, struct kinfo_proc *kp)
 	kp->ki_swtime = p->p_swtime;
 	kp->ki_pid = p->p_pid;
 	kp->ki_nice = p->p_nice;
-	bintime2timeval(&p->p_rux.rux_runtime, &tv);
-	kp->ki_runtime = tv.tv_sec * (u_int64_t)1000000 + tv.tv_usec;
+	kp->ki_runtime = p->p_rux.rux_runtime * 1000000 / cpu_tickrate();
 	mtx_unlock_spin(&sched_lock);
 	if ((p->p_sflag & PS_INMEM) && p->p_stats != NULL) {
 		kp->ki_start = p->p_stats->p_start;
diff --git a/sys/kern/kern_resource.c b/sys/kern/kern_resource.c
index 397a7c5..8ecac2e 100644
--- a/sys/kern/kern_resource.c
+++ b/sys/kern/kern_resource.c
@@ -704,7 +704,7 @@ calcru(p, up, sp)
 	struct timeval *up;
 	struct timeval *sp;
 {
-	struct bintime bt;
+	uint64_t bt;
 	struct rusage_ext rux;
 	struct thread *td;
 	int bt_valid;
@@ -712,6 +712,7 @@ calcru(p, up, sp)
 	PROC_LOCK_ASSERT(p, MA_OWNED);
 	mtx_assert(&sched_lock, MA_NOTOWNED);
 	bt_valid = 0;
+	bt = 0;
 	mtx_lock_spin(&sched_lock);
 	rux = p->p_rux;
 	FOREACH_THREAD_IN_PROC(p, td) {
@@ -725,12 +726,16 @@ calcru(p, up, sp)
 			KASSERT(td->td_oncpu != NOCPU,
 			    ("%s: running thread has no CPU", __func__));
 			if (!bt_valid) {
-				binuptime(&bt);
+				bt = cpu_ticks();
 				bt_valid = 1;
 			}
-			bintime_add(&rux.rux_runtime, &bt);
-			bintime_sub(&rux.rux_runtime,
-			    &pcpu_find(td->td_oncpu)->pc_switchtime);
+			/*
+			 * XXX: Doesn't this mean that this quantum will
+			 * XXX: get counted twice if calcru() is called
+			 * XXX: from SIGINFO ?
+			 */
+			rux.rux_runtime +=
+			    (bt - pcpu_find(td->td_oncpu)->pc_switchtime);
 		}
 	}
 	mtx_unlock_spin(&sched_lock);
@@ -758,7 +763,6 @@ calcru1(p, ruxp, up, sp)
 	struct timeval *up;
 	struct timeval *sp;
 {
-	struct timeval tv;
 	/* {user, system, interrupt, total} {ticks, usec}; previous tu: */
 	u_int64_t ut, uu, st, su, it, iu, tt, tu, ptu;
 
@@ -770,8 +774,7 @@ calcru1(p, ruxp, up, sp)
 		st = 1;
 		tt = 1;
 	}
-	bintime2timeval(&ruxp->rux_runtime, &tv);
-	tu = (u_int64_t)tv.tv_sec * 1000000 + tv.tv_usec;
+	tu = (ruxp->rux_runtime * 1000000LL) / cpu_tickrate();
 	ptu = ruxp->rux_uu + ruxp->rux_su + ruxp->rux_iu;
 	if (tu < ptu) {
 		printf(
@@ -884,7 +887,7 @@ ruadd(ru, rux, ru2, rux2)
 	register long *ip, *ip2;
 	register int i;
 
-	bintime_add(&rux->rux_runtime, &rux2->rux_runtime);
+	rux->rux_runtime += rux2->rux_runtime;
 	rux->rux_uticks += rux2->rux_uticks;
 	rux->rux_sticks += rux2->rux_sticks;
 	rux->rux_iticks += rux2->rux_iticks;
diff --git a/sys/kern/kern_synch.c b/sys/kern/kern_synch.c
index 6764e35..3ccbc15 100644
--- a/sys/kern/kern_synch.c
+++ b/sys/kern/kern_synch.c
@@ -355,7 +355,7 @@ wakeup_one(ident)
 void
 mi_switch(int flags, struct thread *newtd)
 {
-	struct bintime new_switchtime;
+	uint64_t new_switchtime;
 	struct thread *td;
 	struct proc *p;
 
@@ -384,9 +384,8 @@ mi_switch(int flags, struct thread *newtd)
 	 * Compute the amount of time during which the current
 	 * process was running, and add that to its total so far.
 	 */
-	binuptime(&new_switchtime);
-	bintime_add(&p->p_rux.rux_runtime, &new_switchtime);
-	bintime_sub(&p->p_rux.rux_runtime, PCPU_PTR(switchtime));
+	new_switchtime = cpu_ticks();
+	p->p_rux.rux_runtime += (new_switchtime - PCPU_GET(switchtime));
 
 	td->td_generation++;	/* bump preempt-detect counter */
 
@@ -405,7 +404,7 @@ mi_switch(int flags, struct thread *newtd)
 	 * it reaches the max, arrange to kill the process in ast().
 	 */
 	if (p->p_cpulimit != RLIM_INFINITY &&
-	    p->p_rux.rux_runtime.sec >= p->p_cpulimit) {
+	    p->p_rux.rux_runtime >= p->p_cpulimit * cpu_tickrate()) {
 		p->p_sflag |= PS_XCPU;
 		td->td_flags |= TDF_ASTPENDING;
 	}
diff --git a/sys/kern/kern_tc.c b/sys/kern/kern_tc.c
index 929e570..397fd2f 100644
--- a/sys/kern/kern_tc.c
+++ b/sys/kern/kern_tc.c
@@ -131,6 +131,7 @@ sysctl_kern_boottime(SYSCTL_HANDLER_ARGS)
 #endif
 		return SYSCTL_OUT(req, &boottime, sizeof(boottime));
 }
+
 /*
  * Return the difference between the timehands' counter value now and what
  * was when we copied it to the timehands' offset_count.
@@ -782,3 +783,23 @@ inittimecounter(void *dummy)
 }
 
 SYSINIT(timecounter, SI_SUB_CLOCKS, SI_ORDER_SECOND, inittimecounter, NULL)
+
+static
+uint64_t
+tc_cpu_ticks(void)
+{
+	static uint64_t base;
+	static unsigned last;
+	uint64_t u;
+	struct timecounter *tc;
+
+	tc = timehands->th_counter;
+	u = tc->tc_get_timecount(tc) & tc->tc_counter_mask;
+	if (u < last)
+		base += tc->tc_counter_mask + 1;
+	last = u;
+	return (u + base);
+}
+
+uint64_t (*cpu_ticks)(void) = tc_cpu_ticks;
+uint64_t (*cpu_tickrate)(void) = tc_getfrequency;
diff --git a/sys/kern/subr_trap.c b/sys/kern/subr_trap.c
index 7b6c5b8..f4e0a53 100644
--- a/sys/kern/subr_trap.c
+++ b/sys/kern/subr_trap.c
@@ -238,7 +238,7 @@ ast(struct trapframe *framep)
 		PROC_LOCK(p);
 		lim_rlimit(p, RLIMIT_CPU, &rlim);
 		mtx_lock_spin(&sched_lock);
-		if (p->p_rux.rux_runtime.sec >= rlim.rlim_max) {
+		if (p->p_rux.rux_runtime >= rlim.rlim_max * cpu_tickrate()) {
 			mtx_unlock_spin(&sched_lock);
 			killproc(p, "exceeded maximum CPU limit");
 		} else {
diff --git a/sys/sparc64/sparc64/mp_machdep.c b/sys/sparc64/sparc64/mp_machdep.c
index ff2a0d4..e4e8280 100644
--- a/sys/sparc64/sparc64/mp_machdep.c
+++ b/sys/sparc64/sparc64/mp_machdep.c
@@ -361,7 +361,7 @@ cpu_mp_bootstrap(struct pcpu *pc)
 	/* ok, now grab sched_lock and enter the scheduler */
 	mtx_lock_spin(&sched_lock);
 	spinlock_exit();
-	binuptime(PCPU_PTR(switchtime));
+	PCPU_SET(switchtime, cpu_ticks());
 	PCPU_SET(switchticks, ticks);
 	cpu_throw(NULL, choosethread());	/* doesn't return */
 }
diff --git a/sys/sparc64/sparc64/tick.c b/sys/sparc64/sparc64/tick.c
index 09ba1f1..6f9f1a8 100644
--- a/sys/sparc64/sparc64/tick.c
+++ b/sys/sparc64/sparc64/tick.c
@@ -65,6 +65,20 @@ SYSCTL_INT(_machdep_tick, OID_AUTO, adjust_ticks, CTLFLAG_RD, &adjust_ticks,
 
 static void tick_hardclock(struct trapframe *);
 
+static uint64_t
+tick_cputicks(void)
+{
+
+	return (rd(tick));
+}
+
+static uint64_t
+tick_cputickrate(void)
+{
+
+	return (tick_freq);
+}
+
 void
 cpu_initclocks(void)
 {
@@ -156,6 +170,9 @@ tick_init(u_long clock)
 	 * handled.
 	 */
 	tick_stop();
+
+	cpu_ticks = tick_cputicks;
+	cpu_tickrate = tick_cputickrate;
 }
 
 void
diff --git a/sys/sys/pcpu.h b/sys/sys/pcpu.h
index c5c5944..00a3cac 100644
--- a/sys/sys/pcpu.h
+++ b/sys/sys/pcpu.h
@@ -60,7 +60,7 @@ struct pcpu {
 	struct thread	*pc_fpcurthread;	/* Fp state owner */
 	struct thread	*pc_deadthread;		/* Zombie thread or NULL */
 	struct pcb	*pc_curpcb;		/* Current pcb */
-	struct bintime	pc_switchtime;	
+	uint64_t	pc_switchtime;	
 	int		pc_switchticks;
 	u_int		pc_cpuid;		/* This cpu number */
 	cpumask_t	pc_cpumask;		/* This cpu mask */
diff --git a/sys/sys/proc.h b/sys/sys/proc.h
index 101c395..d2a0727 100644
--- a/sys/sys/proc.h
+++ b/sys/sys/proc.h
@@ -505,7 +505,7 @@ struct ksegrp {
  * Locking: (cj) means (j) for p_rux and (c) for p_crux.
  */
 struct rusage_ext {
-	struct bintime	rux_runtime;    /* (cj) Real time. */
+	u_int64_t	rux_runtime;    /* (cj) Real time. */
 	u_int64_t	rux_uticks;     /* (cj) Statclock hits in user mode. */
 	u_int64_t	rux_sticks;     /* (cj) Statclock hits in sys mode. */
 	u_int64_t	rux_iticks;     /* (cj) Statclock hits in intr mode. */
diff --git a/sys/sys/systm.h b/sys/sys/systm.h
index 235e8dd..17967d2 100644
--- a/sys/sys/systm.h
+++ b/sys/sys/systm.h
@@ -238,6 +238,9 @@ int	setenv(const char *name, const char *value);
 int	unsetenv(const char *name);
 int	testenv(const char *name);
 
+extern uint64_t	(*cpu_ticks)(void);
+extern uint64_t	(*cpu_tickrate)(void);
+
 #ifdef APM_FIXUP_CALLTODO
 struct timeval;
 void	adjust_timeout_calltodo(struct timeval *time_change);