Switch to the event timers infrastructure. This includes:

o   Setting td_intr_frame to the XIVs trap frame because it's referenced
    by the ET event handler.
o   Signal EOI to the CPU before calling the registered XIV handlers.
    This prevents lost ITC interrupts, which cause starvation in one-shot
    mode.
o   Adding support for IPI_HARDCLOCK with corresponding per-CPU counters.
o   Have the APs call cpu_initclocks() so as to limited the scattering of
    clock related initialization. cpu_initclocks() calls the <self>_bsp()
    or <self>_ap() version accordingly.
o   Uncomment the ET clock handling in cpu_idle().
o   Update the DDB 'show pcpu' output for the new MD fields.
o   Entirely rewritten ia64_ih_clock(). Note that we don't create as many
    clock XIVs as we have CPUs, as is done on PowerPC. It doesn't scale.
    We can only have 240 XIVs and we can have more CPUs than that. There's
    a single intrcnt index for the cumulative clock ticks and we keep per
    CPU counts in the PCPU stats structure.
o   Register the ITC by hooking SI_SUB_CONFIGURE (2nd order).

Open issues:
o   Clock interrupts can still be lost. Some tweaking is still necessary.

Thanks to: mav@ for his support, feedback and explanations.

ET stats while committing:
eris% sysctl machdep.cpu | grep nclks

machdep.cpu.0.nclks: 24007
machdep.cpu.1.nclks: 22895
machdep.cpu.2.nclks: 13523
machdep.cpu.3.nclks: 9342
machdep.cpu.4.nclks: 9103
machdep.cpu.5.nclks: 9298
machdep.cpu.6.nclks: 10039
machdep.cpu.7.nclks: 9479
eris% vmstat -i | grep clock
clock                      108599         50

7 files changed, 136 insertions, 106 deletions

diff --git a/sys/ia64/ia64/clock.c b/sys/ia64/ia64/clock.c
index 33dbb2e..5f0e2fe 100644
--- a/sys/ia64/ia64/clock.c
+++ b/sys/ia64/ia64/clock.c
@@ -32,9 +32,11 @@ __FBSDID("$FreeBSD$");
 #include <sys/bus.h>
 #include <sys/interrupt.h>
 #include <sys/priority.h>
+#include <sys/proc.h>
 #include <sys/queue.h>
 #include <sys/sysctl.h>
 #include <sys/systm.h>
+#include <sys/timeet.h>
 #include <sys/timetc.h>
 #include <sys/pcpu.h>
 
@@ -45,26 +47,12 @@ __FBSDID("$FreeBSD$");
 #include <machine/md_var.h>
 #include <machine/smp.h>
 
-SYSCTL_NODE(_debug, OID_AUTO, clock, CTLFLAG_RW, 0, "clock statistics");
-
-static int adjust_edges = 0;
-SYSCTL_INT(_debug_clock, OID_AUTO, adjust_edges, CTLFLAG_RD,
-    &adjust_edges, 0, "Number of times ITC got more than 12.5% behind");
-
-static int adjust_excess = 0;
-SYSCTL_INT(_debug_clock, OID_AUTO, adjust_excess, CTLFLAG_RD,
-    &adjust_excess, 0, "Total number of ignored ITC interrupts");
-
-static int adjust_lost = 0;
-SYSCTL_INT(_debug_clock, OID_AUTO, adjust_lost, CTLFLAG_RD,
-    &adjust_lost, 0, "Total number of lost ITC interrupts");
-
-static int adjust_ticks = 0;
-SYSCTL_INT(_debug_clock, OID_AUTO, adjust_ticks, CTLFLAG_RD,
-    &adjust_ticks, 0, "Total number of ITC interrupts with adjustment");
+#define	CLOCK_ET_OFF		0
+#define	CLOCK_ET_PERIODIC	1
+#define	CLOCK_ET_ONESHOT	2
 
+static struct eventtimer ia64_clock_et;
 static u_int ia64_clock_xiv;
-static uint64_t ia64_clock_reload;
 
 #ifndef SMP
 static timecounter_get_t ia64_get_timecount;
@@ -87,75 +75,100 @@ ia64_get_timecount(struct timecounter* tc)
 static u_int
 ia64_ih_clock(struct thread *td, u_int xiv, struct trapframe *tf)
 {
-	uint64_t adj, clk, itc;
-	int64_t delta;
-	int count;
+	struct eventtimer *et;
+	uint64_t itc, load;
+	uint32_t mode;
 
 	PCPU_INC(md.stats.pcs_nclks);
+	intrcnt[INTRCNT_CLOCK]++;
 
-	if (PCPU_GET(cpuid) == 0) {
-		/*
-		 * Clock processing on the BSP.
-		 */
-		intrcnt[INTRCNT_CLOCK]++;
+	itc = ia64_get_itc();
+	PCPU_SET(md.clock, itc);
 
-		itc = ia64_get_itc();
+	mode = PCPU_GET(md.clock_mode);
+	if (mode == CLOCK_ET_PERIODIC) {
+		load = PCPU_GET(md.clock_load);
+		ia64_set_itm(itc + load);
+	} else
+		ia64_set_itv((1 << 16) | xiv);
+	ia64_srlz_d();
 
-		adj = PCPU_GET(md.clockadj);
-		clk = PCPU_GET(md.clock);
+	et = &ia64_clock_et;
+	if (et->et_active)
+		et->et_event_cb(et, et->et_arg);
+	return (0);
+}
 
-		delta = itc - clk;
-		count = 0;
-		while (delta >= ia64_clock_reload) {
-#ifdef SMP
-			ipi_all_but_self(ia64_clock_xiv);
-#endif
-			hardclock(TRAPF_USERMODE(tf), TRAPF_PC(tf));
-			if (profprocs != 0)
-				profclock(TRAPF_USERMODE(tf), TRAPF_PC(tf));
-			statclock(TRAPF_USERMODE(tf));
-			delta -= ia64_clock_reload;
-			clk += ia64_clock_reload;
-			if (adj != 0)
-				adjust_ticks++;
-			count++;
-		}
-		ia64_set_itm(ia64_get_itc() + ia64_clock_reload - adj);
-		ia64_srlz_d();
-		if (count > 0) {
-			adjust_lost += count - 1;
-			if (delta > (ia64_clock_reload >> 3)) {
-				if (adj == 0)
-					adjust_edges++;
-				adj = ia64_clock_reload >> 4;
-			} else
-				adj = 0;
-		} else {
-			adj = 0;
-			adjust_excess++;
-		}
-		PCPU_SET(md.clock, clk);
-		PCPU_SET(md.clockadj, adj);
+/*
+ * Event timer start method.
+ */
+static int
+ia64_clock_start(struct eventtimer *et, struct bintime *first,
+    struct bintime *period)
+{
+	u_long itc, load;
+	register_t is;
+
+	if (period != NULL) {
+		PCPU_SET(md.clock_mode, CLOCK_ET_PERIODIC);
+		load = (et->et_frequency * (period->frac >> 32)) >> 32;
+		if (period->sec > 0)
+			load += et->et_frequency * period->sec;
 	} else {
-		/*
-		 * Clock processing on the BSP.
-		 */
-		hardclock_cpu(TRAPF_USERMODE(tf));
-		if (profprocs != 0)
-			profclock(TRAPF_USERMODE(tf), TRAPF_PC(tf));
-		statclock(TRAPF_USERMODE(tf));
+		PCPU_SET(md.clock_mode, CLOCK_ET_ONESHOT);
+		load = 0;
 	}
 
+	PCPU_SET(md.clock_load, load);
+
+	if (first != NULL) {
+		load = (et->et_frequency * (first->frac >> 32)) >> 32;
+		if (first->sec > 0)
+			load += et->et_frequency * first->sec;
+	}
+
+	is = intr_disable();
+	itc = ia64_get_itc();
+	ia64_set_itm(itc + load);
+	ia64_set_itv(ia64_clock_xiv);
+	ia64_srlz_d();
+	intr_restore(is);
 	return (0);
 }
 
 /*
- * Start the real-time and statistics clocks. We use ar.itc and cr.itm
- * to implement a 1000hz clock.
+ * Event timer stop method.
+ */
+static int
+ia64_clock_stop(struct eventtimer *et)
+{
+
+	ia64_set_itv((1 << 16) | ia64_clock_xiv);
+	ia64_srlz_d();
+	PCPU_SET(md.clock_mode, CLOCK_ET_OFF);
+	PCPU_SET(md.clock_load, 0);
+	return (0);
+}
+
+/*
+ * We call cpu_initclocks() on the APs as well. It allows us to
+ * group common initialization in the same function.
  */
 void
 cpu_initclocks()
 {
+
+	ia64_clock_stop(NULL);
+	if (PCPU_GET(cpuid) == 0)
+		cpu_initclocks_bsp();
+	else
+		cpu_initclocks_ap();
+}
+
+static void
+clock_configure(void *dummy)
+{
+	struct eventtimer *et;
 	u_long itc_freq;
 
 	ia64_clock_xiv = ia64_xiv_alloc(PI_REALTIME, IA64_XIV_IPI,
@@ -165,31 +178,23 @@ cpu_initclocks()
 
 	itc_freq = (u_long)ia64_itc_freq() * 1000000ul;
 
-	stathz = hz;
-	ia64_clock_reload = (itc_freq + hz/2) / hz;
+	et = &ia64_clock_et;
+	et->et_name = "ITC";
+	et->et_flags = ET_FLAGS_PERIODIC | ET_FLAGS_ONESHOT | ET_FLAGS_PERCPU;
+	et->et_quality = 1000;
+	et->et_frequency = itc_freq;
+	et->et_min_period.sec = 0;
+	et->et_min_period.frac = ((1ul << 32) / itc_freq) << 32;
+	et->et_max_period.sec = 0xfffffff0 / itc_freq;
+	et->et_max_period.frac = ((0xfffffffeul << 32) / itc_freq) << 32;
+	et->et_start = ia64_clock_start;
+	et->et_stop = ia64_clock_stop;
+	et->et_priv = NULL;
+	et_register(et);
 
 #ifndef SMP
 	ia64_timecounter.tc_frequency = itc_freq;
 	tc_init(&ia64_timecounter);
 #endif
-
-	PCPU_SET(md.clockadj, 0);
-	PCPU_SET(md.clock, ia64_get_itc());
-	ia64_set_itm(PCPU_GET(md.clock) + ia64_clock_reload);
-	ia64_set_itv(ia64_clock_xiv);
-	ia64_srlz_d();
-}
-
-void
-cpu_startprofclock(void)
-{
-
-	/* nothing to do */
-}
-
-void
-cpu_stopprofclock(void)
-{
-
-	/* nothing to do */
 }
+SYSINIT(clkcfg, SI_SUB_CONFIGURE, SI_ORDER_SECOND, clock_configure, NULL);
diff --git a/sys/ia64/ia64/db_machdep.c b/sys/ia64/ia64/db_machdep.c
index 9d583a1..b689a3c 100644
--- a/sys/ia64/ia64/db_machdep.c
+++ b/sys/ia64/ia64/db_machdep.c
@@ -578,11 +578,13 @@ db_show_mdpcpu(struct pcpu *pc)
 {
 	struct pcpu_md *md = &pc->pc_md;
 
-	db_printf("MD: vhpt     = %#lx\n", md->vhpt);
-	db_printf("MD: lid      = %#lx\n", md->lid);
-	db_printf("MD: clock    = %#lx/%#lx\n", md->clock, md->clockadj);
-	db_printf("MD: stats    = %p\n", &md->stats);
-	db_printf("MD: pmap     = %p\n", md->current_pmap);
+	db_printf("MD: vhpt       = %#lx\n", md->vhpt);
+	db_printf("MD: lid        = %#lx\n", md->lid);
+	db_printf("MD: clock      = %#lx\n", md->clock);
+	db_printf("MD: clock_mode = %u\n", md->clock_mode);
+	db_printf("MD: clock_load = %#lx\n", md->clock_load);
+	db_printf("MD: stats      = %p\n", &md->stats);
+	db_printf("MD: pmap       = %p\n", md->current_pmap);
 }
 
 void
diff --git a/sys/ia64/ia64/interrupt.c b/sys/ia64/ia64/interrupt.c
index adb16ec..f4e5594 100644
--- a/sys/ia64/ia64/interrupt.c
+++ b/sys/ia64/ia64/interrupt.c
@@ -309,6 +309,7 @@ void
 ia64_handle_intr(struct trapframe *tf)
 {
 	struct thread *td;
+	struct trapframe *stf;
 	u_int xiv;
 
 	td = curthread;
@@ -323,17 +324,20 @@ ia64_handle_intr(struct trapframe *tf)
 	}
 
 	critical_enter();
+	stf = td->td_intr_frame;
+	td->td_intr_frame = tf;
 
 	do {
+		ia64_set_eoi(0);
+		ia64_srlz_d();
 		CTR2(KTR_INTR, "INTR: ITC=%u, XIV=%u",
 		    (u_int)tf->tf_special.ifa, xiv);
 		(ia64_handler[xiv])(td, xiv, tf);
-		ia64_set_eoi(0);
-		ia64_srlz_d();
 		xiv = ia64_get_ivr();
 		ia64_srlz_d();
 	} while (xiv != 15);
 
+	td->td_intr_frame = stf;
 	critical_exit();
 
  out:
diff --git a/sys/ia64/ia64/machdep.c b/sys/ia64/ia64/machdep.c
index 185e13c..2bfd62e 100644
--- a/sys/ia64/ia64/machdep.c
+++ b/sys/ia64/ia64/machdep.c
@@ -347,6 +347,11 @@ cpu_startup(void *dummy)
 
 		SYSCTL_ADD_ULONG(&pc->pc_md.sysctl_ctx,
 		    SYSCTL_CHILDREN(pc->pc_md.sysctl_tree), OID_AUTO,
+		    "nhardclocks", CTLFLAG_RD, &pcs->pcs_nhardclocks,
+		    "Number of IPI_HARDCLOCK interrupts");
+
+		SYSCTL_ADD_ULONG(&pc->pc_md.sysctl_ctx,
+		    SYSCTL_CHILDREN(pc->pc_md.sysctl_tree), OID_AUTO,
 		    "nhighfps", CTLFLAG_RD, &pcs->pcs_nhighfps,
 		    "Number of IPI_HIGH_FP interrupts");
 
@@ -416,12 +421,10 @@ cpu_idle(int busy)
 {
 	register_t ie;
 
-#if 0
 	if (!busy) {
 		critical_enter();
 		cpu_idleclock();
 	}
-#endif
 
 	ie = intr_disable();
 	KASSERT(ie != 0, ("%s called with interrupts disabled\n", __func__));
@@ -436,12 +439,10 @@ cpu_idle(int busy)
 		ia64_enable_intr();
 	}
 
-#if 0
 	if (!busy) {
 		cpu_activeclock();
 		critical_exit();
 	}
-#endif
 }
 
 int
diff --git a/sys/ia64/ia64/mp_machdep.c b/sys/ia64/ia64/mp_machdep.c
index 15afea0..59b14d0 100644
--- a/sys/ia64/ia64/mp_machdep.c
+++ b/sys/ia64/ia64/mp_machdep.c
@@ -77,6 +77,7 @@ void ia64_ap_startup(void);
 struct ia64_ap_state ia64_ap_state;
 
 int ia64_ipi_ast;
+int ia64_ipi_hardclock;
 int ia64_ipi_highfp;
 int ia64_ipi_nmi;
 int ia64_ipi_preempt;
@@ -108,6 +109,16 @@ ia64_ih_ast(struct thread *td, u_int xiv, struct trapframe *tf)
 }
 
 static u_int
+ia64_ih_hardclock(struct thread *td, u_int xiv, struct trapframe *tf)
+{
+
+	PCPU_INC(md.stats.pcs_nhardclocks);
+	CTR1(KTR_SMP, "IPI_HARDCLOCK, cpuid=%d", PCPU_GET(cpuid));
+	hardclockintr();
+	return (0);
+}
+
+static u_int
 ia64_ih_highfp(struct thread *td, u_int xiv, struct trapframe *tf)
 {
 
@@ -233,10 +244,11 @@ ia64_ap_startup(void)
 
 	CTR1(KTR_SMP, "SMP: cpu%d launched", PCPU_GET(cpuid));
 
-	/* Mask interval timer interrupts on APs. */
-	ia64_set_itv(0x10000);
+	cpu_initclocks();
+
 	ia64_set_tpr(0);
 	ia64_srlz_d();
+
 	ia64_enable_intr();
 
 	sched_throw(NULL);
@@ -413,6 +425,8 @@ cpu_mp_unleash(void *dummy)
 
 	/* Allocate XIVs for IPIs */
 	ia64_ipi_ast = ia64_xiv_alloc(PI_DULL, IA64_XIV_IPI, ia64_ih_ast);
+	ia64_ipi_hardclock = ia64_xiv_alloc(PI_REALTIME, IA64_XIV_IPI,
+	    ia64_ih_hardclock);
 	ia64_ipi_highfp = ia64_xiv_alloc(PI_AV, IA64_XIV_IPI, ia64_ih_highfp);
 	ia64_ipi_preempt = ia64_xiv_alloc(PI_SOFT, IA64_XIV_IPI,
 	    ia64_ih_preempt);
diff --git a/sys/ia64/include/pcpu.h b/sys/ia64/include/pcpu.h
index 05e2cc1..5ad61ba 100644
--- a/sys/ia64/include/pcpu.h
+++ b/sys/ia64/include/pcpu.h
@@ -37,6 +37,7 @@ struct pcpu_stats {
 	u_long		pcs_nasts;		/* IPI_AST counter. */
 	u_long		pcs_nclks;		/* Clock interrupt counter. */
 	u_long		pcs_nextints;		/* ExtINT counter. */
+	u_long		pcs_nhardclocks;	/* IPI_HARDCLOCK counter. */
 	u_long		pcs_nhighfps;		/* IPI_HIGH_FP counter. */
 	u_long		pcs_nhwints;		/* Hardware int. counter. */
 	u_long		pcs_npreempts;		/* IPI_PREEMPT counter. */
@@ -51,7 +52,8 @@ struct pcpu_md {
 	vm_offset_t	vhpt;			/* Address of VHPT */
 	uint64_t	lid;			/* local CPU ID */
 	uint64_t	clock;			/* Clock counter. */
-	uint64_t	clockadj;		/* Clock adjust. */
+	uint64_t	clock_load;		/* Clock reload value. */
+	uint32_t	clock_mode;		/* Clock ET mode */
 	uint32_t	awake:1;		/* CPU is awake? */
 	struct pcpu_stats stats;		/* Interrupt stats. */
 #ifdef _KERNEL
diff --git a/sys/ia64/include/smp.h b/sys/ia64/include/smp.h
index d2aff76..b80d6a0 100644
--- a/sys/ia64/include/smp.h
+++ b/sys/ia64/include/smp.h
@@ -7,6 +7,7 @@
 #ifdef _KERNEL
 
 #define	IPI_AST			ia64_ipi_ast
+#define	IPI_HARDCLOCK		ia64_ipi_hardclock
 #define	IPI_PREEMPT		ia64_ipi_preempt
 #define	IPI_RENDEZVOUS		ia64_ipi_rndzvs
 #define	IPI_STOP		ia64_ipi_stop
@@ -37,6 +38,7 @@ struct ia64_ap_state {
 };
 
 extern int ia64_ipi_ast;
+extern int ia64_ipi_hardclock;
 extern int ia64_ipi_highfp;
 extern int ia64_ipi_nmi;
 extern int ia64_ipi_preempt;