Initialize DTrace hrtimer frequency during SI_SUB_CPU on i386 and amd64.

This allows the hrtimer to be used earlier during boot. This is required
for boot-time DTrace: anonymous enablings are created during
SI_SUB_DTRACE_ANON, which runs before APs are started. In particular,
the DTrace deadman timer requires that the hrtimer be functional.

MFC after:	2 weeks

2 files changed, 60 insertions, 36 deletions

diff --git a/sys/cddl/dev/dtrace/amd64/dtrace_subr.c b/sys/cddl/dev/dtrace/amd64/dtrace_subr.c
index f6577d5..2a26b65 100644
--- a/sys/cddl/dev/dtrace/amd64/dtrace_subr.c
+++ b/sys/cddl/dev/dtrace/amd64/dtrace_subr.c
@@ -246,24 +246,14 @@ static uint64_t	nsec_scale;
 /* See below for the explanation of this macro. */
 #define SCALE_SHIFT	28
 
+/*
+ * Get the frequency and scale factor as early as possible so that they can be
+ * used for boot-time tracing.
+ */
 static void
-dtrace_gethrtime_init_cpu(void *arg)
-{
-	uintptr_t cpu = (uintptr_t) arg;
-
-	if (cpu == curcpu)
-		tgt_cpu_tsc = rdtsc();
-	else
-		hst_cpu_tsc = rdtsc();
-}
-
-static void
-dtrace_gethrtime_init(void *arg)
+dtrace_gethrtime_init_early(void *arg)
 {
-	struct pcpu *pc;
 	uint64_t tsc_f;
-	cpuset_t map;
-	int i;
 
 	/*
 	 * Get TSC frequency known at this moment.
@@ -279,7 +269,8 @@ dtrace_gethrtime_init(void *arg)
 	 * another 32-bit integer without overflowing 64-bit.
 	 * Thus minimum supported TSC frequency is 62.5MHz.
 	 */
-	KASSERT(tsc_f > (NANOSEC >> (32 - SCALE_SHIFT)), ("TSC frequency is too low"));
+	KASSERT(tsc_f > (NANOSEC >> (32 - SCALE_SHIFT)),
+	    ("TSC frequency is too low"));
 
 	/*
 	 * We scale up NANOSEC/tsc_f ratio to preserve as much precision
@@ -291,6 +282,27 @@ dtrace_gethrtime_init(void *arg)
 	 *   (terahertz) values;
 	 */
 	nsec_scale = ((uint64_t)NANOSEC << SCALE_SHIFT) / tsc_f;
+}
+SYSINIT(dtrace_gethrtime_init_early, SI_SUB_CPU, SI_ORDER_ANY,
+    dtrace_gethrtime_init_early, NULL);
+
+static void
+dtrace_gethrtime_init_cpu(void *arg)
+{
+	uintptr_t cpu = (uintptr_t) arg;
+
+	if (cpu == curcpu)
+		tgt_cpu_tsc = rdtsc();
+	else
+		hst_cpu_tsc = rdtsc();
+}
+
+static void
+dtrace_gethrtime_init(void *arg)
+{
+	struct pcpu *pc;
+	cpuset_t map;
+	int i;
 
 	/* The current CPU is the reference one. */
 	sched_pin();
@@ -311,8 +323,8 @@ dtrace_gethrtime_init(void *arg)
 	}
 	sched_unpin();
 }
-
-SYSINIT(dtrace_gethrtime_init, SI_SUB_SMP, SI_ORDER_ANY, dtrace_gethrtime_init, NULL);
+SYSINIT(dtrace_gethrtime_init, SI_SUB_SMP, SI_ORDER_ANY, dtrace_gethrtime_init,
+    NULL);
 
 /*
  * DTrace needs a high resolution time function which can
diff --git a/sys/cddl/dev/dtrace/i386/dtrace_subr.c b/sys/cddl/dev/dtrace/i386/dtrace_subr.c
index be5bd4b..daca7dd 100644
--- a/sys/cddl/dev/dtrace/i386/dtrace_subr.c
+++ b/sys/cddl/dev/dtrace/i386/dtrace_subr.c
@@ -248,24 +248,14 @@ static uint64_t	nsec_scale;
 /* See below for the explanation of this macro. */
 #define SCALE_SHIFT	28
 
+/*
+ * Get the frequency and scale factor as early as possible so that they can be
+ * used for boot-time tracing.
+ */
 static void
-dtrace_gethrtime_init_cpu(void *arg)
-{
-	uintptr_t cpu = (uintptr_t) arg;
-
-	if (cpu == curcpu)
-		tgt_cpu_tsc = rdtsc();
-	else
-		hst_cpu_tsc = rdtsc();
-}
-
-static void
-dtrace_gethrtime_init(void *arg)
+dtrace_gethrtime_init_early(void *arg)
 {
-	cpuset_t map;
-	struct pcpu *pc;
 	uint64_t tsc_f;
-	int i;
 
 	/*
 	 * Get TSC frequency known at this moment.
@@ -281,7 +271,8 @@ dtrace_gethrtime_init(void *arg)
 	 * another 32-bit integer without overflowing 64-bit.
 	 * Thus minimum supported TSC frequency is 62.5MHz.
 	 */
-	KASSERT(tsc_f > (NANOSEC >> (32 - SCALE_SHIFT)), ("TSC frequency is too low"));
+	KASSERT(tsc_f > (NANOSEC >> (32 - SCALE_SHIFT)),
+	    ("TSC frequency is too low"));
 
 	/*
 	 * We scale up NANOSEC/tsc_f ratio to preserve as much precision
@@ -293,6 +284,27 @@ dtrace_gethrtime_init(void *arg)
 	 *   (terahertz) values;
 	 */
 	nsec_scale = ((uint64_t)NANOSEC << SCALE_SHIFT) / tsc_f;
+}
+SYSINIT(dtrace_gethrtime_init_early, SI_SUB_CPU, SI_ORDER_ANY,
+    dtrace_gethrtime_init_early, NULL);
+
+static void
+dtrace_gethrtime_init_cpu(void *arg)
+{
+	uintptr_t cpu = (uintptr_t) arg;
+
+	if (cpu == curcpu)
+		tgt_cpu_tsc = rdtsc();
+	else
+		hst_cpu_tsc = rdtsc();
+}
+
+static void
+dtrace_gethrtime_init(void *arg)
+{
+	cpuset_t map;
+	struct pcpu *pc;
+	int i;
 
 	/* The current CPU is the reference one. */
 	sched_pin();
@@ -313,8 +325,8 @@ dtrace_gethrtime_init(void *arg)
 	}
 	sched_unpin();
 }
-
-SYSINIT(dtrace_gethrtime_init, SI_SUB_SMP, SI_ORDER_ANY, dtrace_gethrtime_init, NULL);
+SYSINIT(dtrace_gethrtime_init, SI_SUB_SMP, SI_ORDER_ANY, dtrace_gethrtime_init,
+    NULL);
 
 /*
  * DTrace needs a high resolution time function which can