MFC 310048,310101

310048
    hyperv: Implement "enlightened" time counter, which is rdtsc based.

    Reviewed by:    kib
    Sponsored by:   Microsoft
    Differential Revision:  https://reviews.freebsd.org/D8763

310101
    hyperv: Allow userland to ro-mmap reference TSC page

    This paves way to implement VDSO for the enlightened time counter.

    Reviewed by:    kib
    Sponsored by:   Microsoft
    Differential Revision:  https://reviews.freebsd.org/D8768

4 files changed, 204 insertions, 5 deletions

diff --git a/include/Makefile b/include/Makefile
index b5214e8..abf71aa 100644
--- a/include/Makefile
+++ b/include/Makefile
@@ -184,6 +184,9 @@ copies:
 	${INSTALL} -C -o ${BINOWN} -g ${BINGRP} -m 444 nand_dev.h \
 	    ${DESTDIR}${INCLUDEDIR}/dev/nand
 .endif
+	cd ${.CURDIR}/../sys/dev/hyperv/include; \
+	${INSTALL} -C ${TAG_ARGS} -o ${BINOWN} -g ${BINGRP} -m 444 hyperv.h \
+	    ${DESTDIR}${INCLUDEDIR}/dev/hyperv
 	cd ${.CURDIR}/../sys/dev/hyperv/utilities; \
 	${INSTALL} -C ${TAG_ARGS} -o ${BINOWN} -g ${BINGRP} -m 444 hv_snapshot.h \
 	    ${DESTDIR}${INCLUDEDIR}/dev/hyperv
@@ -287,6 +290,11 @@ symlinks:
 		    ${DESTDIR}${INCLUDEDIR}/dev/nand; \
 	done
 .endif
+	cd ${.CURDIR}/../sys/dev/hyperv/include; \
+	for h in hyperv.h; do \
+		${INSTALL_SYMLINK} ${TAG_ARGS} ../../../../sys/dev/hyperv/include/$$h \
+		    ${DESTDIR}${INCLUDEDIR}/dev/hyperv; \
+	done
 	cd ${.CURDIR}/../sys/dev/hyperv/utilities; \
 	for h in hv_snapshot.h; do \
 		${INSTALL_SYMLINK} ${TAG_ARGS} ../../../../sys/dev/hyperv/utilities/$$h \
diff --git a/sys/dev/hyperv/include/hyperv.h b/sys/dev/hyperv/include/hyperv.h
index c5e7dec..0eda9f0 100644
--- a/sys/dev/hyperv/include/hyperv.h
+++ b/sys/dev/hyperv/include/hyperv.h
@@ -31,10 +31,10 @@
 #ifndef _HYPERV_H_
 #define _HYPERV_H_
 
-#include <sys/param.h>
+#ifdef _KERNEL
 
-#include <vm/vm.h>
-#include <vm/pmap.h>
+#include <sys/param.h>
+#include <sys/systm.h>
 
 #define MSR_HV_TIME_REF_COUNT		0x40000020
 
@@ -45,6 +45,7 @@
 #define CPUID_HV_MSR_HYPERCALL		0x0020	/* MSR_HV_GUEST_OS_ID
 						 * MSR_HV_HYPERCALL */
 #define CPUID_HV_MSR_VP_INDEX		0x0040	/* MSR_HV_VP_INDEX */
+#define CPUID_HV_MSR_REFERENCE_TSC	0x0200	/* MSR_HV_REFERENCE_TSC */
 #define CPUID_HV_MSR_GUEST_IDLE		0x0400	/* MSR_HV_GUEST_IDLE */
 
 #ifndef NANOSEC
@@ -53,14 +54,35 @@
 #define HYPERV_TIMER_NS_FACTOR		100ULL
 #define HYPERV_TIMER_FREQ		(NANOSEC / HYPERV_TIMER_NS_FACTOR)
 
+#endif	/* _KERNEL */
+
+#define HYPERV_REFTSC_DEVNAME		"hv_tsc"
+
+/*
+ * Hyper-V Reference TSC
+ */
+struct hyperv_reftsc {
+	volatile uint32_t		tsc_seq;
+	volatile uint32_t		tsc_rsvd1;
+	volatile uint64_t		tsc_scale;
+	volatile int64_t		tsc_ofs;
+} __packed __aligned(PAGE_SIZE);
+#ifdef CTASSERT
+CTASSERT(sizeof(struct hyperv_reftsc) == PAGE_SIZE);
+#endif
+
+#ifdef _KERNEL
+
 struct hyperv_guid {
-	uint8_t		hv_guid[16];
+	uint8_t				hv_guid[16];
 } __packed;
 
-#define HYPERV_GUID_STRLEN	40
+#define HYPERV_GUID_STRLEN		40
 
 int		hyperv_guid2str(const struct hyperv_guid *, char *, size_t);
 
 extern u_int	hyperv_features;	/* CPUID_HV_MSR_ */
 
+#endif	/* _KERNEL */
+
 #endif  /* _HYPERV_H_ */
diff --git a/sys/dev/hyperv/vmbus/amd64/hyperv_machdep.c b/sys/dev/hyperv/vmbus/amd64/hyperv_machdep.c
index 5b5f205..9caed81 100644
--- a/sys/dev/hyperv/vmbus/amd64/hyperv_machdep.c
+++ b/sys/dev/hyperv/vmbus/amd64/hyperv_machdep.c
@@ -28,7 +28,52 @@
 __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
+#include <sys/conf.h>
+#include <sys/fcntl.h>
+#include <sys/kernel.h>
+#include <sys/systm.h>
+#include <sys/timetc.h>
+
+#include <machine/cpufunc.h>
+#include <machine/cputypes.h>
+#include <machine/md_var.h>
+#include <machine/specialreg.h>
+
+#include <vm/vm.h>
+
+#include <dev/hyperv/include/hyperv.h>
+#include <dev/hyperv/include/hyperv_busdma.h>
 #include <dev/hyperv/vmbus/hyperv_machdep.h>
+#include <dev/hyperv/vmbus/hyperv_reg.h>
+#include <dev/hyperv/vmbus/hyperv_var.h>
+
+struct hyperv_reftsc_ctx {
+	struct hyperv_reftsc	*tsc_ref;
+	struct hyperv_dma	tsc_ref_dma;
+};
+
+static d_open_t			hyperv_tsc_open;
+static d_mmap_t			hyperv_tsc_mmap;
+
+static struct timecounter	hyperv_tsc_timecounter = {
+	.tc_get_timecount	= NULL,	/* based on CPU vendor. */
+	.tc_poll_pps		= NULL,
+	.tc_counter_mask	= 0xffffffff,
+	.tc_frequency		= HYPERV_TIMER_FREQ,
+	.tc_name		= "Hyper-V-TSC",
+	.tc_quality		= 3000,
+	.tc_flags		= 0,
+	.tc_priv		= NULL
+};
+
+static struct cdevsw		hyperv_tsc_cdevsw = {
+	.d_version		= D_VERSION,
+	.d_open			= hyperv_tsc_open,
+	.d_mmap			= hyperv_tsc_mmap,
+	.d_name			= HYPERV_REFTSC_DEVNAME
+};
+
+static struct hyperv_reftsc_ctx	hyperv_ref_tsc;
 
 uint64_t
 hypercall_md(volatile void *hc_addr, uint64_t in_val,
@@ -41,3 +86,122 @@ hypercall_md(volatile void *hc_addr, uint64_t in_val,
 	    "c" (in_val), "d" (in_paddr), "m" (hc_addr));
 	return (status);
 }
+
+static int
+hyperv_tsc_open(struct cdev *dev __unused, int oflags, int devtype __unused,
+    struct thread *td __unused)
+{
+
+	if (oflags & FWRITE)
+		return (EPERM);
+	return (0);
+}
+
+static int
+hyperv_tsc_mmap(struct cdev *dev __unused, vm_ooffset_t offset,
+    vm_paddr_t *paddr, int nprot __unused, vm_memattr_t *memattr __unused)
+{
+
+	KASSERT(hyperv_ref_tsc.tsc_ref != NULL, ("reftsc has not been setup"));
+
+	/*
+	 * NOTE:
+	 * 'nprot' does not contain information interested to us;
+	 * WR-open is blocked by d_open.
+	 */
+
+	if (offset != 0)
+		return (EOPNOTSUPP);
+
+	*paddr = hyperv_ref_tsc.tsc_ref_dma.hv_paddr;
+	return (0);
+}
+
+#define HYPERV_TSC_TIMECOUNT(fence)					\
+static u_int								\
+hyperv_tsc_timecount_##fence(struct timecounter *tc)			\
+{									\
+	struct hyperv_reftsc *tsc_ref = hyperv_ref_tsc.tsc_ref;		\
+	uint32_t seq;							\
+									\
+	while ((seq = tsc_ref->tsc_seq) != 0) {				\
+		uint64_t disc, ret, tsc, scale;				\
+		int64_t ofs;						\
+									\
+		__compiler_membar();					\
+		scale = tsc_ref->tsc_scale;				\
+		ofs = tsc_ref->tsc_ofs;					\
+									\
+		fence();						\
+		tsc = rdtsc();						\
+									\
+		/* ret = ((tsc * scale) >> 64) + ofs */			\
+		__asm__ __volatile__ ("mulq %3" :			\
+		    "=d" (ret), "=a" (disc) :				\
+		    "a" (tsc), "r" (scale));				\
+		ret += ofs;						\
+									\
+		__compiler_membar();					\
+		if (tsc_ref->tsc_seq == seq)				\
+			return (ret);					\
+									\
+		/* Sequence changed; re-sync. */			\
+	}								\
+	/* Fallback to the generic timecounter, i.e. rdmsr. */		\
+	return (rdmsr(MSR_HV_TIME_REF_COUNT));				\
+}									\
+struct __hack
+
+HYPERV_TSC_TIMECOUNT(lfence);
+HYPERV_TSC_TIMECOUNT(mfence);
+
+static void
+hyperv_tsc_tcinit(void *dummy __unused)
+{
+	uint64_t val, orig;
+
+	if ((hyperv_features &
+	     (CPUID_HV_MSR_TIME_REFCNT | CPUID_HV_MSR_REFERENCE_TSC)) !=
+	    (CPUID_HV_MSR_TIME_REFCNT | CPUID_HV_MSR_REFERENCE_TSC) ||
+	    (cpu_feature & CPUID_SSE2) == 0)	/* SSE2 for mfence/lfence */
+		return;
+
+	switch (cpu_vendor_id) {
+	case CPU_VENDOR_AMD:
+		hyperv_tsc_timecounter.tc_get_timecount =
+		    hyperv_tsc_timecount_mfence;
+		break;
+
+	case CPU_VENDOR_INTEL:
+		hyperv_tsc_timecounter.tc_get_timecount =
+		    hyperv_tsc_timecount_lfence;
+		break;
+
+	default:
+		/* Unsupport CPU vendors. */
+		return;
+	}
+
+	hyperv_ref_tsc.tsc_ref = hyperv_dmamem_alloc(NULL, PAGE_SIZE, 0,
+	    sizeof(struct hyperv_reftsc), &hyperv_ref_tsc.tsc_ref_dma,
+	    BUS_DMA_WAITOK | BUS_DMA_ZERO);
+	if (hyperv_ref_tsc.tsc_ref == NULL) {
+		printf("hyperv: reftsc page allocation failed\n");
+		return;
+	}
+
+	orig = rdmsr(MSR_HV_REFERENCE_TSC);
+	val = MSR_HV_REFTSC_ENABLE | (orig & MSR_HV_REFTSC_RSVD_MASK) |
+	    ((hyperv_ref_tsc.tsc_ref_dma.hv_paddr >> PAGE_SHIFT) <<
+	     MSR_HV_REFTSC_PGSHIFT);
+	wrmsr(MSR_HV_REFERENCE_TSC, val);
+
+	/* Register "enlightened" timecounter. */
+	tc_init(&hyperv_tsc_timecounter);
+
+	/* Add device for mmap(2). */
+	make_dev(&hyperv_tsc_cdevsw, 0, UID_ROOT, GID_WHEEL, 0444,
+	    HYPERV_REFTSC_DEVNAME);
+}
+SYSINIT(hyperv_tsc_init, SI_SUB_DRIVERS, SI_ORDER_FIRST, hyperv_tsc_tcinit,
+    NULL);
diff --git a/sys/dev/hyperv/vmbus/hyperv_reg.h b/sys/dev/hyperv/vmbus/hyperv_reg.h
index 4fb1228..b3b133c 100644
--- a/sys/dev/hyperv/vmbus/hyperv_reg.h
+++ b/sys/dev/hyperv/vmbus/hyperv_reg.h
@@ -57,6 +57,11 @@
 
 #define MSR_HV_VP_INDEX			0x40000002
 
+#define MSR_HV_REFERENCE_TSC		0x40000021
+#define MSR_HV_REFTSC_ENABLE		0x0001ULL
+#define MSR_HV_REFTSC_RSVD_MASK		0x0ffeULL
+#define MSR_HV_REFTSC_PGSHIFT		12
+
 #define MSR_HV_SCONTROL			0x40000080
 #define MSR_HV_SCTRL_ENABLE		0x0001ULL
 #define MSR_HV_SCTRL_RSVD_MASK		0xfffffffffffffffeULL