7 files changed, 292 insertions, 5 deletions

diff --git a/include/Makefile b/include/Makefile
index 29f076774ede..2fb8e3ca6e8a 100644
--- a/include/Makefile
+++ b/include/Makefile
@@ -184,6 +184,9 @@ copies: .PHONY .META
 	    ${DESTDIR}${INCLUDEDIR}/dev/evdev; \
 	${INSTALL} -C -o ${BINOWN} -g ${BINGRP} -m 444 uinput.h \
 	    ${DESTDIR}${INCLUDEDIR}/dev/evdev
+	cd ${.CURDIR}/../sys/dev/hyperv/include; \
+	${INSTALL} -C ${TAG_ARGS} -o ${BINOWN} -g ${BINGRP} -m 444 hyperv.h \
+	    ${DESTDIR}${INCLUDEDIR}/dev/hyperv
 	cd ${.CURDIR}/../sys/dev/hyperv/utilities; \
 	${INSTALL} -C ${TAG_ARGS} -o ${BINOWN} -g ${BINGRP} -m 444 hv_snapshot.h \
 	    ${DESTDIR}${INCLUDEDIR}/dev/hyperv
@@ -292,6 +295,11 @@ symlinks: .PHONY .META
 		ln -fs ../../../../sys/dev/evdev/$$h \
 		    ${DESTDIR}${INCLUDEDIR}/dev/evdev; \
 	done
+	cd ${.CURDIR}/../sys/dev/hyperv/include; \
+	for h in hyperv.h; do \
+		${INSTALL_SYMLINK} ${TAG_ARGS} ../../../../sys/dev/hyperv/include/$$h \
+		    ${DESTDIR}${INCLUDEDIR}/dev/hyperv; \
+	done
 	cd ${.CURDIR}/../sys/dev/hyperv/utilities; \
 	for h in hv_snapshot.h; do \
 		${INSTALL_SYMLINK} ${TAG_ARGS} ../../../../sys/dev/hyperv/utilities/$$h \
diff --git a/lib/libc/x86/sys/__vdso_gettc.c b/lib/libc/x86/sys/__vdso_gettc.c
index 743b3ad32779..5097ad515650 100644
--- a/lib/libc/x86/sys/__vdso_gettc.c
+++ b/lib/libc/x86/sys/__vdso_gettc.c
@@ -45,6 +45,10 @@ __FBSDID("$FreeBSD$");
 #include <machine/cpufunc.h>
 #include <machine/specialreg.h>
 #include <dev/acpica/acpi_hpet.h>
+#ifdef __amd64__
+#include <machine/atomic.h>
+#include <dev/hyperv/hyperv.h>
+#endif
 #include "libc_private.h"
 
 static void
@@ -144,6 +148,67 @@ __vdso_init_hpet(uint32_t u)
 	_close(fd);
 }
 
+#ifdef __amd64__
+
+#define HYPERV_REFTSC_DEVPATH	"/dev/" HYPERV_REFTSC_DEVNAME
+
+/*
+ * NOTE:
+ * We use 'NULL' for this variable to indicate that initialization
+ * is required.  And if this variable is 'MAP_FAILED', then Hyper-V
+ * reference TSC can not be used, e.g. in misconfigured jail.
+ */
+static struct hyperv_reftsc *hyperv_ref_tsc;
+
+static void
+__vdso_init_hyperv_tsc(void)
+{
+	int fd;
+
+	fd = _open(HYPERV_REFTSC_DEVPATH, O_RDONLY);
+	if (fd < 0) {
+		/* Prevent the caller from re-entering. */
+		hyperv_ref_tsc = MAP_FAILED;
+		return;
+	}
+	hyperv_ref_tsc = mmap(NULL, sizeof(*hyperv_ref_tsc), PROT_READ,
+	    MAP_SHARED, fd, 0);
+	_close(fd);
+}
+
+static int
+__vdso_hyperv_tsc(struct hyperv_reftsc *tsc_ref, u_int *tc)
+{
+	uint64_t disc, ret, tsc, scale;
+	uint32_t seq;
+	int64_t ofs;
+
+	while ((seq = atomic_load_acq_int(&tsc_ref->tsc_seq)) != 0) {
+		scale = tsc_ref->tsc_scale;
+		ofs = tsc_ref->tsc_ofs;
+
+		lfence_mb();
+		tsc = rdtsc();
+
+		/* ret = ((tsc * scale) >> 64) + ofs */
+		__asm__ __volatile__ ("mulq %3" :
+		    "=d" (ret), "=a" (disc) :
+		    "a" (tsc), "r" (scale));
+		ret += ofs;
+
+		atomic_thread_fence_acq();
+		if (tsc_ref->tsc_seq == seq) {
+			*tc = ret;
+			return (0);
+		}
+
+		/* Sequence changed; re-sync. */
+	}
+	return (ENOSYS);
+}
+
+#endif	/* __amd64__ */
+
 #pragma weak __vdso_gettc
 int
 __vdso_gettc(const struct vdso_timehands *th, u_int *tc)
@@ -165,6 +230,14 @@ __vdso_gettc(const struct vdso_timehands *th, u_int *tc)
 			return (ENOSYS);
 		*tc = *(volatile uint32_t *)(hpet_dev_map + HPET_MAIN_COUNTER);
 		return (0);
+#ifdef __amd64__
+	case VDSO_TH_ALGO_X86_HVTSC:
+		if (hyperv_ref_tsc == NULL)
+			__vdso_init_hyperv_tsc();
+		if (hyperv_ref_tsc == MAP_FAILED)
+			return (ENOSYS);
+		return (__vdso_hyperv_tsc(hyperv_ref_tsc, tc));
+#endif
 	default:
 		return (ENOSYS);
 	}
diff --git a/sys/dev/hyperv/include/hyperv.h b/sys/dev/hyperv/include/hyperv.h
index c5e7dec1f680..0eda9f0691a3 100644
--- a/sys/dev/hyperv/include/hyperv.h
+++ b/sys/dev/hyperv/include/hyperv.h
@@ -31,10 +31,10 @@
 #ifndef _HYPERV_H_
 #define _HYPERV_H_
 
-#include <sys/param.h>
+#ifdef _KERNEL
 
-#include <vm/vm.h>
-#include <vm/pmap.h>
+#include <sys/param.h>
+#include <sys/systm.h>
 
 #define MSR_HV_TIME_REF_COUNT		0x40000020
 
@@ -45,6 +45,7 @@
 #define CPUID_HV_MSR_HYPERCALL		0x0020	/* MSR_HV_GUEST_OS_ID
 						 * MSR_HV_HYPERCALL */
 #define CPUID_HV_MSR_VP_INDEX		0x0040	/* MSR_HV_VP_INDEX */
+#define CPUID_HV_MSR_REFERENCE_TSC	0x0200	/* MSR_HV_REFERENCE_TSC */
 #define CPUID_HV_MSR_GUEST_IDLE		0x0400	/* MSR_HV_GUEST_IDLE */
 
 #ifndef NANOSEC
@@ -53,14 +54,35 @@
 #define HYPERV_TIMER_NS_FACTOR		100ULL
 #define HYPERV_TIMER_FREQ		(NANOSEC / HYPERV_TIMER_NS_FACTOR)
 
+#endif	/* _KERNEL */
+
+#define HYPERV_REFTSC_DEVNAME		"hv_tsc"
+
+/*
+ * Hyper-V Reference TSC
+ */
+struct hyperv_reftsc {
+	volatile uint32_t		tsc_seq;
+	volatile uint32_t		tsc_rsvd1;
+	volatile uint64_t		tsc_scale;
+	volatile int64_t		tsc_ofs;
+} __packed __aligned(PAGE_SIZE);
+#ifdef CTASSERT
+CTASSERT(sizeof(struct hyperv_reftsc) == PAGE_SIZE);
+#endif
+
+#ifdef _KERNEL
+
 struct hyperv_guid {
-	uint8_t		hv_guid[16];
+	uint8_t				hv_guid[16];
 } __packed;
 
-#define HYPERV_GUID_STRLEN	40
+#define HYPERV_GUID_STRLEN		40
 
 int		hyperv_guid2str(const struct hyperv_guid *, char *, size_t);
 
 extern u_int	hyperv_features;	/* CPUID_HV_MSR_ */
 
+#endif	/* _KERNEL */
+
 #endif  /* _HYPERV_H_ */
diff --git a/sys/dev/hyperv/vmbus/amd64/hyperv_machdep.c b/sys/dev/hyperv/vmbus/amd64/hyperv_machdep.c
index 5b5f205f333e..cb815bb163c5 100644
--- a/sys/dev/hyperv/vmbus/amd64/hyperv_machdep.c
+++ b/sys/dev/hyperv/vmbus/amd64/hyperv_machdep.c
@@ -28,7 +28,55 @@
 __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
+#include <sys/conf.h>
+#include <sys/fcntl.h>
+#include <sys/kernel.h>
+#include <sys/systm.h>
+#include <sys/timetc.h>
+#include <sys/vdso.h>
+
+#include <machine/cpufunc.h>
+#include <machine/cputypes.h>
+#include <machine/md_var.h>
+#include <machine/specialreg.h>
+
+#include <vm/vm.h>
+
+#include <dev/hyperv/include/hyperv.h>
+#include <dev/hyperv/include/hyperv_busdma.h>
 #include <dev/hyperv/vmbus/hyperv_machdep.h>
+#include <dev/hyperv/vmbus/hyperv_reg.h>
+#include <dev/hyperv/vmbus/hyperv_var.h>
+
+struct hyperv_reftsc_ctx {
+	struct hyperv_reftsc	*tsc_ref;
+	struct hyperv_dma	tsc_ref_dma;
+};
+
+static uint32_t			hyperv_tsc_vdso_timehands(
+				    struct vdso_timehands *,
+				    struct timecounter *);
+
+static d_open_t			hyperv_tsc_open;
+static d_mmap_t			hyperv_tsc_mmap;
+
+static struct timecounter	hyperv_tsc_timecounter = {
+	.tc_get_timecount	= NULL,	/* based on CPU vendor. */
+	.tc_counter_mask	= 0xffffffff,
+	.tc_frequency		= HYPERV_TIMER_FREQ,
+	.tc_name		= "Hyper-V-TSC",
+	.tc_quality		= 3000,
+	.tc_fill_vdso_timehands = hyperv_tsc_vdso_timehands,
+};
+
+static struct cdevsw		hyperv_tsc_cdevsw = {
+	.d_version		= D_VERSION,
+	.d_open			= hyperv_tsc_open,
+	.d_mmap			= hyperv_tsc_mmap,
+	.d_name			= HYPERV_REFTSC_DEVNAME
+};
+
+static struct hyperv_reftsc_ctx	hyperv_ref_tsc;
 
 uint64_t
 hypercall_md(volatile void *hc_addr, uint64_t in_val,
@@ -41,3 +89,131 @@ hypercall_md(volatile void *hc_addr, uint64_t in_val,
 	    "c" (in_val), "d" (in_paddr), "m" (hc_addr));
 	return (status);
 }
+
+static int
+hyperv_tsc_open(struct cdev *dev __unused, int oflags, int devtype __unused,
+    struct thread *td __unused)
+{
+
+	if (oflags & FWRITE)
+		return (EPERM);
+	return (0);
+}
+
+static int
+hyperv_tsc_mmap(struct cdev *dev __unused, vm_ooffset_t offset,
+    vm_paddr_t *paddr, int nprot __unused, vm_memattr_t *memattr __unused)
+{
+
+	KASSERT(hyperv_ref_tsc.tsc_ref != NULL, ("reftsc has not been setup"));
+
+	/*
+	 * NOTE:
+	 * 'nprot' does not contain information interested to us;
+	 * WR-open is blocked by d_open.
+	 */
+
+	if (offset != 0)
+		return (EOPNOTSUPP);
+
+	*paddr = hyperv_ref_tsc.tsc_ref_dma.hv_paddr;
+	return (0);
+}
+
+static uint32_t
+hyperv_tsc_vdso_timehands(struct vdso_timehands *vdso_th,
+    struct timecounter *tc __unused)
+{
+
+	vdso_th->th_algo = VDSO_TH_ALGO_X86_HVTSC;
+	vdso_th->th_x86_shift = 0;
+	vdso_th->th_x86_hpet_idx = 0;
+	bzero(vdso_th->th_res, sizeof(vdso_th->th_res));
+	return (1);
+}
+
+#define HYPERV_TSC_TIMECOUNT(fence)					\
+static u_int								\
+hyperv_tsc_timecount_##fence(struct timecounter *tc)			\
+{									\
+	struct hyperv_reftsc *tsc_ref = hyperv_ref_tsc.tsc_ref;		\
+	uint32_t seq;							\
+									\
+	while ((seq = atomic_load_acq_int(&tsc_ref->tsc_seq)) != 0) {	\
+		uint64_t disc, ret, tsc;				\
+		uint64_t scale = tsc_ref->tsc_scale;			\
+		int64_t ofs = tsc_ref->tsc_ofs;				\
+									\
+		fence();						\
+		tsc = rdtsc();						\
+									\
+		/* ret = ((tsc * scale) >> 64) + ofs */			\
+		__asm__ __volatile__ ("mulq %3" :			\
+		    "=d" (ret), "=a" (disc) :				\
+		    "a" (tsc), "r" (scale));				\
+		ret += ofs;						\
+									\
+		atomic_thread_fence_acq();				\
+		if (tsc_ref->tsc_seq == seq)				\
+			return (ret);					\
+									\
+		/* Sequence changed; re-sync. */			\
+	}								\
+	/* Fallback to the generic timecounter, i.e. rdmsr. */		\
+	return (rdmsr(MSR_HV_TIME_REF_COUNT));				\
+}									\
+struct __hack
+
+HYPERV_TSC_TIMECOUNT(lfence);
+HYPERV_TSC_TIMECOUNT(mfence);
+
+static void
+hyperv_tsc_tcinit(void *dummy __unused)
+{
+	uint64_t val, orig;
+
+	if ((hyperv_features &
+	     (CPUID_HV_MSR_TIME_REFCNT | CPUID_HV_MSR_REFERENCE_TSC)) !=
+	    (CPUID_HV_MSR_TIME_REFCNT | CPUID_HV_MSR_REFERENCE_TSC) ||
+	    (cpu_feature & CPUID_SSE2) == 0)	/* SSE2 for mfence/lfence */
+		return;
+
+	switch (cpu_vendor_id) {
+	case CPU_VENDOR_AMD:
+		hyperv_tsc_timecounter.tc_get_timecount =
+		    hyperv_tsc_timecount_mfence;
+		break;
+
+	case CPU_VENDOR_INTEL:
+		hyperv_tsc_timecounter.tc_get_timecount =
+		    hyperv_tsc_timecount_lfence;
+		break;
+
+	default:
+		/* Unsupport CPU vendors. */
+		return;
+	}
+
+	hyperv_ref_tsc.tsc_ref = hyperv_dmamem_alloc(NULL, PAGE_SIZE, 0,
+	    sizeof(struct hyperv_reftsc), &hyperv_ref_tsc.tsc_ref_dma,
+	    BUS_DMA_WAITOK | BUS_DMA_ZERO);
+	if (hyperv_ref_tsc.tsc_ref == NULL) {
+		printf("hyperv: reftsc page allocation failed\n");
+		return;
+	}
+
+	orig = rdmsr(MSR_HV_REFERENCE_TSC);
+	val = MSR_HV_REFTSC_ENABLE | (orig & MSR_HV_REFTSC_RSVD_MASK) |
+	    ((hyperv_ref_tsc.tsc_ref_dma.hv_paddr >> PAGE_SHIFT) <<
+	     MSR_HV_REFTSC_PGSHIFT);
+	wrmsr(MSR_HV_REFERENCE_TSC, val);
+
+	/* Register "enlightened" timecounter. */
+	tc_init(&hyperv_tsc_timecounter);
+
+	/* Add device for mmap(2). */
+	make_dev(&hyperv_tsc_cdevsw, 0, UID_ROOT, GID_WHEEL, 0444,
+	    HYPERV_REFTSC_DEVNAME);
+}
+SYSINIT(hyperv_tsc_init, SI_SUB_DRIVERS, SI_ORDER_FIRST, hyperv_tsc_tcinit,
+    NULL);
diff --git a/sys/dev/hyperv/vmbus/hyperv_reg.h b/sys/dev/hyperv/vmbus/hyperv_reg.h
index 4fb12286be26..b3b133c84881 100644
--- a/sys/dev/hyperv/vmbus/hyperv_reg.h
+++ b/sys/dev/hyperv/vmbus/hyperv_reg.h
@@ -57,6 +57,11 @@
 
 #define MSR_HV_VP_INDEX			0x40000002
 
+#define MSR_HV_REFERENCE_TSC		0x40000021
+#define MSR_HV_REFTSC_ENABLE		0x0001ULL
+#define MSR_HV_REFTSC_RSVD_MASK		0x0ffeULL
+#define MSR_HV_REFTSC_PGSHIFT		12
+
 #define MSR_HV_SCONTROL			0x40000080
 #define MSR_HV_SCTRL_ENABLE		0x0001ULL
 #define MSR_HV_SCTRL_RSVD_MASK		0xfffffffffffffffeULL
diff --git a/sys/sys/vdso.h b/sys/sys/vdso.h
index db7eea77bf4f..25c5e6695eea 100644
--- a/sys/sys/vdso.h
+++ b/sys/sys/vdso.h
@@ -54,6 +54,8 @@ struct vdso_timekeep {
 #define	VDSO_TK_VER_CURR	VDSO_TK_VER_1
 #define	VDSO_TH_ALGO_1		0x1
 #define	VDSO_TH_ALGO_2		0x2
+#define	VDSO_TH_ALGO_3		0x3
+#define	VDSO_TH_ALGO_4		0x4
 
 #ifndef _KERNEL
 
diff --git a/sys/x86/include/vdso.h b/sys/x86/include/vdso.h
index 074466507880..24f8407437af 100644
--- a/sys/x86/include/vdso.h
+++ b/sys/x86/include/vdso.h
@@ -39,6 +39,7 @@
 
 #define	VDSO_TH_ALGO_X86_TSC	VDSO_TH_ALGO_1
 #define	VDSO_TH_ALGO_X86_HPET	VDSO_TH_ALGO_2
+#define	VDSO_TH_ALGO_X86_HVTSC	VDSO_TH_ALGO_3	/* Hyper-V ref. TSC */
 
 #ifdef _KERNEL
 #ifdef COMPAT_FREEBSD32