4 files changed, 112 insertions, 29 deletions

diff --git a/lib/libc/amd64/sys/__vdso_gettc.c b/lib/libc/amd64/sys/__vdso_gettc.c
index c6f2dfb3556a..1899b213e2f6 100644
--- a/lib/libc/amd64/sys/__vdso_gettc.c
+++ b/lib/libc/amd64/sys/__vdso_gettc.c
@@ -36,19 +36,29 @@ __FBSDID("$FreeBSD$");
 static u_int
 __vdso_gettc_low(const struct vdso_timehands *th)
 {
-	uint32_t rv;
+	u_int rv;
 
-	__asm __volatile("rdtsc; shrd %%cl, %%edx, %0"
+	__asm __volatile("lfence; rdtsc; shrd %%cl, %%edx, %0"
 	    : "=a" (rv) : "c" (th->th_x86_shift) : "edx");
 	return (rv);
 }
 
+static u_int
+__vdso_rdtsc32(void)
+{
+	u_int rv;
+
+	__asm __volatile("lfence;rdtsc" : "=a" (rv) : : "edx");
+	return (rv);
+}
+
 #pragma weak __vdso_gettc
 u_int
 __vdso_gettc(const struct vdso_timehands *th)
 {
 
-	return (th->th_x86_shift > 0 ? __vdso_gettc_low(th) : rdtsc32());
+	return (th->th_x86_shift > 0 ? __vdso_gettc_low(th) :
+	    __vdso_rdtsc32());
 }
 
 #pragma weak __vdso_gettimekeep
diff --git a/lib/libc/i386/sys/__vdso_gettc.c b/lib/libc/i386/sys/__vdso_gettc.c
index c6f2dfb3556a..1454f16699d2 100644
--- a/lib/libc/i386/sys/__vdso_gettc.c
+++ b/lib/libc/i386/sys/__vdso_gettc.c
@@ -31,24 +31,78 @@ __FBSDID("$FreeBSD$");
 #include <sys/time.h>
 #include <sys/vdso.h>
 #include <machine/cpufunc.h>
+#include <machine/specialreg.h>
 #include "libc_private.h"
 
+static int lfence_works = -1;
+
+static int
+get_lfence_usage(void)
+{
+	u_int cpuid_supported, p[4];
+
+	if (lfence_works == -1) {
+		__asm __volatile(
+		    "	pushfl\n"
+		    "	popl	%%eax\n"
+		    "	movl    %%eax,%%ecx\n"
+		    "	xorl    $0x200000,%%eax\n"
+		    "	pushl	%%eax\n"
+		    "	popfl\n"
+		    "	pushfl\n"
+		    "	popl    %%eax\n"
+		    "	xorl    %%eax,%%ecx\n"
+		    "	je	1f\n"
+		    "	movl	$1,%0\n"
+		    "	jmp	2f\n"
+		    "1:	movl	$0,%0\n"
+		    "2:\n"
+		    : "=r" (cpuid_supported) : : "eax", "ecx");
+		if (cpuid_supported) {
+			__asm __volatile(
+			    "	pushl	%%ebx\n"
+			    "	cpuid\n"
+			    "	movl	%%ebx,%1\n"
+			    "	popl	%%ebx\n"
+			    : "=a" (p[0]), "=r" (p[1]), "=c" (p[2]), "=d" (p[3])
+			    :  "0" (0x1));
+			lfence_works = (p[3] & CPUID_SSE2) != 0;
+		} else
+			lfence_works = 0;
+	}
+	return (lfence_works);
+}
+
 static u_int
 __vdso_gettc_low(const struct vdso_timehands *th)
 {
-	uint32_t rv;
+	u_int rv;
 
+	if (get_lfence_usage() == 1)
+		lfence();
 	__asm __volatile("rdtsc; shrd %%cl, %%edx, %0"
 	    : "=a" (rv) : "c" (th->th_x86_shift) : "edx");
 	return (rv);
 }
 
+static u_int
+__vdso_rdtsc32(void)
+{
+	u_int rv;
+
+	if (get_lfence_usage() == 1)
+		lfence();
+	rv = rdtsc32();
+	return (rv);
+}
+
 #pragma weak __vdso_gettc
 u_int
 __vdso_gettc(const struct vdso_timehands *th)
 {
 
-	return (th->th_x86_shift > 0 ? __vdso_gettc_low(th) : rdtsc32());
+	return (th->th_x86_shift > 0 ? __vdso_gettc_low(th) :
+	    __vdso_rdtsc32());
 }
 
 #pragma weak __vdso_gettimekeep
diff --git a/lib/libc/sys/__vdso_gettimeofday.c b/lib/libc/sys/__vdso_gettimeofday.c
index a305173b3ed8..b3527fa4de53 100644
--- a/lib/libc/sys/__vdso_gettimeofday.c
+++ b/lib/libc/sys/__vdso_gettimeofday.c
@@ -42,6 +42,15 @@ tc_delta(const struct vdso_timehands *th)
 	    th->th_counter_mask);
 }
 
+/*
+ * Calculate the absolute or boot-relative time from the
+ * machine-specific fast timecounter and the published timehands
+ * structure read from the shared page.
+ *
+ * The lockless reading scheme is similar to the one used to read the
+ * in-kernel timehands, see sys/kern/kern_tc.c:binuptime().  This code
+ * is based on the kernel implementation.
+ */
 static int
 binuptime(struct bintime *bt, struct vdso_timekeep *tk, int abs)
 {
@@ -52,27 +61,21 @@ binuptime(struct bintime *bt, struct vdso_timekeep *tk, int abs)
 		if (!tk->tk_enabled)
 			return (ENOSYS);
 
-		/*
-		 * XXXKIB. The load of tk->tk_current should use
-		 * atomic_load_acq_32 to provide load barrier. But
-		 * since tk points to r/o mapped page, x86
-		 * implementation of atomic_load_acq faults.
-		 */
-		curr = tk->tk_current;
-		rmb();
+		curr = atomic_load_acq_32(&tk->tk_current);
 		th = &tk->tk_th[curr];
 		if (th->th_algo != VDSO_TH_ALGO_1)
 			return (ENOSYS);
-		gen = th->th_gen;
+		gen = atomic_load_acq_32(&th->th_gen);
 		*bt = th->th_offset;
 		bintime_addx(bt, th->th_scale * tc_delta(th));
 		if (abs)
 			bintime_add(bt, &th->th_boottime);
 
 		/*
-		 * Barrier for load of both tk->tk_current and th->th_gen.
+		 * Ensure that the load of th_offset is completed
+		 * before the load of th_gen.
 		 */
-		rmb();
+		atomic_thread_fence_acq();
 	} while (curr != tk->tk_current || gen == 0 || gen != th->th_gen);
 	return (0);
 }
diff --git a/sys/kern/kern_sharedpage.c b/sys/kern/kern_sharedpage.c
index fd619cd32e08..6ad2ed8bde4b 100644
--- a/sys/kern/kern_sharedpage.c
+++ b/sys/kern/kern_sharedpage.c
@@ -119,6 +119,13 @@ shared_page_init(void *dummy __unused)
 SYSINIT(shp, SI_SUB_EXEC, SI_ORDER_FIRST, (sysinit_cfunc_t)shared_page_init,
     NULL);
 
+/*
+ * Push the timehands update to the shared page.
+ *
+ * The lockless update scheme is similar to the one used to update the
+ * in-kernel timehands, see sys/kern/kern_tc.c:tc_windup() (which
+ * calls us after the timehands are updated).
+ */
 static void
 timehands_update(struct sysentvec *sv)
 {
@@ -127,47 +134,56 @@ timehands_update(struct sysentvec *sv)
 	uint32_t enabled, idx;
 
 	enabled = tc_fill_vdso_timehands(&th);
-	tk = (struct vdso_timekeep *)(shared_page_mapping +
-	    sv->sv_timekeep_off);
+	th.th_gen = 0;
 	idx = sv->sv_timekeep_curr;
-	atomic_store_rel_32(&tk->tk_th[idx].th_gen, 0);
 	if (++idx >= VDSO_TH_NUM)
 		idx = 0;
 	sv->sv_timekeep_curr = idx;
 	if (++sv->sv_timekeep_gen == 0)
 		sv->sv_timekeep_gen = 1;
-	th.th_gen = 0;
+
+	tk = (struct vdso_timekeep *)(shared_page_mapping +
+	    sv->sv_timekeep_off);
+	tk->tk_th[idx].th_gen = 0;
+	atomic_thread_fence_rel();
 	if (enabled)
 		tk->tk_th[idx] = th;
-	tk->tk_enabled = enabled;
 	atomic_store_rel_32(&tk->tk_th[idx].th_gen, sv->sv_timekeep_gen);
-	tk->tk_current = idx;
+	atomic_store_rel_32(&tk->tk_current, idx);
+
+	/*
+	 * The ordering of the assignment to tk_enabled relative to
+	 * the update of the vdso_timehands is not important.
+	 */
+	tk->tk_enabled = enabled;
 }
 
 #ifdef COMPAT_FREEBSD32
 static void
 timehands_update32(struct sysentvec *sv)
 {
-	struct vdso_timekeep32 *tk;
 	struct vdso_timehands32 th;
+	struct vdso_timekeep32 *tk;
 	uint32_t enabled, idx;
 
 	enabled = tc_fill_vdso_timehands32(&th);
-	tk = (struct vdso_timekeep32 *)(shared_page_mapping +
-	    sv->sv_timekeep_off);
+	th.th_gen = 0;
 	idx = sv->sv_timekeep_curr;
-	atomic_store_rel_32(&tk->tk_th[idx].th_gen, 0);
 	if (++idx >= VDSO_TH_NUM)
 		idx = 0;
 	sv->sv_timekeep_curr = idx;
 	if (++sv->sv_timekeep_gen == 0)
 		sv->sv_timekeep_gen = 1;
-	th.th_gen = 0;
+
+	tk = (struct vdso_timekeep32 *)(shared_page_mapping +
+	    sv->sv_timekeep_off);
+	tk->tk_th[idx].th_gen = 0;
+	atomic_thread_fence_rel();
 	if (enabled)
 		tk->tk_th[idx] = th;
-	tk->tk_enabled = enabled;
 	atomic_store_rel_32(&tk->tk_th[idx].th_gen, sv->sv_timekeep_gen);
-	tk->tk_current = idx;
+	atomic_store_rel_32(&tk->tk_current, idx);
+	tk->tk_enabled = enabled;
 }
 #endif