5 files changed, 55 insertions, 1 deletions

diff --git a/sys/dev/acpica/acpi.c b/sys/dev/acpica/acpi.c
index b29b3c415277..cbba9bf35dd2 100644
--- a/sys/dev/acpica/acpi.c
+++ b/sys/dev/acpica/acpi.c
@@ -2856,11 +2856,18 @@ acpi_EnterSleepState(struct acpi_softc *sc, int state)
     stop_all_proc();
     EVENTHANDLER_INVOKE(power_suspend);
 
+#ifdef EARLY_AP_STARTUP
+    MPASS(mp_ncpus == 1 || smp_started);
+    thread_lock(curthread);
+    sched_bind(curthread, 0);
+    thread_unlock(curthread);
+#else
     if (smp_started) {
 	thread_lock(curthread);
 	sched_bind(curthread, 0);
 	thread_unlock(curthread);
     }
+#endif
 
     /*
      * Be sure to hold Giant across DEVICE_SUSPEND/RESUME since non-MPSAFE
@@ -2991,11 +2998,17 @@ backout:
 
     mtx_unlock(&Giant);
 
+#ifdef EARLY_AP_STARTUP
+    thread_lock(curthread);
+    sched_unbind(curthread);
+    thread_unlock(curthread);
+#else
     if (smp_started) {
 	thread_lock(curthread);
 	sched_unbind(curthread);
 	thread_unlock(curthread);
     }
+#endif
 
     resume_all_proc();
 
diff --git a/sys/dev/acpica/acpi_cpu.c b/sys/dev/acpica/acpi_cpu.c
index 218da4e36f68..00c951f06624 100644
--- a/sys/dev/acpica/acpi_cpu.c
+++ b/sys/dev/acpica/acpi_cpu.c
@@ -439,8 +439,12 @@ acpi_cpu_postattach(void *unused __unused)
     free(devices, M_TEMP);
 
     if (attached) {
+#ifdef EARLY_AP_STARTUP
+	acpi_cpu_startup(NULL);
+#else
 	/* Queue post cpu-probing task handler */
 	AcpiOsExecute(OSL_NOTIFY_HANDLER, acpi_cpu_startup, NULL);
+#endif
     }
 }
 
diff --git a/sys/dev/hwpmc/hwpmc_mod.c b/sys/dev/hwpmc/hwpmc_mod.c
index c0caab6de584..48bb23e62c88 100644
--- a/sys/dev/hwpmc/hwpmc_mod.c
+++ b/sys/dev/hwpmc/hwpmc_mod.c
@@ -334,7 +334,11 @@ static moduledata_t pmc_mod = {
 	&pmc_syscall_mod
 };
 
+#ifdef EARLY_AP_STARTUP
+DECLARE_MODULE(pmc, pmc_mod, SI_SUB_SYSCALLS, SI_ORDER_ANY);
+#else
 DECLARE_MODULE(pmc, pmc_mod, SI_SUB_SMP, SI_ORDER_ANY);
+#endif
 MODULE_VERSION(pmc, PMC_VERSION);
 
 #ifdef	HWPMC_DEBUG
diff --git a/sys/dev/hyperv/vmbus/hv_vmbus_drv_freebsd.c b/sys/dev/hyperv/vmbus/hv_vmbus_drv_freebsd.c
index bcdf0cb56184..d0bdfc47e0a7 100644
--- a/sys/dev/hyperv/vmbus/hv_vmbus_drv_freebsd.c
+++ b/sys/dev/hyperv/vmbus/hv_vmbus_drv_freebsd.c
@@ -519,6 +519,7 @@ vmbus_attach(device_t dev)
 		device_printf(dev, "VMBUS: attach dev: %p\n", dev);
 	vmbus_devp = dev;
 
+#ifndef EARLY_AP_STARTUP
 	/* 
 	 * If the system has already booted and thread
 	 * scheduling is possible indicated by the global
@@ -526,6 +527,7 @@ vmbus_attach(device_t dev)
 	 * initialization directly.
 	 */
 	if (!cold)
+#endif
 		vmbus_bus_init();
 
 	bus_generic_probe(dev);
@@ -538,6 +540,7 @@ vmbus_init(void)
 	if (vm_guest != VM_GUEST_HV)
 		return;
 
+#ifndef EARLY_AP_STARTUP
 	/* 
 	 * If the system has already booted and thread
 	 * scheduling is possible, as indicated by the
@@ -545,6 +548,7 @@ vmbus_init(void)
 	 * initialization directly.
 	 */
 	if (!cold) 
+#endif
 		vmbus_bus_init();
 }
 
@@ -611,6 +615,9 @@ vmbus_modevent(module_t mod, int what, void *arg)
 	switch (what) {
 
 	case MOD_LOAD:
+#ifdef EARLY_AP_STARTUP
+		vmbus_init();
+#endif
 		vmbus_mod_load();
 		break;
 	case MOD_UNLOAD:
@@ -649,6 +656,7 @@ DRIVER_MODULE(vmbus, acpi, vmbus_driver, vmbus_devclass, vmbus_modevent, 0);
 MODULE_DEPEND(vmbus, acpi, 1, 1, 1);
 MODULE_VERSION(vmbus, 1);
 
+#ifndef EARLY_AP_STARTUP
 /* We want to be started after SMP is initialized */
 SYSINIT(vmb_init, SI_SUB_SMP + 1, SI_ORDER_FIRST, vmbus_init, NULL);
-
+#endif
diff --git a/sys/dev/xen/control/control.c b/sys/dev/xen/control/control.c
index a62b2cae45fa..ae13c6c10078 100644
--- a/sys/dev/xen/control/control.c
+++ b/sys/dev/xen/control/control.c
@@ -202,11 +202,18 @@ xctrl_suspend()
 	stop_all_proc();
 	EVENTHANDLER_INVOKE(power_suspend);
 
+#ifdef EARLY_AP_STARTUP
+	MPASS(mp_ncpus == 1 || smp_started);
+	thread_lock(curthread);
+	sched_bind(curthread, 0);
+	thread_unlock(curthread);
+#else
 	if (smp_started) {
 		thread_lock(curthread);
 		sched_bind(curthread, 0);
 		thread_unlock(curthread);
 	}
+#endif
 	KASSERT((PCPU_GET(cpuid) == 0), ("Not running on CPU#0"));
 
 	/*
@@ -227,6 +234,17 @@ xctrl_suspend()
 	}
 
 #ifdef SMP
+#ifdef EARLY_AP_STARTUP
+	/*
+	 * Suspend other CPUs. This prevents IPIs while we
+	 * are resuming, and will allow us to reset per-cpu
+	 * vcpu_info on resume.
+	 */
+	cpu_suspend_map = all_cpus;
+	CPU_CLR(PCPU_GET(cpuid), &cpu_suspend_map);
+	if (!CPU_EMPTY(&cpu_suspend_map))
+		suspend_cpus(cpu_suspend_map);
+#else
 	CPU_ZERO(&cpu_suspend_map);	/* silence gcc */
 	if (smp_started) {
 		/*
@@ -240,6 +258,7 @@ xctrl_suspend()
 			suspend_cpus(cpu_suspend_map);
 	}
 #endif
+#endif
 
 	/*
 	 * Prevent any races with evtchn_interrupt() handler.
@@ -285,11 +304,17 @@ xctrl_suspend()
 	timecounter->tc_get_timecount(timecounter);
 	inittodr(time_second);
 
+#ifdef EARLY_AP_STARTUP
+	thread_lock(curthread);
+	sched_unbind(curthread);
+	thread_unlock(curthread);
+#else
 	if (smp_started) {
 		thread_lock(curthread);
 		sched_unbind(curthread);
 		thread_unlock(curthread);
 	}
+#endif
 
 	resume_all_proc();