10 files changed, 451 insertions, 431 deletions

diff --git a/sys/amd64/amd64/db_interface.c b/sys/amd64/amd64/db_interface.c
index 7d5f34125847..1de589083909 100644
--- a/sys/amd64/amd64/db_interface.c
+++ b/sys/amd64/amd64/db_interface.c
@@ -23,7 +23,7 @@
  * any improvements or extensions that they make and grant Carnegie the
  * rights to redistribute these changes.
  *
- *	$Id: db_interface.c,v 1.30 1997/06/27 23:24:38 fsmp Exp $
+ *	$Id: db_interface.c,v 1.31 1997/07/06 23:25:46 fsmp Exp $
  */
 
 /*
@@ -139,7 +139,7 @@ kdb_trap(type, code, regs)
 	cnpollc(TRUE);
 
 #if defined(SMP) && defined(TEST_CPUSTOP)
-	/* we stopp all CPUs except ourselves (obviously) */
+	/* we stop all CPUs except ourselves (obviously) */
 	stop_cpus(other_cpus);
 #endif  /* SMP && TEST_CPUSTOP */
 
diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c
index 8085f0e39dfc..bf1b9a9627ae 100644
--- a/sys/amd64/amd64/mp_machdep.c
+++ b/sys/amd64/amd64/mp_machdep.c
@@ -22,7 +22,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: mp_machdep.c,v 1.6 1997/07/07 00:02:55 smp Exp smp $
+ *	$Id: mp_machdep.c,v 1.7 1997/07/08 23:42:28 smp Exp smp $
  */
 
 #include "opt_smp.h"
@@ -51,10 +51,14 @@
 
 #include <i386/i386/cons.h>	/* cngetc() */
 
+#if defined(TEST_CPUSTOP)
+void	db_printf __P((const char *fmt, ...));
+#endif  /* TEST_CPUSTOP */
+
 #if defined(APIC_IO)
-#include <machine/md_var.h>	/* setidt() */
-#include <i386/isa/icu.h>		/* Xinvltlb() */
-#include <i386/isa/intr_machdep.h>	/* Xinvltlb() */
+#include <machine/md_var.h>		/* setidt() */
+#include <i386/isa/icu.h>		/* IPIs */
+#include <i386/isa/intr_machdep.h>	/* IPIs */
 #endif	/* APIC_IO */
 
 #define WARMBOOT_TARGET		0
@@ -189,12 +193,6 @@ typedef struct BASETABLE_ENTRY {
 /*
  * Values to send to the POST hardware.
  */
-#ifndef POSTCODE
-#define POSTCODE(X)
-#define POSTCODE_LO(X)
-#define POSTCODE_HI(X)
-#endif
-
 #define MP_BOOTADDRESS_POST	0x10
 #define MP_PROBE_POST		0x11
 #define MP_START_POST		0x12
@@ -206,10 +204,10 @@ typedef struct BASETABLE_ENTRY {
 #define INSTALL_AP_TRAMP_POST	0x18
 #define START_AP_POST		0x19
 
-/* XXX FIXME: where does this really belong, isa.h/isa.c perhaps? */
+/** XXX FIXME: where does this really belong, isa.h/isa.c perhaps? */
 int	current_postcode;
 
-/** FIXME: what system files declare these??? */
+/** XXX FIXME: what system files declare these??? */
 extern struct region_descriptor r_gdt, r_idt;
 
 int	mp_ncpus;		/* # of CPUs, including BSP */
@@ -240,11 +238,11 @@ u_int *bootPTD;
 /* Hotwire a 0->4MB V==P mapping */
 extern pt_entry_t KPTphys;
 
-/* virtual address of per-cpu common_tss */
+/* Virtual address of per-cpu common_tss */
 extern struct i386tss common_tss;
 
 /*
- * look for MP compliant motherboard.
+ * Local data and functions.
  */
 
 static int	mp_capable;
@@ -265,7 +263,7 @@ static int	start_ap(int logicalCpu, u_int boot_addr);
 
 
 /*
- * calculate usable address in base memory for AP trampoline code
+ * Calculate usable address in base memory for AP trampoline code.
  */
 u_int
 mp_bootaddress(u_int basemem)
@@ -282,6 +280,9 @@ mp_bootaddress(u_int basemem)
 }
 
 
+/*
+ * Look for an Intel MP spec table (ie, SMP capable hardware).
+ */
 int
 mp_probe(void)
 {
@@ -314,7 +315,7 @@ mp_probe(void)
 	mp_capable = 0;
 	return 0;
 
-found:				/* please forgive the 'goto'! */
+found:
 	/* calculate needed resources */
 	mpfps = (mpfps_t)x;
 	if (mptable_pass1())
@@ -327,7 +328,7 @@ found:				/* please forgive the 'goto'! */
 
 
 /*
- * startup the SMP processors
+ * Startup the SMP processors.
  */
 void
 mp_start(void)
@@ -343,7 +344,7 @@ mp_start(void)
 
 
 /*
- * print various information about the SMP system hardware and setup
+ * Print various information about the SMP system hardware and setup.
  */
 void
 mp_announce(void)
@@ -383,7 +384,7 @@ init_secondary(void)
 
 	r_gdt.rd_limit = sizeof(gdt[0]) * (NGDT + NCPU) - 1;
 	r_gdt.rd_base = (int) gdt;
-	lgdt(&r_gdt);		/* does magic intra-segment return */
+	lgdt(&r_gdt);			/* does magic intra-segment return */
 	lidt(&r_idt);
 	lldt(_default_ldt);
 
@@ -395,7 +396,7 @@ init_secondary(void)
 	common_tss.tss_ioopt = (sizeof common_tss) << 16;
 	ltr(gsel_tss);
 
-	load_cr0(0x8005003b);	/* XXX! */
+	load_cr0(0x8005003b);		/* XXX! */
 
 	PTD[0] = 0;
 	invltlb();
@@ -403,44 +404,44 @@ init_secondary(void)
 
 
 #if defined(APIC_IO)
+/*
+ * Final configuration of the BSP's local APIC:
+ *  - disable 'pic mode'.
+ *  - disable 'virtual wire mode'.
+ *  - enable NMI.
+ */
 void
-configure_local_apic(void)
+bsp_apic_configure(void)
 {
-	u_char  byte;
-	u_int32_t temp;
+	u_char		byte;
+	u_int32_t	temp;
 
+	/* leave 'pic mode' if necessary */
 	if (picmode) {
 		outb(0x22, 0x70);	/* select IMCR */
 		byte = inb(0x23);	/* current contents */
-		byte |= 0x01;	/* mask external INTR */
+		byte |= 0x01;		/* mask external INTR */
 		outb(0x23, byte);	/* disconnect 8259s/NMI */
 	}
 
 	/* mask lint0 (the 8259 'virtual wire' connection) */
 	temp = lapic.lvt_lint0;
-	temp |= APIC_LVT_M;
+	temp |= APIC_LVT_M;		/* set the mask */
 	lapic.lvt_lint0 = temp;
 
         /* setup lint1 to handle NMI */
-#if 1
-        /** XXX FIXME: 
-         *      should we arrange for ALL CPUs to catch NMI???
-         *      it would probably crash, so for now only the BSP
-         *      will catch it
-         */
-        if (cpuid != 0)
-                return;
-#endif /* 0/1 */
-
         temp = lapic.lvt_lint1;
-
-        /* clear fields of interest, preserve undefined fields */
-        temp &= ~(0x1f000 | APIC_LVT_DM | APIC_LVT_VECTOR);
-
-        /* setup for NMI, edge trigger, active hi */
-        temp |= (APIC_LVT_DM_NMI | APIC_LVT_IIPP_INTAHI);
-
+        temp &= ~APIC_LVT_M;		/* clear the mask */
         lapic.lvt_lint1 = temp;
+
+#if defined(TEST_CPUSTOP)
+	printf(">>> CPU%02d bsp_apic_configure() lint0: 0x%08x\n",
+	       cpuid, lapic.lvt_lint0);
+	printf(">>>                            lint1: 0x%08x\n",
+	       lapic.lvt_lint1);
+	printf(">>>                            TPR:   0x%08x\n", lapic.tpr);
+	printf(">>>                            SVR:   0x%08x\n", lapic.svr);
+#endif  /* TEST_CPUSTOP */
 }
 #endif  /* APIC_IO */
 
@@ -463,7 +464,7 @@ mp_enable(u_int boot_addr)
 
 	POSTCODE(MP_ENABLE_POST);
 
-	/* Turn on 4MB of V == P addressing so we can get to MP table */
+	/* turn on 4MB of V == P addressing so we can get to MP table */
 	*(int *)PTD = PG_V | PG_RW | ((u_long)KPTphys & PG_FRAME);
 	invltlb();
 
@@ -1439,7 +1440,7 @@ start_all_aps(u_int boot_addr)
 	mp_lock = 1;	/* this uses a LOGICAL cpu ID, ie BSP == 0 */
 
 	/* initialize BSP's local APIC */
-	apic_initialize(1);
+	apic_initialize();
 
 	/* install the AP 1st level boot code */
 	install_ap_tramp(boot_addr);
@@ -1814,6 +1815,7 @@ stop_cpus( u_int map )
 
 #if defined(DEBUG_CPUSTOP)
 	db_printf("  spun\nstopped, sihits: %d\n", sihits);
+	cngetc();
 #endif /* DEBUG_CPUSTOP */
 
 	return 1;
diff --git a/sys/amd64/amd64/mptable.c b/sys/amd64/amd64/mptable.c
index 8085f0e39dfc..bf1b9a9627ae 100644
--- a/sys/amd64/amd64/mptable.c
+++ b/sys/amd64/amd64/mptable.c
@@ -22,7 +22,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: mp_machdep.c,v 1.6 1997/07/07 00:02:55 smp Exp smp $
+ *	$Id: mp_machdep.c,v 1.7 1997/07/08 23:42:28 smp Exp smp $
  */
 
 #include "opt_smp.h"
@@ -51,10 +51,14 @@
 
 #include <i386/i386/cons.h>	/* cngetc() */
 
+#if defined(TEST_CPUSTOP)
+void	db_printf __P((const char *fmt, ...));
+#endif  /* TEST_CPUSTOP */
+
 #if defined(APIC_IO)
-#include <machine/md_var.h>	/* setidt() */
-#include <i386/isa/icu.h>		/* Xinvltlb() */
-#include <i386/isa/intr_machdep.h>	/* Xinvltlb() */
+#include <machine/md_var.h>		/* setidt() */
+#include <i386/isa/icu.h>		/* IPIs */
+#include <i386/isa/intr_machdep.h>	/* IPIs */
 #endif	/* APIC_IO */
 
 #define WARMBOOT_TARGET		0
@@ -189,12 +193,6 @@ typedef struct BASETABLE_ENTRY {
 /*
  * Values to send to the POST hardware.
  */
-#ifndef POSTCODE
-#define POSTCODE(X)
-#define POSTCODE_LO(X)
-#define POSTCODE_HI(X)
-#endif
-
 #define MP_BOOTADDRESS_POST	0x10
 #define MP_PROBE_POST		0x11
 #define MP_START_POST		0x12
@@ -206,10 +204,10 @@ typedef struct BASETABLE_ENTRY {
 #define INSTALL_AP_TRAMP_POST	0x18
 #define START_AP_POST		0x19
 
-/* XXX FIXME: where does this really belong, isa.h/isa.c perhaps? */
+/** XXX FIXME: where does this really belong, isa.h/isa.c perhaps? */
 int	current_postcode;
 
-/** FIXME: what system files declare these??? */
+/** XXX FIXME: what system files declare these??? */
 extern struct region_descriptor r_gdt, r_idt;
 
 int	mp_ncpus;		/* # of CPUs, including BSP */
@@ -240,11 +238,11 @@ u_int *bootPTD;
 /* Hotwire a 0->4MB V==P mapping */
 extern pt_entry_t KPTphys;
 
-/* virtual address of per-cpu common_tss */
+/* Virtual address of per-cpu common_tss */
 extern struct i386tss common_tss;
 
 /*
- * look for MP compliant motherboard.
+ * Local data and functions.
  */
 
 static int	mp_capable;
@@ -265,7 +263,7 @@ static int	start_ap(int logicalCpu, u_int boot_addr);
 
 
 /*
- * calculate usable address in base memory for AP trampoline code
+ * Calculate usable address in base memory for AP trampoline code.
  */
 u_int
 mp_bootaddress(u_int basemem)
@@ -282,6 +280,9 @@ mp_bootaddress(u_int basemem)
 }
 
 
+/*
+ * Look for an Intel MP spec table (ie, SMP capable hardware).
+ */
 int
 mp_probe(void)
 {
@@ -314,7 +315,7 @@ mp_probe(void)
 	mp_capable = 0;
 	return 0;
 
-found:				/* please forgive the 'goto'! */
+found:
 	/* calculate needed resources */
 	mpfps = (mpfps_t)x;
 	if (mptable_pass1())
@@ -327,7 +328,7 @@ found:				/* please forgive the 'goto'! */
 
 
 /*
- * startup the SMP processors
+ * Startup the SMP processors.
  */
 void
 mp_start(void)
@@ -343,7 +344,7 @@ mp_start(void)
 
 
 /*
- * print various information about the SMP system hardware and setup
+ * Print various information about the SMP system hardware and setup.
  */
 void
 mp_announce(void)
@@ -383,7 +384,7 @@ init_secondary(void)
 
 	r_gdt.rd_limit = sizeof(gdt[0]) * (NGDT + NCPU) - 1;
 	r_gdt.rd_base = (int) gdt;
-	lgdt(&r_gdt);		/* does magic intra-segment return */
+	lgdt(&r_gdt);			/* does magic intra-segment return */
 	lidt(&r_idt);
 	lldt(_default_ldt);
 
@@ -395,7 +396,7 @@ init_secondary(void)
 	common_tss.tss_ioopt = (sizeof common_tss) << 16;
 	ltr(gsel_tss);
 
-	load_cr0(0x8005003b);	/* XXX! */
+	load_cr0(0x8005003b);		/* XXX! */
 
 	PTD[0] = 0;
 	invltlb();
@@ -403,44 +404,44 @@ init_secondary(void)
 
 
 #if defined(APIC_IO)
+/*
+ * Final configuration of the BSP's local APIC:
+ *  - disable 'pic mode'.
+ *  - disable 'virtual wire mode'.
+ *  - enable NMI.
+ */
 void
-configure_local_apic(void)
+bsp_apic_configure(void)
 {
-	u_char  byte;
-	u_int32_t temp;
+	u_char		byte;
+	u_int32_t	temp;
 
+	/* leave 'pic mode' if necessary */
 	if (picmode) {
 		outb(0x22, 0x70);	/* select IMCR */
 		byte = inb(0x23);	/* current contents */
-		byte |= 0x01;	/* mask external INTR */
+		byte |= 0x01;		/* mask external INTR */
 		outb(0x23, byte);	/* disconnect 8259s/NMI */
 	}
 
 	/* mask lint0 (the 8259 'virtual wire' connection) */
 	temp = lapic.lvt_lint0;
-	temp |= APIC_LVT_M;
+	temp |= APIC_LVT_M;		/* set the mask */
 	lapic.lvt_lint0 = temp;
 
         /* setup lint1 to handle NMI */
-#if 1
-        /** XXX FIXME: 
-         *      should we arrange for ALL CPUs to catch NMI???
-         *      it would probably crash, so for now only the BSP
-         *      will catch it
-         */
-        if (cpuid != 0)
-                return;
-#endif /* 0/1 */
-
         temp = lapic.lvt_lint1;
-
-        /* clear fields of interest, preserve undefined fields */
-        temp &= ~(0x1f000 | APIC_LVT_DM | APIC_LVT_VECTOR);
-
-        /* setup for NMI, edge trigger, active hi */
-        temp |= (APIC_LVT_DM_NMI | APIC_LVT_IIPP_INTAHI);
-
+        temp &= ~APIC_LVT_M;		/* clear the mask */
         lapic.lvt_lint1 = temp;
+
+#if defined(TEST_CPUSTOP)
+	printf(">>> CPU%02d bsp_apic_configure() lint0: 0x%08x\n",
+	       cpuid, lapic.lvt_lint0);
+	printf(">>>                            lint1: 0x%08x\n",
+	       lapic.lvt_lint1);
+	printf(">>>                            TPR:   0x%08x\n", lapic.tpr);
+	printf(">>>                            SVR:   0x%08x\n", lapic.svr);
+#endif  /* TEST_CPUSTOP */
 }
 #endif  /* APIC_IO */
 
@@ -463,7 +464,7 @@ mp_enable(u_int boot_addr)
 
 	POSTCODE(MP_ENABLE_POST);
 
-	/* Turn on 4MB of V == P addressing so we can get to MP table */
+	/* turn on 4MB of V == P addressing so we can get to MP table */
 	*(int *)PTD = PG_V | PG_RW | ((u_long)KPTphys & PG_FRAME);
 	invltlb();
 
@@ -1439,7 +1440,7 @@ start_all_aps(u_int boot_addr)
 	mp_lock = 1;	/* this uses a LOGICAL cpu ID, ie BSP == 0 */
 
 	/* initialize BSP's local APIC */
-	apic_initialize(1);
+	apic_initialize();
 
 	/* install the AP 1st level boot code */
 	install_ap_tramp(boot_addr);
@@ -1814,6 +1815,7 @@ stop_cpus( u_int map )
 
 #if defined(DEBUG_CPUSTOP)
 	db_printf("  spun\nstopped, sihits: %d\n", sihits);
+	cngetc();
 #endif /* DEBUG_CPUSTOP */
 
 	return 1;
diff --git a/sys/amd64/include/mptable.h b/sys/amd64/include/mptable.h
index 8085f0e39dfc..bf1b9a9627ae 100644
--- a/sys/amd64/include/mptable.h
+++ b/sys/amd64/include/mptable.h
@@ -22,7 +22,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: mp_machdep.c,v 1.6 1997/07/07 00:02:55 smp Exp smp $
+ *	$Id: mp_machdep.c,v 1.7 1997/07/08 23:42:28 smp Exp smp $
  */
 
 #include "opt_smp.h"
@@ -51,10 +51,14 @@
 
 #include <i386/i386/cons.h>	/* cngetc() */
 
+#if defined(TEST_CPUSTOP)
+void	db_printf __P((const char *fmt, ...));
+#endif  /* TEST_CPUSTOP */
+
 #if defined(APIC_IO)
-#include <machine/md_var.h>	/* setidt() */
-#include <i386/isa/icu.h>		/* Xinvltlb() */
-#include <i386/isa/intr_machdep.h>	/* Xinvltlb() */
+#include <machine/md_var.h>		/* setidt() */
+#include <i386/isa/icu.h>		/* IPIs */
+#include <i386/isa/intr_machdep.h>	/* IPIs */
 #endif	/* APIC_IO */
 
 #define WARMBOOT_TARGET		0
@@ -189,12 +193,6 @@ typedef struct BASETABLE_ENTRY {
 /*
  * Values to send to the POST hardware.
  */
-#ifndef POSTCODE
-#define POSTCODE(X)
-#define POSTCODE_LO(X)
-#define POSTCODE_HI(X)
-#endif
-
 #define MP_BOOTADDRESS_POST	0x10
 #define MP_PROBE_POST		0x11
 #define MP_START_POST		0x12
@@ -206,10 +204,10 @@ typedef struct BASETABLE_ENTRY {
 #define INSTALL_AP_TRAMP_POST	0x18
 #define START_AP_POST		0x19
 
-/* XXX FIXME: where does this really belong, isa.h/isa.c perhaps? */
+/** XXX FIXME: where does this really belong, isa.h/isa.c perhaps? */
 int	current_postcode;
 
-/** FIXME: what system files declare these??? */
+/** XXX FIXME: what system files declare these??? */
 extern struct region_descriptor r_gdt, r_idt;
 
 int	mp_ncpus;		/* # of CPUs, including BSP */
@@ -240,11 +238,11 @@ u_int *bootPTD;
 /* Hotwire a 0->4MB V==P mapping */
 extern pt_entry_t KPTphys;
 
-/* virtual address of per-cpu common_tss */
+/* Virtual address of per-cpu common_tss */
 extern struct i386tss common_tss;
 
 /*
- * look for MP compliant motherboard.
+ * Local data and functions.
  */
 
 static int	mp_capable;
@@ -265,7 +263,7 @@ static int	start_ap(int logicalCpu, u_int boot_addr);
 
 
 /*
- * calculate usable address in base memory for AP trampoline code
+ * Calculate usable address in base memory for AP trampoline code.
  */
 u_int
 mp_bootaddress(u_int basemem)
@@ -282,6 +280,9 @@ mp_bootaddress(u_int basemem)
 }
 
 
+/*
+ * Look for an Intel MP spec table (ie, SMP capable hardware).
+ */
 int
 mp_probe(void)
 {
@@ -314,7 +315,7 @@ mp_probe(void)
 	mp_capable = 0;
 	return 0;
 
-found:				/* please forgive the 'goto'! */
+found:
 	/* calculate needed resources */
 	mpfps = (mpfps_t)x;
 	if (mptable_pass1())
@@ -327,7 +328,7 @@ found:				/* please forgive the 'goto'! */
 
 
 /*
- * startup the SMP processors
+ * Startup the SMP processors.
  */
 void
 mp_start(void)
@@ -343,7 +344,7 @@ mp_start(void)
 
 
 /*
- * print various information about the SMP system hardware and setup
+ * Print various information about the SMP system hardware and setup.
  */
 void
 mp_announce(void)
@@ -383,7 +384,7 @@ init_secondary(void)
 
 	r_gdt.rd_limit = sizeof(gdt[0]) * (NGDT + NCPU) - 1;
 	r_gdt.rd_base = (int) gdt;
-	lgdt(&r_gdt);		/* does magic intra-segment return */
+	lgdt(&r_gdt);			/* does magic intra-segment return */
 	lidt(&r_idt);
 	lldt(_default_ldt);
 
@@ -395,7 +396,7 @@ init_secondary(void)
 	common_tss.tss_ioopt = (sizeof common_tss) << 16;
 	ltr(gsel_tss);
 
-	load_cr0(0x8005003b);	/* XXX! */
+	load_cr0(0x8005003b);		/* XXX! */
 
 	PTD[0] = 0;
 	invltlb();
@@ -403,44 +404,44 @@ init_secondary(void)
 
 
 #if defined(APIC_IO)
+/*
+ * Final configuration of the BSP's local APIC:
+ *  - disable 'pic mode'.
+ *  - disable 'virtual wire mode'.
+ *  - enable NMI.
+ */
 void
-configure_local_apic(void)
+bsp_apic_configure(void)
 {
-	u_char  byte;
-	u_int32_t temp;
+	u_char		byte;
+	u_int32_t	temp;
 
+	/* leave 'pic mode' if necessary */
 	if (picmode) {
 		outb(0x22, 0x70);	/* select IMCR */
 		byte = inb(0x23);	/* current contents */
-		byte |= 0x01;	/* mask external INTR */
+		byte |= 0x01;		/* mask external INTR */
 		outb(0x23, byte);	/* disconnect 8259s/NMI */
 	}
 
 	/* mask lint0 (the 8259 'virtual wire' connection) */
 	temp = lapic.lvt_lint0;
-	temp |= APIC_LVT_M;
+	temp |= APIC_LVT_M;		/* set the mask */
 	lapic.lvt_lint0 = temp;
 
         /* setup lint1 to handle NMI */
-#if 1
-        /** XXX FIXME: 
-         *      should we arrange for ALL CPUs to catch NMI???
-         *      it would probably crash, so for now only the BSP
-         *      will catch it
-         */
-        if (cpuid != 0)
-                return;
-#endif /* 0/1 */
-
         temp = lapic.lvt_lint1;
-
-        /* clear fields of interest, preserve undefined fields */
-        temp &= ~(0x1f000 | APIC_LVT_DM | APIC_LVT_VECTOR);
-
-        /* setup for NMI, edge trigger, active hi */
-        temp |= (APIC_LVT_DM_NMI | APIC_LVT_IIPP_INTAHI);
-
+        temp &= ~APIC_LVT_M;		/* clear the mask */
         lapic.lvt_lint1 = temp;
+
+#if defined(TEST_CPUSTOP)
+	printf(">>> CPU%02d bsp_apic_configure() lint0: 0x%08x\n",
+	       cpuid, lapic.lvt_lint0);
+	printf(">>>                            lint1: 0x%08x\n",
+	       lapic.lvt_lint1);
+	printf(">>>                            TPR:   0x%08x\n", lapic.tpr);
+	printf(">>>                            SVR:   0x%08x\n", lapic.svr);
+#endif  /* TEST_CPUSTOP */
 }
 #endif  /* APIC_IO */
 
@@ -463,7 +464,7 @@ mp_enable(u_int boot_addr)
 
 	POSTCODE(MP_ENABLE_POST);
 
-	/* Turn on 4MB of V == P addressing so we can get to MP table */
+	/* turn on 4MB of V == P addressing so we can get to MP table */
 	*(int *)PTD = PG_V | PG_RW | ((u_long)KPTphys & PG_FRAME);
 	invltlb();
 
@@ -1439,7 +1440,7 @@ start_all_aps(u_int boot_addr)
 	mp_lock = 1;	/* this uses a LOGICAL cpu ID, ie BSP == 0 */
 
 	/* initialize BSP's local APIC */
-	apic_initialize(1);
+	apic_initialize();
 
 	/* install the AP 1st level boot code */
 	install_ap_tramp(boot_addr);
@@ -1814,6 +1815,7 @@ stop_cpus( u_int map )
 
 #if defined(DEBUG_CPUSTOP)
 	db_printf("  spun\nstopped, sihits: %d\n", sihits);
+	cngetc();
 #endif /* DEBUG_CPUSTOP */
 
 	return 1;
diff --git a/sys/i386/i386/db_interface.c b/sys/i386/i386/db_interface.c
index 7d5f34125847..1de589083909 100644
--- a/sys/i386/i386/db_interface.c
+++ b/sys/i386/i386/db_interface.c
@@ -23,7 +23,7 @@
  * any improvements or extensions that they make and grant Carnegie the
  * rights to redistribute these changes.
  *
- *	$Id: db_interface.c,v 1.30 1997/06/27 23:24:38 fsmp Exp $
+ *	$Id: db_interface.c,v 1.31 1997/07/06 23:25:46 fsmp Exp $
  */
 
 /*
@@ -139,7 +139,7 @@ kdb_trap(type, code, regs)
 	cnpollc(TRUE);
 
 #if defined(SMP) && defined(TEST_CPUSTOP)
-	/* we stopp all CPUs except ourselves (obviously) */
+	/* we stop all CPUs except ourselves (obviously) */
 	stop_cpus(other_cpus);
 #endif  /* SMP && TEST_CPUSTOP */
 
diff --git a/sys/i386/i386/mp_machdep.c b/sys/i386/i386/mp_machdep.c
index 8085f0e39dfc..bf1b9a9627ae 100644
--- a/sys/i386/i386/mp_machdep.c
+++ b/sys/i386/i386/mp_machdep.c
@@ -22,7 +22,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: mp_machdep.c,v 1.6 1997/07/07 00:02:55 smp Exp smp $
+ *	$Id: mp_machdep.c,v 1.7 1997/07/08 23:42:28 smp Exp smp $
  */
 
 #include "opt_smp.h"
@@ -51,10 +51,14 @@
 
 #include <i386/i386/cons.h>	/* cngetc() */
 
+#if defined(TEST_CPUSTOP)
+void	db_printf __P((const char *fmt, ...));
+#endif  /* TEST_CPUSTOP */
+
 #if defined(APIC_IO)
-#include <machine/md_var.h>	/* setidt() */
-#include <i386/isa/icu.h>		/* Xinvltlb() */
-#include <i386/isa/intr_machdep.h>	/* Xinvltlb() */
+#include <machine/md_var.h>		/* setidt() */
+#include <i386/isa/icu.h>		/* IPIs */
+#include <i386/isa/intr_machdep.h>	/* IPIs */
 #endif	/* APIC_IO */
 
 #define WARMBOOT_TARGET		0
@@ -189,12 +193,6 @@ typedef struct BASETABLE_ENTRY {
 /*
  * Values to send to the POST hardware.
  */
-#ifndef POSTCODE
-#define POSTCODE(X)
-#define POSTCODE_LO(X)
-#define POSTCODE_HI(X)
-#endif
-
 #define MP_BOOTADDRESS_POST	0x10
 #define MP_PROBE_POST		0x11
 #define MP_START_POST		0x12
@@ -206,10 +204,10 @@ typedef struct BASETABLE_ENTRY {
 #define INSTALL_AP_TRAMP_POST	0x18
 #define START_AP_POST		0x19
 
-/* XXX FIXME: where does this really belong, isa.h/isa.c perhaps? */
+/** XXX FIXME: where does this really belong, isa.h/isa.c perhaps? */
 int	current_postcode;
 
-/** FIXME: what system files declare these??? */
+/** XXX FIXME: what system files declare these??? */
 extern struct region_descriptor r_gdt, r_idt;
 
 int	mp_ncpus;		/* # of CPUs, including BSP */
@@ -240,11 +238,11 @@ u_int *bootPTD;
 /* Hotwire a 0->4MB V==P mapping */
 extern pt_entry_t KPTphys;
 
-/* virtual address of per-cpu common_tss */
+/* Virtual address of per-cpu common_tss */
 extern struct i386tss common_tss;
 
 /*
- * look for MP compliant motherboard.
+ * Local data and functions.
  */
 
 static int	mp_capable;
@@ -265,7 +263,7 @@ static int	start_ap(int logicalCpu, u_int boot_addr);
 
 
 /*
- * calculate usable address in base memory for AP trampoline code
+ * Calculate usable address in base memory for AP trampoline code.
  */
 u_int
 mp_bootaddress(u_int basemem)
@@ -282,6 +280,9 @@ mp_bootaddress(u_int basemem)
 }
 
 
+/*
+ * Look for an Intel MP spec table (ie, SMP capable hardware).
+ */
 int
 mp_probe(void)
 {
@@ -314,7 +315,7 @@ mp_probe(void)
 	mp_capable = 0;
 	return 0;
 
-found:				/* please forgive the 'goto'! */
+found:
 	/* calculate needed resources */
 	mpfps = (mpfps_t)x;
 	if (mptable_pass1())
@@ -327,7 +328,7 @@ found:				/* please forgive the 'goto'! */
 
 
 /*
- * startup the SMP processors
+ * Startup the SMP processors.
  */
 void
 mp_start(void)
@@ -343,7 +344,7 @@ mp_start(void)
 
 
 /*
- * print various information about the SMP system hardware and setup
+ * Print various information about the SMP system hardware and setup.
  */
 void
 mp_announce(void)
@@ -383,7 +384,7 @@ init_secondary(void)
 
 	r_gdt.rd_limit = sizeof(gdt[0]) * (NGDT + NCPU) - 1;
 	r_gdt.rd_base = (int) gdt;
-	lgdt(&r_gdt);		/* does magic intra-segment return */
+	lgdt(&r_gdt);			/* does magic intra-segment return */
 	lidt(&r_idt);
 	lldt(_default_ldt);
 
@@ -395,7 +396,7 @@ init_secondary(void)
 	common_tss.tss_ioopt = (sizeof common_tss) << 16;
 	ltr(gsel_tss);
 
-	load_cr0(0x8005003b);	/* XXX! */
+	load_cr0(0x8005003b);		/* XXX! */
 
 	PTD[0] = 0;
 	invltlb();
@@ -403,44 +404,44 @@ init_secondary(void)
 
 
 #if defined(APIC_IO)
+/*
+ * Final configuration of the BSP's local APIC:
+ *  - disable 'pic mode'.
+ *  - disable 'virtual wire mode'.
+ *  - enable NMI.
+ */
 void
-configure_local_apic(void)
+bsp_apic_configure(void)
 {
-	u_char  byte;
-	u_int32_t temp;
+	u_char		byte;
+	u_int32_t	temp;
 
+	/* leave 'pic mode' if necessary */
 	if (picmode) {
 		outb(0x22, 0x70);	/* select IMCR */
 		byte = inb(0x23);	/* current contents */
-		byte |= 0x01;	/* mask external INTR */
+		byte |= 0x01;		/* mask external INTR */
 		outb(0x23, byte);	/* disconnect 8259s/NMI */
 	}
 
 	/* mask lint0 (the 8259 'virtual wire' connection) */
 	temp = lapic.lvt_lint0;
-	temp |= APIC_LVT_M;
+	temp |= APIC_LVT_M;		/* set the mask */
 	lapic.lvt_lint0 = temp;
 
         /* setup lint1 to handle NMI */
-#if 1
-        /** XXX FIXME: 
-         *      should we arrange for ALL CPUs to catch NMI???
-         *      it would probably crash, so for now only the BSP
-         *      will catch it
-         */
-        if (cpuid != 0)
-                return;
-#endif /* 0/1 */
-
         temp = lapic.lvt_lint1;
-
-        /* clear fields of interest, preserve undefined fields */
-        temp &= ~(0x1f000 | APIC_LVT_DM | APIC_LVT_VECTOR);
-
-        /* setup for NMI, edge trigger, active hi */
-        temp |= (APIC_LVT_DM_NMI | APIC_LVT_IIPP_INTAHI);
-
+        temp &= ~APIC_LVT_M;		/* clear the mask */
         lapic.lvt_lint1 = temp;
+
+#if defined(TEST_CPUSTOP)
+	printf(">>> CPU%02d bsp_apic_configure() lint0: 0x%08x\n",
+	       cpuid, lapic.lvt_lint0);
+	printf(">>>                            lint1: 0x%08x\n",
+	       lapic.lvt_lint1);
+	printf(">>>                            TPR:   0x%08x\n", lapic.tpr);
+	printf(">>>                            SVR:   0x%08x\n", lapic.svr);
+#endif  /* TEST_CPUSTOP */
 }
 #endif  /* APIC_IO */
 
@@ -463,7 +464,7 @@ mp_enable(u_int boot_addr)
 
 	POSTCODE(MP_ENABLE_POST);
 
-	/* Turn on 4MB of V == P addressing so we can get to MP table */
+	/* turn on 4MB of V == P addressing so we can get to MP table */
 	*(int *)PTD = PG_V | PG_RW | ((u_long)KPTphys & PG_FRAME);
 	invltlb();
 
@@ -1439,7 +1440,7 @@ start_all_aps(u_int boot_addr)
 	mp_lock = 1;	/* this uses a LOGICAL cpu ID, ie BSP == 0 */
 
 	/* initialize BSP's local APIC */
-	apic_initialize(1);
+	apic_initialize();
 
 	/* install the AP 1st level boot code */
 	install_ap_tramp(boot_addr);
@@ -1814,6 +1815,7 @@ stop_cpus( u_int map )
 
 #if defined(DEBUG_CPUSTOP)
 	db_printf("  spun\nstopped, sihits: %d\n", sihits);
+	cngetc();
 #endif /* DEBUG_CPUSTOP */
 
 	return 1;
diff --git a/sys/i386/i386/mpapic.c b/sys/i386/i386/mpapic.c
index 35fd22bdf90e..67dd9e755e01 100644
--- a/sys/i386/i386/mpapic.c
+++ b/sys/i386/i386/mpapic.c
@@ -22,7 +22,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: mpapic.c,v 1.8 1997/07/06 23:59:31 fsmp Exp $
+ *	$Id: mpapic.c,v 1.6 1997/07/08 23:42:28 smp Exp smp $
  */
 
 #include "opt_smp.h"
@@ -39,9 +39,13 @@
 
 #include <i386/isa/intr_machdep.h>	/* Xspuriousint() */
 
+#if defined(TEST_CPUSTOP)
+void	db_printf __P((const char *fmt, ...));
+#endif  /* TEST_CPUSTOP */
+
 /* EISA Edge/Level trigger control registers */
-#define ELCR0	0x4d0		/* eisa irq 0-7 */
-#define ELCR1	0x4d1		/* eisa irq 8-15 */
+#define ELCR0	0x4d0			/* eisa irq 0-7 */
+#define ELCR1	0x4d1			/* eisa irq 8-15 */
 
 /*
  * pointers to pmapped apic hardware.
@@ -52,54 +56,71 @@ volatile ioapic_t	*ioapic[NAPIC];
 #endif	/* APIC_IO */
 
 /*
- * enable APIC, configure interrupts
+ * Enable APIC, configure interrupts.
+ *
+ * XXX FIXME: remove the magic numbers.
  */
 void
-apic_initialize(int is_bsp)
+apic_initialize(void)
 {
 	u_int   temp;
 
-	if (is_bsp) {
-		/* setup LVT1 as ExtINT */
-		temp = lapic.lvt_lint0;
-		temp &= 0xfffe58ff;
-		temp |= 0x00000700;
-		lapic.lvt_lint0 = temp;
-	}
-	/* setup LVT2 as NMI */
+	/* setup LVT1 as ExtINT */
+	temp = lapic.lvt_lint0;
+	temp &= 0xfffe58ff;		/* preserve undefined fields */
+	if (cpuid == 0)
+		temp |= 0x00000700;	/* process ExtInts */
+	else
+		temp |= 0x00010700;	/* mask ExtInts */
+	lapic.lvt_lint0 = temp;
+
+	/* setup LVT2 as NMI, masked till later... */
 	temp = lapic.lvt_lint1;
-	temp &= 0xfffe58ff;
-	temp |= 0xffff0400;
+	temp &= 0xfffe58ff;		/* preserve undefined fields */
+	temp |= 0x00010400;		/* masked, edge trigger, active hi */
+
 	lapic.lvt_lint1 = temp;
 
 	/* set the Task Priority Register as needed */
 	temp = lapic.tpr;
-	temp &= ~APIC_TPR_PRIO;	/* clear priority field */
+	temp &= ~APIC_TPR_PRIO;		/* clear priority field */
 
 #if defined(TEST_LOPRIO)
-	if (is_bsp)
-		temp |= 0x10;	/* allow INT arbitration */
+#if 1
+	/* The new order of startup since private pages makes this possible. */
+	temp |= 0x10;			/* allow INT arbitration */
+#else
+	if (cpuid == 0)
+		temp |= 0x10;		/* allow INT arbitration */
 	else
-		temp |= 0xff;	/* disallow INT arbitration */
+		temp |= 0xff;		/* disallow INT arbitration */
+#endif
 #endif	/* TEST_LOPRIO */
 
 	lapic.tpr = temp;
 
-	/* enable the beast */
+	/* enable the local APIC */
 	temp = lapic.svr;
-	temp |= APIC_SVR_SWEN;	/* software enable APIC */
-	temp &= ~APIC_SVR_FOCUS;/* enable 'focus processor' */
+	temp |= APIC_SVR_SWEN;		/* software enable APIC */
+	temp &= ~APIC_SVR_FOCUS;	/* enable 'focus processor' */
+
 #if defined(TEST_CPUSTOP)
-	if ((XSPURIOUSINT_OFFSET & 0xf) != 0xf)
+	if ((XSPURIOUSINT_OFFSET & APIC_SVR_VEC_FIX) != APIC_SVR_VEC_FIX)
 		panic("bad XSPURIOUSINT_OFFSET: 0x%08x", XSPURIOUSINT_OFFSET);
-	temp &= ~0xff;		/* clear vector field */
-	temp |= XSPURIOUSINT_OFFSET;
-	printf(">>> SVR: 0x%08x\n", temp);
+	temp &= ~APIC_SVR_VEC_PROG;	/* clear (programmable) vector field */
+	temp |= (XSPURIOUSINT_OFFSET & APIC_SVR_VEC_PROG);
 #endif  /* TEST_CPUSTOP */
-#if 0
-	temp |= 0x20;		/** FIXME: 2f == strayIRQ15 */
-#endif
+
 	lapic.svr = temp;
+
+#if defined(TEST_CPUSTOP)
+	printf(">>> CPU%02d apic_initialize()    lint0: 0x%08x\n",
+	       cpuid, lapic.lvt_lint0);
+	printf(">>>                            lint1: 0x%08x\n",
+	       lapic.lvt_lint1);
+	printf(">>>                            TPR:   0x%08x\n", lapic.tpr);
+	printf(">>>                            SVR:   0x%08x\n", lapic.svr);
+#endif  /* TEST_CPUSTOP */
 }
 
 
@@ -117,30 +138,6 @@ static int trigger __P((int apic, int pin, u_int32_t * flags));
 static void polarity __P((int apic, int pin, u_int32_t * flags, int level));
 
 
-/*
- * setup I/O APIC
- */
-
-#ifdef HOW_8259_IS_PROGRAMMED
-outb(IO_ICU1, 0x11);		/* reset; program device, four bytes */
-outb(IO_ICU1 + 1, NRSVIDT);	/* starting at this vector index */
-outb(IO_ICU1 + 1, 1 << 2);	/* slave on line 2 */
-outb(IO_ICU1 + 1, 1);		/* 8086 mode */
-
-outb(IO_ICU1 + 1, 0xff);	/* leave interrupts masked */
-outb(IO_ICU1, 0x0a);		/* default to IRR on read */
-outb(IO_ICU1, 0xc0 | (3 - 1));	/* pri order 3-7, 0-2 (com2 first) */
-
-outb(IO_ICU2, 0x11);		/* reset; program device, four bytes */
-outb(IO_ICU2 + 1, NRSVIDT + 8);	/* staring at this vector index */
-outb(IO_ICU2 + 1, 2);		/* my slave id is 2 */
-outb(IO_ICU2 + 1, 1);		/* 8086 mode */
-
-outb(IO_ICU2 + 1, 0xff);	/* leave interrupts masked */
-outb(IO_ICU2, 0x0a);		/* default to IRR on read */
-#endif	/* HOW_8259_IS_PROGRAMMED */
-
-
 #if defined(TEST_LOPRIO)
 #define DEFAULT_FLAGS		\
 	((u_int32_t)		\
@@ -172,7 +169,7 @@ outb(IO_ICU2, 0x0a);		/* default to IRR on read */
 #endif	/* TEST_LOPRIO */
 
 /*
- *
+ * Setup the IO APIC.
  */
 int
 io_apic_setup(int apic)
@@ -191,7 +188,7 @@ io_apic_setup(int apic)
 #endif	/* TEST_LOPRIO */
 
 	if (apic == 0) {
-		maxpin = REDIRCNT_IOAPIC(apic);/* pins-1 in this part */
+		maxpin = REDIRCNT_IOAPIC(apic);		/* pins-1 in APIC */
 		for (pin = 0; pin < maxpin; ++pin) {
 			int bus, bustype;
 
@@ -218,13 +215,15 @@ io_apic_setup(int apic)
 			}
 
 			/* program the appropriate registers */
-			select = pin * 2 + IOAPIC_REDTBL0;/* register */
-			vector = NRSVIDT + pin;/* IDT vec */
+			select = pin * 2 + IOAPIC_REDTBL0;	/* register */
+			vector = NRSVIDT + pin;			/* IDT vec */
 			io_apic_write(apic, select, flags | vector);
 			io_apic_write(apic, select + 1, target);
 		}
         }
-        else {	/* program entry according to MP table. */
+
+	/* program entry according to MP table. */
+        else {
 #if defined(MULTIPLE_IOAPICS)
 #error MULTIPLE_IOAPICSXXX
 #else
@@ -248,7 +247,7 @@ io_apic_setup(int apic)
 	  IOART_DELEXINT))
 
 /*
- *
+ * Setup the source of External INTerrupts.
  */
 int
 ext_int_setup(int apic, int intr)
@@ -261,8 +260,8 @@ ext_int_setup(int apic, int intr)
 	if (apic_int_type(apic, intr) != 3)
 		return -1;
 
-/** FIXME: where should we steer ExtInts ??? */
-#if defined(BROADCAST_EXTINT)
+/** XXX FIXME: changed on 970708, make default if no complaints */
+#if 1
 	target = IOART_DEST;
 #else
 	target = boot_cpu_id << 24;
@@ -281,7 +280,7 @@ ext_int_setup(int apic, int intr)
 
 
 /*
- *
+ * Set the trigger level for an IO APIC pin.
  */
 static int
 trigger(int apic, int pin, u_int32_t * flags)
@@ -319,18 +318,23 @@ trigger(int apic, int pin, u_int32_t * flags)
 
 	case EISA:
 		eirq = apic_src_bus_irq(apic, pin);
+
 		if (eirq < 0 || eirq > 15) {
 			printf("EISA IRQ %d?!?!\n", eirq);
 			goto bad;
 		}
+
 		if (intcontrol == -1) {
 			intcontrol = inb(ELCR1) << 8;
 			intcontrol |= inb(ELCR0);
 			printf("EISA INTCONTROL = %08x\n", intcontrol);
 		}
-		/* EISA IRQ's are identical to ISA irq's, regardless of
-		 * whether they are edge or level since they go through the
-		 * level/polarity converter gadget. */
+
+		/*
+		 * EISA IRQ's are identical to ISA irq's, regardless of
+		 * whether they are edge or level since they go through
+		 * the level/polarity converter gadget.
+		 */
 		level = 0;
 
 		if (level)
@@ -355,7 +359,7 @@ bad:
 
 
 /*
- *
+ * Set the polarity value for an IO APIC pin.
  */
 static void
 polarity(int apic, int pin, u_int32_t * flags, int level)
@@ -400,10 +404,10 @@ polarity(int apic, int pin, u_int32_t * flags, int level)
 		 * whether they are edge or level since they go through the
 		 * level/polarity converter gadget. */
 
-		if (level == 1)	/* XXX Always false */
-			pol = 0;/* If level, active low */
+		if (level == 1)			/* XXX Always false */
+			pol = 0;		/* if level, active low */
 		else
-			pol = 1;/* If edge, high edge */
+			pol = 1;		/* if edge, high edge */
 
 		if (pol == 0)
 			*flags |= IOART_INTALO;
@@ -425,10 +429,11 @@ bad:
 	panic("bad APIC IO INT flags");
 }
 
+
 /*
- * set INT mask bit for each bit set in 'mask'.
- * clear INT mask bit for all others.
- * only consider lower 24 bits in mask.
+ * Set INT mask bit for each bit set in 'mask'.
+ * Clear INT mask bit for all others.
+ * Only consider lower 24 bits in mask.
  */
 #if defined(MULTIPLE_IOAPICS)
 #error MULTIPLE_IOAPICSXXX
@@ -440,19 +445,19 @@ void
 write_io_apic_mask24(int apic, u_int32_t mask)
 {
 	int     x, y;
-	u_char  select;		/* the select register is 8 bits */
-	u_int32_t low_reg;	/* the window register is 32 bits */
+	u_char  select;			/* the select register is 8 bits */
+	u_int32_t low_reg;		/* the window register is 32 bits */
 	u_int32_t diffs;
 
-	mask &= IO_FIELD;	/* safety valve, only use 24 bits */
-	if (mask == IO_MASK)	/* check for same value as current */
+	mask &= IO_FIELD;		/* safety valve, only use 24 bits */
+	if (mask == IO_MASK)		/* check for same value as current */
 		return;
 
-	diffs = mask ^ IO_MASK;	/* record differences */
+	diffs = mask ^ IO_MASK;		/* record differences */
 
 	for (x = 0, y = REDIRCNT_IOAPIC(apic); x < y; ++x) {
 		if (!(diffs & (1 << x)))
-			continue;	/* no change, skip */
+			continue;			/* no change, skip */
 
 		select = IOAPIC_REDTBL + (x * 2);	/* calculate addr */
 		low_reg = io_apic_read(apic, select);	/* read contents */
@@ -470,7 +475,7 @@ write_io_apic_mask24(int apic, u_int32_t mask)
 
 #if defined(READY)
 /*
- * read current IRQ0 -IRQ23 masks
+ * Read current IRQ0 -IRQ23 masks.
  */
 #if defined(MULTIPLE_IOAPICS)
 #error MULTIPLE_IOAPICSXXX
@@ -486,20 +491,20 @@ read_io_apic_mask24(int apic)
 
 #if defined(READY)
 /*
- * set INT mask bit for each bit set in 'mask'.
- * ignore INT mask bit for all others.
- * only consider lower 24 bits in mask.
+ * Set INT mask bit for each bit set in 'mask'.
+ * Ignore INT mask bit for all others.
+ * Only consider lower 24 bits in mask.
  */
 void
 set_io_apic_mask24(apic, u_int32_t bits)
 {
 	int     x, y;
-	u_char  select;		/* the select register is 8 bits */
-	u_int32_t low_reg;	/* the window register is 32 bits */
+	u_char  select;			/* the select register is 8 bits */
+	u_int32_t low_reg;		/* the window register is 32 bits */
 	u_int32_t diffs;
 
-	bits &= IO_FIELD;	/* safety valve, only use 24 bits */
-	diffs = bits & ~IO_MASK;/* clear AND needing 'set'ing */
+	bits &= IO_FIELD;		/* safety valve, only use 24 bits */
+	diffs = bits & ~IO_MASK;	/* clear AND needing 'set'ing */
 	if (!diffs)
 		return;
 
@@ -507,12 +512,12 @@ set_io_apic_mask24(apic, u_int32_t bits)
 
 	for (x = 0, y = REDIRCNT_IOAPIC(apic); x < y; ++x) {
 		if (!(diffs & (1 << x)))
-			continue;	/* no change, skip */
+			continue;			/* no change, skip */
 
 		select = IOAPIC_REDTBL + (x * 2);	/* calculate addr */
 		low_reg = io_apic_read(apic, select);	/* read contents */
 
-		lowReg |= IOART_INTMASK;	/* set mask */
+		lowReg |= IOART_INTMASK;		/* set mask */
 
 		io_apic_write(apic, select, low_reg);	/* new value */
 	}
@@ -522,20 +527,20 @@ set_io_apic_mask24(apic, u_int32_t bits)
 
 #if defined(READY)
 /*
- * clear INT mask bit for each bit set in 'mask'.
- * ignore INT mask bit for all others.
- * only consider lower 24 bits in mask.
+ * Clear INT mask bit for each bit set in 'mask'.
+ * Ignore INT mask bit for all others.
+ * Only consider lower 24 bits in mask.
  */
 void
 clr_io_apic_mask24(int apic, u_int32_t bits)
 {
 	int     x, y;
-	u_char  select;		/* the select register is 8 bits */
-	u_int32_t low_reg;	/* the window register is 32 bits */
+	u_char  select;			/* the select register is 8 bits */
+	u_int32_t low_reg;		/* the window register is 32 bits */
 	u_int32_t diffs;
 
-	bits &= IO_FIELD;	/* safety valve, only use 24 bits */
-	diffs = bits & IO_MASK;	/* set AND needing 'clr'ing */
+	bits &= IO_FIELD;		/* safety valve, only use 24 bits */
+	diffs = bits & IO_MASK;		/* set AND needing 'clr'ing */
 	if (!diffs)
 		return;
 
@@ -543,12 +548,12 @@ clr_io_apic_mask24(int apic, u_int32_t bits)
 
 	for (x = 0, y = REDIRCNT_IOAPIC(apic); x < y; ++x) {
 		if (!(diffs & (1 << x)))
-			continue;	/* no change, skip */
+			continue;			/* no change, skip */
 
 		select = IOAPIC_REDTBL + (x * 2);	/* calculate addr */
 		low_reg = io_apic_read(apic, select);	/* read contents */
 
-		low_reg &= ~IOART_INTMASK;	/* clear mask */
+		low_reg &= ~IOART_INTMASK;		/* clear mask */
 
 		io_apic_write(apic, select, low_reg);	/* new value */
 	}
@@ -565,7 +570,7 @@ clr_io_apic_mask24(int apic, u_int32_t bits)
 
 
 /*
- * send APIC IPI 'vector' to 'destType' via 'deliveryMode'
+ * Send APIC IPI 'vector' to 'destType' via 'deliveryMode'.
  *
  *  destType is 1 of: APIC_DEST_SELF, APIC_DEST_ALLISELF, APIC_DEST_ALLESELF
  *  vector is any valid SYSTEM INT vector
@@ -613,13 +618,13 @@ apic_ipi(int dest_type, int vector, int delivery_mode)
 		 /* spin */ ;
 #endif  /* DETECT_DEADLOCK */
 
-	/** FIXME: return result */
+	/** XXX FIXME: return result */
 	return 0;
 }
 
 
 /*
- * send APIC IPI 'vector' to 'target's via 'delivery_mode'
+ * Send APIC IPI 'vector' to 'target's via 'delivery_mode'.
  *
  *  target contains a bitfield with a bit set for selected APICs.
  *  vector is any valid SYSTEM INT vector
@@ -641,9 +646,9 @@ selected_apic_ipi(u_int target, int vector, int delivery_mode)
 			icr_hi = lapic.icr_hi & ~APIC_ID_MASK;
 			icr_hi |= (CPU_TO_ID(x) << 24);
 			lapic.icr_hi = icr_hi;
-#if defined(DEBUG_CPUSTOP)
+#if defined(TEST_CPUSTOP)
 			db_printf( "icr_hi: 0x%08x\n", lapic.icr_hi );
-#endif /* DEBUG_CPUSTOP */
+#endif  /* TEST_CPUSTOP */
 			/* send the IPI */
 			if (apic_ipi(APIC_DEST_DESTFLD, vector,
 				     delivery_mode) == -1)
@@ -655,7 +660,7 @@ selected_apic_ipi(u_int target, int vector, int delivery_mode)
 
 #if defined(READY)
 /*
- * send an IPI INTerrupt containing 'vector' to CPU 'target'
+ * Send an IPI INTerrupt containing 'vector' to CPU 'target'
  *   NOTE: target is a LOGICAL APIC ID
  */
 int
@@ -678,7 +683,7 @@ selected_proc_ipi(int target, int vector)
 	while (lapic.icr_lo & APIC_DELSTAT_MASK)
 		/* spin */ ;
 
-	return 0;	/** FIXME: return result */
+	return 0;	/** XXX FIXME: return result */
 }
 #endif /* READY */
 
@@ -687,11 +692,11 @@ selected_proc_ipi(int target, int vector)
 
 
 /*
- * timer code, in development...
- * suggested by rgrimes@gndrsh.aac.dev.com
+ * Timer code, in development...
+ *  - suggested by rgrimes@gndrsh.aac.dev.com
  */
 
-/** FIXME: temp hack till we can determin bus clock */
+/** XXX FIXME: temp hack till we can determin bus clock */
 #ifndef BUS_CLOCK
 #define BUS_CLOCK	66000000
 #define bus_clock()	66000000
@@ -702,7 +707,7 @@ int acquire_apic_timer __P((void));
 int release_apic_timer __P((void));
 
 /*
- * acquire the APIC timer for exclusive use
+ * Acquire the APIC timer for exclusive use.
  */
 int
 acquire_apic_timer(void)
@@ -710,14 +715,14 @@ acquire_apic_timer(void)
 #if 1
 	return 0;
 #else
-	/** FIXME: make this really do something */
+	/** XXX FIXME: make this really do something */
 	panic("APIC timer in use when attempting to aquire");
 #endif
 }
 
 
 /*
- * return the APIC timer
+ * Return the APIC timer.
  */
 int
 release_apic_timer(void)
@@ -725,7 +730,7 @@ release_apic_timer(void)
 #if 1
 	return 0;
 #else
-	/** FIXME: make this really do something */
+	/** XXX FIXME: make this really do something */
 	panic("APIC timer was already released");
 #endif
 }
@@ -733,7 +738,7 @@ release_apic_timer(void)
 
 
 /*
- * load a 'downcount time' in uSeconds
+ * Load a 'downcount time' in uSeconds.
  */
 void
 set_apic_timer(int value)
@@ -741,10 +746,11 @@ set_apic_timer(int value)
 	u_long  lvtt;
 	long    ticks_per_microsec;
 
-	/* calculate divisor and count from value:
+	/*
+	 * Calculate divisor and count from value:
 	 * 
-	 * timeBase == CPU bus clock divisor == [1,2,4,8,16,32,64,128] value ==
-	 * time in uS
+	 *  timeBase == CPU bus clock divisor == [1,2,4,8,16,32,64,128]
+	 *  value == time in uS
 	 */
 	lapic.dcr_timer = APIC_TDCR_1;
 	ticks_per_microsec = bus_clock() / 1000000;
@@ -752,7 +758,7 @@ set_apic_timer(int value)
 	/* configure timer as one-shot */
 	lvtt = lapic.lvt_timer;
 	lvtt &= ~(APIC_LVTT_VECTOR | APIC_LVTT_DS | APIC_LVTT_M | APIC_LVTT_TM);
-	lvtt |= APIC_LVTT_M;	/* no INT, one-shot */
+	lvtt |= APIC_LVTT_M;			/* no INT, one-shot */
 	lapic.lvt_timer = lvtt;
 
 	/* */
@@ -761,13 +767,13 @@ set_apic_timer(int value)
 
 
 /*
- * read remaining time in timer
+ * Read remaining time in timer.
  */
 int
 read_apic_timer(void)
 {
 #if 0
-	/** FIXME: we need to return the actual remaining time,
+	/** XXX FIXME: we need to return the actual remaining time,
          *         for now we just return the remaining count.
          */
 #else
@@ -777,7 +783,7 @@ read_apic_timer(void)
 
 
 /*
- * spin-style delay, set delay time in uS, spin till it drains
+ * Spin-style delay, set delay time in uS, spin till it drains.
  */
 void
 u_sleep(int count)
diff --git a/sys/i386/i386/mptable.c b/sys/i386/i386/mptable.c
index 8085f0e39dfc..bf1b9a9627ae 100644
--- a/sys/i386/i386/mptable.c
+++ b/sys/i386/i386/mptable.c
@@ -22,7 +22,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: mp_machdep.c,v 1.6 1997/07/07 00:02:55 smp Exp smp $
+ *	$Id: mp_machdep.c,v 1.7 1997/07/08 23:42:28 smp Exp smp $
  */
 
 #include "opt_smp.h"
@@ -51,10 +51,14 @@
 
 #include <i386/i386/cons.h>	/* cngetc() */
 
+#if defined(TEST_CPUSTOP)
+void	db_printf __P((const char *fmt, ...));
+#endif  /* TEST_CPUSTOP */
+
 #if defined(APIC_IO)
-#include <machine/md_var.h>	/* setidt() */
-#include <i386/isa/icu.h>		/* Xinvltlb() */
-#include <i386/isa/intr_machdep.h>	/* Xinvltlb() */
+#include <machine/md_var.h>		/* setidt() */
+#include <i386/isa/icu.h>		/* IPIs */
+#include <i386/isa/intr_machdep.h>	/* IPIs */
 #endif	/* APIC_IO */
 
 #define WARMBOOT_TARGET		0
@@ -189,12 +193,6 @@ typedef struct BASETABLE_ENTRY {
 /*
  * Values to send to the POST hardware.
  */
-#ifndef POSTCODE
-#define POSTCODE(X)
-#define POSTCODE_LO(X)
-#define POSTCODE_HI(X)
-#endif
-
 #define MP_BOOTADDRESS_POST	0x10
 #define MP_PROBE_POST		0x11
 #define MP_START_POST		0x12
@@ -206,10 +204,10 @@ typedef struct BASETABLE_ENTRY {
 #define INSTALL_AP_TRAMP_POST	0x18
 #define START_AP_POST		0x19
 
-/* XXX FIXME: where does this really belong, isa.h/isa.c perhaps? */
+/** XXX FIXME: where does this really belong, isa.h/isa.c perhaps? */
 int	current_postcode;
 
-/** FIXME: what system files declare these??? */
+/** XXX FIXME: what system files declare these??? */
 extern struct region_descriptor r_gdt, r_idt;
 
 int	mp_ncpus;		/* # of CPUs, including BSP */
@@ -240,11 +238,11 @@ u_int *bootPTD;
 /* Hotwire a 0->4MB V==P mapping */
 extern pt_entry_t KPTphys;
 
-/* virtual address of per-cpu common_tss */
+/* Virtual address of per-cpu common_tss */
 extern struct i386tss common_tss;
 
 /*
- * look for MP compliant motherboard.
+ * Local data and functions.
  */
 
 static int	mp_capable;
@@ -265,7 +263,7 @@ static int	start_ap(int logicalCpu, u_int boot_addr);
 
 
 /*
- * calculate usable address in base memory for AP trampoline code
+ * Calculate usable address in base memory for AP trampoline code.
  */
 u_int
 mp_bootaddress(u_int basemem)
@@ -282,6 +280,9 @@ mp_bootaddress(u_int basemem)
 }
 
 
+/*
+ * Look for an Intel MP spec table (ie, SMP capable hardware).
+ */
 int
 mp_probe(void)
 {
@@ -314,7 +315,7 @@ mp_probe(void)
 	mp_capable = 0;
 	return 0;
 
-found:				/* please forgive the 'goto'! */
+found:
 	/* calculate needed resources */
 	mpfps = (mpfps_t)x;
 	if (mptable_pass1())
@@ -327,7 +328,7 @@ found:				/* please forgive the 'goto'! */
 
 
 /*
- * startup the SMP processors
+ * Startup the SMP processors.
  */
 void
 mp_start(void)
@@ -343,7 +344,7 @@ mp_start(void)
 
 
 /*
- * print various information about the SMP system hardware and setup
+ * Print various information about the SMP system hardware and setup.
  */
 void
 mp_announce(void)
@@ -383,7 +384,7 @@ init_secondary(void)
 
 	r_gdt.rd_limit = sizeof(gdt[0]) * (NGDT + NCPU) - 1;
 	r_gdt.rd_base = (int) gdt;
-	lgdt(&r_gdt);		/* does magic intra-segment return */
+	lgdt(&r_gdt);			/* does magic intra-segment return */
 	lidt(&r_idt);
 	lldt(_default_ldt);
 
@@ -395,7 +396,7 @@ init_secondary(void)
 	common_tss.tss_ioopt = (sizeof common_tss) << 16;
 	ltr(gsel_tss);
 
-	load_cr0(0x8005003b);	/* XXX! */
+	load_cr0(0x8005003b);		/* XXX! */
 
 	PTD[0] = 0;
 	invltlb();
@@ -403,44 +404,44 @@ init_secondary(void)
 
 
 #if defined(APIC_IO)
+/*
+ * Final configuration of the BSP's local APIC:
+ *  - disable 'pic mode'.
+ *  - disable 'virtual wire mode'.
+ *  - enable NMI.
+ */
 void
-configure_local_apic(void)
+bsp_apic_configure(void)
 {
-	u_char  byte;
-	u_int32_t temp;
+	u_char		byte;
+	u_int32_t	temp;
 
+	/* leave 'pic mode' if necessary */
 	if (picmode) {
 		outb(0x22, 0x70);	/* select IMCR */
 		byte = inb(0x23);	/* current contents */
-		byte |= 0x01;	/* mask external INTR */
+		byte |= 0x01;		/* mask external INTR */
 		outb(0x23, byte);	/* disconnect 8259s/NMI */
 	}
 
 	/* mask lint0 (the 8259 'virtual wire' connection) */
 	temp = lapic.lvt_lint0;
-	temp |= APIC_LVT_M;
+	temp |= APIC_LVT_M;		/* set the mask */
 	lapic.lvt_lint0 = temp;
 
         /* setup lint1 to handle NMI */
-#if 1
-        /** XXX FIXME: 
-         *      should we arrange for ALL CPUs to catch NMI???
-         *      it would probably crash, so for now only the BSP
-         *      will catch it
-         */
-        if (cpuid != 0)
-                return;
-#endif /* 0/1 */
-
         temp = lapic.lvt_lint1;
-
-        /* clear fields of interest, preserve undefined fields */
-        temp &= ~(0x1f000 | APIC_LVT_DM | APIC_LVT_VECTOR);
-
-        /* setup for NMI, edge trigger, active hi */
-        temp |= (APIC_LVT_DM_NMI | APIC_LVT_IIPP_INTAHI);
-
+        temp &= ~APIC_LVT_M;		/* clear the mask */
         lapic.lvt_lint1 = temp;
+
+#if defined(TEST_CPUSTOP)
+	printf(">>> CPU%02d bsp_apic_configure() lint0: 0x%08x\n",
+	       cpuid, lapic.lvt_lint0);
+	printf(">>>                            lint1: 0x%08x\n",
+	       lapic.lvt_lint1);
+	printf(">>>                            TPR:   0x%08x\n", lapic.tpr);
+	printf(">>>                            SVR:   0x%08x\n", lapic.svr);
+#endif  /* TEST_CPUSTOP */
 }
 #endif  /* APIC_IO */
 
@@ -463,7 +464,7 @@ mp_enable(u_int boot_addr)
 
 	POSTCODE(MP_ENABLE_POST);
 
-	/* Turn on 4MB of V == P addressing so we can get to MP table */
+	/* turn on 4MB of V == P addressing so we can get to MP table */
 	*(int *)PTD = PG_V | PG_RW | ((u_long)KPTphys & PG_FRAME);
 	invltlb();
 
@@ -1439,7 +1440,7 @@ start_all_aps(u_int boot_addr)
 	mp_lock = 1;	/* this uses a LOGICAL cpu ID, ie BSP == 0 */
 
 	/* initialize BSP's local APIC */
-	apic_initialize(1);
+	apic_initialize();
 
 	/* install the AP 1st level boot code */
 	install_ap_tramp(boot_addr);
@@ -1814,6 +1815,7 @@ stop_cpus( u_int map )
 
 #if defined(DEBUG_CPUSTOP)
 	db_printf("  spun\nstopped, sihits: %d\n", sihits);
+	cngetc();
 #endif /* DEBUG_CPUSTOP */
 
 	return 1;
diff --git a/sys/i386/include/mptable.h b/sys/i386/include/mptable.h
index 8085f0e39dfc..bf1b9a9627ae 100644
--- a/sys/i386/include/mptable.h
+++ b/sys/i386/include/mptable.h
@@ -22,7 +22,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: mp_machdep.c,v 1.6 1997/07/07 00:02:55 smp Exp smp $
+ *	$Id: mp_machdep.c,v 1.7 1997/07/08 23:42:28 smp Exp smp $
  */
 
 #include "opt_smp.h"
@@ -51,10 +51,14 @@
 
 #include <i386/i386/cons.h>	/* cngetc() */
 
+#if defined(TEST_CPUSTOP)
+void	db_printf __P((const char *fmt, ...));
+#endif  /* TEST_CPUSTOP */
+
 #if defined(APIC_IO)
-#include <machine/md_var.h>	/* setidt() */
-#include <i386/isa/icu.h>		/* Xinvltlb() */
-#include <i386/isa/intr_machdep.h>	/* Xinvltlb() */
+#include <machine/md_var.h>		/* setidt() */
+#include <i386/isa/icu.h>		/* IPIs */
+#include <i386/isa/intr_machdep.h>	/* IPIs */
 #endif	/* APIC_IO */
 
 #define WARMBOOT_TARGET		0
@@ -189,12 +193,6 @@ typedef struct BASETABLE_ENTRY {
 /*
  * Values to send to the POST hardware.
  */
-#ifndef POSTCODE
-#define POSTCODE(X)
-#define POSTCODE_LO(X)
-#define POSTCODE_HI(X)
-#endif
-
 #define MP_BOOTADDRESS_POST	0x10
 #define MP_PROBE_POST		0x11
 #define MP_START_POST		0x12
@@ -206,10 +204,10 @@ typedef struct BASETABLE_ENTRY {
 #define INSTALL_AP_TRAMP_POST	0x18
 #define START_AP_POST		0x19
 
-/* XXX FIXME: where does this really belong, isa.h/isa.c perhaps? */
+/** XXX FIXME: where does this really belong, isa.h/isa.c perhaps? */
 int	current_postcode;
 
-/** FIXME: what system files declare these??? */
+/** XXX FIXME: what system files declare these??? */
 extern struct region_descriptor r_gdt, r_idt;
 
 int	mp_ncpus;		/* # of CPUs, including BSP */
@@ -240,11 +238,11 @@ u_int *bootPTD;
 /* Hotwire a 0->4MB V==P mapping */
 extern pt_entry_t KPTphys;
 
-/* virtual address of per-cpu common_tss */
+/* Virtual address of per-cpu common_tss */
 extern struct i386tss common_tss;
 
 /*
- * look for MP compliant motherboard.
+ * Local data and functions.
  */
 
 static int	mp_capable;
@@ -265,7 +263,7 @@ static int	start_ap(int logicalCpu, u_int boot_addr);
 
 
 /*
- * calculate usable address in base memory for AP trampoline code
+ * Calculate usable address in base memory for AP trampoline code.
  */
 u_int
 mp_bootaddress(u_int basemem)
@@ -282,6 +280,9 @@ mp_bootaddress(u_int basemem)
 }
 
 
+/*
+ * Look for an Intel MP spec table (ie, SMP capable hardware).
+ */
 int
 mp_probe(void)
 {
@@ -314,7 +315,7 @@ mp_probe(void)
 	mp_capable = 0;
 	return 0;
 
-found:				/* please forgive the 'goto'! */
+found:
 	/* calculate needed resources */
 	mpfps = (mpfps_t)x;
 	if (mptable_pass1())
@@ -327,7 +328,7 @@ found:				/* please forgive the 'goto'! */
 
 
 /*
- * startup the SMP processors
+ * Startup the SMP processors.
  */
 void
 mp_start(void)
@@ -343,7 +344,7 @@ mp_start(void)
 
 
 /*
- * print various information about the SMP system hardware and setup
+ * Print various information about the SMP system hardware and setup.
  */
 void
 mp_announce(void)
@@ -383,7 +384,7 @@ init_secondary(void)
 
 	r_gdt.rd_limit = sizeof(gdt[0]) * (NGDT + NCPU) - 1;
 	r_gdt.rd_base = (int) gdt;
-	lgdt(&r_gdt);		/* does magic intra-segment return */
+	lgdt(&r_gdt);			/* does magic intra-segment return */
 	lidt(&r_idt);
 	lldt(_default_ldt);
 
@@ -395,7 +396,7 @@ init_secondary(void)
 	common_tss.tss_ioopt = (sizeof common_tss) << 16;
 	ltr(gsel_tss);
 
-	load_cr0(0x8005003b);	/* XXX! */
+	load_cr0(0x8005003b);		/* XXX! */
 
 	PTD[0] = 0;
 	invltlb();
@@ -403,44 +404,44 @@ init_secondary(void)
 
 
 #if defined(APIC_IO)
+/*
+ * Final configuration of the BSP's local APIC:
+ *  - disable 'pic mode'.
+ *  - disable 'virtual wire mode'.
+ *  - enable NMI.
+ */
 void
-configure_local_apic(void)
+bsp_apic_configure(void)
 {
-	u_char  byte;
-	u_int32_t temp;
+	u_char		byte;
+	u_int32_t	temp;
 
+	/* leave 'pic mode' if necessary */
 	if (picmode) {
 		outb(0x22, 0x70);	/* select IMCR */
 		byte = inb(0x23);	/* current contents */
-		byte |= 0x01;	/* mask external INTR */
+		byte |= 0x01;		/* mask external INTR */
 		outb(0x23, byte);	/* disconnect 8259s/NMI */
 	}
 
 	/* mask lint0 (the 8259 'virtual wire' connection) */
 	temp = lapic.lvt_lint0;
-	temp |= APIC_LVT_M;
+	temp |= APIC_LVT_M;		/* set the mask */
 	lapic.lvt_lint0 = temp;
 
         /* setup lint1 to handle NMI */
-#if 1
-        /** XXX FIXME: 
-         *      should we arrange for ALL CPUs to catch NMI???
-         *      it would probably crash, so for now only the BSP
-         *      will catch it
-         */
-        if (cpuid != 0)
-                return;
-#endif /* 0/1 */
-
         temp = lapic.lvt_lint1;
-
-        /* clear fields of interest, preserve undefined fields */
-        temp &= ~(0x1f000 | APIC_LVT_DM | APIC_LVT_VECTOR);
-
-        /* setup for NMI, edge trigger, active hi */
-        temp |= (APIC_LVT_DM_NMI | APIC_LVT_IIPP_INTAHI);
-
+        temp &= ~APIC_LVT_M;		/* clear the mask */
         lapic.lvt_lint1 = temp;
+
+#if defined(TEST_CPUSTOP)
+	printf(">>> CPU%02d bsp_apic_configure() lint0: 0x%08x\n",
+	       cpuid, lapic.lvt_lint0);
+	printf(">>>                            lint1: 0x%08x\n",
+	       lapic.lvt_lint1);
+	printf(">>>                            TPR:   0x%08x\n", lapic.tpr);
+	printf(">>>                            SVR:   0x%08x\n", lapic.svr);
+#endif  /* TEST_CPUSTOP */
 }
 #endif  /* APIC_IO */
 
@@ -463,7 +464,7 @@ mp_enable(u_int boot_addr)
 
 	POSTCODE(MP_ENABLE_POST);
 
-	/* Turn on 4MB of V == P addressing so we can get to MP table */
+	/* turn on 4MB of V == P addressing so we can get to MP table */
 	*(int *)PTD = PG_V | PG_RW | ((u_long)KPTphys & PG_FRAME);
 	invltlb();
 
@@ -1439,7 +1440,7 @@ start_all_aps(u_int boot_addr)
 	mp_lock = 1;	/* this uses a LOGICAL cpu ID, ie BSP == 0 */
 
 	/* initialize BSP's local APIC */
-	apic_initialize(1);
+	apic_initialize();
 
 	/* install the AP 1st level boot code */
 	install_ap_tramp(boot_addr);
@@ -1814,6 +1815,7 @@ stop_cpus( u_int map )
 
 #if defined(DEBUG_CPUSTOP)
 	db_printf("  spun\nstopped, sihits: %d\n", sihits);
+	cngetc();
 #endif /* DEBUG_CPUSTOP */
 
 	return 1;
diff --git a/sys/kern/subr_smp.c b/sys/kern/subr_smp.c
index 8085f0e39dfc..bf1b9a9627ae 100644
--- a/sys/kern/subr_smp.c
+++ b/sys/kern/subr_smp.c
@@ -22,7 +22,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: mp_machdep.c,v 1.6 1997/07/07 00:02:55 smp Exp smp $
+ *	$Id: mp_machdep.c,v 1.7 1997/07/08 23:42:28 smp Exp smp $
  */
 
 #include "opt_smp.h"
@@ -51,10 +51,14 @@
 
 #include <i386/i386/cons.h>	/* cngetc() */
 
+#if defined(TEST_CPUSTOP)
+void	db_printf __P((const char *fmt, ...));
+#endif  /* TEST_CPUSTOP */
+
 #if defined(APIC_IO)
-#include <machine/md_var.h>	/* setidt() */
-#include <i386/isa/icu.h>		/* Xinvltlb() */
-#include <i386/isa/intr_machdep.h>	/* Xinvltlb() */
+#include <machine/md_var.h>		/* setidt() */
+#include <i386/isa/icu.h>		/* IPIs */
+#include <i386/isa/intr_machdep.h>	/* IPIs */
 #endif	/* APIC_IO */
 
 #define WARMBOOT_TARGET		0
@@ -189,12 +193,6 @@ typedef struct BASETABLE_ENTRY {
 /*
  * Values to send to the POST hardware.
  */
-#ifndef POSTCODE
-#define POSTCODE(X)
-#define POSTCODE_LO(X)
-#define POSTCODE_HI(X)
-#endif
-
 #define MP_BOOTADDRESS_POST	0x10
 #define MP_PROBE_POST		0x11
 #define MP_START_POST		0x12
@@ -206,10 +204,10 @@ typedef struct BASETABLE_ENTRY {
 #define INSTALL_AP_TRAMP_POST	0x18
 #define START_AP_POST		0x19
 
-/* XXX FIXME: where does this really belong, isa.h/isa.c perhaps? */
+/** XXX FIXME: where does this really belong, isa.h/isa.c perhaps? */
 int	current_postcode;
 
-/** FIXME: what system files declare these??? */
+/** XXX FIXME: what system files declare these??? */
 extern struct region_descriptor r_gdt, r_idt;
 
 int	mp_ncpus;		/* # of CPUs, including BSP */
@@ -240,11 +238,11 @@ u_int *bootPTD;
 /* Hotwire a 0->4MB V==P mapping */
 extern pt_entry_t KPTphys;
 
-/* virtual address of per-cpu common_tss */
+/* Virtual address of per-cpu common_tss */
 extern struct i386tss common_tss;
 
 /*
- * look for MP compliant motherboard.
+ * Local data and functions.
  */
 
 static int	mp_capable;
@@ -265,7 +263,7 @@ static int	start_ap(int logicalCpu, u_int boot_addr);
 
 
 /*
- * calculate usable address in base memory for AP trampoline code
+ * Calculate usable address in base memory for AP trampoline code.
  */
 u_int
 mp_bootaddress(u_int basemem)
@@ -282,6 +280,9 @@ mp_bootaddress(u_int basemem)
 }
 
 
+/*
+ * Look for an Intel MP spec table (ie, SMP capable hardware).
+ */
 int
 mp_probe(void)
 {
@@ -314,7 +315,7 @@ mp_probe(void)
 	mp_capable = 0;
 	return 0;
 
-found:				/* please forgive the 'goto'! */
+found:
 	/* calculate needed resources */
 	mpfps = (mpfps_t)x;
 	if (mptable_pass1())
@@ -327,7 +328,7 @@ found:				/* please forgive the 'goto'! */
 
 
 /*
- * startup the SMP processors
+ * Startup the SMP processors.
  */
 void
 mp_start(void)
@@ -343,7 +344,7 @@ mp_start(void)
 
 
 /*
- * print various information about the SMP system hardware and setup
+ * Print various information about the SMP system hardware and setup.
  */
 void
 mp_announce(void)
@@ -383,7 +384,7 @@ init_secondary(void)
 
 	r_gdt.rd_limit = sizeof(gdt[0]) * (NGDT + NCPU) - 1;
 	r_gdt.rd_base = (int) gdt;
-	lgdt(&r_gdt);		/* does magic intra-segment return */
+	lgdt(&r_gdt);			/* does magic intra-segment return */
 	lidt(&r_idt);
 	lldt(_default_ldt);
 
@@ -395,7 +396,7 @@ init_secondary(void)
 	common_tss.tss_ioopt = (sizeof common_tss) << 16;
 	ltr(gsel_tss);
 
-	load_cr0(0x8005003b);	/* XXX! */
+	load_cr0(0x8005003b);		/* XXX! */
 
 	PTD[0] = 0;
 	invltlb();
@@ -403,44 +404,44 @@ init_secondary(void)
 
 
 #if defined(APIC_IO)
+/*
+ * Final configuration of the BSP's local APIC:
+ *  - disable 'pic mode'.
+ *  - disable 'virtual wire mode'.
+ *  - enable NMI.
+ */
 void
-configure_local_apic(void)
+bsp_apic_configure(void)
 {
-	u_char  byte;
-	u_int32_t temp;
+	u_char		byte;
+	u_int32_t	temp;
 
+	/* leave 'pic mode' if necessary */
 	if (picmode) {
 		outb(0x22, 0x70);	/* select IMCR */
 		byte = inb(0x23);	/* current contents */
-		byte |= 0x01;	/* mask external INTR */
+		byte |= 0x01;		/* mask external INTR */
 		outb(0x23, byte);	/* disconnect 8259s/NMI */
 	}
 
 	/* mask lint0 (the 8259 'virtual wire' connection) */
 	temp = lapic.lvt_lint0;
-	temp |= APIC_LVT_M;
+	temp |= APIC_LVT_M;		/* set the mask */
 	lapic.lvt_lint0 = temp;
 
         /* setup lint1 to handle NMI */
-#if 1
-        /** XXX FIXME: 
-         *      should we arrange for ALL CPUs to catch NMI???
-         *      it would probably crash, so for now only the BSP
-         *      will catch it
-         */
-        if (cpuid != 0)
-                return;
-#endif /* 0/1 */
-
         temp = lapic.lvt_lint1;
-
-        /* clear fields of interest, preserve undefined fields */
-        temp &= ~(0x1f000 | APIC_LVT_DM | APIC_LVT_VECTOR);
-
-        /* setup for NMI, edge trigger, active hi */
-        temp |= (APIC_LVT_DM_NMI | APIC_LVT_IIPP_INTAHI);
-
+        temp &= ~APIC_LVT_M;		/* clear the mask */
         lapic.lvt_lint1 = temp;
+
+#if defined(TEST_CPUSTOP)
+	printf(">>> CPU%02d bsp_apic_configure() lint0: 0x%08x\n",
+	       cpuid, lapic.lvt_lint0);
+	printf(">>>                            lint1: 0x%08x\n",
+	       lapic.lvt_lint1);
+	printf(">>>                            TPR:   0x%08x\n", lapic.tpr);
+	printf(">>>                            SVR:   0x%08x\n", lapic.svr);
+#endif  /* TEST_CPUSTOP */
 }
 #endif  /* APIC_IO */
 
@@ -463,7 +464,7 @@ mp_enable(u_int boot_addr)
 
 	POSTCODE(MP_ENABLE_POST);
 
-	/* Turn on 4MB of V == P addressing so we can get to MP table */
+	/* turn on 4MB of V == P addressing so we can get to MP table */
 	*(int *)PTD = PG_V | PG_RW | ((u_long)KPTphys & PG_FRAME);
 	invltlb();
 
@@ -1439,7 +1440,7 @@ start_all_aps(u_int boot_addr)
 	mp_lock = 1;	/* this uses a LOGICAL cpu ID, ie BSP == 0 */
 
 	/* initialize BSP's local APIC */
-	apic_initialize(1);
+	apic_initialize();
 
 	/* install the AP 1st level boot code */
 	install_ap_tramp(boot_addr);
@@ -1814,6 +1815,7 @@ stop_cpus( u_int map )
 
 #if defined(DEBUG_CPUSTOP)
 	db_printf("  spun\nstopped, sihits: %d\n", sihits);
+	cngetc();
 #endif /* DEBUG_CPUSTOP */
 
 	return 1;