15 files changed, 74 insertions, 67 deletions

diff --git a/sys/amd64/amd64/apic_vector.S b/sys/amd64/amd64/apic_vector.S
index 3d7a687de3fe..6147b310a58d 100644
--- a/sys/amd64/amd64/apic_vector.S
+++ b/sys/amd64/amd64/apic_vector.S
@@ -7,8 +7,6 @@
 #include <machine/apic.h>
 #include <machine/smp.h>
 
-#include "i386/isa/intr_machdep.h"
-
 /* convert an absolute IRQ# into a bitmask */
 #define IRQ_BIT(irq_num)	(1 << (irq_num))
 
@@ -67,7 +65,7 @@ IDTVEC(vec_name) ;							\
 #define REDIRIDX(irq_num) CNAME(int_to_apicintpin) + 16 * (irq_num) + 12
 	
 #define MASK_IRQ(irq_num)						\
-	IMASK_LOCK ;				/* into critical reg */	\
+	ICU_LOCK ;				/* into critical reg */	\
 	testl	$IRQ_BIT(irq_num), apic_imen ;				\
 	jne	7f ;			/* masked, don't mask */	\
 	orl	$IRQ_BIT(irq_num), apic_imen ;	/* set the mask bit */	\
@@ -78,7 +76,7 @@ IDTVEC(vec_name) ;							\
 	orl	$IOART_INTMASK, %eax ;		/* set the mask */	\
 	movl	%eax, IOAPIC_WINDOW(%ecx) ;	/* new value */		\
 7: ;						/* already masked */	\
-	IMASK_UNLOCK
+	ICU_UNLOCK
 /*
  * Test to see whether we are handling an edge or level triggered INT.
  *  Level-triggered INTs must still be masked as we don't clear the source,
@@ -113,7 +111,7 @@ IDTVEC(vec_name) ;							\
  * Test to see if the source is currently masked, clear if so.
  */
 #define UNMASK_IRQ(irq_num)					\
-	IMASK_LOCK ;				/* into critical reg */	\
+	ICU_LOCK ;				/* into critical reg */	\
 	testl	$IRQ_BIT(irq_num), _apic_imen ;				\
 	je	7f ;			/* bit clear, not masked */	\
 	andl	$~IRQ_BIT(irq_num), _apic_imen ;/* clear mask bit */	\
@@ -124,7 +122,7 @@ IDTVEC(vec_name) ;							\
 	andl	$~IOART_INTMASK, %eax ;		/* clear the mask */	\
 	movl	%eax, IOAPIC_WINDOW(%ecx) ;	/* new value */		\
 7: ;						/* already unmasked */	\
-	IMASK_UNLOCK
+	ICU_UNLOCK
 
 /* 
  * Slow, threaded interrupts.
diff --git a/sys/amd64/amd64/fpu.c b/sys/amd64/amd64/fpu.c
index 34a822a230a7..5d44aaa2fab2 100644
--- a/sys/amd64/amd64/fpu.c
+++ b/sys/amd64/amd64/fpu.c
@@ -82,6 +82,7 @@
 #include <i386/isa/isa.h>
 #endif
 #endif
+#include <i386/isa/intr_machdep.h>
 #include <isa/isavar.h>
 
 /*
@@ -425,7 +426,9 @@ no_irq13:
 	 * XXX hack around brokenness of bus_teardown_intr().  If we left the
 	 * irq active then we would get it instead of exception 16.
 	 */
+	mtx_lock_spin(&icu_lock);
 	INTRDIS(1 << irq_num);
+	mtx_unlock_spin(&icu_lock);
 
 	bus_release_resource(dev, SYS_RES_IRQ, irq_rid, irq_res);
 	bus_release_resource(dev, SYS_RES_IOPORT, ioport_rid, ioport_res);
diff --git a/sys/amd64/amd64/machdep.c b/sys/amd64/amd64/machdep.c
index c2ef5dcf526c..349a14084f9f 100644
--- a/sys/amd64/amd64/machdep.c
+++ b/sys/amd64/amd64/machdep.c
@@ -211,6 +211,7 @@ static struct pcpu __pcpu;
 
 struct mtx sched_lock;
 struct mtx Giant;
+struct mtx icu_lock;
 
 static void
 cpu_startup(dummy)
@@ -1744,9 +1745,7 @@ init386(first)
 	mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_RECURSE);
 	mtx_init(&proc0.p_mtx, "process lock", MTX_DEF);
 	mtx_init(&clock_lock, "clk", MTX_SPIN | MTX_RECURSE);
-#ifdef SMP
-	mtx_init(&imen_mtx, "imen", MTX_SPIN);
-#endif
+	mtx_init(&icu_lock, "icu", MTX_SPIN);
 	mtx_lock(&Giant);
 
 	/* make ldt memory segments */
diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c
index e8da25b887fe..1feb66af173c 100644
--- a/sys/amd64/amd64/mp_machdep.c
+++ b/sys/amd64/amd64/mp_machdep.c
@@ -320,9 +320,6 @@ static void	release_aps(void *dummy);
  * initialize all the SMP locks
  */
 
-/* critical region around IO APIC, apic_imen */
-struct mtx		imen_mtx;
-
 /* lock region used by kernel profiling */
 int	mcount_lock;
 
diff --git a/sys/amd64/amd64/mptable.c b/sys/amd64/amd64/mptable.c
index e8da25b887fe..1feb66af173c 100644
--- a/sys/amd64/amd64/mptable.c
+++ b/sys/amd64/amd64/mptable.c
@@ -320,9 +320,6 @@ static void	release_aps(void *dummy);
  * initialize all the SMP locks
  */
 
-/* critical region around IO APIC, apic_imen */
-struct mtx		imen_mtx;
-
 /* lock region used by kernel profiling */
 int	mcount_lock;
 
diff --git a/sys/amd64/amd64/tsc.c b/sys/amd64/amd64/tsc.c
index a7aaea0b7a5e..a39f7731be6a 100644
--- a/sys/amd64/amd64/tsc.c
+++ b/sys/amd64/amd64/tsc.c
@@ -1026,7 +1026,9 @@ cpu_initclocks()
 
 	inthand_add("clk", apic_8254_intr, (driver_intr_t *)clkintr, NULL,
 	    INTR_TYPE_CLK | INTR_FAST, &clkdesc);
+	mtx_lock_spin(&icu_lock);
 	INTREN(1 << apic_8254_intr);
+	mtx_unlock_spin(&icu_lock);
 
 #else /* APIC_IO */
 
@@ -1037,7 +1039,9 @@ cpu_initclocks()
 	 */
 	inthand_add("clk", 0, (driver_intr_t *)clkintr, NULL,
 	    INTR_TYPE_CLK | INTR_FAST, NULL);
+	mtx_lock_spin(&icu_lock);
 	INTREN(IRQ0);
+	mtx_unlock_spin(&icu_lock);
 
 #endif /* APIC_IO */
 
@@ -1060,11 +1064,13 @@ cpu_initclocks()
 	inthand_add("rtc", 8, (driver_intr_t *)rtcintr, NULL,
 	    INTR_TYPE_CLK | INTR_FAST, NULL);
 
+	mtx_lock_spin(&icu_lock);
 #ifdef APIC_IO
 	INTREN(APIC_IRQ8);
 #else
 	INTREN(IRQ8);
 #endif /* APIC_IO */
+	mtx_unlock_spin(&icu_lock);
 
 	writertc(RTC_STATUSB, rtc_statusb);
 
@@ -1081,7 +1087,9 @@ cpu_initclocks()
 			 * on the IO APIC.
 			 * Workaround: Limited variant of mixed mode.
 			 */
+			mtx_lock_spin(&icu_lock);
 			INTRDIS(1 << apic_8254_intr);
+			mtx_unlock_spin(&icu_lock);
 			inthand_remove(clkdesc);
 			printf("APIC_IO: Broken MP table detected: "
 			       "8254 is not connected to "
@@ -1104,7 +1112,9 @@ cpu_initclocks()
 			inthand_add("clk", apic_8254_intr,
 				    (driver_intr_t *)clkintr, NULL,
 				    INTR_TYPE_CLK | INTR_FAST, NULL);
+			mtx_lock_spin(&icu_lock);
 			INTREN(1 << apic_8254_intr);
+			mtx_unlock_spin(&icu_lock);
 		}
 		
 	}
diff --git a/sys/amd64/include/apicreg.h b/sys/amd64/include/apicreg.h
index 6345ce7e60e8..70b8011fceab 100644
--- a/sys/amd64/include/apicreg.h
+++ b/sys/amd64/include/apicreg.h
@@ -458,28 +458,4 @@ typedef struct IOAPIC ioapic_t;
 
 #define IOART_INTVEC	0x000000ff	/* R/W: INTerrupt vector field */
 
-#ifdef LOCORE
-
-#ifdef SMP
-
-/*
- * Protects the IO APIC and apic_imen as a critical region.
- */
-#define IMASK_LOCK	MTX_LOCK_SPIN(imen_mtx, 0)
-#define IMASK_UNLOCK	MTX_UNLOCK_SPIN(imen_mtx)
-
-#else  /* SMP */
-
-#define IMASK_LOCK				/* NOP */
-#define IMASK_UNLOCK				/* NOP */
-
-#endif /* SMP */
-
-#else /* LOCORE */
-
-/* global data in mp_machdep.c */
-extern struct mtx	imen_mtx;
-
-#endif /* LOCORE */
-
 #endif /* _MACHINE_APIC_H_ */
diff --git a/sys/amd64/include/mptable.h b/sys/amd64/include/mptable.h
index e8da25b887fe..1feb66af173c 100644
--- a/sys/amd64/include/mptable.h
+++ b/sys/amd64/include/mptable.h
@@ -320,9 +320,6 @@ static void	release_aps(void *dummy);
  * initialize all the SMP locks
  */
 
-/* critical region around IO APIC, apic_imen */
-struct mtx		imen_mtx;
-
 /* lock region used by kernel profiling */
 int	mcount_lock;
 
diff --git a/sys/amd64/isa/clock.c b/sys/amd64/isa/clock.c
index a7aaea0b7a5e..a39f7731be6a 100644
--- a/sys/amd64/isa/clock.c
+++ b/sys/amd64/isa/clock.c
@@ -1026,7 +1026,9 @@ cpu_initclocks()
 
 	inthand_add("clk", apic_8254_intr, (driver_intr_t *)clkintr, NULL,
 	    INTR_TYPE_CLK | INTR_FAST, &clkdesc);
+	mtx_lock_spin(&icu_lock);
 	INTREN(1 << apic_8254_intr);
+	mtx_unlock_spin(&icu_lock);
 
 #else /* APIC_IO */
 
@@ -1037,7 +1039,9 @@ cpu_initclocks()
 	 */
 	inthand_add("clk", 0, (driver_intr_t *)clkintr, NULL,
 	    INTR_TYPE_CLK | INTR_FAST, NULL);
+	mtx_lock_spin(&icu_lock);
 	INTREN(IRQ0);
+	mtx_unlock_spin(&icu_lock);
 
 #endif /* APIC_IO */
 
@@ -1060,11 +1064,13 @@ cpu_initclocks()
 	inthand_add("rtc", 8, (driver_intr_t *)rtcintr, NULL,
 	    INTR_TYPE_CLK | INTR_FAST, NULL);
 
+	mtx_lock_spin(&icu_lock);
 #ifdef APIC_IO
 	INTREN(APIC_IRQ8);
 #else
 	INTREN(IRQ8);
 #endif /* APIC_IO */
+	mtx_unlock_spin(&icu_lock);
 
 	writertc(RTC_STATUSB, rtc_statusb);
 
@@ -1081,7 +1087,9 @@ cpu_initclocks()
 			 * on the IO APIC.
 			 * Workaround: Limited variant of mixed mode.
 			 */
+			mtx_lock_spin(&icu_lock);
 			INTRDIS(1 << apic_8254_intr);
+			mtx_unlock_spin(&icu_lock);
 			inthand_remove(clkdesc);
 			printf("APIC_IO: Broken MP table detected: "
 			       "8254 is not connected to "
@@ -1104,7 +1112,9 @@ cpu_initclocks()
 			inthand_add("clk", apic_8254_intr,
 				    (driver_intr_t *)clkintr, NULL,
 				    INTR_TYPE_CLK | INTR_FAST, NULL);
+			mtx_lock_spin(&icu_lock);
 			INTREN(1 << apic_8254_intr);
+			mtx_unlock_spin(&icu_lock);
 		}
 		
 	}
diff --git a/sys/amd64/isa/intr_machdep.c b/sys/amd64/isa/intr_machdep.c
index c17806f961d0..3d20f9f7b5f0 100644
--- a/sys/amd64/isa/intr_machdep.c
+++ b/sys/amd64/isa/intr_machdep.c
@@ -289,14 +289,13 @@ isa_nmi(cd)
 void icu_reinit()
 {
 	int i;
-	u_int32_t eflags;
-	eflags = read_eflags();
-	disable_intr();
+
+	mtx_lock_spin(&icu_lock);
 	init_i8259();
 	for(i=0;i<ICU_LEN;i++)
 		if(intr_handler[i] != isa_strayintr)
 			INTREN(1<<i);
-	write_eflags(eflags);
+	mtx_unlock_spin(&icu_lock);
 }
 
 /*
@@ -312,7 +311,9 @@ isa_defaultirq()
 	/* icu vectors */
 	for (i = 0; i < ICU_LEN; i++)
 		icu_unset(i, (driver_intr_t *)NULL);
+	mtx_lock_spin(&icu_lock);
 	init_i8259();
+	mtx_unlock_spin(&icu_lock);
 }
 
 
@@ -406,8 +407,10 @@ isa_irq_pending()
 	u_char irr1;
 	u_char irr2;
 
+	mtx_lock_spin(&icu_lock);
 	irr1 = inb(IO_ICU1);
 	irr2 = inb(IO_ICU2);
+	mtx_unlock_spin(&icu_lock);
 	return ((irr2 << 8) | irr1);
 }
 #endif
@@ -473,7 +476,6 @@ icu_setup(int intr, driver_intr_t *handler, void *arg, int flags)
 	int		vector;
 	u_int32_t	value;		/* the window register is 32 bits */
 #endif /* FAST_HI */
-	u_long	ef;
 
 #if defined(APIC_IO)
 	if ((u_int)intr >= ICU_LEN)	/* no 8259 SLAVE to ignore */
@@ -486,8 +488,7 @@ icu_setup(int intr, driver_intr_t *handler, void *arg, int flags)
 		return (EBUSY);
 #endif
 
-	ef = read_eflags();
-	disable_intr();
+	mtx_lock_spin(&icu_lock);
 	intr_handler[intr] = handler;
 	intr_unit[intr] = arg;
 #ifdef FAST_HI
@@ -528,7 +529,7 @@ icu_setup(int intr, driver_intr_t *handler, void *arg, int flags)
 	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
 #endif /* FAST_HI */
 	INTREN(1 << intr);
-	write_eflags(ef);
+	mtx_unlock_spin(&icu_lock);
 	return (0);
 }
 
@@ -542,14 +543,12 @@ icu_unset(intr, handler)
 	int	intr;
 	driver_intr_t *handler;
 {
-	u_long	ef;
 
 	if ((u_int)intr >= ICU_LEN || handler != intr_handler[intr])
 		return (EINVAL);
 
+	mtx_lock_spin(&icu_lock);
 	INTRDIS(1 << intr);
-	ef = read_eflags();
-	disable_intr();
 	intr_countp[intr] = &intrcnt[1 + intr];
 	intr_handler[intr] = isa_strayintr;
 	intr_unit[intr] = &intr_unit[intr];
@@ -564,7 +563,7 @@ icu_unset(intr, handler)
 	setidt(ICU_OFFSET + intr, slowintr[intr], SDT_SYS386IGT, SEL_KPL,
 	    GSEL(GCODE_SEL, SEL_KPL));
 #endif /* FAST_HI */
-	write_eflags(ef);
+	mtx_unlock_spin(&icu_lock);
 	return (0);
 }
 
@@ -580,14 +579,18 @@ static void
 ithread_enable(int vector)
 {
 
+	mtx_lock_spin(&icu_lock);
 	INTREN(1 << vector);
+	mtx_unlock_spin(&icu_lock);
 }
 
 static void
 ithread_disable(int vector)
 {
 
+	mtx_lock_spin(&icu_lock);
 	INTRDIS(1 << vector);
+	mtx_unlock_spin(&icu_lock);
 }
 
 int
diff --git a/sys/amd64/isa/intr_machdep.h b/sys/amd64/isa/intr_machdep.h
index 39bffb993dd2..17266358155e 100644
--- a/sys/amd64/isa/intr_machdep.h
+++ b/sys/amd64/isa/intr_machdep.h
@@ -126,7 +126,15 @@
 
 #endif /* SMP || APIC_IO */
 
-#ifndef	LOCORE
+#ifdef LOCORE
+
+/*
+ * Protects the IO APIC, 8259 PIC, imen, and apic_imen
+ */
+#define ICU_LOCK	MTX_LOCK_SPIN(icu_lock, 0)
+#define ICU_UNLOCK	MTX_UNLOCK_SPIN(icu_lock)
+
+#else /* LOCORE */
 
 /*
  * Type of the first (asm) part of an interrupt handler.
@@ -139,6 +147,7 @@ extern u_long *intr_countp[];	/* pointers into intrcnt[] */
 extern driver_intr_t *intr_handler[];	/* C entry points of intr handlers */
 extern struct ithd *ithds[];
 extern void *intr_unit[];	/* cookies to pass to intr handlers */
+extern struct mtx icu_lock;
 
 inthand_t
 	IDTVEC(fastintr0), IDTVEC(fastintr1),
diff --git a/sys/amd64/isa/nmi.c b/sys/amd64/isa/nmi.c
index c17806f961d0..3d20f9f7b5f0 100644
--- a/sys/amd64/isa/nmi.c
+++ b/sys/amd64/isa/nmi.c
@@ -289,14 +289,13 @@ isa_nmi(cd)
 void icu_reinit()
 {
 	int i;
-	u_int32_t eflags;
-	eflags = read_eflags();
-	disable_intr();
+
+	mtx_lock_spin(&icu_lock);
 	init_i8259();
 	for(i=0;i<ICU_LEN;i++)
 		if(intr_handler[i] != isa_strayintr)
 			INTREN(1<<i);
-	write_eflags(eflags);
+	mtx_unlock_spin(&icu_lock);
 }
 
 /*
@@ -312,7 +311,9 @@ isa_defaultirq()
 	/* icu vectors */
 	for (i = 0; i < ICU_LEN; i++)
 		icu_unset(i, (driver_intr_t *)NULL);
+	mtx_lock_spin(&icu_lock);
 	init_i8259();
+	mtx_unlock_spin(&icu_lock);
 }
 
 
@@ -406,8 +407,10 @@ isa_irq_pending()
 	u_char irr1;
 	u_char irr2;
 
+	mtx_lock_spin(&icu_lock);
 	irr1 = inb(IO_ICU1);
 	irr2 = inb(IO_ICU2);
+	mtx_unlock_spin(&icu_lock);
 	return ((irr2 << 8) | irr1);
 }
 #endif
@@ -473,7 +476,6 @@ icu_setup(int intr, driver_intr_t *handler, void *arg, int flags)
 	int		vector;
 	u_int32_t	value;		/* the window register is 32 bits */
 #endif /* FAST_HI */
-	u_long	ef;
 
 #if defined(APIC_IO)
 	if ((u_int)intr >= ICU_LEN)	/* no 8259 SLAVE to ignore */
@@ -486,8 +488,7 @@ icu_setup(int intr, driver_intr_t *handler, void *arg, int flags)
 		return (EBUSY);
 #endif
 
-	ef = read_eflags();
-	disable_intr();
+	mtx_lock_spin(&icu_lock);
 	intr_handler[intr] = handler;
 	intr_unit[intr] = arg;
 #ifdef FAST_HI
@@ -528,7 +529,7 @@ icu_setup(int intr, driver_intr_t *handler, void *arg, int flags)
 	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
 #endif /* FAST_HI */
 	INTREN(1 << intr);
-	write_eflags(ef);
+	mtx_unlock_spin(&icu_lock);
 	return (0);
 }
 
@@ -542,14 +543,12 @@ icu_unset(intr, handler)
 	int	intr;
 	driver_intr_t *handler;
 {
-	u_long	ef;
 
 	if ((u_int)intr >= ICU_LEN || handler != intr_handler[intr])
 		return (EINVAL);
 
+	mtx_lock_spin(&icu_lock);
 	INTRDIS(1 << intr);
-	ef = read_eflags();
-	disable_intr();
 	intr_countp[intr] = &intrcnt[1 + intr];
 	intr_handler[intr] = isa_strayintr;
 	intr_unit[intr] = &intr_unit[intr];
@@ -564,7 +563,7 @@ icu_unset(intr, handler)
 	setidt(ICU_OFFSET + intr, slowintr[intr], SDT_SYS386IGT, SEL_KPL,
 	    GSEL(GCODE_SEL, SEL_KPL));
 #endif /* FAST_HI */
-	write_eflags(ef);
+	mtx_unlock_spin(&icu_lock);
 	return (0);
 }
 
@@ -580,14 +579,18 @@ static void
 ithread_enable(int vector)
 {
 
+	mtx_lock_spin(&icu_lock);
 	INTREN(1 << vector);
+	mtx_unlock_spin(&icu_lock);
 }
 
 static void
 ithread_disable(int vector)
 {
 
+	mtx_lock_spin(&icu_lock);
 	INTRDIS(1 << vector);
+	mtx_unlock_spin(&icu_lock);
 }
 
 int
diff --git a/sys/amd64/isa/npx.c b/sys/amd64/isa/npx.c
index 34a822a230a7..5d44aaa2fab2 100644
--- a/sys/amd64/isa/npx.c
+++ b/sys/amd64/isa/npx.c
@@ -82,6 +82,7 @@
 #include <i386/isa/isa.h>
 #endif
 #endif
+#include <i386/isa/intr_machdep.h>
 #include <isa/isavar.h>
 
 /*
@@ -425,7 +426,9 @@ no_irq13:
 	 * XXX hack around brokenness of bus_teardown_intr().  If we left the
 	 * irq active then we would get it instead of exception 16.
 	 */
+	mtx_lock_spin(&icu_lock);
 	INTRDIS(1 << irq_num);
+	mtx_unlock_spin(&icu_lock);
 
 	bus_release_resource(dev, SYS_RES_IRQ, irq_rid, irq_res);
 	bus_release_resource(dev, SYS_RES_IOPORT, ioport_rid, ioport_res);
diff --git a/sys/amd64/isa/vector.S b/sys/amd64/isa/vector.S
index d3b7a894c978..0bed76fdb083 100644
--- a/sys/amd64/isa/vector.S
+++ b/sys/amd64/isa/vector.S
@@ -15,6 +15,7 @@
 #else
 #include <i386/isa/isa.h>
 #endif
+#include <i386/isa/intr_machdep.h>
 
 #define FAST_INTR_HANDLER_USES_ES 1
 #ifdef FAST_INTR_HANDLER_USES_ES
diff --git a/sys/amd64/isa/vector.s b/sys/amd64/isa/vector.s
index d3b7a894c978..0bed76fdb083 100644
--- a/sys/amd64/isa/vector.s
+++ b/sys/amd64/isa/vector.s
@@ -15,6 +15,7 @@
 #else
 #include <i386/isa/isa.h>
 #endif
+#include <i386/isa/intr_machdep.h>
 
 #define FAST_INTR_HANDLER_USES_ES 1
 #ifdef FAST_INTR_HANDLER_USES_ES