9 files changed, 35 insertions, 17 deletions

diff --git a/sys/amd64/amd64/cpu_switch.S b/sys/amd64/amd64/cpu_switch.S
index 20fe50a87ada8..97154df09a7f7 100644
--- a/sys/amd64/amd64/cpu_switch.S
+++ b/sys/amd64/amd64/cpu_switch.S
@@ -48,7 +48,6 @@
 #include <machine/pmap.h>
 #include <machine/apic.h>
 #include <machine/smptests.h>		/** GRAB_LOPRIO */
-#include <machine/lock.h>
 #endif /* SMP */
 
 #include "assym.s"
diff --git a/sys/amd64/amd64/exception.S b/sys/amd64/amd64/exception.S
index 51662ba81e98b..33ed923448a13 100644
--- a/sys/amd64/amd64/exception.S
+++ b/sys/amd64/amd64/exception.S
@@ -38,7 +38,7 @@
 #include <machine/asmacros.h>
 #include <sys/ipl.h>
 #include <sys/mutex.h>
-#include <machine/lock.h>
+#include <machine/atomic.h>
 #include <machine/psl.h>
 #include <machine/trap.h>
 #ifdef SMP
diff --git a/sys/amd64/amd64/exception.s b/sys/amd64/amd64/exception.s
index 51662ba81e98b..33ed923448a13 100644
--- a/sys/amd64/amd64/exception.s
+++ b/sys/amd64/amd64/exception.s
@@ -38,7 +38,7 @@
 #include <machine/asmacros.h>
 #include <sys/ipl.h>
 #include <sys/mutex.h>
-#include <machine/lock.h>
+#include <machine/atomic.h>
 #include <machine/psl.h>
 #include <machine/trap.h>
 #ifdef SMP
diff --git a/sys/amd64/amd64/swtch.s b/sys/amd64/amd64/swtch.s
index 20fe50a87ada8..97154df09a7f7 100644
--- a/sys/amd64/amd64/swtch.s
+++ b/sys/amd64/amd64/swtch.s
@@ -48,7 +48,6 @@
 #include <machine/pmap.h>
 #include <machine/apic.h>
 #include <machine/smptests.h>		/** GRAB_LOPRIO */
-#include <machine/lock.h>
 #endif /* SMP */
 
 #include "assym.s"
diff --git a/sys/amd64/include/apicreg.h b/sys/amd64/include/apicreg.h
index be9de4f8fe1ff..3d84792ed5046 100644
--- a/sys/amd64/include/apicreg.h
+++ b/sys/amd64/include/apicreg.h
@@ -463,4 +463,28 @@ typedef struct IOAPIC ioapic_t;
 
 #define IOART_INTVEC	0x000000ff	/* R/W: INTerrupt vector field */
 
+#ifdef LOCORE
+
+#ifdef SMP
+
+/*
+ * Protects the IO APIC and apic_imen as a critical region.
+ */
+#define IMASK_LOCK	MTX_LOCK_SPIN(_imen_mtx, 0)
+#define IMASK_UNLOCK	MTX_UNLOCK_SPIN(_imen_mtx)
+
+#else  /* SMP */
+
+#define IMASK_LOCK				/* NOP */
+#define IMASK_UNLOCK				/* NOP */
+
+#endif /* SMP */
+
+#else /* LOCORE */
+
+/* global data in mp_machdep.c */
+extern struct mtx	imen_mtx;
+
+#endif /* LOCORE */
+
 #endif /* _MACHINE_APIC_H_ */
diff --git a/sys/amd64/include/atomic.h b/sys/amd64/include/atomic.h
index b5035b89e4d47..cb899a62d62f3 100644
--- a/sys/amd64/include/atomic.h
+++ b/sys/amd64/include/atomic.h
@@ -74,16 +74,18 @@ u_##TYPE	atomic_load_acq_##TYPE(volatile u_##TYPE *p);	\
 void		atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v);
 
 #else /* !KLD_MODULE */
+
 #if defined(SMP)
 #if defined(LOCORE)
 #define	MPLOCKED	lock ;
-#else
+#else /* !LOCORE */
 #define MPLOCKED	"lock ; "
-#endif
-#else
+#endif /* LOCORE */
+#else /* !SMP */
 #define MPLOCKED
-#endif
+#endif /* SMP */
 
+#if !defined(LOCORE)
 /*
  * The assembly is volatilized to demark potential before-and-after side
  * effects if an interrupt or SMP collision were to occur.
@@ -201,8 +203,10 @@ atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\
 	: : "memory");				 	\
 }
 #endif	/* defined(I386_CPU) */
+#endif	/* !defined(LOCORE) */
 #endif /* KLD_MODULE */
 
+#if !defined(LOCORE)
 ATOMIC_ASM(set,	     char,  "orb %b2,%0",   v)
 ATOMIC_ASM(clear,    char,  "andb %b2,%0", ~v)
 ATOMIC_ASM(add,	     char,  "addb %b2,%0",  v)
@@ -401,4 +405,5 @@ atomic_readandclear_long(volatile u_long *addr)
 	return (result);
 }
 #endif	/* !defined(WANT_FUNCTIONS) */
+#endif	/* !defined(LOCORE) */
 #endif /* ! _MACHINE_ATOMIC_H_ */
diff --git a/sys/amd64/isa/intr_machdep.c b/sys/amd64/isa/intr_machdep.c
index 2dd96d7259af2..9fcab8553934a 100644
--- a/sys/amd64/isa/intr_machdep.c
+++ b/sys/amd64/isa/intr_machdep.c
@@ -43,9 +43,6 @@
 
 #include <sys/param.h>
 #include <sys/bus.h>
-#ifndef SMP
-#include <machine/lock.h>
-#endif
 #include <sys/proc.h>
 #include <sys/systm.h>
 #include <sys/syslog.h>
diff --git a/sys/amd64/isa/ithread.c b/sys/amd64/isa/ithread.c
index f3b0109af0c7a..728d75fb0c5eb 100644
--- a/sys/amd64/isa/ithread.c
+++ b/sys/amd64/isa/ithread.c
@@ -36,9 +36,6 @@
 #include <sys/param.h>
 #include <sys/bus.h>
 #include <sys/rtprio.h>			/* change this name XXX */
-#ifndef SMP
-#include <machine/lock.h>
-#endif
 #include <sys/proc.h>
 #include <sys/systm.h>
 #include <sys/syslog.h>
diff --git a/sys/amd64/isa/nmi.c b/sys/amd64/isa/nmi.c
index 2dd96d7259af2..9fcab8553934a 100644
--- a/sys/amd64/isa/nmi.c
+++ b/sys/amd64/isa/nmi.c
@@ -43,9 +43,6 @@
 
 #include <sys/param.h>
 #include <sys/bus.h>
-#ifndef SMP
-#include <machine/lock.h>
-#endif
 #include <sys/proc.h>
 #include <sys/systm.h>
 #include <sys/syslog.h>