13 files changed, 102 insertions, 11 deletions

diff --git a/share/man/man9/swi.9 b/share/man/man9/swi.9
index b9ac0ca47b8b..52a39fe61c86 100644
--- a/share/man/man9/swi.9
+++ b/share/man/man9/swi.9
@@ -23,7 +23,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd April 19, 2012
+.Dd July 25, 2020
 .Dt SWI 9
 .Os
 .Sh NAME
@@ -132,7 +132,7 @@ The
 .Fa flags
 argument specifies how and when the handler should be run and is a mask of one
 or more of the following flags:
-.Bl -tag -width SWI_DELAY
+.Bl -tag -width SWI_FROMNMI
 .It Dv SWI_DELAY
 Specifies that the kernel should mark the specified handler as needing to run,
 but the kernel should not schedule the software interrupt thread to run.
@@ -146,6 +146,13 @@ functionality performed by
 .Fn setdelayed
 in earlier versions of
 .Fx .
+.It Dv SWI_FROMNMI
+Specifies that
+.Fn swi_sched
+is called from NMI context and should be careful about used KPIs.
+On platforms allowing IPI sending from NMI context it immediately wakes
+.Va clk_intr_event
+via the IPI, otherwise it works just like SWI_DELAY.
 .El
 .Pp
 The
diff --git a/sys/amd64/amd64/apic_vector.S b/sys/amd64/amd64/apic_vector.S
index 8d73717b03b3..4de39283f92f 100644
--- a/sys/amd64/amd64/apic_vector.S
+++ b/sys/amd64/amd64/apic_vector.S
@@ -206,6 +206,16 @@ IDTVEC(spuriousint)
 	jmp	doreti
 
 /*
+ * Executed by a CPU when it receives an IPI_SWI.
+ */
+	INTR_HANDLER ipi_swi
+	call	as_lapic_eoi
+	FAKE_MCOUNT(TF_RIP(%rsp))
+	call	ipi_swi_handler
+	MEXITCOUNT
+	jmp	doreti
+
+/*
  * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU.
  *
  * - Calls the generic rendezvous action function.
diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c
index b4cc7ab829a4..d46362ba9f9c 100644
--- a/sys/amd64/amd64/mp_machdep.c
+++ b/sys/amd64/amd64/mp_machdep.c
@@ -223,6 +223,10 @@ cpu_mp_start(void)
 	setidt(IPI_SUSPEND, pti ? IDTVEC(cpususpend_pti) : IDTVEC(cpususpend),
 	    SDT_SYSIGT, SEL_KPL, 0);
 
+	/* Install an IPI for calling delayed SWI */
+	setidt(IPI_SWI, pti ? IDTVEC(ipi_swi_pti) : IDTVEC(ipi_swi),
+	    SDT_SYSIGT, SEL_KPL, 0);
+
 	/* Set boot_cpu_id if needed. */
 	if (boot_cpu_id == -1) {
 		boot_cpu_id = PCPU_GET(apic_id);
diff --git a/sys/amd64/include/smp.h b/sys/amd64/include/smp.h
index d5b5fa9c5b81..8fbd89da0e57 100644
--- a/sys/amd64/include/smp.h
+++ b/sys/amd64/include/smp.h
@@ -32,6 +32,7 @@ inthand_t
 	IDTVEC(invlop_pti),
 	IDTVEC(invlop),
 	IDTVEC(ipi_intr_bitmap_handler_pti),
+	IDTVEC(ipi_swi_pti),
 	IDTVEC(cpustop_pti),
 	IDTVEC(cpususpend_pti),
 	IDTVEC(rendezvous_pti);
diff --git a/sys/i386/i386/apic_vector.s b/sys/i386/i386/apic_vector.s
index 7587f6bb5f88..6c5f3667de71 100644
--- a/sys/i386/i386/apic_vector.s
+++ b/sys/i386/i386/apic_vector.s
@@ -309,6 +309,23 @@ IDTVEC(cpususpend)
 	jmp	doreti
 
 /*
+ * Executed by a CPU when it receives an IPI_SWI.
+ */
+	.text
+	SUPERALIGN_TEXT
+IDTVEC(ipi_swi)
+	PUSH_FRAME
+	SET_KERNEL_SREGS
+	cld
+	KENTER
+	call	as_lapic_eoi
+	FAKE_MCOUNT(TF_EIP(%esp))
+	movl	$ipi_swi_handler, %eax
+	call	*%eax
+	MEXITCOUNT
+	jmp	doreti
+
+/*
  * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU.
  *
  * - Calls the generic rendezvous action function.
diff --git a/sys/i386/i386/mp_machdep.c b/sys/i386/i386/mp_machdep.c
index acdb40197433..8631d56f4854 100644
--- a/sys/i386/i386/mp_machdep.c
+++ b/sys/i386/i386/mp_machdep.c
@@ -188,6 +188,10 @@ cpu_mp_start(void)
 	setidt(IPI_SUSPEND, IDTVEC(cpususpend),
 	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
 
+	/* Install an IPI for calling delayed SWI */
+	setidt(IPI_SWI, IDTVEC(ipi_swi),
+	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
+
 	/* Set boot_cpu_id if needed. */
 	if (boot_cpu_id == -1) {
 		boot_cpu_id = PCPU_GET(apic_id);
diff --git a/sys/kern/kern_clock.c b/sys/kern/kern_clock.c
index 1009ee1cd4b6..e91eac05a2dc 100644
--- a/sys/kern/kern_clock.c
+++ b/sys/kern/kern_clock.c
@@ -508,6 +508,7 @@ hardclock(int cnt, int usermode)
 			if (i > 0 && i <= newticks)
 				watchdog_fire();
 		}
+		intr_event_handle(clk_intr_event, NULL);
 	}
 	if (curcpu == CPU_FIRST())
 		cpu_tick_calibration();
diff --git a/sys/kern/kern_intr.c b/sys/kern/kern_intr.c
index 5c96f41eda8b..0e11af2123e2 100644
--- a/sys/kern/kern_intr.c
+++ b/sys/kern/kern_intr.c
@@ -60,6 +60,7 @@ __FBSDID("$FreeBSD$");
 #include <machine/atomic.h>
 #include <machine/cpu.h>
 #include <machine/md_var.h>
+#include <machine/smp.h>
 #include <machine/stdarg.h>
 #ifdef DDB
 #include <ddb/ddb.h>
@@ -85,6 +86,7 @@ struct	intr_entropy {
 	uintptr_t event;
 };
 
+struct	intr_event *clk_intr_event;
 struct	intr_event *tty_intr_event;
 void	*vm_ih;
 struct proc *intrproc;
@@ -1018,7 +1020,7 @@ swi_add(struct intr_event **eventp, const char *name, driver_intr_t handler,
 	    void *arg, int pri, enum intr_type flags, void **cookiep)
 {
 	struct intr_event *ie;
-	int error;
+	int error = 0;
 
 	if (flags & INTR_ENTROPY)
 		return (EINVAL);
@@ -1036,8 +1038,10 @@ swi_add(struct intr_event **eventp, const char *name, driver_intr_t handler,
 		if (eventp != NULL)
 			*eventp = ie;
 	}
-	error = intr_event_add_handler(ie, name, NULL, handler, arg,
-	    PI_SWI(pri), flags, cookiep);
+	if (handler != NULL) {
+		error = intr_event_add_handler(ie, name, NULL, handler, arg,
+		    PI_SWI(pri), flags, cookiep);
+	}
 	return (error);
 }
 
@@ -1055,9 +1059,11 @@ swi_sched(void *cookie, int flags)
 	CTR3(KTR_INTR, "swi_sched: %s %s need=%d", ie->ie_name, ih->ih_name,
 	    ih->ih_need);
 
-	entropy.event = (uintptr_t)ih;
-	entropy.td = curthread;
-	random_harvest_queue(&entropy, sizeof(entropy), RANDOM_SWI);
+	if ((flags & SWI_FROMNMI) == 0) {
+		entropy.event = (uintptr_t)ih;
+		entropy.td = curthread;
+		random_harvest_queue(&entropy, sizeof(entropy), RANDOM_SWI);
+	}
 
 	/*
 	 * Set ih_need for this handler so that if the ithread is already
@@ -1066,7 +1072,16 @@ swi_sched(void *cookie, int flags)
 	 */
 	ih->ih_need = 1;
 
-	if (!(flags & SWI_DELAY)) {
+	if (flags & SWI_DELAY)
+		return;
+
+	if (flags & SWI_FROMNMI) {
+#if defined(SMP) && (defined(__i386__) || defined(__amd64__))
+		KASSERT(ie == clk_intr_event,
+		    ("SWI_FROMNMI used not with clk_intr_event"));
+		ipi_self_from_nmi(IPI_SWI);
+#endif
+	} else {
 		VM_CNT_INC(v_soft);
 		error = intr_event_schedule_thread(ie);
 		KASSERT(error == 0, ("stray software interrupt"));
@@ -1346,6 +1361,8 @@ intr_event_handle(struct intr_event *ie, struct trapframe *frame)
 	CK_SLIST_FOREACH(ih, &ie->ie_handlers, ih_next) {
 		if ((ih->ih_flags & IH_SUSP) != 0)
 			continue;
+		if ((ie->ie_flags & IE_SOFT) != 0 && ih->ih_need == 0)
+			continue;
 		if (ih->ih_filter == NULL) {
 			thread = true;
 			continue;
@@ -1570,6 +1587,9 @@ static void
 start_softintr(void *dummy)
 {
 
+	if (swi_add(&clk_intr_event, "clk", NULL, NULL, SWI_CLOCK,
+	    INTR_MPSAFE, NULL))
+		panic("died while creating clk swi ithread");
 	if (swi_add(NULL, "vm", swi_vm, NULL, SWI_VM, INTR_MPSAFE, &vm_ih))
 		panic("died while creating vm swi ithread");
 }
diff --git a/sys/sys/interrupt.h b/sys/sys/interrupt.h
index 3492b17bf8d4..138e99495af2 100644
--- a/sys/sys/interrupt.h
+++ b/sys/sys/interrupt.h
@@ -133,7 +133,8 @@ struct intr_event {
 #define	IE_SOFT		0x000001	/* Software interrupt. */
 #define	IE_ADDING_THREAD 0x000004	/* Currently building an ithread. */
 
-/* Flags to pass to sched_swi. */
+/* Flags to pass to swi_sched. */
+#define	SWI_FROMNMI	0x1
 #define	SWI_DELAY	0x2
 
 /*
@@ -151,6 +152,7 @@ struct intr_event {
 
 struct proc;
 
+extern struct	intr_event *clk_intr_event;
 extern struct	intr_event *tty_intr_event;
 extern void	*vm_ih;
 
diff --git a/sys/x86/include/apicvar.h b/sys/x86/include/apicvar.h
index 866dafe6dca4..0f4961c1d631 100644
--- a/sys/x86/include/apicvar.h
+++ b/sys/x86/include/apicvar.h
@@ -130,7 +130,8 @@
 
 #define	IPI_STOP	(APIC_IPI_INTS + 6)	/* Stop CPU until restarted. */
 #define	IPI_SUSPEND	(APIC_IPI_INTS + 7)	/* Suspend CPU until restarted. */
-#define	IPI_DYN_FIRST	(APIC_IPI_INTS + 8)
+#define	IPI_SWI		(APIC_IPI_INTS + 8)	/* Run clk_intr_event. */
+#define	IPI_DYN_FIRST	(APIC_IPI_INTS + 9)
 #define	IPI_DYN_LAST	(254)			/* IPIs allocated at runtime */
 
 /*
diff --git a/sys/x86/include/x86_smp.h b/sys/x86/include/x86_smp.h
index e2b59a2b4703..e01f869a58f6 100644
--- a/sys/x86/include/x86_smp.h
+++ b/sys/x86/include/x86_smp.h
@@ -76,6 +76,7 @@ extern u_long *ipi_rendezvous_counts[MAXCPU];
 /* IPI handlers */
 inthand_t
 	IDTVEC(ipi_intr_bitmap_handler), /* Bitmap based IPIs */ 
+	IDTVEC(ipi_swi),	/* Runs delayed SWI */
 	IDTVEC(cpustop),	/* CPU stops & waits to be restarted */
 	IDTVEC(cpususpend),	/* CPU suspends & waits to be resumed */
 	IDTVEC(rendezvous);	/* handle CPU rendezvous */
@@ -96,6 +97,7 @@ void	ipi_all_but_self(u_int ipi);
 void 	ipi_bitmap_handler(struct trapframe frame);
 void	ipi_cpu(int cpu, u_int ipi);
 int	ipi_nmi_handler(void);
+void	ipi_swi_handler(struct trapframe frame);
 void	ipi_selected(cpuset_t cpus, u_int ipi);
 void	ipi_self_from_nmi(u_int vector);
 void	set_interrupt_apic_ids(void);
diff --git a/sys/x86/x86/mp_x86.c b/sys/x86/x86/mp_x86.c
index cfcbca53a1c9..df17cfd6b137 100644
--- a/sys/x86/x86/mp_x86.c
+++ b/sys/x86/x86/mp_x86.c
@@ -47,6 +47,7 @@ __FBSDID("$FreeBSD$");
 #ifdef GPROF 
 #include <sys/gmon.h>
 #endif
+#include <sys/interrupt.h>
 #include <sys/kdb.h>
 #include <sys/kernel.h>
 #include <sys/ktr.h>
@@ -1621,6 +1622,16 @@ cpususpend_handler(void)
 }
 
 /*
+ * Handle an IPI_SWI by waking delayed SWI thread.
+ */
+void
+ipi_swi_handler(struct trapframe frame)
+{
+
+	intr_event_handle(clk_intr_event, &frame);
+}
+
+/*
  * This is called once the rest of the system is up and running and we're
  * ready to let the AP's out of the pen.
  */
diff --git a/sys/x86/xen/xen_apic.c b/sys/x86/xen/xen_apic.c
index 7d23f0a50417..78471eb1d997 100644
--- a/sys/x86/xen/xen_apic.c
+++ b/sys/x86/xen/xen_apic.c
@@ -76,6 +76,7 @@ static driver_filter_t xen_invlcache;
 static driver_filter_t xen_ipi_bitmap_handler;
 static driver_filter_t xen_cpustop_handler;
 static driver_filter_t xen_cpususpend_handler;
+static driver_filter_t xen_ipi_swi_handler;
 #endif
 
 /*---------------------------------- Macros ----------------------------------*/
@@ -103,6 +104,7 @@ static struct xen_ipi_handler xen_ipis[] =
 	[IPI_TO_IDX(IPI_BITMAP_VECTOR)] = { xen_ipi_bitmap_handler,	"b"   },
 	[IPI_TO_IDX(IPI_STOP)]		= { xen_cpustop_handler,	"st"  },
 	[IPI_TO_IDX(IPI_SUSPEND)]	= { xen_cpususpend_handler,	"sp"  },
+	[IPI_TO_IDX(IPI_SWI)]		= { xen_ipi_swi_handler,	"sw"  },
 };
 #endif
 
@@ -522,6 +524,15 @@ xen_cpususpend_handler(void *arg)
 	return (FILTER_HANDLED);
 }
 
+static int
+xen_ipi_swi_handler(void *arg)
+{
+	struct trapframe *frame = arg;
+
+	ipi_swi_handler(*frame);
+	return (FILTER_HANDLED);
+}
+
 /*----------------------------- XEN PV IPI setup -----------------------------*/
 /*
  * Those functions are provided outside of the Xen PV APIC implementation