5 files changed, 252 insertions, 156 deletions

diff --git a/sys/dev/acpica/Osd/OsdSchedule.c b/sys/dev/acpica/Osd/OsdSchedule.c
index f1e84e4de48b..1e8aecab2526 100644
--- a/sys/dev/acpica/Osd/OsdSchedule.c
+++ b/sys/dev/acpica/Osd/OsdSchedule.c
@@ -65,31 +65,8 @@ struct acpi_task_ctx {
     void 			*at_context;
 };
 
-/*
- * Private task queue definition for ACPI
- */
-static struct proc *
-acpi_task_start_threads(struct taskqueue **tqp)
-{
-    struct proc	*acpi_kthread_proc;
-    int	err, i;
-
-    KASSERT(*tqp != NULL, ("acpi taskqueue not created before threads"));
-
-    /* Start one or more threads to service our taskqueue. */
-    for (i = 0; i < acpi_max_threads; i++) {
-	err = kthread_create(taskqueue_thread_loop, tqp, &acpi_kthread_proc,
-	    0, 0, "acpi_task%d", i);
-	if (err) {
-	    printf("%s: kthread_create failed (%d)\n", __func__, err);
-	    break;
-	}
-    }
-    return (acpi_kthread_proc);
-}
-
 TASKQUEUE_DEFINE(acpi, taskqueue_thread_enqueue, &taskqueue_acpi,
-    taskqueue_acpi_proc = acpi_task_start_threads(&taskqueue_acpi));
+    taskqueue_start_threads(&taskqueue_acpi, 3, PWAIT, "acpi_task"));
 
 /*
  * Bounce through this wrapper function since ACPI-CA doesn't understand
diff --git a/sys/kern/kern_synch.c b/sys/kern/kern_synch.c
index 2b9d31fda61f..6a5bff2c820a 100644
--- a/sys/kern/kern_synch.c
+++ b/sys/kern/kern_synch.c
@@ -221,6 +221,88 @@ msleep(ident, mtx, priority, wmesg, timo)
 	return (rval);
 }
 
+int
+msleep_spin(ident, mtx, wmesg, timo)
+	void *ident;
+	struct mtx *mtx;
+	const char *wmesg;
+	int timo;
+{
+	struct thread *td;
+	struct proc *p;
+	int rval;
+	WITNESS_SAVE_DECL(mtx);
+
+	td = curthread;
+	p = td->td_proc;
+	KASSERT(mtx != NULL, ("sleeping without a mutex"));
+	KASSERT(p != NULL, ("msleep1"));
+	KASSERT(ident != NULL && TD_IS_RUNNING(td), ("msleep"));
+
+	if (cold) {
+		/*
+		 * During autoconfiguration, just return;
+		 * don't run any other threads or panic below,
+		 * in case this is the idle thread and already asleep.
+		 * XXX: this used to do "s = splhigh(); splx(safepri);
+		 * splx(s);" to give interrupts a chance, but there is
+		 * no way to give interrupts a chance now.
+		 */
+		return (0);
+	}
+
+	sleepq_lock(ident);
+	CTR5(KTR_PROC, "msleep_spin: thread %p (pid %ld, %s) on %s (%p)",
+	    (void *)td, (long)p->p_pid, p->p_comm, wmesg, ident);
+
+	DROP_GIANT();
+	mtx_assert(mtx, MA_OWNED | MA_NOTRECURSED);
+	WITNESS_SAVE(&mtx->mtx_object, mtx);
+	mtx_unlock_spin(mtx);
+
+	/*
+	 * We put ourselves on the sleep queue and start our timeout.
+	 */
+	sleepq_add(ident, mtx, wmesg, SLEEPQ_MSLEEP);
+	if (timo)
+		sleepq_set_timeout(ident, timo);
+
+	/*
+	 * Can't call ktrace with any spin locks held so it can lock the
+	 * ktrace_mtx lock, and WITNESS_WARN considers it an error to hold
+	 * any spin lock.  Thus, we have to drop the sleepq spin lock while
+	 * we handle those requests.  This is safe since we have placed our
+	 * thread on the sleep queue already.
+	 */
+#ifdef KTRACE
+	if (KTRPOINT(td, KTR_CSW)) {
+		sleepq_release(ident);
+		ktrcsw(1, 0);
+		sleepq_lock(ident);
+	}
+#endif
+#ifdef WITNESS
+	sleepq_release(ident);
+	WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, "Sleeping on \"%s\"",
+	    wmesg);
+	sleepq_lock(ident);
+#endif
+	if (timo)
+		rval = sleepq_timedwait(ident);
+	else {
+		sleepq_wait(ident);
+		rval = 0;
+	}
+#ifdef KTRACE
+	if (KTRPOINT(td, KTR_CSW))
+		ktrcsw(0, 0);
+#endif
+	PICKUP_GIANT();
+	mtx_lock_spin(mtx);
+	WITNESS_RESTORE(&mtx->mtx_object, mtx);
+	return (rval);
+}
+
 /*
  * Make all threads sleeping on the specified identifier runnable.
  */
diff --git a/sys/kern/subr_taskqueue.c b/sys/kern/subr_taskqueue.c
index f7781878f029..a13e75ef2445 100644
--- a/sys/kern/subr_taskqueue.c
+++ b/sys/kern/subr_taskqueue.c
@@ -37,8 +37,10 @@ __FBSDID("$FreeBSD$");
 #include <sys/malloc.h>
 #include <sys/mutex.h>
 #include <sys/proc.h>
+#include <sys/sched.h>
 #include <sys/taskqueue.h>
 #include <sys/unistd.h>
+#include <machine/stdarg.h>
 
 static MALLOC_DEFINE(M_TASKQUEUE, "taskqueue", "Task Queues");
 static void	*taskqueue_giant_ih;
@@ -55,10 +57,42 @@ struct taskqueue {
 	struct task		*tq_running;
 	struct mtx		tq_mutex;
 	struct proc		**tq_pproc;
+	int			tq_pcount;
+	int			tq_spin;
+	int			tq_flags;
 };
 
+#define	TQ_FLAGS_ACTIVE		(1 << 0)
+
+static __inline void
+TQ_LOCK(struct taskqueue *tq)
+{
+	if (tq->tq_spin)
+		mtx_lock_spin(&tq->tq_mutex);
+	else
+		mtx_lock(&tq->tq_mutex);
+}
+
+static __inline void
+TQ_UNLOCK(struct taskqueue *tq)
+{
+	if (tq->tq_spin)
+		mtx_unlock_spin(&tq->tq_mutex);
+	else
+		mtx_unlock(&tq->tq_mutex);
+}
+
 static void	init_taskqueue_list(void *data);
 
+static __inline int
+TQ_SLEEP(struct taskqueue *tq, void *p, struct mtx *m, int pri, const char *wm,
+    int t)
+{
+	if (tq->tq_spin)
+		return (msleep_spin(p, m, wm, t));
+	return (msleep(p, m, pri, wm, t));
+}
+
 static void
 init_taskqueue_list(void *data __unused)
 {
@@ -69,10 +103,10 @@ init_taskqueue_list(void *data __unused)
 SYSINIT(taskqueue_list, SI_SUB_INTRINSIC, SI_ORDER_ANY, init_taskqueue_list,
     NULL);
 
-struct taskqueue *
-taskqueue_create(const char *name, int mflags,
+static struct taskqueue *
+_taskqueue_create(const char *name, int mflags,
 		 taskqueue_enqueue_fn enqueue, void *context,
-		 struct proc **pp)
+		 int mtxflags, const char *mtxname)
 {
 	struct taskqueue *queue;
 
@@ -84,8 +118,9 @@ taskqueue_create(const char *name, int mflags,
 	queue->tq_name = name;
 	queue->tq_enqueue = enqueue;
 	queue->tq_context = context;
-	queue->tq_pproc = pp;
-	mtx_init(&queue->tq_mutex, "taskqueue", NULL, MTX_DEF);
+	queue->tq_spin = (mtxflags & MTX_SPIN) != 0;
+	queue->tq_flags |= TQ_FLAGS_ACTIVE;
+	mtx_init(&queue->tq_mutex, mtxname, NULL, mtxflags);
 
 	mtx_lock(&taskqueue_queues_mutex);
 	STAILQ_INSERT_TAIL(&taskqueue_queues, queue, tq_link);
@@ -94,23 +129,26 @@ taskqueue_create(const char *name, int mflags,
 	return queue;
 }
 
+struct taskqueue *
+taskqueue_create(const char *name, int mflags,
+		 taskqueue_enqueue_fn enqueue, void *context,
+		 struct proc **pp)
+{
+	(void) pp;
+	return _taskqueue_create(name, mflags, enqueue, context,
+			MTX_DEF, "taskqueue");
+}
+
 /*
  * Signal a taskqueue thread to terminate.
  */
 static void
 taskqueue_terminate(struct proc **pp, struct taskqueue *tq)
 {
-	struct proc *p;
 
-	p = *pp;
-	*pp = NULL;
-	if (p) {
-		wakeup_one(tq);
-		PROC_LOCK(p);		   /* NB: insure we don't miss wakeup */
-		mtx_unlock(&tq->tq_mutex); /* let taskqueue thread run */
-		msleep(p, &p->p_mtx, PWAIT, "taskqueue_destroy", 0);
-		PROC_UNLOCK(p);
-		mtx_lock(&tq->tq_mutex);
+	while (tq->tq_pcount > 0) {
+		wakeup(tq);
+		TQ_SLEEP(tq, pp, &tq->tq_mutex, PWAIT, "taskqueue_destroy", 0);
 	}
 }
 
@@ -122,10 +160,12 @@ taskqueue_free(struct taskqueue *queue)
 	STAILQ_REMOVE(&taskqueue_queues, queue, taskqueue, tq_link);
 	mtx_unlock(&taskqueue_queues_mutex);
 
-	mtx_lock(&queue->tq_mutex);
+	TQ_LOCK(queue);
+	queue->tq_flags &= ~TQ_FLAGS_ACTIVE;
 	taskqueue_run(queue);
 	taskqueue_terminate(queue->tq_pproc, queue);
 	mtx_destroy(&queue->tq_mutex);
+	free(queue->tq_pproc, M_TASKQUEUE);
 	free(queue, M_TASKQUEUE);
 }
 
@@ -140,7 +180,7 @@ taskqueue_find(const char *name)
 	mtx_lock(&taskqueue_queues_mutex);
 	STAILQ_FOREACH(queue, &taskqueue_queues, tq_link) {
 		if (strcmp(queue->tq_name, name) == 0) {
-			mtx_lock(&queue->tq_mutex);
+			TQ_LOCK(queue);
 			mtx_unlock(&taskqueue_queues_mutex);
 			return queue;
 		}
@@ -155,14 +195,14 @@ taskqueue_enqueue(struct taskqueue *queue, struct task *task)
 	struct task *ins;
 	struct task *prev;
 
-	mtx_lock(&queue->tq_mutex);
+	TQ_LOCK(queue);
 
 	/*
 	 * Count multiple enqueues.
 	 */
 	if (task->ta_pending) {
 		task->ta_pending++;
-		mtx_unlock(&queue->tq_mutex);
+		TQ_UNLOCK(queue);
 		return 0;
 	}
 
@@ -188,7 +228,7 @@ taskqueue_enqueue(struct taskqueue *queue, struct task *task)
 	task->ta_pending = 1;
 	queue->tq_enqueue(queue->tq_context);
 
-	mtx_unlock(&queue->tq_mutex);
+	TQ_UNLOCK(queue);
 
 	return 0;
 }
@@ -201,7 +241,7 @@ taskqueue_run(struct taskqueue *queue)
 
 	owned = mtx_owned(&queue->tq_mutex);
 	if (!owned)
-		mtx_lock(&queue->tq_mutex);
+		TQ_LOCK(queue);
 	while (STAILQ_FIRST(&queue->tq_queue)) {
 		/*
 		 * Carefully remove the first task from the queue and
@@ -212,11 +252,11 @@ taskqueue_run(struct taskqueue *queue)
 		pending = task->ta_pending;
 		task->ta_pending = 0;
 		queue->tq_running = task;
-		mtx_unlock(&queue->tq_mutex);
+		TQ_UNLOCK(queue);
 
 		task->ta_func(task->ta_context, pending);
 
-		mtx_lock(&queue->tq_mutex);
+		TQ_LOCK(queue);
 		queue->tq_running = NULL;
 		wakeup(task);
 	}
@@ -226,18 +266,25 @@ taskqueue_run(struct taskqueue *queue)
 	 * on entry, although this opens a race window.
 	 */
 	if (!owned)
-		mtx_unlock(&queue->tq_mutex);
+		TQ_UNLOCK(queue);
 }
 
 void
 taskqueue_drain(struct taskqueue *queue, struct task *task)
 {
-	WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, "taskqueue_drain");
+	if (queue->tq_spin) {		/* XXX */
+		mtx_lock_spin(&queue->tq_mutex);
+		while (task->ta_pending != 0 || task == queue->tq_running)
+			msleep_spin(task, &queue->tq_mutex, "-", 0);
+		mtx_unlock_spin(&queue->tq_mutex);
+	} else {
+		WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, __func__);
 
-	mtx_lock(&queue->tq_mutex);
-	while (task->ta_pending != 0 || task == queue->tq_running)
-		msleep(task, &queue->tq_mutex, PWAIT, "-", 0);
-	mtx_unlock(&queue->tq_mutex);
+		mtx_lock(&queue->tq_mutex);
+		while (task->ta_pending != 0 || task == queue->tq_running)
+			msleep(task, &queue->tq_mutex, PWAIT, "-", 0);
+		mtx_unlock(&queue->tq_mutex);
+	}
 }
 
 static void
@@ -264,6 +311,43 @@ taskqueue_swi_giant_run(void *dummy)
 	taskqueue_run(taskqueue_swi_giant);
 }
 
+int
+taskqueue_start_threads(struct taskqueue **tqp, int count, int pri,
+			const char *name, ...)
+{
+	va_list ap;
+	struct taskqueue *tq;
+	char ktname[MAXCOMLEN];
+	int i;
+
+	if (count <= 0)
+		return (EINVAL);
+	tq = *tqp;
+
+	if ((tq->tq_pproc = malloc(sizeof(struct proc *) * count, M_TASKQUEUE,
+	    M_NOWAIT | M_ZERO)) == NULL)
+		return (ENOMEM);
+	
+	va_start(ap, name);
+	vsnprintf(ktname, MAXCOMLEN, name, ap);
+	va_end(ap);
+
+	for (i = 0; i < count; i++) {
+		if (count == 1)
+			kthread_create(taskqueue_thread_loop, tqp,
+			    &tq->tq_pproc[i], 0, 0, ktname);
+		else
+			kthread_create(taskqueue_thread_loop, tqp,
+			    &tq->tq_pproc[i], 0, 0, "%s_%d", ktname, i);
+		mtx_lock_spin(&sched_lock);
+		sched_prio(FIRST_THREAD_IN_PROC(tq->tq_pproc[i]), pri);
+		mtx_unlock_spin(&sched_lock);
+		tq->tq_pcount++;
+	}
+
+	return (0);
+}
+
 void
 taskqueue_thread_loop(void *arg)
 {
@@ -271,15 +355,16 @@ taskqueue_thread_loop(void *arg)
 
 	tqp = arg;
 	tq = *tqp;
-	mtx_lock(&tq->tq_mutex);
+	TQ_LOCK(tq);
 	do {
 		taskqueue_run(tq);
-		msleep(tq, &tq->tq_mutex, PWAIT, "-", 0); 
-	} while (*tq->tq_pproc != NULL);
+		TQ_SLEEP(tq, tq, &tq->tq_mutex, curthread->td_priority, "-", 0);
+	} while ((tq->tq_flags & TQ_FLAGS_ACTIVE) != 0);
 
 	/* rendezvous with thread that asked us to terminate */
-	wakeup_one(tq);
-	mtx_unlock(&tq->tq_mutex);
+	tq->tq_pcount--;
+	wakeup_one(tq->tq_pproc);
+	TQ_UNLOCK(tq);
 	kthread_exit(0);
 }
 
@@ -300,85 +385,30 @@ TASKQUEUE_DEFINE(swi, taskqueue_swi_enqueue, 0,
 		     INTR_MPSAFE, &taskqueue_ih)); 
 
 TASKQUEUE_DEFINE(swi_giant, taskqueue_swi_giant_enqueue, 0,
-		 swi_add(NULL, "Giant task queue", taskqueue_swi_giant_run,
+		 swi_add(NULL, "Giant taskq", taskqueue_swi_giant_run,
 		     NULL, SWI_TQ_GIANT, 0, &taskqueue_giant_ih)); 
 
 TASKQUEUE_DEFINE_THREAD(thread);
 
-int
-taskqueue_enqueue_fast(struct taskqueue *queue, struct task *task)
+struct taskqueue *
+taskqueue_create_fast(const char *name, int mflags,
+		 taskqueue_enqueue_fn enqueue, void *context)
 {
-	struct task *ins;
-	struct task *prev;
-
-	mtx_lock_spin(&queue->tq_mutex);
-
-	/*
-	 * Count multiple enqueues.
-	 */
-	if (task->ta_pending) {
-		task->ta_pending++;
-		mtx_unlock_spin(&queue->tq_mutex);
-		return 0;
-	}
-
-	/*
-	 * Optimise the case when all tasks have the same priority.
-	 */
-	prev = STAILQ_LAST(&queue->tq_queue, task, ta_link);
-	if (!prev || prev->ta_priority >= task->ta_priority) {
-		STAILQ_INSERT_TAIL(&queue->tq_queue, task, ta_link);
-	} else {
-		prev = 0;
-		for (ins = STAILQ_FIRST(&queue->tq_queue); ins;
-		     prev = ins, ins = STAILQ_NEXT(ins, ta_link))
-			if (ins->ta_priority < task->ta_priority)
-				break;
-
-		if (prev)
-			STAILQ_INSERT_AFTER(&queue->tq_queue, prev, task, ta_link);
-		else
-			STAILQ_INSERT_HEAD(&queue->tq_queue, task, ta_link);
-	}
-
-	task->ta_pending = 1;
-	queue->tq_enqueue(queue->tq_context);
-
-	mtx_unlock_spin(&queue->tq_mutex);
-
-	return 0;
+	return _taskqueue_create(name, mflags, enqueue, context,
+			MTX_SPIN, "fast_taskqueue");
 }
 
-static void
-taskqueue_run_fast(struct taskqueue *queue)
+/* NB: for backwards compatibility */
+int
+taskqueue_enqueue_fast(struct taskqueue *queue, struct task *task)
 {
-	struct task *task;
-	int pending;
-
-	mtx_lock_spin(&queue->tq_mutex);
-	while (STAILQ_FIRST(&queue->tq_queue)) {
-		/*
-		 * Carefully remove the first task from the queue and
-		 * zero its pending count.
-		 */
-		task = STAILQ_FIRST(&queue->tq_queue);
-		STAILQ_REMOVE_HEAD(&queue->tq_queue, ta_link);
-		pending = task->ta_pending;
-		task->ta_pending = 0;
-		mtx_unlock_spin(&queue->tq_mutex);
-
-		task->ta_func(task->ta_context, pending);
-
-		mtx_lock_spin(&queue->tq_mutex);
-	}
-	mtx_unlock_spin(&queue->tq_mutex);
+	return taskqueue_enqueue(queue, task);
 }
 
-struct taskqueue *taskqueue_fast;
 static void	*taskqueue_fast_ih;
 
 static void
-taskqueue_fast_schedule(void *context)
+taskqueue_fast_enqueue(void *context)
 {
 	swi_sched(taskqueue_fast_ih, 0);
 }
@@ -386,31 +416,9 @@ taskqueue_fast_schedule(void *context)
 static void
 taskqueue_fast_run(void *dummy)
 {
-	taskqueue_run_fast(taskqueue_fast);
+	taskqueue_run(taskqueue_fast);
 }
 
-static void
-taskqueue_define_fast(void *arg)
-{
-
-	taskqueue_fast = malloc(sizeof(struct taskqueue), M_TASKQUEUE,
-	    M_NOWAIT | M_ZERO);
-	if (!taskqueue_fast) {
-		printf("%s: Unable to allocate fast task queue!\n", __func__);
-		return;
-	}
-
-	STAILQ_INIT(&taskqueue_fast->tq_queue);
-	taskqueue_fast->tq_name = "fast";
-	taskqueue_fast->tq_enqueue = taskqueue_fast_schedule;
-	mtx_init(&taskqueue_fast->tq_mutex, "taskqueue_fast", NULL, MTX_SPIN);
-
-	mtx_lock(&taskqueue_queues_mutex);
-	STAILQ_INSERT_TAIL(&taskqueue_queues, taskqueue_fast, tq_link);
-	mtx_unlock(&taskqueue_queues_mutex);
-
-	swi_add(NULL, "Fast task queue", taskqueue_fast_run,
-		NULL, SWI_TQ_FAST, INTR_MPSAFE, &taskqueue_fast_ih);
-}
-SYSINIT(taskqueue_fast, SI_SUB_CONFIGURE, SI_ORDER_SECOND,
-    taskqueue_define_fast, NULL);
+TASKQUEUE_FAST_DEFINE(fast, taskqueue_fast_enqueue, 0,
+	swi_add(NULL, "Fast task queue", taskqueue_fast_run, NULL,
+	SWI_TQ_FAST, INTR_MPSAFE, &taskqueue_fast_ih));
diff --git a/sys/sys/systm.h b/sys/sys/systm.h
index d41944857924..3530a425fdee 100644
--- a/sys/sys/systm.h
+++ b/sys/sys/systm.h
@@ -296,6 +296,7 @@ static __inline void		splx(intrmask_t ipl __unused)	{ return; }
  */
 int	msleep(void *chan, struct mtx *mtx, int pri, const char *wmesg,
 	    int timo);
+int	msleep_spin(void *chan, struct mtx *mtx, const char *wmesg, int timo);
 #define	tsleep(chan, pri, wmesg, timo)	msleep(chan, NULL, pri, wmesg, timo)
 void	wakeup(void *chan) __nonnull(1);
 void	wakeup_one(void *chan) __nonnull(1);
diff --git a/sys/sys/taskqueue.h b/sys/sys/taskqueue.h
index 1a69e58a1a22..67a3d7a4460b 100644
--- a/sys/sys/taskqueue.h
+++ b/sys/sys/taskqueue.h
@@ -51,6 +51,8 @@ struct proc;
 struct taskqueue *taskqueue_create(const char *name, int mflags,
 				    taskqueue_enqueue_fn enqueue,
 				    void *context, struct proc **);
+int	taskqueue_start_threads(struct taskqueue **tqp, int count, int pri,
+				const char *name, ...) __printflike(4, 5);
 int	taskqueue_enqueue(struct taskqueue *queue, struct task *task);
 void	taskqueue_drain(struct taskqueue *queue, struct task *task);
 struct taskqueue *taskqueue_find(const char *name);
@@ -80,7 +82,7 @@ void	taskqueue_thread_enqueue(void *context);
 extern struct taskqueue *taskqueue_##name
 
 /*
- * Define and initialise a taskqueue.
+ * Define and initialise a global taskqueue that uses sleep mutexes.
  */
 #define TASKQUEUE_DEFINE(name, enqueue, context, init)			\
 									\
@@ -89,10 +91,8 @@ struct taskqueue *taskqueue_##name;					\
 static void								\
 taskqueue_define_##name(void *arg)					\
 {									\
-	static struct proc *taskqueue_##name##_proc;			\
 	taskqueue_##name =						\
-	    taskqueue_create(#name, M_NOWAIT, (enqueue), (context),	\
-	    &taskqueue_##name##_proc);					\
+	    taskqueue_create(#name, M_NOWAIT, (enqueue), (context), NULL);\
 	init;								\
 }									\
 									\
@@ -102,8 +102,33 @@ SYSINIT(taskqueue_##name, SI_SUB_CONFIGURE, SI_ORDER_SECOND,		\
 struct __hack
 #define TASKQUEUE_DEFINE_THREAD(name)					\
 TASKQUEUE_DEFINE(name, taskqueue_thread_enqueue, &taskqueue_##name,	\
-	kthread_create(taskqueue_thread_loop, &taskqueue_##name,	\
-	&taskqueue_##name##_proc, 0, 0, #name " taskq"))
+	taskqueue_start_threads(&taskqueue_##name, 1, PWAIT,		\
+	"%s taskq", #name))
+
+/*
+ * Define and initialise a global taskqueue that uses spin mutexes.
+ */
+#define TASKQUEUE_FAST_DEFINE(name, enqueue, context, init)		\
+									\
+struct taskqueue *taskqueue_##name;					\
+									\
+static void								\
+taskqueue_define_##name(void *arg)					\
+{									\
+	taskqueue_##name =						\
+	    taskqueue_create_fast(#name, M_NOWAIT, (enqueue),		\
+	    (context));							\
+	init;								\
+}									\
+									\
+SYSINIT(taskqueue_##name, SI_SUB_CONFIGURE, SI_ORDER_SECOND,		\
+	taskqueue_define_##name, NULL)					\
+									\
+struct __hack
+#define TASKQUEUE_FAST_DEFINE_THREAD(name)				\
+TASKQUEUE_FAST_DEFINE(name, taskqueue_thread_enqueue,			\
+	&taskqueue_##name, taskqueue_start_threads(&taskqueue_##name	\
+	1, PWAIT, "%s taskq", #name))
 
 /*
  * These queues are serviced by software interrupt handlers.  To enqueue
@@ -127,5 +152,8 @@ TASKQUEUE_DECLARE(thread);
  */
 TASKQUEUE_DECLARE(fast);
 int	taskqueue_enqueue_fast(struct taskqueue *queue, struct task *task);
+struct taskqueue *taskqueue_create_fast(const char *name, int mflags,
+				    taskqueue_enqueue_fn enqueue,
+				    void *context);
 
 #endif /* !_SYS_TASKQUEUE_H_ */