14 files changed, 595 insertions, 560 deletions

diff --git a/sys/dev/virtio/balloon/virtio_balloon.c b/sys/dev/virtio/balloon/virtio_balloon.c
index 9b454591a9d0..9a87cf24d5e9 100644
--- a/sys/dev/virtio/balloon/virtio_balloon.c
+++ b/sys/dev/virtio/balloon/virtio_balloon.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2011, Bryan Venteicher <bryanv@daemoninthecloset.org>
+ * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -70,7 +70,7 @@ struct vtballoon_softc {
 	uint32_t		 vtballoon_current_npages;
 	TAILQ_HEAD(,vm_page)	 vtballoon_pages;
 
-	struct proc		*vtballoon_kproc;
+	struct thread		*vtballoon_td;
 	uint32_t		*vtballoon_page_frames;
 	int			 vtballoon_timeout;
 };
@@ -90,7 +90,7 @@ static int	vtballoon_config_change(device_t);
 static void	vtballoon_negotiate_features(struct vtballoon_softc *);
 static int	vtballoon_alloc_virtqueues(struct vtballoon_softc *);
 
-static int	vtballoon_vq_intr(void *);
+static void	vtballoon_vq_intr(void *);
 
 static void	vtballoon_inflate(struct vtballoon_softc *, int);
 static void	vtballoon_deflate(struct vtballoon_softc *, int);
@@ -127,9 +127,9 @@ CTASSERT(VTBALLOON_PAGES_PER_REQUEST * sizeof(uint32_t) <= PAGE_SIZE);
 
 #define VTBALLOON_MTX(_sc)		&(_sc)->vtballoon_mtx
 #define VTBALLOON_LOCK_INIT(_sc, _name)	mtx_init(VTBALLOON_MTX((_sc)), _name, \
-					    "VirtIO Balloon Lock", MTX_SPIN)
-#define VTBALLOON_LOCK(_sc)		mtx_lock_spin(VTBALLOON_MTX((_sc)))
-#define VTBALLOON_UNLOCK(_sc)		mtx_unlock_spin(VTBALLOON_MTX((_sc)))
+					    "VirtIO Balloon Lock", MTX_DEF)
+#define VTBALLOON_LOCK(_sc)		mtx_lock(VTBALLOON_MTX((_sc)))
+#define VTBALLOON_UNLOCK(_sc)		mtx_unlock(VTBALLOON_MTX((_sc)))
 #define VTBALLOON_LOCK_DESTROY(_sc)	mtx_destroy(VTBALLOON_MTX((_sc)))
 
 static device_method_t vtballoon_methods[] = {
@@ -206,10 +206,10 @@ vtballoon_attach(device_t dev)
 		goto fail;
 	}
 
-	error = kproc_create(vtballoon_thread, sc, &sc->vtballoon_kproc,
+	error = kthread_add(vtballoon_thread, sc, NULL, &sc->vtballoon_td,
 	    0, 0, "virtio_balloon");
 	if (error) {
-		device_printf(dev, "cannot create balloon kproc\n");
+		device_printf(dev, "cannot create balloon kthread\n");
 		goto fail;
 	}
 
@@ -230,15 +230,14 @@ vtballoon_detach(device_t dev)
 
 	sc = device_get_softc(dev);
 
-	if (sc->vtballoon_kproc != NULL) {
+	if (sc->vtballoon_td != NULL) {
 		VTBALLOON_LOCK(sc);
 		sc->vtballoon_flags |= VTBALLOON_FLAG_DETACH;
 		wakeup_one(sc);
-		msleep_spin(sc->vtballoon_kproc, VTBALLOON_MTX(sc),
-		    "vtbdth", 0);
+		msleep(sc->vtballoon_td, VTBALLOON_MTX(sc), 0, "vtbdth", 0);
 		VTBALLOON_UNLOCK(sc);
 
-		sc->vtballoon_kproc = NULL;
+		sc->vtballoon_td = NULL;
 	}
 
 	if (device_is_attached(dev)) {
@@ -300,7 +299,7 @@ vtballoon_alloc_virtqueues(struct vtballoon_softc *sc)
 	return (virtio_alloc_virtqueues(dev, 0, nvqs, vq_info));
 }
 
-static int
+static void
 vtballoon_vq_intr(void *xsc)
 {
 	struct vtballoon_softc *sc;
@@ -310,8 +309,6 @@ vtballoon_vq_intr(void *xsc)
 	VTBALLOON_LOCK(sc);
 	wakeup_one(sc);
 	VTBALLOON_UNLOCK(sc);
-
-	return (1);
 }
 
 static void
@@ -322,28 +319,26 @@ vtballoon_inflate(struct vtballoon_softc *sc, int npages)
 	int i;
 
 	vq = sc->vtballoon_inflate_vq;
-	m = NULL;
 
 	if (npages > VTBALLOON_PAGES_PER_REQUEST)
 		npages = VTBALLOON_PAGES_PER_REQUEST;
-	KASSERT(npages > 0, ("balloon doesn't need inflating?"));
 
 	for (i = 0; i < npages; i++) {
-		if ((m = vtballoon_alloc_page(sc)) == NULL)
+		if ((m = vtballoon_alloc_page(sc)) == NULL) {
+			sc->vtballoon_timeout = VTBALLOON_LOWMEM_TIMEOUT;
 			break;
+		}
 
 		sc->vtballoon_page_frames[i] =
 		    VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT;
 
-		KASSERT(m->queue == PQ_NONE, ("allocated page on queue"));
+		KASSERT(m->queue == PQ_NONE,
+		    ("%s: allocated page %p on queue", __func__, m));
 		TAILQ_INSERT_TAIL(&sc->vtballoon_pages, m, pageq);
 	}
 
 	if (i > 0)
 		vtballoon_send_page_frames(sc, vq, i);
-
-	if (m == NULL)
-		sc->vtballoon_timeout = VTBALLOON_LOWMEM_TIMEOUT;
 }
 
 static void
@@ -359,11 +354,10 @@ vtballoon_deflate(struct vtballoon_softc *sc, int npages)
 
 	if (npages > VTBALLOON_PAGES_PER_REQUEST)
 		npages = VTBALLOON_PAGES_PER_REQUEST;
-	KASSERT(npages > 0, ("balloon doesn't need deflating?"));
 
 	for (i = 0; i < npages; i++) {
 		m = TAILQ_FIRST(&sc->vtballoon_pages);
-		KASSERT(m != NULL, ("no more pages to deflate"));
+		KASSERT(m != NULL, ("%s: no more pages to deflate", __func__));
 
 		sc->vtballoon_page_frames[i] =
 		    VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT;
@@ -385,7 +379,9 @@ vtballoon_deflate(struct vtballoon_softc *sc, int npages)
 	KASSERT((TAILQ_EMPTY(&sc->vtballoon_pages) &&
 	    sc->vtballoon_current_npages == 0) ||
 	    (!TAILQ_EMPTY(&sc->vtballoon_pages) &&
-	    sc->vtballoon_current_npages != 0), ("balloon empty?"));
+	    sc->vtballoon_current_npages != 0),
+	    ("%s: bogus page count %d", __func__,
+	    sc->vtballoon_current_npages));
 }
 
 static void
@@ -413,7 +409,7 @@ vtballoon_send_page_frames(struct vtballoon_softc *sc, struct virtqueue *vq,
 	 */
 	VTBALLOON_LOCK(sc);
 	while ((c = virtqueue_dequeue(vq, NULL)) == NULL)
-		msleep_spin(sc, VTBALLOON_MTX(sc), "vtbspf", 0);
+		msleep(sc, VTBALLOON_MTX(sc), 0, "vtbspf", 0);
 	VTBALLOON_UNLOCK(sc);
 
 	KASSERT(c == vq, ("unexpected balloon operation response"));
@@ -512,7 +508,7 @@ vtballoon_sleep(struct vtballoon_softc *sc)
 		if (current < desired && timeout == 0)
 			break;
 
-		msleep_spin(sc, VTBALLOON_MTX(sc), "vtbslp", timeout);
+		msleep(sc, VTBALLOON_MTX(sc), 0, "vtbslp", timeout);
 	}
 	VTBALLOON_UNLOCK(sc);
 
@@ -544,7 +540,7 @@ vtballoon_thread(void *xsc)
 		}
 	}
 
-	kproc_exit(0);
+	kthread_exit();
 }
 
 static void
diff --git a/sys/dev/virtio/block/virtio_blk.c b/sys/dev/virtio/block/virtio_blk.c
index 21d7703dd4f1..7e2f8c109397 100644
--- a/sys/dev/virtio/block/virtio_blk.c
+++ b/sys/dev/virtio/block/virtio_blk.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2011, Bryan Venteicher <bryanv@daemoninthecloset.org>
+ * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -36,10 +36,10 @@ __FBSDID("$FreeBSD$");
 #include <sys/malloc.h>
 #include <sys/module.h>
 #include <sys/sglist.h>
+#include <sys/sysctl.h>
 #include <sys/lock.h>
 #include <sys/mutex.h>
 #include <sys/queue.h>
-#include <sys/taskqueue.h>
 
 #include <geom/geom_disk.h>
 
@@ -62,6 +62,12 @@ struct vtblk_request {
 	TAILQ_ENTRY(vtblk_request)	 vbr_link;
 };
 
+enum vtblk_cache_mode {
+	VTBLK_CACHE_WRITETHROUGH,
+	VTBLK_CACHE_WRITEBACK,
+	VTBLK_CACHE_MAX
+};
+
 struct vtblk_softc {
 	device_t		 vtblk_dev;
 	struct mtx		 vtblk_mtx;
@@ -73,6 +79,7 @@ struct vtblk_softc {
 #define VTBLK_FLAG_SUSPEND	0x0008
 #define VTBLK_FLAG_DUMPING	0x0010
 #define VTBLK_FLAG_BARRIER	0x0020
+#define VTBLK_FLAG_WC_CONFIG	0x0040
 
 	struct virtqueue	*vtblk_vq;
 	struct sglist		*vtblk_sglist;
@@ -85,11 +92,9 @@ struct vtblk_softc {
 				 vtblk_req_ready;
 	struct vtblk_request	*vtblk_req_ordered;
 
-	struct taskqueue	*vtblk_tq;
-	struct task		 vtblk_intr_task;
-
 	int			 vtblk_max_nsegs;
 	int			 vtblk_request_count;
+	enum vtblk_cache_mode	 vtblk_write_cache;
 
 	struct vtblk_request	 vtblk_dump_request;
 };
@@ -102,8 +107,9 @@ static struct virtio_feature_desc vtblk_feature_desc[] = {
 	{ VIRTIO_BLK_F_RO,		"ReadOnly"	},
 	{ VIRTIO_BLK_F_BLK_SIZE,	"BlockSize"	},
 	{ VIRTIO_BLK_F_SCSI,		"SCSICmds"	},
-	{ VIRTIO_BLK_F_FLUSH,		"FlushCmd"	},
+	{ VIRTIO_BLK_F_WCE,		"WriteCache"	},
 	{ VIRTIO_BLK_F_TOPOLOGY,	"Topology"	},
+	{ VIRTIO_BLK_F_CONFIG_WCE,	"ConfigWCE"	},
 
 	{ 0, NULL }
 };
@@ -116,6 +122,7 @@ static int	vtblk_detach(device_t);
 static int	vtblk_suspend(device_t);
 static int	vtblk_resume(device_t);
 static int	vtblk_shutdown(device_t);
+static int	vtblk_config_change(device_t);
 
 static int	vtblk_open(struct disk *);
 static int	vtblk_close(struct disk *);
@@ -128,6 +135,10 @@ static void	vtblk_negotiate_features(struct vtblk_softc *);
 static int	vtblk_maximum_segments(struct vtblk_softc *,
 		    struct virtio_blk_config *);
 static int	vtblk_alloc_virtqueue(struct vtblk_softc *);
+static void	vtblk_set_write_cache(struct vtblk_softc *, int);
+static int	vtblk_write_cache_enabled(struct vtblk_softc *sc,
+		    struct virtio_blk_config *);
+static int	vtblk_write_cache_sysctl(SYSCTL_HANDLER_ARGS);
 static void	vtblk_alloc_disk(struct vtblk_softc *,
 		    struct virtio_blk_config *);
 static void	vtblk_create_disk(struct vtblk_softc *);
@@ -138,11 +149,12 @@ static struct vtblk_request * vtblk_bio_request(struct vtblk_softc *);
 static int	vtblk_execute_request(struct vtblk_softc *,
 		    struct vtblk_request *);
 
-static int	vtblk_vq_intr(void *);
-static void	vtblk_intr_task(void *, int);
+static void	vtblk_vq_intr(void *);
 
 static void	vtblk_stop(struct vtblk_softc *);
 
+static void	vtblk_read_config(struct vtblk_softc *,
+		    struct virtio_blk_config *);
 static void	vtblk_get_ident(struct vtblk_softc *);
 static void	vtblk_prepare_dump(struct vtblk_softc *);
 static int	vtblk_write_dump(struct vtblk_softc *, void *, off_t, size_t);
@@ -167,9 +179,14 @@ static void	vtblk_enqueue_ready(struct vtblk_softc *,
 static int	vtblk_request_error(struct vtblk_request *);
 static void	vtblk_finish_bio(struct bio *, int);
 
+static void	vtblk_setup_sysctl(struct vtblk_softc *);
+static int	vtblk_tunable_int(struct vtblk_softc *, const char *, int);
+
 /* Tunables. */
 static int vtblk_no_ident = 0;
 TUNABLE_INT("hw.vtblk.no_ident", &vtblk_no_ident);
+static int vtblk_writecache_mode = -1;
+TUNABLE_INT("hw.vtblk.writecache_mode", &vtblk_writecache_mode);
 
 /* Features desired/implemented by this driver. */
 #define VTBLK_FEATURES \
@@ -179,13 +196,14 @@ TUNABLE_INT("hw.vtblk.no_ident", &vtblk_no_ident);
      VIRTIO_BLK_F_GEOMETRY		| \
      VIRTIO_BLK_F_RO			| \
      VIRTIO_BLK_F_BLK_SIZE		| \
-     VIRTIO_BLK_F_FLUSH			| \
+     VIRTIO_BLK_F_WCE			| \
+     VIRTIO_BLK_F_CONFIG_WCE		| \
      VIRTIO_RING_F_INDIRECT_DESC)
 
 #define VTBLK_MTX(_sc)		&(_sc)->vtblk_mtx
 #define VTBLK_LOCK_INIT(_sc, _name) \
 				mtx_init(VTBLK_MTX((_sc)), (_name), \
-				    "VTBLK Lock", MTX_DEF)
+				    "VirtIO Block Lock", MTX_DEF)
 #define VTBLK_LOCK(_sc)		mtx_lock(VTBLK_MTX((_sc)))
 #define VTBLK_UNLOCK(_sc)	mtx_unlock(VTBLK_MTX((_sc)))
 #define VTBLK_LOCK_DESTROY(_sc)	mtx_destroy(VTBLK_MTX((_sc)))
@@ -211,6 +229,9 @@ static device_method_t vtblk_methods[] = {
 	DEVMETHOD(device_resume,	vtblk_resume),
 	DEVMETHOD(device_shutdown,	vtblk_shutdown),
 
+	/* VirtIO methods. */
+	DEVMETHOD(virtio_config_change,	vtblk_config_change),
+
 	DEVMETHOD_END
 };
 
@@ -284,10 +305,13 @@ vtblk_attach(device_t dev)
 		sc->vtblk_flags |= VTBLK_FLAG_READONLY;
 	if (virtio_with_feature(dev, VIRTIO_BLK_F_BARRIER))
 		sc->vtblk_flags |= VTBLK_FLAG_BARRIER;
+	if (virtio_with_feature(dev, VIRTIO_BLK_F_CONFIG_WCE))
+		sc->vtblk_flags |= VTBLK_FLAG_WC_CONFIG;
+
+	vtblk_setup_sysctl(sc);
 
 	/* Get local copy of config. */
-	virtio_read_device_config(dev, 0, &blkcfg,
-	    sizeof(struct virtio_blk_config));
+	vtblk_read_config(sc, &blkcfg);
 
 	/*
 	 * With the current sglist(9) implementation, it is not easy
@@ -333,24 +357,12 @@ vtblk_attach(device_t dev)
 
 	vtblk_alloc_disk(sc, &blkcfg);
 
-	TASK_INIT(&sc->vtblk_intr_task, 0, vtblk_intr_task, sc);
-	sc->vtblk_tq = taskqueue_create_fast("vtblk_taskq", M_NOWAIT,
-	    taskqueue_thread_enqueue, &sc->vtblk_tq);
-	if (sc->vtblk_tq == NULL) {
-		error = ENOMEM;
-		device_printf(dev, "cannot allocate taskqueue\n");
-		goto fail;
-	}
-
 	error = virtio_setup_intr(dev, INTR_TYPE_BIO | INTR_ENTROPY);
 	if (error) {
 		device_printf(dev, "cannot setup virtqueue interrupt\n");
 		goto fail;
 	}
 
-	taskqueue_start_threads(&sc->vtblk_tq, 1, PI_DISK, "%s taskq",
-	    device_get_nameunit(dev));
-
 	vtblk_create_disk(sc);
 
 	virtqueue_enable_intr(sc->vtblk_vq);
@@ -375,12 +387,6 @@ vtblk_detach(device_t dev)
 		vtblk_stop(sc);
 	VTBLK_UNLOCK(sc);
 
-	if (sc->vtblk_tq != NULL) {
-		taskqueue_drain(sc->vtblk_tq, &sc->vtblk_intr_task);
-		taskqueue_free(sc->vtblk_tq);
-		sc->vtblk_tq = NULL;
-	}
-
 	vtblk_drain(sc);
 
 	if (sc->vtblk_disk != NULL) {
@@ -441,6 +447,13 @@ vtblk_shutdown(device_t dev)
 }
 
 static int
+vtblk_config_change(device_t dev)
+{
+
+	return (0);
+}
+
+static int
 vtblk_open(struct disk *dp)
 {
 	struct vtblk_softc *sc;
@@ -541,8 +554,8 @@ vtblk_strategy(struct bio *bp)
 		max_nsegs = sc->vtblk_max_nsegs - VTBLK_MIN_SEGMENTS;
 
 		KASSERT(nsegs <= max_nsegs,
-		    ("bio %p spanned too many segments: %d, max: %d",
-		    bp, nsegs, max_nsegs));
+		    ("%s: bio %p spanned too many segments: %d, max: %d",
+		    __func__, bp, nsegs, max_nsegs));
 	}
 #endif
 
@@ -606,6 +619,59 @@ vtblk_alloc_virtqueue(struct vtblk_softc *sc)
 }
 
 static void
+vtblk_set_write_cache(struct vtblk_softc *sc, int wc)
+{
+
+	/* Set either writeback (1) or writethrough (0) mode. */
+	virtio_write_dev_config_1(sc->vtblk_dev,
+	    offsetof(struct virtio_blk_config, writeback), wc);
+}
+
+static int
+vtblk_write_cache_enabled(struct vtblk_softc *sc,
+    struct virtio_blk_config *blkcfg)
+{
+	int wc;
+
+	if (sc->vtblk_flags & VTBLK_FLAG_WC_CONFIG) {
+		wc = vtblk_tunable_int(sc, "writecache_mode",
+		    vtblk_writecache_mode);
+		if (wc >= 0 && wc < VTBLK_CACHE_MAX)
+			vtblk_set_write_cache(sc, wc);
+		else
+			wc = blkcfg->writeback;
+	} else
+		wc = virtio_with_feature(sc->vtblk_dev, VIRTIO_BLK_F_WCE);
+
+	return (wc);
+}
+
+static int
+vtblk_write_cache_sysctl(SYSCTL_HANDLER_ARGS)
+{
+	struct vtblk_softc *sc;
+	int wc, error;
+
+	sc = oidp->oid_arg1;
+	wc = sc->vtblk_write_cache;
+
+	error = sysctl_handle_int(oidp, &wc, 0, req);
+	if (error || req->newptr == NULL)
+		return (error);
+	if ((sc->vtblk_flags & VTBLK_FLAG_WC_CONFIG) == 0)
+		return (EPERM);
+	if (wc < 0 || wc >= VTBLK_CACHE_MAX)
+		return (EINVAL);
+
+	VTBLK_LOCK(sc);
+	sc->vtblk_write_cache = wc;
+	vtblk_set_write_cache(sc, sc->vtblk_write_cache);
+	VTBLK_UNLOCK(sc);
+
+	return (0);
+}
+
+static void
 vtblk_alloc_disk(struct vtblk_softc *sc, struct virtio_blk_config *blkcfg)
 {
 	device_t dev;
@@ -621,6 +687,11 @@ vtblk_alloc_disk(struct vtblk_softc *sc, struct virtio_blk_config *blkcfg)
 	dp->d_name = VTBLK_DISK_NAME;
 	dp->d_unit = device_get_unit(dev);
 	dp->d_drv1 = sc;
+	dp->d_flags = DISKFLAG_CANFLUSHCACHE;
+	dp->d_hba_vendor = virtio_get_vendor(dev);
+	dp->d_hba_device = virtio_get_device(dev);
+	dp->d_hba_subvendor = virtio_get_subvendor(dev);
+	dp->d_hba_subdevice = virtio_get_subdevice(dev);
 
 	if ((sc->vtblk_flags & VTBLK_FLAG_READONLY) == 0)
 		dp->d_dump = vtblk_dump;
@@ -656,8 +727,18 @@ vtblk_alloc_disk(struct vtblk_softc *sc, struct virtio_blk_config *blkcfg)
 		dp->d_fwheads = blkcfg->geometry.heads;
 	}
 
-	if (virtio_with_feature(dev, VIRTIO_BLK_F_FLUSH))
-		dp->d_flags |= DISKFLAG_CANFLUSHCACHE;
+	if (virtio_with_feature(dev, VIRTIO_BLK_F_TOPOLOGY)) {
+		dp->d_stripesize = dp->d_sectorsize *
+		    (1 << blkcfg->topology.physical_block_exp);
+		dp->d_stripeoffset = (dp->d_stripesize -
+		    blkcfg->topology.alignment_offset * dp->d_sectorsize) %
+		    dp->d_stripesize;
+	}
+
+	if (vtblk_write_cache_enabled(sc, blkcfg) != 0)
+		sc->vtblk_write_cache = VTBLK_CACHE_WRITEBACK;
+	else
+		sc->vtblk_write_cache = VTBLK_CACHE_WRITETHROUGH;
 }
 
 static void
@@ -765,8 +846,7 @@ vtblk_bio_request(struct vtblk_softc *sc)
 		req->vbr_hdr.sector = bp->bio_offset / 512;
 		break;
 	default:
-		panic("%s: bio with unhandled cmd: %d", __FUNCTION__,
-		    bp->bio_cmd);
+		panic("%s: bio with unhandled cmd: %d", __func__, bp->bio_cmd);
 	}
 
 	return (req);
@@ -809,14 +889,13 @@ vtblk_execute_request(struct vtblk_softc *sc, struct vtblk_request *req)
 	}
 
 	sglist_reset(sg);
-
 	sglist_append(sg, &req->vbr_hdr, sizeof(struct virtio_blk_outhdr));
 
 	if (bp->bio_cmd == BIO_READ || bp->bio_cmd == BIO_WRITE) {
 		error = sglist_append(sg, bp->bio_data, bp->bio_bcount);
 		if (error || sg->sg_nseg == sg->sg_maxseg)
 			panic("%s: data buffer too big bio:%p error:%d",
-			    __FUNCTION__, bp, error);
+			    __func__, bp, error);
 
 		/* BIO_READ means the host writes into our buffer. */
 		if (bp->bio_cmd == BIO_READ)
@@ -834,28 +913,16 @@ vtblk_execute_request(struct vtblk_softc *sc, struct vtblk_request *req)
 	return (error);
 }
 
-static int
-vtblk_vq_intr(void *xsc)
-{
-	struct vtblk_softc *sc;
-
-	sc = xsc;
-
-	virtqueue_disable_intr(sc->vtblk_vq);
-	taskqueue_enqueue_fast(sc->vtblk_tq, &sc->vtblk_intr_task);
-
-	return (1);
-}
-
 static void
-vtblk_intr_task(void *arg, int pending)
+vtblk_vq_intr(void *xsc)
 {
 	struct vtblk_softc *sc;
 	struct virtqueue *vq;
 
-	sc = arg;
+	sc = xsc;
 	vq = sc->vtblk_vq;
 
+again:
 	VTBLK_LOCK(sc);
 	if (sc->vtblk_flags & VTBLK_FLAG_DETACH) {
 		VTBLK_UNLOCK(sc);
@@ -872,9 +939,7 @@ vtblk_intr_task(void *arg, int pending)
 	if (virtqueue_enable_intr(vq) != 0) {
 		virtqueue_disable_intr(vq);
 		VTBLK_UNLOCK(sc);
-		taskqueue_enqueue_fast(sc->vtblk_tq,
-		    &sc->vtblk_intr_task);
-		return;
+		goto again;
 	}
 
 	VTBLK_UNLOCK(sc);
@@ -888,6 +953,37 @@ vtblk_stop(struct vtblk_softc *sc)
 	virtio_stop(sc->vtblk_dev);
 }
 
+#define VTBLK_GET_CONFIG(_dev, _feature, _field, _cfg)			\
+	if (virtio_with_feature(_dev, _feature)) {			\
+		virtio_read_device_config(_dev,				\
+		    offsetof(struct virtio_blk_config, _field),		\
+		    &(_cfg)->_field, sizeof((_cfg)->_field));		\
+	}
+
+static void
+vtblk_read_config(struct vtblk_softc *sc, struct virtio_blk_config *blkcfg)
+{
+	device_t dev;
+
+	dev = sc->vtblk_dev;
+
+	bzero(blkcfg, sizeof(struct virtio_blk_config));
+
+	/* The capacity is always available. */
+	virtio_read_device_config(dev, offsetof(struct virtio_blk_config,
+	    capacity), &blkcfg->capacity, sizeof(blkcfg->capacity));
+
+	/* Read the configuration if the feature was negotiated. */
+	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_SIZE_MAX, size_max, blkcfg);
+	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_SEG_MAX, seg_max, blkcfg);
+	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_GEOMETRY, geometry, blkcfg);
+	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_BLK_SIZE, blk_size, blkcfg);
+	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_TOPOLOGY, topology, blkcfg);
+	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_CONFIG_WCE, writeback, blkcfg);
+}
+
+#undef VTBLK_GET_CONFIG
+
 static void
 vtblk_get_ident(struct vtblk_softc *sc)
 {
@@ -899,7 +995,7 @@ vtblk_get_ident(struct vtblk_softc *sc)
 	dp = sc->vtblk_disk;
 	len = MIN(VIRTIO_BLK_ID_BYTES, DISK_IDENT_SIZE);
 
-	if (vtblk_no_ident != 0)
+	if (vtblk_tunable_int(sc, "no_ident", vtblk_no_ident) != 0)
 		return;
 
 	req = vtblk_dequeue_request(sc);
@@ -949,8 +1045,10 @@ vtblk_prepare_dump(struct vtblk_softc *sc)
 	 */
 	vtblk_drain_vq(sc, 1);
 
-	if (virtio_reinit(dev, sc->vtblk_features) != 0)
-		panic("cannot reinit VirtIO block device during dump");
+	if (virtio_reinit(dev, sc->vtblk_features) != 0) {
+		panic("%s: cannot reinit VirtIO block device during dump",
+		    device_get_nameunit(dev));
+	}
 
 	virtqueue_disable_intr(vq);
 	virtio_reinit_complete(dev);
@@ -1003,7 +1101,6 @@ static int
 vtblk_poll_request(struct vtblk_softc *sc, struct vtblk_request *req)
 {
 	struct virtqueue *vq;
-	struct vtblk_request *r;
 	int error;
 
 	vq = sc->vtblk_vq;
@@ -1016,14 +1113,12 @@ vtblk_poll_request(struct vtblk_softc *sc, struct vtblk_request *req)
 		return (error);
 
 	virtqueue_notify(vq);
-
-	r = virtqueue_poll(vq, NULL);
-	KASSERT(r == req, ("unexpected request response: %p/%p", r, req));
+	virtqueue_poll(vq, NULL);
 
 	error = vtblk_request_error(req);
 	if (error && bootverbose) {
 		device_printf(sc->vtblk_dev,
-		    "%s: IO error: %d\n", __FUNCTION__, error);
+		    "%s: IO error: %d\n", __func__, error);
 	}
 
 	return (error);
@@ -1154,7 +1249,7 @@ vtblk_free_requests(struct vtblk_softc *sc)
 	struct vtblk_request *req;
 
 	KASSERT(TAILQ_EMPTY(&sc->vtblk_req_ready),
-	    ("ready requests left on queue"));
+	    ("%s: ready requests left on queue", __func__));
 
 	while ((req = vtblk_dequeue_request(sc)) != NULL) {
 		sc->vtblk_request_count--;
@@ -1162,7 +1257,7 @@ vtblk_free_requests(struct vtblk_softc *sc)
 	}
 
 	KASSERT(sc->vtblk_request_count == 0,
-	    ("leaked requests: %d", sc->vtblk_request_count));
+	    ("%s: leaked %d requests", __func__, sc->vtblk_request_count));
 }
 
 static struct vtblk_request *
@@ -1236,3 +1331,33 @@ vtblk_finish_bio(struct bio *bp, int error)
 
 	biodone(bp);
 }
+
+static void
+vtblk_setup_sysctl(struct vtblk_softc *sc)
+{
+	device_t dev;
+	struct sysctl_ctx_list *ctx;
+	struct sysctl_oid *tree;
+	struct sysctl_oid_list *child;
+
+	dev = sc->vtblk_dev;
+	ctx = device_get_sysctl_ctx(dev);
+	tree = device_get_sysctl_tree(dev);
+	child = SYSCTL_CHILDREN(tree);
+
+	SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "writecache_mode",
+	    CTLTYPE_INT | CTLFLAG_RW, sc, 0, vtblk_write_cache_sysctl,
+	    "I", "Write cache mode (writethrough (0) or writeback (1))");
+}
+
+static int
+vtblk_tunable_int(struct vtblk_softc *sc, const char *knob, int def)
+{
+	char path[64];
+
+	snprintf(path, sizeof(path),
+	    "hw.vtblk.%d.%s", device_get_unit(sc->vtblk_dev), knob);
+	TUNABLE_INT_FETCH(path, &def);
+
+	return (def);
+}
diff --git a/sys/dev/virtio/block/virtio_blk.h b/sys/dev/virtio/block/virtio_blk.h
index fdac9e5e5f89..cdb8d3fde27a 100644
--- a/sys/dev/virtio/block/virtio_blk.h
+++ b/sys/dev/virtio/block/virtio_blk.h
@@ -39,8 +39,9 @@
 #define VIRTIO_BLK_F_RO		0x0020	/* Disk is read-only */
 #define VIRTIO_BLK_F_BLK_SIZE	0x0040	/* Block size of disk is available*/
 #define VIRTIO_BLK_F_SCSI	0x0080	/* Supports scsi command passthru */
-#define VIRTIO_BLK_F_FLUSH	0x0200	/* Cache flush command support */
+#define VIRTIO_BLK_F_WCE	0x0200	/* Writeback mode enabled after reset */
 #define VIRTIO_BLK_F_TOPOLOGY	0x0400	/* Topology information is available */
+#define VIRTIO_BLK_F_CONFIG_WCE 0x0800	/* Writeback mode available in config */
 
 #define VIRTIO_BLK_ID_BYTES	20	/* ID string length */
 
@@ -51,15 +52,27 @@ struct virtio_blk_config {
 	uint32_t size_max;
 	/* The maximum number of segments (if VIRTIO_BLK_F_SEG_MAX) */
 	uint32_t seg_max;
-	/* geometry the device (if VIRTIO_BLK_F_GEOMETRY) */
+	/* Geometry of the device (if VIRTIO_BLK_F_GEOMETRY) */
 	struct virtio_blk_geometry {
 		uint16_t cylinders;
 		uint8_t heads;
 		uint8_t sectors;
 	} geometry;
 
-	/* block size of device (if VIRTIO_BLK_F_BLK_SIZE) */
+	/* Block size of device (if VIRTIO_BLK_F_BLK_SIZE) */
 	uint32_t blk_size;
+
+	/* Topology of the device (if VIRTIO_BLK_F_TOPOLOGY) */
+	struct virtio_blk_topology {
+		uint8_t physical_block_exp;
+		uint8_t alignment_offset;
+		uint16_t min_io_size;
+		uint16_t opt_io_size;
+	} topology;
+
+	/* Writeback mode (if VIRTIO_BLK_F_CONFIG_WCE) */
+	uint8_t writeback;
+
 } __packed;
 
 /*
diff --git a/sys/dev/virtio/network/if_vtnet.c b/sys/dev/virtio/network/if_vtnet.c
index 8e2de46b4427..89604d1b26b0 100644
--- a/sys/dev/virtio/network/if_vtnet.c
+++ b/sys/dev/virtio/network/if_vtnet.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2011, Bryan Venteicher <bryanv@daemoninthecloset.org>
+ * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -42,7 +42,6 @@ __FBSDID("$FreeBSD$");
 #include <sys/module.h>
 #include <sys/socket.h>
 #include <sys/sysctl.h>
-#include <sys/taskqueue.h>
 #include <sys/random.h>
 #include <sys/sglist.h>
 #include <sys/lock.h>
@@ -97,7 +96,6 @@ static void	vtnet_set_hwaddr(struct vtnet_softc *);
 static int	vtnet_is_link_up(struct vtnet_softc *);
 static void	vtnet_update_link_status(struct vtnet_softc *);
 static void	vtnet_watchdog(struct vtnet_softc *);
-static void	vtnet_config_change_task(void *, int);
 static int	vtnet_change_mtu(struct vtnet_softc *, int);
 static int	vtnet_ioctl(struct ifnet *, u_long, caddr_t);
 
@@ -123,8 +121,7 @@ static int	vtnet_rx_csum(struct vtnet_softc *, struct mbuf *,
 		    struct virtio_net_hdr *);
 static int	vtnet_rxeof_merged(struct vtnet_softc *, struct mbuf *, int);
 static int	vtnet_rxeof(struct vtnet_softc *, int, int *);
-static void	vtnet_rx_intr_task(void *, int);
-static int	vtnet_rx_vq_intr(void *);
+static void	vtnet_rx_vq_intr(void *);
 
 static void	vtnet_txeof(struct vtnet_softc *);
 static struct mbuf * vtnet_tx_offload(struct vtnet_softc *, struct mbuf *,
@@ -135,8 +132,7 @@ static int	vtnet_encap(struct vtnet_softc *, struct mbuf **);
 static void	vtnet_start_locked(struct ifnet *);
 static void	vtnet_start(struct ifnet *);
 static void	vtnet_tick(void *);
-static void	vtnet_tx_intr_task(void *, int);
-static int	vtnet_tx_vq_intr(void *);
+static void	vtnet_tx_vq_intr(void *);
 
 static void	vtnet_stop(struct vtnet_softc *);
 static int	vtnet_reinit(struct vtnet_softc *);
@@ -427,19 +423,6 @@ vtnet_attach(device_t dev)
 	ifp->if_capabilities |= IFCAP_POLLING;
 #endif
 
-	TASK_INIT(&sc->vtnet_rx_intr_task, 0, vtnet_rx_intr_task, sc);
-	TASK_INIT(&sc->vtnet_tx_intr_task, 0, vtnet_tx_intr_task, sc);
-	TASK_INIT(&sc->vtnet_cfgchg_task, 0, vtnet_config_change_task, sc);
-
-	sc->vtnet_tq = taskqueue_create_fast("vtnet_taskq", M_NOWAIT,
-	    taskqueue_thread_enqueue, &sc->vtnet_tq);
-	if (sc->vtnet_tq == NULL) {
-		error = ENOMEM;
-		device_printf(dev, "cannot allocate taskqueue\n");
-		ether_ifdetach(ifp);
-		goto fail;
-	}
-
 	error = virtio_setup_intr(dev, INTR_TYPE_NET);
 	if (error) {
 		device_printf(dev, "cannot setup virtqueue interrupts\n");
@@ -447,9 +430,6 @@ vtnet_attach(device_t dev)
 		goto fail;
 	}
 
-	taskqueue_start_threads(&sc->vtnet_tq, 1, PI_NET, "%s taskq",
-	    device_get_nameunit(dev));
-
 	/*
 	 * Device defaults to promiscuous mode for backwards
 	 * compatibility. Turn it off if possible.
@@ -495,18 +475,10 @@ vtnet_detach(device_t dev)
 		VTNET_UNLOCK(sc);
 
 		callout_drain(&sc->vtnet_tick_ch);
-		taskqueue_drain(taskqueue_fast, &sc->vtnet_cfgchg_task);
 
 		ether_ifdetach(ifp);
 	}
 
-	if (sc->vtnet_tq != NULL) {
-		taskqueue_drain(sc->vtnet_tq, &sc->vtnet_rx_intr_task);
-		taskqueue_drain(sc->vtnet_tq, &sc->vtnet_tx_intr_task);
-		taskqueue_free(sc->vtnet_tq);
-		sc->vtnet_tq = NULL;
-	}
-
 	if (sc->vtnet_vlan_attach != NULL) {
 		EVENTHANDLER_DEREGISTER(vlan_config, sc->vtnet_vlan_attach);
 		sc->vtnet_vlan_attach = NULL;
@@ -590,9 +562,11 @@ vtnet_config_change(device_t dev)
 
 	sc = device_get_softc(dev);
 
-	taskqueue_enqueue_fast(taskqueue_fast, &sc->vtnet_cfgchg_task);
+	VTNET_LOCK(sc);
+	vtnet_update_link_status(sc);
+	VTNET_UNLOCK(sc);
 
-	return (1);
+	return (0);
 }
 
 static void
@@ -788,18 +762,6 @@ vtnet_watchdog(struct vtnet_softc *sc)
 	vtnet_init_locked(sc);
 }
 
-static void
-vtnet_config_change_task(void *arg, int pending)
-{
-	struct vtnet_softc *sc;
-
-	sc = arg;
-
-	VTNET_LOCK(sc);
-	vtnet_update_link_status(sc);
-	VTNET_UNLOCK(sc);
-}
-
 static int
 vtnet_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 {
@@ -1705,15 +1667,16 @@ vtnet_rxeof(struct vtnet_softc *sc, int count, int *rx_npktsp)
 }
 
 static void
-vtnet_rx_intr_task(void *arg, int pending)
+vtnet_rx_vq_intr(void *xsc)
 {
 	struct vtnet_softc *sc;
 	struct ifnet *ifp;
 	int more;
 
-	sc = arg;
+	sc = xsc;
 	ifp = sc->vtnet_ifp;
 
+again:
 	VTNET_LOCK(sc);
 
 #ifdef DEVICE_POLLING
@@ -1730,31 +1693,15 @@ vtnet_rx_intr_task(void *arg, int pending)
 	}
 
 	more = vtnet_rxeof(sc, sc->vtnet_rx_process_limit, NULL);
-	if (!more && vtnet_enable_rx_intr(sc) != 0) {
-		vtnet_disable_rx_intr(sc);
-		more = 1;
-	}
-
-	VTNET_UNLOCK(sc);
-
-	if (more) {
+	if (more || vtnet_enable_rx_intr(sc) != 0) {
+		if (!more)
+			vtnet_disable_rx_intr(sc);
 		sc->vtnet_stats.rx_task_rescheduled++;
-		taskqueue_enqueue_fast(sc->vtnet_tq,
-		    &sc->vtnet_rx_intr_task);
+		VTNET_UNLOCK(sc);
+		goto again;
 	}
-}
-
-static int
-vtnet_rx_vq_intr(void *xsc)
-{
-	struct vtnet_softc *sc;
-
-	sc = xsc;
-
-	vtnet_disable_rx_intr(sc);
-	taskqueue_enqueue_fast(sc->vtnet_tq, &sc->vtnet_rx_intr_task);
 
-	return (1);
+	VTNET_UNLOCK(sc);
 }
 
 static void
@@ -1800,7 +1747,6 @@ vtnet_tx_offload(struct vtnet_softc *sc, struct mbuf *m,
 	uint8_t ip_proto, gso_type;
 
 	ifp = sc->vtnet_ifp;
-	M_ASSERTPKTHDR(m);
 
 	ip_offset = sizeof(struct ether_header);
 	if (m->m_len < ip_offset) {
@@ -1918,7 +1864,7 @@ vtnet_enqueue_txbuf(struct vtnet_softc *sc, struct mbuf **m_head,
 	sglist_init(&sg, VTNET_MAX_TX_SEGS, segs);
 	error = sglist_append(&sg, &txhdr->vth_uhdr, sc->vtnet_hdr_size);
 	KASSERT(error == 0 && sg.sg_nseg == 1,
-	    ("cannot add header to sglist"));
+	    ("%s: cannot add header to sglist error %d", __func__, error));
 
 again:
 	error = sglist_append_mbuf(&sg, m);
@@ -1955,6 +1901,7 @@ vtnet_encap(struct vtnet_softc *sc, struct mbuf **m_head)
 	int error;
 
 	m = *m_head;
+	M_ASSERTPKTHDR(m);
 
 	txhdr = uma_zalloc(vtnet_tx_header_zone, M_NOWAIT | M_ZERO);
 	if (txhdr == NULL) {
@@ -2077,14 +2024,15 @@ vtnet_tick(void *xsc)
 }
 
 static void
-vtnet_tx_intr_task(void *arg, int pending)
+vtnet_tx_vq_intr(void *xsc)
 {
 	struct vtnet_softc *sc;
 	struct ifnet *ifp;
 
-	sc = arg;
+	sc = xsc;
 	ifp = sc->vtnet_ifp;
 
+again:
 	VTNET_LOCK(sc);
 
 #ifdef DEVICE_POLLING
@@ -2109,26 +2057,12 @@ vtnet_tx_intr_task(void *arg, int pending)
 		vtnet_disable_tx_intr(sc);
 		sc->vtnet_stats.tx_task_rescheduled++;
 		VTNET_UNLOCK(sc);
-		taskqueue_enqueue_fast(sc->vtnet_tq, &sc->vtnet_tx_intr_task);
-		return;
+		goto again;
 	}
 
 	VTNET_UNLOCK(sc);
 }
 
-static int
-vtnet_tx_vq_intr(void *xsc)
-{
-	struct vtnet_softc *sc;
-
-	sc = xsc;
-
-	vtnet_disable_tx_intr(sc);
-	taskqueue_enqueue_fast(sc->vtnet_tq, &sc->vtnet_tx_intr_task);
-
-	return (1);
-}
-
 static void
 vtnet_stop(struct vtnet_softc *sc)
 {
diff --git a/sys/dev/virtio/network/if_vtnetvar.h b/sys/dev/virtio/network/if_vtnetvar.h
index 184870e4d764..d870436e4542 100644
--- a/sys/dev/virtio/network/if_vtnetvar.h
+++ b/sys/dev/virtio/network/if_vtnetvar.h
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2011, Bryan Venteicher <bryanv@daemoninthecloset.org>
+ * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -79,11 +79,6 @@ struct vtnet_softc {
 	int			 vtnet_watchdog_timer;
 	uint64_t		 vtnet_features;
 
-	struct taskqueue	*vtnet_tq;
-	struct task		 vtnet_rx_intr_task;
-	struct task		 vtnet_tx_intr_task;
-	struct task		 vtnet_cfgchg_task;
-
 	struct vtnet_statistics	 vtnet_stats;
 
 	struct callout		 vtnet_tick_ch;
diff --git a/sys/dev/virtio/pci/virtio_pci.c b/sys/dev/virtio/pci/virtio_pci.c
index 0f9ffc747e5b..cf424238672e 100644
--- a/sys/dev/virtio/pci/virtio_pci.c
+++ b/sys/dev/virtio/pci/virtio_pci.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2011, Bryan Venteicher <bryanv@daemoninthecloset.org>
+ * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -51,6 +51,17 @@ __FBSDID("$FreeBSD$");
 #include "virtio_bus_if.h"
 #include "virtio_if.h"
 
+struct vtpci_interrupt {
+	struct resource		*vti_irq;
+	int			 vti_rid;
+	void			*vti_handler;
+};
+
+struct vtpci_virtqueue {
+	struct virtqueue	*vtv_vq;
+	int			 vtv_no_intr;
+};
+
 struct vtpci_softc {
 	device_t			 vtpci_dev;
 	struct resource			*vtpci_res;
@@ -69,40 +80,22 @@ struct vtpci_softc {
 	device_t			 vtpci_child_dev;
 	struct virtio_feature_desc	*vtpci_child_feat_desc;
 
-	/*
-	 * Ideally, each virtqueue that the driver provides a callback for
-	 * will receive its own MSIX vector. If there are not sufficient
-	 * vectors available, we will then attempt to have all the VQs
-	 * share one vector. Note that when using MSIX, the configuration
-	 * changed notifications must be on their own vector.
-	 *
-	 * If MSIX is not available, we will attempt to have the whole
-	 * device share one MSI vector, and then, finally, one legacy
-	 * interrupt.
-	 */
 	int				 vtpci_nvqs;
-	struct vtpci_virtqueue {
-		struct virtqueue *vq;
-		/* Device did not provide a callback for this virtqueue. */
-		int		  no_intr;
-		/* Index into vtpci_intr_res[] below. Unused, then -1. */
-		int		  ires_idx;
-	} vtpci_vqx[VIRTIO_MAX_VIRTQUEUES];
+	struct vtpci_virtqueue		*vtpci_vqs;
 
 	/*
-	 * When using MSIX interrupts, the first element of vtpci_intr_res[]
-	 * is always the configuration changed notifications. The remaining
-	 * element(s) are used for the virtqueues.
+	 * Ideally, each virtqueue that the driver provides a callback for will
+	 * receive its own MSIX vector. If there are not sufficient vectors
+	 * available, then attempt to have all the VQs share one vector. For
+	 * MSIX, the configuration changed notifications must be on their own
+	 * vector.
 	 *
-	 * With MSI and legacy interrupts, only the first element of
-	 * vtpci_intr_res[] is used.
+	 * If MSIX is not available, we will attempt to have the whole device
+	 * share one MSI vector, and then, finally, one legacy interrupt.
 	 */
-	int				 vtpci_nintr_res;
-	struct vtpci_intr_resource {
-		struct resource	*irq;
-		int		 rid;
-		void		*intrhand;
-	} vtpci_intr_res[1 + VIRTIO_MAX_VIRTQUEUES];
+	struct vtpci_interrupt		 vtpci_device_interrupt;
+	struct vtpci_interrupt		*vtpci_msix_vq_interrupts;
+	int				 vtpci_nmsix_resources;
 };
 
 static int	vtpci_probe(device_t);
@@ -134,36 +127,45 @@ static void	vtpci_describe_features(struct vtpci_softc *, const char *,
 		    uint64_t);
 static void	vtpci_probe_and_attach_child(struct vtpci_softc *);
 
-static int 	vtpci_alloc_msix(struct vtpci_softc *, int);
-static int 	vtpci_alloc_msi(struct vtpci_softc *);
-static int 	vtpci_alloc_intr_msix_pervq(struct vtpci_softc *);
-static int 	vtpci_alloc_intr_msix_shared(struct vtpci_softc *);
-static int 	vtpci_alloc_intr_msi(struct vtpci_softc *);
-static int 	vtpci_alloc_intr_legacy(struct vtpci_softc *);
+static int	vtpci_alloc_msix(struct vtpci_softc *, int);
+static int	vtpci_alloc_msi(struct vtpci_softc *);
+static int	vtpci_alloc_intr_msix_pervq(struct vtpci_softc *);
+static int	vtpci_alloc_intr_msix_shared(struct vtpci_softc *);
+static int	vtpci_alloc_intr_msi(struct vtpci_softc *);
+static int	vtpci_alloc_intr_legacy(struct vtpci_softc *);
+static int	vtpci_alloc_interrupt(struct vtpci_softc *, int, int,
+		    struct vtpci_interrupt *);
 static int	vtpci_alloc_intr_resources(struct vtpci_softc *);
 
-static int 	vtpci_setup_legacy_interrupt(struct vtpci_softc *,
+static int	vtpci_setup_legacy_interrupt(struct vtpci_softc *,
+		    enum intr_type);
+static int	vtpci_setup_pervq_msix_interrupts(struct vtpci_softc *,
 		    enum intr_type);
-static int 	vtpci_setup_msix_interrupts(struct vtpci_softc *,
+static int	vtpci_setup_msix_interrupts(struct vtpci_softc *,
 		    enum intr_type);
-static int 	vtpci_setup_interrupts(struct vtpci_softc *, enum intr_type);
+static int	vtpci_setup_interrupts(struct vtpci_softc *, enum intr_type);
 
-static int	vtpci_register_msix_vector(struct vtpci_softc *, int, int);
-static int 	vtpci_set_host_msix_vectors(struct vtpci_softc *);
-static int 	vtpci_reinit_virtqueue(struct vtpci_softc *, int);
+static int	vtpci_register_msix_vector(struct vtpci_softc *, int,
+		    struct vtpci_interrupt *);
+static int	vtpci_set_host_msix_vectors(struct vtpci_softc *);
+static int	vtpci_reinit_virtqueue(struct vtpci_softc *, int);
 
+static void	vtpci_free_interrupt(struct vtpci_softc *,
+		    struct vtpci_interrupt *);
 static void	vtpci_free_interrupts(struct vtpci_softc *);
 static void	vtpci_free_virtqueues(struct vtpci_softc *);
-static void 	vtpci_cleanup_setup_intr_attempt(struct vtpci_softc *);
 static void	vtpci_release_child_resources(struct vtpci_softc *);
+static void	vtpci_cleanup_setup_intr_attempt(struct vtpci_softc *);
 static void	vtpci_reset(struct vtpci_softc *);
 
 static void	vtpci_select_virtqueue(struct vtpci_softc *, int);
 
-static int	vtpci_legacy_intr(void *);
-static int	vtpci_vq_shared_intr(void *);
-static int	vtpci_vq_intr(void *);
-static int	vtpci_config_intr(void *);
+static void	vtpci_legacy_intr(void *);
+static int	vtpci_vq_shared_intr_filter(void *);
+static void	vtpci_vq_shared_intr(void *);
+static int	vtpci_vq_intr_filter(void *);
+static void	vtpci_vq_intr(void *);
+static void	vtpci_config_intr(void *);
 
 #define vtpci_setup_msi_interrupt vtpci_setup_legacy_interrupt
 
@@ -478,18 +480,19 @@ vtpci_alloc_virtqueues(device_t dev, int flags, int nvqs,
 	uint16_t size;
 
 	sc = device_get_softc(dev);
-	error = 0;
 
 	if (sc->vtpci_nvqs != 0)
 		return (EALREADY);
-	if (nvqs <= 0 || nvqs > VIRTIO_MAX_VIRTQUEUES)
+	if (nvqs <= 0)
 		return (EINVAL);
 
-	if (flags & VIRTIO_ALLOC_VQS_DISABLE_MSIX)
-		sc->vtpci_flags |= VTPCI_FLAG_NO_MSIX;
+	sc->vtpci_vqs = malloc(nvqs * sizeof(struct vtpci_virtqueue),
+	    M_DEVBUF, M_NOWAIT | M_ZERO);
+	if (sc->vtpci_vqs == NULL)
+		return (ENOMEM);
 
 	for (idx = 0; idx < nvqs; idx++) {
-		vqx = &sc->vtpci_vqx[idx];
+		vqx = &sc->vtpci_vqs[idx];
 		info = &vq_info[idx];
 
 		vtpci_select_virtqueue(sc, idx);
@@ -506,12 +509,15 @@ vtpci_alloc_virtqueues(device_t dev, int flags, int nvqs,
 		vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN,
 		    virtqueue_paddr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT);
 
-		vqx->vq = *info->vqai_vq = vq;
-		vqx->no_intr = info->vqai_intr == NULL;
+		vqx->vtv_vq = *info->vqai_vq = vq;
+		vqx->vtv_no_intr = info->vqai_intr == NULL;
 
 		sc->vtpci_nvqs++;
 	}
 
+	if (error)
+		vtpci_free_virtqueues(sc);
+
 	return (error);
 }
 
@@ -772,7 +778,7 @@ vtpci_alloc_msix(struct vtpci_softc *sc, int nvectors)
 
 	cnt = required;
 	if (pci_alloc_msix(dev, &cnt) == 0 && cnt >= required) {
-		sc->vtpci_nintr_res = required;
+		sc->vtpci_nmsix_resources = required;
 		return (0);
 	}
 
@@ -795,10 +801,8 @@ vtpci_alloc_msi(struct vtpci_softc *sc)
 		return (1);
 
 	cnt = required;
-	if (pci_alloc_msi(dev, &cnt) == 0 && cnt >= required) {
-		sc->vtpci_nintr_res = required;
+	if (pci_alloc_msi(dev, &cnt) == 0 && cnt >= required)
 		return (0);
-	}
 
 	pci_release_msi(dev);
 
@@ -815,7 +819,7 @@ vtpci_alloc_intr_msix_pervq(struct vtpci_softc *sc)
 		return (ENOTSUP);
 
 	for (nvectors = 0, i = 0; i < sc->vtpci_nvqs; i++) {
-		if (sc->vtpci_vqx[i].no_intr == 0)
+		if (sc->vtpci_vqs[i].vtv_no_intr == 0)
 			nvectors++;
 	}
 
@@ -869,54 +873,62 @@ vtpci_alloc_intr_legacy(struct vtpci_softc *sc)
 {
 
 	sc->vtpci_flags |= VTPCI_FLAG_LEGACY;
-	sc->vtpci_nintr_res = 1;
 
 	return (0);
 }
 
 static int
-vtpci_alloc_intr_resources(struct vtpci_softc *sc)
+vtpci_alloc_interrupt(struct vtpci_softc *sc, int rid, int flags,
+    struct vtpci_interrupt *intr)
 {
-	device_t dev;
 	struct resource *irq;
-	struct vtpci_virtqueue *vqx;
-	int i, rid, flags, res_idx;
 
-	dev = sc->vtpci_dev;
+	irq = bus_alloc_resource_any(sc->vtpci_dev, SYS_RES_IRQ, &rid, flags);
+	if (irq == NULL)
+		return (ENXIO);
 
-	if (sc->vtpci_flags & VTPCI_FLAG_LEGACY) {
-		rid = 0;
-		flags = RF_ACTIVE | RF_SHAREABLE;
-	} else {
-		rid = 1;
-		flags = RF_ACTIVE;
-	}
+	intr->vti_irq = irq;
+	intr->vti_rid = rid;
 
-	for (i = 0; i < sc->vtpci_nintr_res; i++) {
-		irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, flags);
-		if (irq == NULL)
-			return (ENXIO);
+	return (0);
+}
 
-		sc->vtpci_intr_res[i].irq = irq;
-		sc->vtpci_intr_res[i].rid = rid++;
-	}
+static int
+vtpci_alloc_intr_resources(struct vtpci_softc *sc)
+{
+	struct vtpci_interrupt *intr;
+	int i, rid, flags, nvq_intrs, error;
+
+	rid = 0;
+	flags = RF_ACTIVE;
+
+	if (sc->vtpci_flags & VTPCI_FLAG_LEGACY)
+		flags |= RF_SHAREABLE;
+	else
+		rid = 1;
 
 	/*
-	 * Map the virtqueue into the correct index in vq_intr_res[]. The
-	 * first index is reserved for configuration changed notifications.
+	 * For legacy and MSI interrupts, this single resource handles all
+	 * interrupts. For MSIX, this resource is used for the configuration
+	 * changed interrupt.
 	 */
-	for (i = 0, res_idx = 1; i < sc->vtpci_nvqs; i++) {
-		vqx = &sc->vtpci_vqx[i];
+	intr = &sc->vtpci_device_interrupt;
+	error = vtpci_alloc_interrupt(sc, rid, flags, intr);
+	if (error || sc->vtpci_flags & (VTPCI_FLAG_LEGACY | VTPCI_FLAG_MSI))
+		return (error);
 
-		if (sc->vtpci_flags & VTPCI_FLAG_MSIX) {
-			if (vqx->no_intr != 0)
-				vqx->ires_idx = -1;
-			else if (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX)
-				vqx->ires_idx = res_idx;
-			else
-				vqx->ires_idx = res_idx++;
-		} else
-			vqx->ires_idx = -1;
+	/* Subtract one for the configuration changed interrupt. */
+	nvq_intrs = sc->vtpci_nmsix_resources - 1;
+
+	intr = sc->vtpci_msix_vq_interrupts = malloc(nvq_intrs *
+	    sizeof(struct vtpci_interrupt), M_DEVBUF, M_NOWAIT | M_ZERO);
+	if (sc->vtpci_msix_vq_interrupts == NULL)
+		return (ENOMEM);
+
+	for (i = 0, rid++; i < nvq_intrs; i++, rid++, intr++) {
+		error = vtpci_alloc_interrupt(sc, rid, flags, intr);
+		if (error)
+			return (error);
 	}
 
 	return (0);
@@ -925,67 +937,67 @@ vtpci_alloc_intr_resources(struct vtpci_softc *sc)
 static int
 vtpci_setup_legacy_interrupt(struct vtpci_softc *sc, enum intr_type type)
 {
-	device_t dev;
-	struct vtpci_intr_resource *ires;
+	struct vtpci_interrupt *intr;
 	int error;
 
-	dev = sc->vtpci_dev;
-
-	ires = &sc->vtpci_intr_res[0];
-	error = bus_setup_intr(dev, ires->irq, type, vtpci_legacy_intr, NULL,
-	    sc, &ires->intrhand);
+	intr = &sc->vtpci_device_interrupt;
+	error = bus_setup_intr(sc->vtpci_dev, intr->vti_irq, type, NULL,
+	    vtpci_legacy_intr, sc, &intr->vti_handler);
 
 	return (error);
 }
 
 static int
-vtpci_setup_msix_interrupts(struct vtpci_softc *sc, enum intr_type type)
+vtpci_setup_pervq_msix_interrupts(struct vtpci_softc *sc, enum intr_type type)
 {
-	device_t dev;
-	struct vtpci_intr_resource *ires;
 	struct vtpci_virtqueue *vqx;
+	struct vtpci_interrupt *intr;
 	int i, error;
 
-	dev = sc->vtpci_dev;
+	intr = sc->vtpci_msix_vq_interrupts;
 
-	/*
-	 * The first resource is used for configuration changed interrupts.
-	 */
-	ires = &sc->vtpci_intr_res[0];
-	error = bus_setup_intr(dev, ires->irq, type, vtpci_config_intr,
-	    NULL, sc, &ires->intrhand);
-	if (error)
-		return (error);
+	for (i = 0; i < sc->vtpci_nvqs; i++) {
+		vqx = &sc->vtpci_vqs[i];
 
-	if (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) {
-		ires = &sc->vtpci_intr_res[1];
+		if (vqx->vtv_no_intr)
+			continue;
 
-		error = bus_setup_intr(dev, ires->irq, type,
-		    vtpci_vq_shared_intr, NULL, sc, &ires->intrhand);
+		error = bus_setup_intr(sc->vtpci_dev, intr->vti_irq, type,
+		    vtpci_vq_intr_filter, vtpci_vq_intr, vqx->vtv_vq,
+		    &intr->vti_handler);
 		if (error)
 			return (error);
-	} else {
-		/*
-		 * Each remaining resource is assigned to a specific virtqueue.
-		 */
-		for (i = 0; i < sc->vtpci_nvqs; i++) {
-			vqx = &sc->vtpci_vqx[i];
-			if (vqx->ires_idx < 1)
-				continue;
 
-			ires = &sc->vtpci_intr_res[vqx->ires_idx];
-			error = bus_setup_intr(dev, ires->irq, type,
-			    vtpci_vq_intr, NULL, vqx->vq, &ires->intrhand);
-			if (error)
-				return (error);
-		}
+		intr++;
 	}
 
-	error = vtpci_set_host_msix_vectors(sc);
+	return (0);
+}
+
+static int
+vtpci_setup_msix_interrupts(struct vtpci_softc *sc, enum intr_type type)
+{
+	device_t dev;
+	struct vtpci_interrupt *intr;
+	int error;
+
+	dev = sc->vtpci_dev;
+	intr = &sc->vtpci_device_interrupt;
+
+	error = bus_setup_intr(dev, intr->vti_irq, type, NULL,
+	    vtpci_config_intr, sc, &intr->vti_handler);
 	if (error)
 		return (error);
 
-	return (0);
+	if (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) {
+		intr = sc->vtpci_msix_vq_interrupts;
+		error = bus_setup_intr(dev, intr->vti_irq, type,
+		    vtpci_vq_shared_intr_filter, vtpci_vq_shared_intr, sc,
+		    &intr->vti_handler);
+	} else
+		error = vtpci_setup_pervq_msix_interrupts(sc, type);
+
+	return (error ? error : vtpci_set_host_msix_vectors(sc));
 }
 
 static int
@@ -995,7 +1007,7 @@ vtpci_setup_interrupts(struct vtpci_softc *sc, enum intr_type type)
 
 	type |= INTR_MPSAFE;
 	KASSERT(sc->vtpci_flags & VTPCI_FLAG_ITYPE_MASK,
-	    ("no interrupt type selected: %#x", sc->vtpci_flags));
+	    ("%s: no interrupt type selected %#x", __func__, sc->vtpci_flags));
 
 	error = vtpci_alloc_intr_resources(sc);
 	if (error)
@@ -1012,34 +1024,24 @@ vtpci_setup_interrupts(struct vtpci_softc *sc, enum intr_type type)
 }
 
 static int
-vtpci_register_msix_vector(struct vtpci_softc *sc, int offset, int res_idx)
+vtpci_register_msix_vector(struct vtpci_softc *sc, int offset,
+    struct vtpci_interrupt *intr)
 {
 	device_t dev;
-	uint16_t vector, rdvector;
+	uint16_t vector;
 
 	dev = sc->vtpci_dev;
 
-	if (res_idx != -1) {
+	if (intr != NULL) {
 		/* Map from guest rid to host vector. */
-		vector = sc->vtpci_intr_res[res_idx].rid - 1;
+		vector = intr->vti_rid - 1;
 	} else
 		vector = VIRTIO_MSI_NO_VECTOR;
 
-	/*
-	 * Assert the first resource is always used for the configuration
-	 * changed interrupts.
-	 */
-	if (res_idx == 0) {
-		KASSERT(vector == 0 && offset == VIRTIO_MSI_CONFIG_VECTOR,
-		    ("bad first res use vector:%d offset:%d", vector, offset));
-	} else
-		KASSERT(offset == VIRTIO_MSI_QUEUE_VECTOR, ("bad offset"));
-
 	vtpci_write_config_2(sc, offset, vector);
 
 	/* Read vector to determine if the host had sufficient resources. */
-	rdvector = vtpci_read_config_2(sc, offset);
-	if (rdvector != vector) {
+	if (vtpci_read_config_2(sc, offset) != vector) {
 		device_printf(dev,
 		    "insufficient host resources for MSIX interrupts\n");
 		return (ENODEV);
@@ -1051,24 +1053,40 @@ vtpci_register_msix_vector(struct vtpci_softc *sc, int offset, int res_idx)
 static int
 vtpci_set_host_msix_vectors(struct vtpci_softc *sc)
 {
-	struct vtpci_virtqueue *vqx;
-	int idx, error;
+	struct vtpci_interrupt *intr, *tintr;
+	int idx, offset, error;
 
-	error = vtpci_register_msix_vector(sc, VIRTIO_MSI_CONFIG_VECTOR, 0);
+	intr = &sc->vtpci_device_interrupt;
+	offset = VIRTIO_MSI_CONFIG_VECTOR;
+
+	error = vtpci_register_msix_vector(sc, offset, intr);
 	if (error)
 		return (error);
 
-	for (idx = 0; idx < sc->vtpci_nvqs; idx++) {
-		vqx = &sc->vtpci_vqx[idx];
+	intr = sc->vtpci_msix_vq_interrupts;
+	offset = VIRTIO_MSI_QUEUE_VECTOR;
 
+	for (idx = 0; idx < sc->vtpci_nvqs; idx++) {
 		vtpci_select_virtqueue(sc, idx);
-		error = vtpci_register_msix_vector(sc, VIRTIO_MSI_QUEUE_VECTOR,
-		    vqx->ires_idx);
+
+		if (sc->vtpci_vqs[idx].vtv_no_intr)
+			tintr = NULL;
+		else
+			tintr = intr;
+
+		error = vtpci_register_msix_vector(sc, offset, tintr);
 		if (error)
-			return (error);
+			break;
+
+		/*
+		 * For shared MSIX, all the virtqueues share the first
+		 * interrupt.
+		 */
+		if ((sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) == 0)
+			intr++;
 	}
 
-	return (0);
+	return (error);
 }
 
 static int
@@ -1079,10 +1097,10 @@ vtpci_reinit_virtqueue(struct vtpci_softc *sc, int idx)
 	int error;
 	uint16_t size;
 
-	vqx = &sc->vtpci_vqx[idx];
-	vq = vqx->vq;
+	vqx = &sc->vtpci_vqs[idx];
+	vq = vqx->vtv_vq;
 
-	KASSERT(vq != NULL, ("vq %d not allocated", idx));
+	KASSERT(vq != NULL, ("%s: vq %d not allocated", __func__, idx));
 
 	vtpci_select_virtqueue(sc, idx);
 	size = vtpci_read_config_2(sc, VIRTIO_PCI_QUEUE_NUM);
@@ -1098,35 +1116,50 @@ vtpci_reinit_virtqueue(struct vtpci_softc *sc, int idx)
 }
 
 static void
-vtpci_free_interrupts(struct vtpci_softc *sc)
+vtpci_free_interrupt(struct vtpci_softc *sc, struct vtpci_interrupt *intr)
 {
 	device_t dev;
-	struct vtpci_intr_resource *ires;
-	int i;
 
 	dev = sc->vtpci_dev;
 
-	for (i = 0; i < sc->vtpci_nintr_res; i++) {
-		ires = &sc->vtpci_intr_res[i];
+	if (intr->vti_handler != NULL) {
+		bus_teardown_intr(dev, intr->vti_irq, intr->vti_handler);
+		intr->vti_handler = NULL;
+	}
 
-		if (ires->intrhand != NULL) {
-			bus_teardown_intr(dev, ires->irq, ires->intrhand);
-			ires->intrhand = NULL;
-		}
+	if (intr->vti_irq != NULL) {
+		bus_release_resource(dev, SYS_RES_IRQ, intr->vti_rid,
+		    intr->vti_irq);
+		intr->vti_irq = NULL;
+		intr->vti_rid = -1;
+	}
+}
 
-		if (ires->irq != NULL) {
-			bus_release_resource(dev, SYS_RES_IRQ, ires->rid,
-			    ires->irq);
-			ires->irq = NULL;
-		}
+static void
+vtpci_free_interrupts(struct vtpci_softc *sc)
+{
+	struct vtpci_interrupt *intr;
+	int i, nvq_intrs;
+
+	vtpci_free_interrupt(sc, &sc->vtpci_device_interrupt);
 
-		ires->rid = -1;
+	if (sc->vtpci_nmsix_resources != 0) {
+		nvq_intrs = sc->vtpci_nmsix_resources - 1;
+		sc->vtpci_nmsix_resources = 0;
+
+		intr = sc->vtpci_msix_vq_interrupts;
+		if (intr != NULL) {
+			for (i = 0; i < nvq_intrs; i++, intr++)
+				vtpci_free_interrupt(sc, intr);
+
+			free(sc->vtpci_msix_vq_interrupts, M_DEVBUF);
+			sc->vtpci_msix_vq_interrupts = NULL;
+		}
 	}
 
 	if (sc->vtpci_flags & (VTPCI_FLAG_MSI | VTPCI_FLAG_MSIX))
-		pci_release_msi(dev);
+		pci_release_msi(sc->vtpci_dev);
 
-	sc->vtpci_nintr_res = 0;
 	sc->vtpci_flags &= ~VTPCI_FLAG_ITYPE_MASK;
 }
 
@@ -1134,19 +1167,32 @@ static void
 vtpci_free_virtqueues(struct vtpci_softc *sc)
 {
 	struct vtpci_virtqueue *vqx;
-	int i;
+	int idx;
 
-	for (i = 0; i < sc->vtpci_nvqs; i++) {
-		vqx = &sc->vtpci_vqx[i];
+	for (idx = 0; idx < sc->vtpci_nvqs; idx++) {
+		vqx = &sc->vtpci_vqs[idx];
 
-		virtqueue_free(vqx->vq);
-		vqx->vq = NULL;
+		vtpci_select_virtqueue(sc, idx);
+		vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN, 0);
+
+		virtqueue_free(vqx->vtv_vq);
+		vqx->vtv_vq = NULL;
 	}
 
+	free(sc->vtpci_vqs, M_DEVBUF);
+	sc->vtpci_vqs = NULL;
 	sc->vtpci_nvqs = 0;
 }
 
 static void
+vtpci_release_child_resources(struct vtpci_softc *sc)
+{
+
+	vtpci_free_interrupts(sc);
+	vtpci_free_virtqueues(sc);
+}
+
+static void
 vtpci_cleanup_setup_intr_attempt(struct vtpci_softc *sc)
 {
 	int idx;
@@ -1166,14 +1212,6 @@ vtpci_cleanup_setup_intr_attempt(struct vtpci_softc *sc)
 }
 
 static void
-vtpci_release_child_resources(struct vtpci_softc *sc)
-{
-
-	vtpci_free_interrupts(sc);
-	vtpci_free_virtqueues(sc);
-}
-
-static void
 vtpci_reset(struct vtpci_softc *sc)
 {
 
@@ -1191,7 +1229,7 @@ vtpci_select_virtqueue(struct vtpci_softc *sc, int idx)
 	vtpci_write_config_2(sc, VIRTIO_PCI_QUEUE_SEL, idx);
 }
 
-static int
+static void
 vtpci_legacy_intr(void *xsc)
 {
 	struct vtpci_softc *sc;
@@ -1200,7 +1238,7 @@ vtpci_legacy_intr(void *xsc)
 	uint8_t isr;
 
 	sc = xsc;
-	vqx = &sc->vtpci_vqx[0];
+	vqx = &sc->vtpci_vqs[0];
 
 	/* Reading the ISR also clears it. */
 	isr = vtpci_read_config_1(sc, VIRTIO_PCI_ISR);
@@ -1208,15 +1246,16 @@ vtpci_legacy_intr(void *xsc)
 	if (isr & VIRTIO_PCI_ISR_CONFIG)
 		vtpci_config_intr(sc);
 
-	if (isr & VIRTIO_PCI_ISR_INTR)
-		for (i = 0; i < sc->vtpci_nvqs; i++, vqx++)
-			virtqueue_intr(vqx->vq);
-
-	return (isr ? FILTER_HANDLED : FILTER_STRAY);
+	if (isr & VIRTIO_PCI_ISR_INTR) {
+		for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) {
+			if (vqx->vtv_no_intr == 0)
+				virtqueue_intr(vqx->vtv_vq);
+		}
+	}
 }
 
 static int
-vtpci_vq_shared_intr(void *xsc)
+vtpci_vq_shared_intr_filter(void *xsc)
 {
 	struct vtpci_softc *sc;
 	struct vtpci_virtqueue *vqx;
@@ -1224,39 +1263,62 @@ vtpci_vq_shared_intr(void *xsc)
 
 	rc = 0;
 	sc = xsc;
-	vqx = &sc->vtpci_vqx[0];
+	vqx = &sc->vtpci_vqs[0];
 
-	for (i = 0; i < sc->vtpci_nvqs; i++, vqx++)
-		rc |= virtqueue_intr(vqx->vq);
+	for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) {
+		if (vqx->vtv_no_intr == 0)
+			rc |= virtqueue_intr_filter(vqx->vtv_vq);
+	}
 
-	return (rc ? FILTER_HANDLED : FILTER_STRAY);
+	return (rc ? FILTER_SCHEDULE_THREAD : FILTER_STRAY);
+}
+
+static void
+vtpci_vq_shared_intr(void *xsc)
+{
+	struct vtpci_softc *sc;
+	struct vtpci_virtqueue *vqx;
+	int i;
+
+	sc = xsc;
+	vqx = &sc->vtpci_vqs[0];
+
+	for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) {
+		if (vqx->vtv_no_intr == 0)
+			virtqueue_intr(vqx->vtv_vq);
+	}
 }
 
 static int
-vtpci_vq_intr(void *xvq)
+vtpci_vq_intr_filter(void *xvq)
 {
 	struct virtqueue *vq;
 	int rc;
 
 	vq = xvq;
-	rc = virtqueue_intr(vq);
+	rc = virtqueue_intr_filter(vq);
 
-	return (rc ? FILTER_HANDLED : FILTER_STRAY);
+	return (rc ? FILTER_SCHEDULE_THREAD : FILTER_STRAY);
 }
 
-static int
+static void
+vtpci_vq_intr(void *xvq)
+{
+	struct virtqueue *vq;
+
+	vq = xvq;
+	virtqueue_intr(vq);
+}
+
+static void
 vtpci_config_intr(void *xsc)
 {
 	struct vtpci_softc *sc;
 	device_t child;
-	int rc;
 
-	rc = 0;
 	sc = xsc;
 	child = sc->vtpci_child_dev;
 
 	if (child != NULL)
-		rc = VIRTIO_CONFIG_CHANGE(child);
-
-	return (rc ? FILTER_HANDLED : FILTER_STRAY);
+		VIRTIO_CONFIG_CHANGE(child);
 }
diff --git a/sys/dev/virtio/scsi/virtio_scsi.c b/sys/dev/virtio/scsi/virtio_scsi.c
index 52f1581c7e53..e5f922befbd4 100644
--- a/sys/dev/virtio/scsi/virtio_scsi.c
+++ b/sys/dev/virtio/scsi/virtio_scsi.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2012, Bryan Venteicher <bryanv@daemoninthecloset.org>
+ * Copyright (c) 2012, Bryan Venteicher <bryanv@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -40,7 +40,6 @@ __FBSDID("$FreeBSD$");
 #include <sys/lock.h>
 #include <sys/mutex.h>
 #include <sys/callout.h>
-#include <sys/taskqueue.h>
 #include <sys/queue.h>
 #include <sys/sbuf.h>
 
@@ -172,13 +171,10 @@ static struct vtscsi_request * vtscsi_dequeue_request(struct vtscsi_softc *);
 
 static void	vtscsi_complete_request(struct vtscsi_request *);
 static void 	vtscsi_complete_vq(struct vtscsi_softc *, struct virtqueue *);
-static void	vtscsi_control_vq_task(void *, int);
-static void	vtscsi_event_vq_task(void *, int);
-static void	vtscsi_request_vq_task(void *, int);
 
-static int	vtscsi_control_vq_intr(void *);
-static int	vtscsi_event_vq_intr(void *);
-static int	vtscsi_request_vq_intr(void *);
+static void	vtscsi_control_vq_intr(void *);
+static void	vtscsi_event_vq_intr(void *);
+static void	vtscsi_request_vq_intr(void *);
 static void 	vtscsi_disable_vqs_intr(struct vtscsi_softc *);
 static void 	vtscsi_enable_vqs_intr(struct vtscsi_softc *);
 
@@ -333,30 +329,12 @@ vtscsi_attach(device_t dev)
 		goto fail;
 	}
 
-	TASK_INIT(&sc->vtscsi_control_intr_task, 0,
-	    vtscsi_control_vq_task, sc);
-	TASK_INIT(&sc->vtscsi_event_intr_task, 0,
-	    vtscsi_event_vq_task, sc);
-	TASK_INIT(&sc->vtscsi_request_intr_task, 0,
-	    vtscsi_request_vq_task, sc);
-
-	sc->vtscsi_tq = taskqueue_create_fast("vtscsi_taskq", M_NOWAIT,
-	    taskqueue_thread_enqueue, &sc->vtscsi_tq);
-	if (sc->vtscsi_tq == NULL) {
-		error = ENOMEM;
-		device_printf(dev, "cannot allocate taskqueue\n");
-		goto fail;
-	}
-
 	error = virtio_setup_intr(dev, INTR_TYPE_CAM);
 	if (error) {
 		device_printf(dev, "cannot setup virtqueue interrupts\n");
 		goto fail;
 	}
 
-	taskqueue_start_threads(&sc->vtscsi_tq, 1, PI_DISK, "%s taskq",
-	    device_get_nameunit(dev));
-
 	vtscsi_enable_vqs_intr(sc);
 
 	/*
@@ -389,14 +367,6 @@ vtscsi_detach(device_t dev)
 		vtscsi_stop(sc);
 	VTSCSI_UNLOCK(sc);
 
-	if (sc->vtscsi_tq != NULL) {
-		taskqueue_drain(sc->vtscsi_tq, &sc->vtscsi_control_intr_task);
-		taskqueue_drain(sc->vtscsi_tq, &sc->vtscsi_event_intr_task);
-		taskqueue_drain(sc->vtscsi_tq, &sc->vtscsi_request_intr_task);
-		taskqueue_free(sc->vtscsi_tq);
-		sc->vtscsi_tq = NULL;
-	}
-
 	vtscsi_complete_vqs(sc);
 	vtscsi_drain_vqs(sc);
 
@@ -572,19 +542,14 @@ vtscsi_register_cam(struct vtscsi_softc *sc)
 		goto fail;
 	}
 
-	VTSCSI_UNLOCK(sc);
-
-	/*
-	 * The async register apparently needs to be done without
-	 * the lock held, otherwise it can recurse on the lock.
-	 */
 	if (vtscsi_register_async(sc) != CAM_REQ_CMP) {
 		error = EIO;
 		device_printf(dev, "cannot register async callback\n");
-		VTSCSI_LOCK(sc);
 		goto fail;
 	}
 
+	VTSCSI_UNLOCK(sc);
+
 	return (0);
 
 fail:
@@ -652,8 +617,6 @@ vtscsi_register_async(struct vtscsi_softc *sc)
 {
 	struct ccb_setasync csa;
 
-	VTSCSI_LOCK_NOTOWNED(sc);
-
 	xpt_setup_ccb(&csa.ccb_h, sc->vtscsi_path, 5);
 	csa.ccb_h.func_code = XPT_SASYNC_CB;
 	csa.event_enable = AC_LOST_DEVICE | AC_FOUND_DEVICE;
@@ -1268,7 +1231,7 @@ vtscsi_scsi_cmd_cam_status(struct virtio_scsi_cmd_resp *cmd_resp)
 		status = CAM_REQ_ABORTED;
 		break;
 	case VIRTIO_SCSI_S_BAD_TARGET:
-		status = CAM_TID_INVALID;
+		status = CAM_SEL_TIMEOUT;
 		break;
 	case VIRTIO_SCSI_S_RESET:
 		status = CAM_SCSI_BUS_RESET;
@@ -2152,14 +2115,15 @@ vtscsi_complete_vq(struct vtscsi_softc *sc, struct virtqueue *vq)
 }
 
 static void
-vtscsi_control_vq_task(void *arg, int pending)
+vtscsi_control_vq_intr(void *xsc)
 {
 	struct vtscsi_softc *sc;
 	struct virtqueue *vq;
 
-	sc = arg;
+	sc = xsc;
 	vq = sc->vtscsi_control_vq;
 
+again:
 	VTSCSI_LOCK(sc);
 
 	vtscsi_complete_vq(sc, sc->vtscsi_control_vq);
@@ -2167,24 +2131,23 @@ vtscsi_control_vq_task(void *arg, int pending)
 	if (virtqueue_enable_intr(vq) != 0) {
 		virtqueue_disable_intr(vq);
 		VTSCSI_UNLOCK(sc);
-		taskqueue_enqueue_fast(sc->vtscsi_tq,
-		    &sc->vtscsi_control_intr_task);
-		return;
+		goto again;
 	}
 
 	VTSCSI_UNLOCK(sc);
 }
 
 static void
-vtscsi_event_vq_task(void *arg, int pending)
+vtscsi_event_vq_intr(void *xsc)
 {
 	struct vtscsi_softc *sc;
 	struct virtqueue *vq;
 	struct virtio_scsi_event *event;
 
-	sc = arg;
+	sc = xsc;
 	vq = sc->vtscsi_event_vq;
 
+again:
 	VTSCSI_LOCK(sc);
 
 	while ((event = virtqueue_dequeue(vq, NULL)) != NULL)
@@ -2193,23 +2156,22 @@ vtscsi_event_vq_task(void *arg, int pending)
 	if (virtqueue_enable_intr(vq) != 0) {
 		virtqueue_disable_intr(vq);
 		VTSCSI_UNLOCK(sc);
-		taskqueue_enqueue_fast(sc->vtscsi_tq,
-		    &sc->vtscsi_control_intr_task);
-		return;
+		goto again;
 	}
 
 	VTSCSI_UNLOCK(sc);
 }
 
 static void
-vtscsi_request_vq_task(void *arg, int pending)
+vtscsi_request_vq_intr(void *xsc)
 {
 	struct vtscsi_softc *sc;
 	struct virtqueue *vq;
 
-	sc = arg;
+	sc = xsc;
 	vq = sc->vtscsi_request_vq;
 
+again:
 	VTSCSI_LOCK(sc);
 
 	vtscsi_complete_vq(sc, sc->vtscsi_request_vq);
@@ -2217,56 +2179,12 @@ vtscsi_request_vq_task(void *arg, int pending)
 	if (virtqueue_enable_intr(vq) != 0) {
 		virtqueue_disable_intr(vq);
 		VTSCSI_UNLOCK(sc);
-		taskqueue_enqueue_fast(sc->vtscsi_tq,
-		    &sc->vtscsi_request_intr_task);
-		return;
+		goto again;
 	}
 
 	VTSCSI_UNLOCK(sc);
 }
 
-static int
-vtscsi_control_vq_intr(void *xsc)
-{
-	struct vtscsi_softc *sc;
-
-	sc = xsc;
-
-	virtqueue_disable_intr(sc->vtscsi_control_vq);
-	taskqueue_enqueue_fast(sc->vtscsi_tq,
-	    &sc->vtscsi_control_intr_task);
-
-	return (1);
-}
-
-static int
-vtscsi_event_vq_intr(void *xsc)
-{
-	struct vtscsi_softc *sc;
-
-	sc = xsc;
-
-	virtqueue_disable_intr(sc->vtscsi_event_vq);
-	taskqueue_enqueue_fast(sc->vtscsi_tq,
-	    &sc->vtscsi_event_intr_task);
-
-	return (1);
-}
-
-static int
-vtscsi_request_vq_intr(void *xsc)
-{
-	struct vtscsi_softc *sc;
-
-	sc = xsc;
-
-	virtqueue_disable_intr(sc->vtscsi_request_vq);
-	taskqueue_enqueue_fast(sc->vtscsi_tq,
-	    &sc->vtscsi_request_intr_task);
-
-	return (1);
-}
-
 static void
 vtscsi_disable_vqs_intr(struct vtscsi_softc *sc)
 {
diff --git a/sys/dev/virtio/scsi/virtio_scsivar.h b/sys/dev/virtio/scsi/virtio_scsivar.h
index 7afe32f869b7..f30f66751b81 100644
--- a/sys/dev/virtio/scsi/virtio_scsivar.h
+++ b/sys/dev/virtio/scsi/virtio_scsivar.h
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2012, Bryan Venteicher <bryanv@daemoninthecloset.org>
+ * Copyright (c) 2012, Bryan Venteicher <bryanv@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -62,11 +62,6 @@ struct vtscsi_softc {
 	struct virtqueue	*vtscsi_event_vq;
 	struct virtqueue	*vtscsi_request_vq;
 
-	struct taskqueue	*vtscsi_tq;
-	struct task		 vtscsi_control_intr_task;
-	struct task		 vtscsi_event_intr_task;
-	struct task		 vtscsi_request_intr_task;
-
 	struct cam_sim		*vtscsi_sim;
 	struct cam_path		*vtscsi_path;
 
diff --git a/sys/dev/virtio/virtio.c b/sys/dev/virtio/virtio.c
index 6c6b0bc056ab..ea2b91acda68 100644
--- a/sys/dev/virtio/virtio.c
+++ b/sys/dev/virtio/virtio.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2011, Bryan Venteicher <bryanv@daemoninthecloset.org>
+ * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -36,7 +36,6 @@ __FBSDID("$FreeBSD$");
 
 #include <machine/bus.h>
 #include <machine/resource.h>
-#include <machine/_inttypes.h>
 #include <sys/bus.h>
 #include <sys/rman.h>
 
@@ -49,8 +48,8 @@ static int virtio_modevent(module_t, int, void *);
 static const char *virtio_feature_name(uint64_t, struct virtio_feature_desc *);
 
 static struct virtio_ident {
-	uint16_t devid;
-	char	*name;
+	uint16_t	devid;
+	const char	*name;
 } virtio_ident_table[] = {
 	{ VIRTIO_ID_NETWORK,	"Network"	},
 	{ VIRTIO_ID_BLOCK,	"Block"		},
@@ -87,9 +86,29 @@ virtio_device_name(uint16_t devid)
 	return (NULL);
 }
 
+static const char *
+virtio_feature_name(uint64_t val, struct virtio_feature_desc *desc)
+{
+	int i, j;
+	struct virtio_feature_desc *descs[2] = { desc,
+	    virtio_common_feature_desc };
+
+	for (i = 0; i < 2; i++) {
+		if (descs[i] == NULL)
+			continue;
+
+		for (j = 0; descs[i][j].vfd_val != 0; j++) {
+			if (val == descs[i][j].vfd_val)
+				return (descs[i][j].vfd_str);
+		}
+	}
+
+	return (NULL);
+}
+
 void
 virtio_describe(device_t dev, const char *msg,
-    uint64_t features, struct virtio_feature_desc *feature_desc)
+    uint64_t features, struct virtio_feature_desc *desc)
 {
 	struct sbuf sb;
 	uint64_t val;
@@ -98,13 +117,12 @@ virtio_describe(device_t dev, const char *msg,
 	int n;
 
 	if ((buf = malloc(512, M_TEMP, M_NOWAIT)) == NULL) {
-		device_printf(dev, "%s features: 0x%"PRIx64"\n", msg,
-		    features);
+		device_printf(dev, "%s features: %#jx\n", msg, (uintmax_t) features);
 		return;
 	}
 
 	sbuf_new(&sb, buf, 512, SBUF_FIXEDLEN);
-	sbuf_printf(&sb, "%s features: 0x%"PRIx64, msg, features);
+	sbuf_printf(&sb, "%s features: %#jx", msg, (uintmax_t) features);
 
 	for (n = 0, val = 1ULL << 63; val != 0; val >>= 1) {
 		/*
@@ -119,15 +137,9 @@ virtio_describe(device_t dev, const char *msg,
 		else
 			sbuf_cat(&sb, ",");
 
-		name = NULL;
-		if (feature_desc != NULL)
-			name = virtio_feature_name(val, feature_desc);
+		name = virtio_feature_name(val, desc);
 		if (name == NULL)
-			name = virtio_feature_name(val,
-			    virtio_common_feature_desc);
-
-		if (name == NULL)
-			sbuf_printf(&sb, "0x%"PRIx64, val);
+			sbuf_printf(&sb, "%#jx", (uintmax_t) val);
 		else
 			sbuf_cat(&sb, name);
 	}
@@ -147,18 +159,6 @@ virtio_describe(device_t dev, const char *msg,
 	free(buf, M_TEMP);
 }
 
-static const char *
-virtio_feature_name(uint64_t val, struct virtio_feature_desc *feature_desc)
-{
-	int i;
-
-	for (i = 0; feature_desc[i].vfd_val != 0; i++)
-		if (val == feature_desc[i].vfd_val)
-			return (feature_desc[i].vfd_str);
-
-	return (NULL);
-}
-
 /*
  * VirtIO bus method wrappers.
  */
@@ -251,13 +251,12 @@ virtio_modevent(module_t mod, int type, void *unused)
 {
 	int error;
 
-	error = 0;
-
 	switch (type) {
 	case MOD_LOAD:
 	case MOD_QUIESCE:
 	case MOD_UNLOAD:
 	case MOD_SHUTDOWN:
+		error = 0;
 		break;
 	default:
 		error = EOPNOTSUPP;
diff --git a/sys/dev/virtio/virtio.h b/sys/dev/virtio/virtio.h
index 4d069dd94cd3..b1334f811acf 100644
--- a/sys/dev/virtio/virtio.h
+++ b/sys/dev/virtio/virtio.h
@@ -71,11 +71,6 @@ struct vq_alloc_info;
 #define VIRTIO_TRANSPORT_F_END		32
 
 /*
- * Maximum number of virtqueues per device.
- */
-#define VIRTIO_MAX_VIRTQUEUES 8
-
-/*
  * Each virtqueue indirect descriptor list must be physically contiguous.
  * To allow us to malloc(9) each list individually, limit the number
  * supported to what will fit in one page. With 4KB pages, this is a limit
@@ -99,7 +94,7 @@ struct vq_alloc_info;
 
 struct virtio_feature_desc {
 	uint64_t	 vfd_val;
-	char		*vfd_str;
+	const char	*vfd_str;
 };
 
 const char *virtio_device_name(uint16_t devid);
diff --git a/sys/dev/virtio/virtio_bus_if.m b/sys/dev/virtio/virtio_bus_if.m
index ec2029de16ac..74341ffe3751 100644
--- a/sys/dev/virtio/virtio_bus_if.m
+++ b/sys/dev/virtio/virtio_bus_if.m
@@ -1,5 +1,5 @@
 #-
-# Copyright (c) 2011, Bryan Venteicher <bryanv@daemoninthecloset.org>
+# Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org>
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -50,9 +50,6 @@ METHOD int alloc_virtqueues {
 	int		nvqs;
 	struct vq_alloc_info *info;
 };
-HEADER {
-#define VIRTIO_ALLOC_VQS_DISABLE_MSIX 0x1
-};
 
 METHOD int setup_intr {
 	device_t	dev;
diff --git a/sys/dev/virtio/virtio_if.m b/sys/dev/virtio/virtio_if.m
index 701678c02fe6..9a99d371f663 100644
--- a/sys/dev/virtio/virtio_if.m
+++ b/sys/dev/virtio/virtio_if.m
@@ -1,5 +1,5 @@
 #-
-# Copyright (c) 2011, Bryan Venteicher <bryanv@daemoninthecloset.org>
+# Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org>
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -33,8 +33,7 @@ CODE {
 	static int
 	virtio_default_config_change(device_t dev)
 	{
-		/* Return that we've handled the change. */
-		return (1);
+		return (0);
 	}
 };
 
diff --git a/sys/dev/virtio/virtqueue.c b/sys/dev/virtio/virtqueue.c
index 7e02deb7c857..a65c870c0b09 100644
--- a/sys/dev/virtio/virtqueue.c
+++ b/sys/dev/virtio/virtqueue.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2011, Bryan Venteicher <bryanv@daemoninthecloset.org>
+ * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -414,18 +414,24 @@ virtqueue_nused(struct virtqueue *vq)
 }
 
 int
-virtqueue_intr(struct virtqueue *vq)
+virtqueue_intr_filter(struct virtqueue *vq)
 {
 
-	if (vq->vq_intrhand == NULL ||
-	    vq->vq_used_cons_idx == vq->vq_ring.used->idx)
+	if (vq->vq_used_cons_idx == vq->vq_ring.used->idx)
 		return (0);
 
-	vq->vq_intrhand(vq->vq_intrhand_arg);
+	virtqueue_disable_intr(vq);
 
 	return (1);
 }
 
+void
+virtqueue_intr(struct virtqueue *vq)
+{
+
+	vq->vq_intrhand(vq->vq_intrhand_arg);
+}
+
 int
 virtqueue_enable_intr(struct virtqueue *vq)
 {
diff --git a/sys/dev/virtio/virtqueue.h b/sys/dev/virtio/virtqueue.h
index 0296b8c2fba4..128a10a79e26 100644
--- a/sys/dev/virtio/virtqueue.h
+++ b/sys/dev/virtio/virtqueue.h
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2011, Bryan Venteicher <bryanv@daemoninthecloset.org>
+ * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -39,7 +39,7 @@ struct sglist;
 #define VIRTIO_RING_F_EVENT_IDX		(1 << 29)
 
 /* Device callback for a virtqueue interrupt. */
-typedef int virtqueue_intr_t(void *);
+typedef void virtqueue_intr_t(void *);
 
 #define VIRTQUEUE_MAX_NAME_SZ	32
 
@@ -70,7 +70,8 @@ void	*virtqueue_drain(struct virtqueue *vq, int *last);
 void	 virtqueue_free(struct virtqueue *vq);
 int	 virtqueue_reinit(struct virtqueue *vq, uint16_t size);
 
-int	 virtqueue_intr(struct virtqueue *vq);
+int	 virtqueue_intr_filter(struct virtqueue *vq);
+void	 virtqueue_intr(struct virtqueue *vq);
 int	 virtqueue_enable_intr(struct virtqueue *vq);
 int	 virtqueue_postpone_intr(struct virtqueue *vq);
 void	 virtqueue_disable_intr(struct virtqueue *vq);