11 files changed, 536 insertions, 325 deletions

diff --git a/sys/kern/kern_cpu.c b/sys/kern/kern_cpu.c
index 724131d713a91..8534b4455e558 100644
--- a/sys/kern/kern_cpu.c
+++ b/sys/kern/kern_cpu.c
@@ -76,6 +76,7 @@ struct cpufreq_softc {
 	int				all_count;
 	int				max_mhz;
 	device_t			dev;
+	device_t			cf_drv_dev;
 	struct sysctl_ctx_list		sysctl_ctx;
 	struct task			startup_task;
 	struct cf_level			*levels_buf;
@@ -142,6 +143,11 @@ SYSCTL_INT(_debug_cpufreq, OID_AUTO, lowest, CTLFLAG_RWTUN, &cf_lowest_freq, 1,
 SYSCTL_INT(_debug_cpufreq, OID_AUTO, verbose, CTLFLAG_RWTUN, &cf_verbose, 1,
     "Print verbose debugging messages");
 
+/*
+ * This is called as the result of a hardware specific frequency control driver
+ * calling cpufreq_register. It provides a general interface for system wide
+ * frequency controls and operates on a per cpu basis.
+ */
 static int
 cpufreq_attach(device_t dev)
 {
@@ -149,7 +155,6 @@ cpufreq_attach(device_t dev)
 	struct pcpu *pc;
 	device_t parent;
 	uint64_t rate;
-	int numdevs;
 
 	CF_DEBUG("initializing %s\n", device_get_nameunit(dev));
 	sc = device_get_softc(dev);
@@ -164,6 +169,7 @@ cpufreq_attach(device_t dev)
 	sc->max_mhz = cpu_get_nominal_mhz(dev);
 	/* If that fails, try to measure the current rate */
 	if (sc->max_mhz <= 0) {
+		CF_DEBUG("Unable to obtain nominal frequency.\n");
 		pc = cpu_get_pcpu(dev);
 		if (cpu_est_clockrate(pc->pc_cpuid, &rate) == 0)
 			sc->max_mhz = rate / 1000000;
@@ -171,15 +177,6 @@ cpufreq_attach(device_t dev)
 			sc->max_mhz = CPUFREQ_VAL_UNKNOWN;
 	}
 
-	/*
-	 * Only initialize one set of sysctls for all CPUs.  In the future,
-	 * if multiple CPUs can have different settings, we can move these
-	 * sysctls to be under every CPU instead of just the first one.
-	 */
-	numdevs = devclass_get_count(cpufreq_dc);
-	if (numdevs > 1)
-		return (0);
-
 	CF_DEBUG("initializing one-time data for %s\n",
 	    device_get_nameunit(dev));
 	sc->levels_buf = malloc(CF_MAX_LEVELS * sizeof(*sc->levels_buf),
@@ -216,7 +213,6 @@ cpufreq_detach(device_t dev)
 {
 	struct cpufreq_softc *sc;
 	struct cf_saved_freq *saved_freq;
-	int numdevs;
 
 	CF_DEBUG("shutdown %s\n", device_get_nameunit(dev));
 	sc = device_get_softc(dev);
@@ -227,12 +223,7 @@ cpufreq_detach(device_t dev)
 		free(saved_freq, M_TEMP);
 	}
 
-	/* Only clean up these resources when the last device is detaching. */
-	numdevs = devclass_get_count(cpufreq_dc);
-	if (numdevs == 1) {
-		CF_DEBUG("final shutdown for %s\n", device_get_nameunit(dev));
-		free(sc->levels_buf, M_DEVBUF);
-	}
+	free(sc->levels_buf, M_DEVBUF);
 
 	return (0);
 }
@@ -422,25 +413,74 @@ out:
 }
 
 static int
+cpufreq_get_frequency(device_t dev)
+{
+	struct cf_setting set;
+
+	if (CPUFREQ_DRV_GET(dev, &set) != 0)
+		return (-1);
+
+	return (set.freq);
+}
+
+/* Returns the index into *levels with the match */
+static int
+cpufreq_get_level(device_t dev, struct cf_level *levels, int count)
+{
+	int i, freq;
+
+	if ((freq = cpufreq_get_frequency(dev)) < 0)
+		return (-1);
+	for (i = 0; i < count; i++)
+		if (freq == levels[i].total_set.freq)
+			return (i);
+
+	return (-1);
+}
+
+/*
+ * Used by the cpufreq core, this function will populate *level with the current
+ * frequency as either determined by a cached value sc->curr_level, or in the
+ * case the lower level driver has set the CPUFREQ_FLAG_UNCACHED flag, it will
+ * obtain the frequency from the driver itself.
+ */
+static int
 cf_get_method(device_t dev, struct cf_level *level)
 {
 	struct cpufreq_softc *sc;
 	struct cf_level *levels;
-	struct cf_setting *curr_set, set;
+	struct cf_setting *curr_set;
 	struct pcpu *pc;
-	device_t *devs;
-	int bdiff, count, diff, error, i, n, numdevs;
+	int bdiff, count, diff, error, i, type;
 	uint64_t rate;
 
 	sc = device_get_softc(dev);
 	error = 0;
 	levels = NULL;
 
-	/* If we already know the current frequency, we're done. */
+	/*
+	 * If we already know the current frequency, and the driver didn't ask
+	 * for uncached usage, we're done.
+	 */
 	CF_MTX_LOCK(&sc->lock);
 	curr_set = &sc->curr_level.total_set;
-	if (curr_set->freq != CPUFREQ_VAL_UNKNOWN) {
+	error = CPUFREQ_DRV_TYPE(sc->cf_drv_dev, &type);
+	if (error == 0 && (type & CPUFREQ_FLAG_UNCACHED)) {
+		struct cf_setting set;
+
+		/*
+		 * If the driver wants to always report back the real frequency,
+		 * first try the driver and if that fails, fall back to
+		 * estimating.
+		 */
+		if (CPUFREQ_DRV_GET(sc->cf_drv_dev, &set) != 0)
+			goto estimate;
+		sc->curr_level.total_set = set;
+		CF_DEBUG("get returning immediate freq %d\n", curr_set->freq);
+		goto out;
+	} else if (curr_set->freq != CPUFREQ_VAL_UNKNOWN) {
 		CF_DEBUG("get returning known freq %d\n", curr_set->freq);
+		error = 0;
 		goto out;
 	}
 	CF_MTX_UNLOCK(&sc->lock);
@@ -461,11 +501,6 @@ cf_get_method(device_t dev, struct cf_level *level)
 		free(levels, M_TEMP);
 		return (error);
 	}
-	error = device_get_children(device_get_parent(dev), &devs, &numdevs);
-	if (error) {
-		free(levels, M_TEMP);
-		return (error);
-	}
 
 	/*
 	 * Reacquire the lock and search for the given level.
@@ -476,24 +511,21 @@ cf_get_method(device_t dev, struct cf_level *level)
 	 * The estimation code below catches this case though.
 	 */
 	CF_MTX_LOCK(&sc->lock);
-	for (n = 0; n < numdevs && curr_set->freq == CPUFREQ_VAL_UNKNOWN; n++) {
-		if (!device_is_attached(devs[n]))
-			continue;
-		if (CPUFREQ_DRV_GET(devs[n], &set) != 0)
-			continue;
-		for (i = 0; i < count; i++) {
-			if (set.freq == levels[i].total_set.freq) {
-				sc->curr_level = levels[i];
-				break;
-			}
-		}
-	}
-	free(devs, M_TEMP);
+	i = cpufreq_get_level(sc->cf_drv_dev, levels, count);
+	if (i >= 0)
+		sc->curr_level = levels[i];
+	else
+		CF_DEBUG("Couldn't find supported level for %s\n",
+		    device_get_nameunit(sc->cf_drv_dev));
+
 	if (curr_set->freq != CPUFREQ_VAL_UNKNOWN) {
 		CF_DEBUG("get matched freq %d from drivers\n", curr_set->freq);
 		goto out;
 	}
 
+estimate:
+	CF_MTX_ASSERT(&sc->lock);
+
 	/*
 	 * We couldn't find an exact match, so attempt to estimate and then
 	 * match against a level.
@@ -525,6 +557,73 @@ out:
 	return (error);
 }
 
+/*
+ * Either directly obtain settings from the cpufreq driver, or build a list of
+ * relative settings to be integrated later against an absolute max.
+ */
+static int
+cpufreq_add_levels(device_t cf_dev, struct cf_setting_lst *rel_sets)
+{
+	struct cf_setting_array *set_arr;
+	struct cf_setting *sets;
+	device_t dev;
+	struct cpufreq_softc *sc;
+	int type, set_count, error;
+
+	sc = device_get_softc(cf_dev);
+	dev = sc->cf_drv_dev;
+
+	/* Skip devices that aren't ready. */
+	if (!device_is_attached(cf_dev))
+		return (0);
+
+	/*
+	 * Get settings, skipping drivers that offer no settings or
+	 * provide settings for informational purposes only.
+	 */
+	error = CPUFREQ_DRV_TYPE(dev, &type);
+	if (error != 0 || (type & CPUFREQ_FLAG_INFO_ONLY)) {
+		if (error == 0) {
+			CF_DEBUG("skipping info-only driver %s\n",
+			    device_get_nameunit(cf_dev));
+		}
+		return (error);
+	}
+
+	sets = malloc(MAX_SETTINGS * sizeof(*sets), M_TEMP, M_NOWAIT);
+	if (sets == NULL)
+		return (ENOMEM);
+
+	set_count = MAX_SETTINGS;
+	error = CPUFREQ_DRV_SETTINGS(dev, sets, &set_count);
+	if (error != 0 || set_count == 0)
+		goto out;
+
+	/* Add the settings to our absolute/relative lists. */
+	switch (type & CPUFREQ_TYPE_MASK) {
+	case CPUFREQ_TYPE_ABSOLUTE:
+		error = cpufreq_insert_abs(sc, sets, set_count);
+		break;
+	case CPUFREQ_TYPE_RELATIVE:
+		CF_DEBUG("adding %d relative settings\n", set_count);
+		set_arr = malloc(sizeof(*set_arr), M_TEMP, M_NOWAIT);
+		if (set_arr == NULL) {
+			error = ENOMEM;
+			goto out;
+		}
+		bcopy(sets, set_arr->sets, set_count * sizeof(*sets));
+		set_arr->count = set_count;
+		TAILQ_INSERT_TAIL(rel_sets, set_arr, link);
+		break;
+	default:
+		error = EINVAL;
+	}
+
+out:
+	free(sets, M_TEMP);
+	return (error);
+}
+
 static int
 cf_levels_method(device_t dev, struct cf_level *levels, int *count)
 {
@@ -532,10 +631,8 @@ cf_levels_method(device_t dev, struct cf_level *levels, int *count)
 	struct cf_setting_lst rel_sets;
 	struct cpufreq_softc *sc;
 	struct cf_level *lev;
-	struct cf_setting *sets;
 	struct pcpu *pc;
-	device_t *devs;
-	int error, i, numdevs, set_count, type;
+	int error, i;
 	uint64_t rate;
 
 	if (levels == NULL || count == NULL)
@@ -543,67 +640,21 @@ cf_levels_method(device_t dev, struct cf_level *levels, int *count)
 
 	TAILQ_INIT(&rel_sets);
 	sc = device_get_softc(dev);
-	error = device_get_children(device_get_parent(dev), &devs, &numdevs);
-	if (error)
-		return (error);
-	sets = malloc(MAX_SETTINGS * sizeof(*sets), M_TEMP, M_NOWAIT);
-	if (sets == NULL) {
-		free(devs, M_TEMP);
-		return (ENOMEM);
-	}
 
-	/* Get settings from all cpufreq drivers. */
 	CF_MTX_LOCK(&sc->lock);
-	for (i = 0; i < numdevs; i++) {
-		/* Skip devices that aren't ready. */
-		if (!device_is_attached(devs[i]))
-			continue;
-
-		/*
-		 * Get settings, skipping drivers that offer no settings or
-		 * provide settings for informational purposes only.
-		 */
-		error = CPUFREQ_DRV_TYPE(devs[i], &type);
-		if (error || (type & CPUFREQ_FLAG_INFO_ONLY)) {
-			if (error == 0) {
-				CF_DEBUG("skipping info-only driver %s\n",
-				    device_get_nameunit(devs[i]));
-			}
-			continue;
-		}
-		set_count = MAX_SETTINGS;
-		error = CPUFREQ_DRV_SETTINGS(devs[i], sets, &set_count);
-		if (error || set_count == 0)
-			continue;
-
-		/* Add the settings to our absolute/relative lists. */
-		switch (type & CPUFREQ_TYPE_MASK) {
-		case CPUFREQ_TYPE_ABSOLUTE:
-			error = cpufreq_insert_abs(sc, sets, set_count);
-			break;
-		case CPUFREQ_TYPE_RELATIVE:
-			CF_DEBUG("adding %d relative settings\n", set_count);
-			set_arr = malloc(sizeof(*set_arr), M_TEMP, M_NOWAIT);
-			if (set_arr == NULL) {
-				error = ENOMEM;
-				goto out;
-			}
-			bcopy(sets, set_arr->sets, set_count * sizeof(*sets));
-			set_arr->count = set_count;
-			TAILQ_INSERT_TAIL(&rel_sets, set_arr, link);
-			break;
-		default:
-			error = EINVAL;
-		}
-		if (error)
-			goto out;
-	}
+	error = cpufreq_add_levels(sc->dev, &rel_sets);
+	if (error)
+		goto out;
 
 	/*
 	 * If there are no absolute levels, create a fake one at 100%.  We
 	 * then cache the clockrate for later use as our base frequency.
 	 */
 	if (TAILQ_EMPTY(&sc->all_levels)) {
+		struct cf_setting set;
+
+		CF_DEBUG("No absolute levels returned by driver\n");
+
 		if (sc->max_mhz == CPUFREQ_VAL_UNKNOWN) {
 			sc->max_mhz = cpu_get_nominal_mhz(dev);
 			/*
@@ -617,10 +668,10 @@ cf_levels_method(device_t dev, struct cf_level *levels, int *count)
 				sc->max_mhz = rate / 1000000;
 			}
 		}
-		memset(&sets[0], CPUFREQ_VAL_UNKNOWN, sizeof(*sets));
-		sets[0].freq = sc->max_mhz;
-		sets[0].dev = NULL;
-		error = cpufreq_insert_abs(sc, sets, 1);
+		memset(&set, CPUFREQ_VAL_UNKNOWN, sizeof(set));
+		set.freq = sc->max_mhz;
+		set.dev = NULL;
+		error = cpufreq_insert_abs(sc, &set, 1);
 		if (error)
 			goto out;
 	}
@@ -665,8 +716,6 @@ out:
 		TAILQ_REMOVE(&rel_sets, set_arr, link);
 		free(set_arr, M_TEMP);
 	}
-	free(devs, M_TEMP);
-	free(sets, M_TEMP);
 	return (error);
 }
 
@@ -1011,11 +1060,24 @@ out:
 	return (error);
 }
 
+static void
+cpufreq_add_freq_driver_sysctl(device_t cf_dev)
+{
+	struct cpufreq_softc *sc;
+
+	sc = device_get_softc(cf_dev);
+	SYSCTL_ADD_CONST_STRING(&sc->sysctl_ctx,
+	    SYSCTL_CHILDREN(device_get_sysctl_tree(cf_dev)), OID_AUTO,
+	    "freq_driver", CTLFLAG_RD, device_get_nameunit(sc->cf_drv_dev),
+	    "cpufreq driver used by this cpu");
+}
+
 int
 cpufreq_register(device_t dev)
 {
 	struct cpufreq_softc *sc;
 	device_t cf_dev, cpu_dev;
+	int error;
 
 	/* Add a sysctl to get each driver's settings separately. */
 	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
@@ -1031,6 +1093,7 @@ cpufreq_register(device_t dev)
 	if ((cf_dev = device_find_child(cpu_dev, "cpufreq", -1))) {
 		sc = device_get_softc(cf_dev);
 		sc->max_mhz = CPUFREQ_VAL_UNKNOWN;
+		MPASS(sc->cf_drv_dev != NULL);
 		return (0);
 	}
 
@@ -1040,40 +1103,36 @@ cpufreq_register(device_t dev)
 		return (ENOMEM);
 	device_quiet(cf_dev);
 
-	return (device_probe_and_attach(cf_dev));
+	error = device_probe_and_attach(cf_dev);
+	if (error)
+		return (error);
+
+	sc = device_get_softc(cf_dev);
+	sc->cf_drv_dev = dev;
+	cpufreq_add_freq_driver_sysctl(cf_dev);
+	return (error);
 }
 
 int
 cpufreq_unregister(device_t dev)
 {
-	device_t cf_dev, *devs;
-	int cfcount, devcount, error, i, type;
+	device_t cf_dev;
+	struct cpufreq_softc *sc;
 
 	/*
 	 * If this is the last cpufreq child device, remove the control
 	 * device as well.  We identify cpufreq children by calling a method
 	 * they support.
 	 */
-	error = device_get_children(device_get_parent(dev), &devs, &devcount);
-	if (error)
-		return (error);
 	cf_dev = device_find_child(device_get_parent(dev), "cpufreq", -1);
 	if (cf_dev == NULL) {
 		device_printf(dev,
 	"warning: cpufreq_unregister called with no cpufreq device active\n");
-		free(devs, M_TEMP);
 		return (0);
 	}
-	cfcount = 0;
-	for (i = 0; i < devcount; i++) {
-		if (!device_is_attached(devs[i]))
-			continue;
-		if (CPUFREQ_DRV_TYPE(devs[i], &type) == 0)
-			cfcount++;
-	}
-	if (cfcount <= 1)
-		device_delete_child(device_get_parent(cf_dev), cf_dev);
-	free(devs, M_TEMP);
+	sc = device_get_softc(cf_dev);
+	MPASS(sc->cf_drv_dev == dev);
+	device_delete_child(device_get_parent(cf_dev), cf_dev);
 
 	return (0);
 }
diff --git a/sys/kern/kern_intr.c b/sys/kern/kern_intr.c
index 67a799ec076f6..523811f38da68 100644
--- a/sys/kern/kern_intr.c
+++ b/sys/kern/kern_intr.c
@@ -48,6 +48,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/mutex.h>
 #include <sys/priv.h>
 #include <sys/proc.h>
+#include <sys/epoch.h>
 #include <sys/random.h>
 #include <sys/resourcevar.h>
 #include <sys/sched.h>
@@ -94,6 +95,9 @@ static int intr_storm_threshold = 0;
 SYSCTL_INT(_hw, OID_AUTO, intr_storm_threshold, CTLFLAG_RWTUN,
     &intr_storm_threshold, 0,
     "Number of consecutive interrupts before storm protection is enabled");
+static int intr_epoch_batch = 1000;
+SYSCTL_INT(_hw, OID_AUTO, intr_epoch_batch, CTLFLAG_RWTUN, &intr_epoch_batch,
+    0, "Maximum interrupt handler executions without re-entering epoch(9)");
 static TAILQ_HEAD(, intr_event) event_list =
     TAILQ_HEAD_INITIALIZER(event_list);
 static struct mtx event_lock;
@@ -190,7 +194,7 @@ intr_event_update(struct intr_event *ie)
 	/* Start off with no entropy and just the name of the event. */
 	mtx_assert(&ie->ie_lock, MA_OWNED);
 	strlcpy(ie->ie_fullname, ie->ie_name, sizeof(ie->ie_fullname));
-	ie->ie_flags &= ~IE_ENTROPY;
+	ie->ie_hflags = 0;
 	missed = 0;
 	space = 1;
 
@@ -203,8 +207,7 @@ intr_event_update(struct intr_event *ie)
 			space = 0;
 		} else
 			missed++;
-		if (ih->ih_flags & IH_ENTROPY)
-			ie->ie_flags |= IE_ENTROPY;
+		ie->ie_hflags |= ih->ih_flags;
 	}
 
 	/*
@@ -588,6 +591,8 @@ intr_event_add_handler(struct intr_event *ie, const char *name,
 		ih->ih_flags |= IH_MPSAFE;
 	if (flags & INTR_ENTROPY)
 		ih->ih_flags |= IH_ENTROPY;
+	if (flags & INTR_TYPE_NET)
+		ih->ih_flags |= IH_NET;
 
 	/* We can only have one exclusive handler in a event. */
 	mtx_lock(&ie->ie_lock);
@@ -958,7 +963,7 @@ intr_event_schedule_thread(struct intr_event *ie)
 	 * If any of the handlers for this ithread claim to be good
 	 * sources of entropy, then gather some.
 	 */
-	if (ie->ie_flags & IE_ENTROPY) {
+	if (ie->ie_hflags & IH_ENTROPY) {
 		entropy.event = (uintptr_t)ie;
 		entropy.td = ctd;
 		random_harvest_queue(&entropy, sizeof(entropy), RANDOM_INTERRUPT);
@@ -1197,11 +1202,12 @@ ithread_execute_handlers(struct proc *p, struct intr_event *ie)
 static void
 ithread_loop(void *arg)
 {
+	struct epoch_tracker et;
 	struct intr_thread *ithd;
 	struct intr_event *ie;
 	struct thread *td;
 	struct proc *p;
-	int wake;
+	int wake, epoch_count;
 
 	td = curthread;
 	p = td->td_proc;
@@ -1236,8 +1242,21 @@ ithread_loop(void *arg)
 		 * that the load of ih_need in ithread_execute_handlers()
 		 * is ordered after the load of it_need here.
 		 */
-		while (atomic_cmpset_acq_int(&ithd->it_need, 1, 0) != 0)
+		if (ie->ie_hflags & IH_NET) {
+			epoch_count = 0;
+			NET_EPOCH_ENTER(et);
+		}
+		while (atomic_cmpset_acq_int(&ithd->it_need, 1, 0) != 0) {
 			ithread_execute_handlers(p, ie);
+			if ((ie->ie_hflags & IH_NET) &&
+			    ++epoch_count >= intr_epoch_batch) {
+				NET_EPOCH_EXIT(et);
+				epoch_count = 0;
+				NET_EPOCH_ENTER(et);
+			}
+		}
+		if (ie->ie_hflags & IH_NET)
+			NET_EPOCH_EXIT(et);
 		WITNESS_WARN(WARN_PANIC, NULL, "suspending ithread");
 		mtx_assert(&Giant, MA_NOTOWNED);
 
@@ -1492,7 +1511,7 @@ db_dump_intr_event(struct intr_event *ie, int handlers)
 		db_printf("(pid %d)", it->it_thread->td_proc->p_pid);
 	else
 		db_printf("(no thread)");
-	if ((ie->ie_flags & (IE_SOFT | IE_ENTROPY | IE_ADDING_THREAD)) != 0 ||
+	if ((ie->ie_flags & (IE_SOFT | IE_ADDING_THREAD)) != 0 ||
 	    (it != NULL && it->it_need)) {
 		db_printf(" {");
 		comma = 0;
@@ -1500,12 +1519,6 @@ db_dump_intr_event(struct intr_event *ie, int handlers)
 			db_printf("SOFT");
 			comma = 1;
 		}
-		if (ie->ie_flags & IE_ENTROPY) {
-			if (comma)
-				db_printf(", ");
-			db_printf("ENTROPY");
-			comma = 1;
-		}
 		if (ie->ie_flags & IE_ADDING_THREAD) {
 			if (comma)
 				db_printf(", ");
diff --git a/sys/kern/kern_kcov.c b/sys/kern/kern_kcov.c
index e84b403cd31eb..1e9ed224f17ec 100644
--- a/sys/kern/kern_kcov.c
+++ b/sys/kern/kern_kcov.c
@@ -383,8 +383,9 @@ kcov_alloc(struct kcov_info *info, size_t entries)
 	VM_OBJECT_WLOCK(info->bufobj);
 	for (n = 0; n < pages; n++) {
 		m = vm_page_grab(info->bufobj, n,
-		    VM_ALLOC_NOBUSY | VM_ALLOC_ZERO | VM_ALLOC_WIRED);
-		m->valid = VM_PAGE_BITS_ALL;
+		    VM_ALLOC_ZERO | VM_ALLOC_WIRED);
+		vm_page_valid(m);
+		vm_page_xunbusy(m);
 		pmap_qenter(info->kvaddr + n * PAGE_SIZE, &m, 1);
 	}
 	VM_OBJECT_WUNLOCK(info->bufobj);
diff --git a/sys/kern/kern_lock.c b/sys/kern/kern_lock.c
index ae3598000edb7..b48bc608e1774 100644
--- a/sys/kern/kern_lock.c
+++ b/sys/kern/kern_lock.c
@@ -209,7 +209,6 @@ static void
 lockmgr_note_shared_release(struct lock *lk, const char *file, int line)
 {
 
-	LOCKSTAT_PROFILE_RELEASE_RWLOCK(lockmgr__release, lk, LOCKSTAT_READER);
 	WITNESS_UNLOCK(&lk->lock_object, 0, file, line);
 	LOCK_LOG_LOCK("SUNLOCK", &lk->lock_object, 0, 0, file, line);
 	TD_LOCKS_DEC(curthread);
@@ -234,11 +233,12 @@ static void
 lockmgr_note_exclusive_release(struct lock *lk, const char *file, int line)
 {
 
-	LOCKSTAT_PROFILE_RELEASE_RWLOCK(lockmgr__release, lk, LOCKSTAT_WRITER);
+	if (LK_HOLDER(lk->lk_lock) != LK_KERNPROC) {
+		WITNESS_UNLOCK(&lk->lock_object, LOP_EXCLUSIVE, file, line);
+		TD_LOCKS_DEC(curthread);
+	}
 	LOCK_LOG_LOCK("XUNLOCK", &lk->lock_object, 0, lk->lk_recurse, file,
 	    line);
-	WITNESS_UNLOCK(&lk->lock_object, LOP_EXCLUSIVE, file, line);
-	TD_LOCKS_DEC(curthread);
 }
 
 static __inline struct thread *
@@ -388,7 +388,7 @@ retry_sleepq:
 		break;
 	}
 
-	lockmgr_note_shared_release(lk, file, line);
+	LOCKSTAT_PROFILE_RELEASE_RWLOCK(lockmgr__release, lk, LOCKSTAT_READER);
 	return (wakeup_swapper);
 }
 
@@ -924,6 +924,7 @@ lockmgr_upgrade(struct lock *lk, u_int flags, struct lock_object *ilk,
 	 * We have been unable to succeed in upgrading, so just
 	 * give up the shared lock.
 	 */
+	lockmgr_note_shared_release(lk, file, line);
 	wakeup_swapper |= wakeupshlk(lk, file, line);
 	error = lockmgr_xlock_hard(lk, flags, ilk, file, line, lwa);
 	flags &= ~LK_INTERLOCK;
@@ -1034,11 +1035,6 @@ lockmgr_xunlock_hard(struct lock *lk, uintptr_t x, u_int flags, struct lock_obje
 	 */
 	if (LK_HOLDER(x) == LK_KERNPROC)
 		tid = LK_KERNPROC;
-	else {
-		WITNESS_UNLOCK(&lk->lock_object, LOP_EXCLUSIVE, file, line);
-		TD_LOCKS_DEC(curthread);
-	}
-	LOCK_LOG_LOCK("XUNLOCK", &lk->lock_object, 0, lk->lk_recurse, file, line);
 
 	/*
 	 * The lock is held in exclusive mode.
@@ -1135,16 +1131,18 @@ lockmgr_unlock_fast_path(struct lock *lk, u_int flags, struct lock_object *ilk)
 	_lockmgr_assert(lk, KA_LOCKED, file, line);
 	x = lk->lk_lock;
 	if (__predict_true(x & LK_SHARE) != 0) {
+		lockmgr_note_shared_release(lk, file, line);
 		if (lockmgr_sunlock_try(lk, &x)) {
-			lockmgr_note_shared_release(lk, file, line);
+			LOCKSTAT_PROFILE_RELEASE_RWLOCK(lockmgr__release, lk, LOCKSTAT_READER);
 		} else {
 			return (lockmgr_sunlock_hard(lk, x, flags, ilk, file, line));
 		}
 	} else {
 		tid = (uintptr_t)curthread;
+		lockmgr_note_exclusive_release(lk, file, line);
 		if (!lockmgr_recursed(lk) &&
 		    atomic_cmpset_rel_ptr(&lk->lk_lock, tid, LK_UNLOCKED)) {
-			lockmgr_note_exclusive_release(lk, file, line);
+			LOCKSTAT_PROFILE_RELEASE_RWLOCK(lockmgr__release, lk, LOCKSTAT_WRITER);
 		} else {
 			return (lockmgr_xunlock_hard(lk, x, flags, ilk, file, line));
 		}
@@ -1221,16 +1219,18 @@ lockmgr_unlock(struct lock *lk)
 	_lockmgr_assert(lk, KA_LOCKED, file, line);
 	x = lk->lk_lock;
 	if (__predict_true(x & LK_SHARE) != 0) {
+		lockmgr_note_shared_release(lk, file, line);
 		if (lockmgr_sunlock_try(lk, &x)) {
-			lockmgr_note_shared_release(lk, file, line);
+			LOCKSTAT_PROFILE_RELEASE_RWLOCK(lockmgr__release, lk, LOCKSTAT_READER);
 		} else {
 			return (lockmgr_sunlock_hard(lk, x, LK_RELEASE, NULL, file, line));
 		}
 	} else {
 		tid = (uintptr_t)curthread;
+		lockmgr_note_exclusive_release(lk, file, line);
 		if (!lockmgr_recursed(lk) &&
 		    atomic_cmpset_rel_ptr(&lk->lk_lock, tid, LK_UNLOCKED)) {
-			lockmgr_note_exclusive_release(lk, file, line);
+			LOCKSTAT_PROFILE_RELEASE_RWLOCK(lockmgr__release, lk, LOCKSTAT_WRITER);
 		} else {
 			return (lockmgr_xunlock_hard(lk, x, LK_RELEASE, NULL, file, line));
 		}
@@ -1347,8 +1347,10 @@ __lockmgr_args(struct lock *lk, u_int flags, struct lock_object *ilk,
 		x = lk->lk_lock;
 
 		if (__predict_true(x & LK_SHARE) != 0) {
+			lockmgr_note_shared_release(lk, file, line);
 			return (lockmgr_sunlock_hard(lk, x, flags, ilk, file, line));
 		} else {
+			lockmgr_note_exclusive_release(lk, file, line);
 			return (lockmgr_xunlock_hard(lk, x, flags, ilk, file, line));
 		}
 		break;
diff --git a/sys/kern/kern_poll.c b/sys/kern/kern_poll.c
index c6e1e7bd09441..3663b7d3df2fe 100644
--- a/sys/kern/kern_poll.c
+++ b/sys/kern/kern_poll.c
@@ -37,6 +37,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/kernel.h>
 #include <sys/kthread.h>
 #include <sys/proc.h>
+#include <sys/epoch.h>
 #include <sys/eventhandler.h>
 #include <sys/resourcevar.h>
 #include <sys/socket.h>			/* needed by net/if.h		*/
@@ -332,6 +333,7 @@ hardclock_device_poll(void)
 static void
 ether_poll(int count)
 {
+	struct epoch_tracker et;
 	int i;
 
 	mtx_lock(&poll_mtx);
@@ -339,8 +341,10 @@ ether_poll(int count)
 	if (count > poll_each_burst)
 		count = poll_each_burst;
 
+	NET_EPOCH_ENTER(et);
 	for (i = 0 ; i < poll_handlers ; i++)
 		pr[i].handler(pr[i].ifp, POLL_ONLY, count);
+	NET_EPOCH_EXIT(et);
 
 	mtx_unlock(&poll_mtx);
 }
@@ -429,6 +433,8 @@ netisr_poll(void)
 	int i, cycles;
 	enum poll_cmd arg = POLL_ONLY;
 
+	NET_EPOCH_ASSERT();
+
 	if (poll_handlers == 0)
 		return;
 
diff --git a/sys/kern/kern_sendfile.c b/sys/kern/kern_sendfile.c
index 106019328b7e4..5b3f15af08289 100644
--- a/sys/kern/kern_sendfile.c
+++ b/sys/kern/kern_sendfile.c
@@ -388,7 +388,7 @@ sendfile_swapin(vm_object_t obj, struct sf_io *sfio, int *nios, off_t off,
 		if (!vm_pager_has_page(obj, OFF_TO_IDX(vmoff(i, off)), NULL,
 		    &a)) {
 			pmap_zero_page(pa[i]);
-			pa[i]->valid = VM_PAGE_BITS_ALL;
+			vm_page_valid(pa[i]);
 			MPASS(pa[i]->dirty == 0);
 			vm_page_xunbusy(pa[i]);
 			i++;
diff --git a/sys/kern/sched_ule.c b/sys/kern/sched_ule.c
index 3c35179e2bee0..7dc48a2336bf5 100644
--- a/sys/kern/sched_ule.c
+++ b/sys/kern/sched_ule.c
@@ -2894,7 +2894,7 @@ sched_throw(struct thread *td)
 	struct thread *newtd;
 	struct tdq *tdq;
 
-	if (td == NULL) {
+	if (__predict_false(td == NULL)) {
 #ifdef SMP
 		PCPU_SET(sched, DPCPU_PTR(tdq));
 #endif
@@ -2912,13 +2912,18 @@ sched_throw(struct thread *td)
 		tdq_load_rem(tdq, td);
 		td->td_lastcpu = td->td_oncpu;
 		td->td_oncpu = NOCPU;
+		thread_lock_block(td);
 	}
 	newtd = choosethread();
 	spinlock_enter();
 	TDQ_UNLOCK(tdq);
 	KASSERT(curthread->td_md.md_spinlock_count == 1,
 	    ("invalid count %d", curthread->td_md.md_spinlock_count));
-	cpu_throw(td, newtd);		/* doesn't return */
+	/* doesn't return */
+	if (__predict_false(td == NULL))
+		cpu_throw(td, newtd);		/* doesn't return */
+	else
+		cpu_switch(td, newtd, TDQ_LOCKPTR(tdq));
 }
 
 /*
diff --git a/sys/kern/uipc_ktls.c b/sys/kern/uipc_ktls.c
index a43ffbbc391e3..a6f272329c717 100644
--- a/sys/kern/uipc_ktls.c
+++ b/sys/kern/uipc_ktls.c
@@ -1141,7 +1141,9 @@ ktls_reset_send_tag(void *context, int pending)
 			if (!(inp->inp_flags & INP_TIMEWAIT) &&
 			    !(inp->inp_flags & INP_DROPPED)) {
 				tp = intotcpcb(inp);
+				CURVNET_SET(tp->t_vnet);
 				tp = tcp_drop(tp, ECONNABORTED);
+				CURVNET_RESTORE();
 				if (tp != NULL)
 					INP_WUNLOCK(inp);
 				counter_u64_add(ktls_ifnet_reset_dropped, 1);
diff --git a/sys/kern/uipc_usrreq.c b/sys/kern/uipc_usrreq.c
index 1a95b289ec086..0bede5162fef5 100644
--- a/sys/kern/uipc_usrreq.c
+++ b/sys/kern/uipc_usrreq.c
@@ -260,7 +260,7 @@ static struct mtx	unp_defers_lock;
 #define	UNP_LINK_LOCK_INIT()		rw_init(&unp_link_rwlock,	\
 					    "unp_link_rwlock")
 
-#define	UNP_LINK_LOCK_ASSERT()	rw_assert(&unp_link_rwlock,	\
+#define	UNP_LINK_LOCK_ASSERT()		rw_assert(&unp_link_rwlock,	\
 					    RA_LOCKED)
 #define	UNP_LINK_UNLOCK_ASSERT()	rw_assert(&unp_link_rwlock,	\
 					    RA_UNLOCKED)
@@ -778,6 +778,8 @@ uipc_detach(struct socket *so)
 
 	UNP_LINK_WLOCK();
 	LIST_REMOVE(unp, unp_link);
+	if (unp->unp_gcflag & UNPGC_DEAD)
+		LIST_REMOVE(unp, unp_dead);
 	unp->unp_gencnt = ++unp_gencnt;
 	--unp_count;
 	UNP_LINK_WUNLOCK();
@@ -2481,49 +2483,60 @@ unp_externalize_fp(struct file *fp)
  * synchronization.
  */
 static int	unp_marked;
-static int	unp_unreachable;
 
 static void
-unp_accessable(struct filedescent **fdep, int fdcount)
+unp_remove_dead_ref(struct filedescent **fdep, int fdcount)
 {
 	struct unpcb *unp;
 	struct file *fp;
 	int i;
 
+	/*
+	 * This function can only be called from the gc task.
+	 */
+	KASSERT(taskqueue_member(taskqueue_thread, curthread) != 0,
+	    ("%s: not on gc callout", __func__));
+	UNP_LINK_LOCK_ASSERT();
+
 	for (i = 0; i < fdcount; i++) {
 		fp = fdep[i]->fde_file;
 		if ((unp = fptounp(fp)) == NULL)
 			continue;
-		if (unp->unp_gcflag & UNPGC_REF)
+		if ((unp->unp_gcflag & UNPGC_DEAD) == 0)
 			continue;
-		unp->unp_gcflag &= ~UNPGC_DEAD;
-		unp->unp_gcflag |= UNPGC_REF;
-		unp_marked++;
+		unp->unp_gcrefs--;
 	}
 }
 
 static void
-unp_gc_process(struct unpcb *unp)
+unp_restore_undead_ref(struct filedescent **fdep, int fdcount)
 {
-	struct socket *so, *soa;
+	struct unpcb *unp;
 	struct file *fp;
-
-	/* Already processed. */
-	if (unp->unp_gcflag & UNPGC_SCANNED)
-		return;
-	fp = unp->unp_file;
+	int i;
 
 	/*
-	 * Check for a socket potentially in a cycle.  It must be in a
-	 * queue as indicated by msgcount, and this must equal the file
-	 * reference count.  Note that when msgcount is 0 the file is NULL.
+	 * This function can only be called from the gc task.
 	 */
-	if ((unp->unp_gcflag & UNPGC_REF) == 0 && fp &&
-	    unp->unp_msgcount != 0 && fp->f_count == unp->unp_msgcount) {
-		unp->unp_gcflag |= UNPGC_DEAD;
-		unp_unreachable++;
-		return;
+	KASSERT(taskqueue_member(taskqueue_thread, curthread) != 0,
+	    ("%s: not on gc callout", __func__));
+	UNP_LINK_LOCK_ASSERT();
+
+	for (i = 0; i < fdcount; i++) {
+		fp = fdep[i]->fde_file;
+		if ((unp = fptounp(fp)) == NULL)
+			continue;
+		if ((unp->unp_gcflag & UNPGC_DEAD) == 0)
+			continue;
+		unp->unp_gcrefs++;
+		unp_marked++;
 	}
+}
+
+static void
+unp_gc_scan(struct unpcb *unp, void (*op)(struct filedescent **, int))
+{
+	struct socket *so, *soa;
 
 	so = unp->unp_socket;
 	SOCK_LOCK(so);
@@ -2535,7 +2548,7 @@ unp_gc_process(struct unpcb *unp)
 			if (sotounpcb(soa)->unp_gcflag & UNPGC_IGNORE_RIGHTS)
 				continue;
 			SOCKBUF_LOCK(&soa->so_rcv);
-			unp_scan(soa->so_rcv.sb_mb, unp_accessable);
+			unp_scan(soa->so_rcv.sb_mb, op);
 			SOCKBUF_UNLOCK(&soa->so_rcv);
 		}
 	} else {
@@ -2544,12 +2557,11 @@ unp_gc_process(struct unpcb *unp)
 		 */
 		if ((unp->unp_gcflag & UNPGC_IGNORE_RIGHTS) == 0) {
 			SOCKBUF_LOCK(&so->so_rcv);
-			unp_scan(so->so_rcv.sb_mb, unp_accessable);
+			unp_scan(so->so_rcv.sb_mb, op);
 			SOCKBUF_UNLOCK(&so->so_rcv);
 		}
 	}
 	SOCK_UNLOCK(so);
-	unp->unp_gcflag |= UNPGC_SCANNED;
 }
 
 static int unp_recycled;
@@ -2560,67 +2572,115 @@ static int unp_taskcount;
 SYSCTL_INT(_net_local, OID_AUTO, taskcount, CTLFLAG_RD, &unp_taskcount, 0, 
     "Number of times the garbage collector has run.");
 
+SYSCTL_UINT(_net_local, OID_AUTO, sockcount, CTLFLAG_RD, &unp_count, 0, 
+    "Number of active local sockets.");
+
 static void
 unp_gc(__unused void *arg, int pending)
 {
 	struct unp_head *heads[] = { &unp_dhead, &unp_shead, &unp_sphead,
 				    NULL };
 	struct unp_head **head;
+	struct unp_head unp_deadhead;	/* List of potentially-dead sockets. */
 	struct file *f, **unref;
-	struct unpcb *unp;
-	int i, total;
+	struct unpcb *unp, *unptmp;
+	int i, total, unp_unreachable;
 
+	LIST_INIT(&unp_deadhead);
 	unp_taskcount++;
 	UNP_LINK_RLOCK();
 	/*
-	 * First clear all gc flags from previous runs, apart from
-	 * UNPGC_IGNORE_RIGHTS.
+	 * First determine which sockets may be in cycles.
 	 */
+	unp_unreachable = 0;
+
 	for (head = heads; *head != NULL; head++)
-		LIST_FOREACH(unp, *head, unp_link)
-			unp->unp_gcflag =
-			    (unp->unp_gcflag & UNPGC_IGNORE_RIGHTS);
+		LIST_FOREACH(unp, *head, unp_link) {
+
+			KASSERT((unp->unp_gcflag & ~UNPGC_IGNORE_RIGHTS) == 0,
+			    ("%s: unp %p has unexpected gc flags 0x%x",
+			    __func__, unp, (unsigned int)unp->unp_gcflag));
+
+			f = unp->unp_file;
+
+			/*
+			 * Check for an unreachable socket potentially in a
+			 * cycle.  It must be in a queue as indicated by
+			 * msgcount, and this must equal the file reference
+			 * count.  Note that when msgcount is 0 the file is
+			 * NULL.
+			 */
+			if (f != NULL && unp->unp_msgcount != 0 &&
+			    f->f_count == unp->unp_msgcount) {
+				LIST_INSERT_HEAD(&unp_deadhead, unp, unp_dead);
+				unp->unp_gcflag |= UNPGC_DEAD;
+				unp->unp_gcrefs = unp->unp_msgcount;
+				unp_unreachable++;
+			}
+		}
 
 	/*
-	 * Scan marking all reachable sockets with UNPGC_REF.  Once a socket
-	 * is reachable all of the sockets it references are reachable.
+	 * Scan all sockets previously marked as potentially being in a cycle
+	 * and remove the references each socket holds on any UNPGC_DEAD
+	 * sockets in its queue.  After this step, all remaining references on
+	 * sockets marked UNPGC_DEAD should not be part of any cycle.
+	 */
+	LIST_FOREACH(unp, &unp_deadhead, unp_dead)
+		unp_gc_scan(unp, unp_remove_dead_ref);
+
+	/*
+	 * If a socket still has a non-negative refcount, it cannot be in a
+	 * cycle.  In this case increment refcount of all children iteratively.
 	 * Stop the scan once we do a complete loop without discovering
 	 * a new reachable socket.
 	 */
 	do {
-		unp_unreachable = 0;
 		unp_marked = 0;
-		for (head = heads; *head != NULL; head++)
-			LIST_FOREACH(unp, *head, unp_link)
-				unp_gc_process(unp);
+		LIST_FOREACH_SAFE(unp, &unp_deadhead, unp_dead, unptmp)
+			if (unp->unp_gcrefs > 0) {
+				unp->unp_gcflag &= ~UNPGC_DEAD;
+				LIST_REMOVE(unp, unp_dead);
+				KASSERT(unp_unreachable > 0,
+				    ("%s: unp_unreachable underflow.",
+				    __func__));
+				unp_unreachable--;
+				unp_gc_scan(unp, unp_restore_undead_ref);
+			}
 	} while (unp_marked);
+
 	UNP_LINK_RUNLOCK();
+
 	if (unp_unreachable == 0)
 		return;
 
 	/*
-	 * Allocate space for a local list of dead unpcbs.
+	 * Allocate space for a local array of dead unpcbs.
+	 * TODO: can this path be simplified by instead using the local
+	 * dead list at unp_deadhead, after taking out references
+	 * on the file object and/or unpcb and dropping the link lock?
 	 */
 	unref = malloc(unp_unreachable * sizeof(struct file *),
 	    M_TEMP, M_WAITOK);
 
 	/*
 	 * Iterate looking for sockets which have been specifically marked
-	 * as as unreachable and store them locally.
+	 * as unreachable and store them locally.
 	 */
 	UNP_LINK_RLOCK();
-	for (total = 0, head = heads; *head != NULL; head++)
-		LIST_FOREACH(unp, *head, unp_link)
-			if ((unp->unp_gcflag & UNPGC_DEAD) != 0) {
-				f = unp->unp_file;
-				if (unp->unp_msgcount == 0 || f == NULL ||
-				    f->f_count != unp->unp_msgcount ||
-				    !fhold(f))
-					continue;
-				unref[total++] = f;
-				KASSERT(total <= unp_unreachable,
-				    ("unp_gc: incorrect unreachable count."));
-			}
+	total = 0;
+	LIST_FOREACH(unp, &unp_deadhead, unp_dead) {
+		KASSERT((unp->unp_gcflag & UNPGC_DEAD) != 0,
+		    ("%s: unp %p not marked UNPGC_DEAD", __func__, unp));
+		unp->unp_gcflag &= ~UNPGC_DEAD;
+		f = unp->unp_file;
+		if (unp->unp_msgcount == 0 || f == NULL ||
+		    f->f_count != unp->unp_msgcount ||
+		    !fhold(f))
+			continue;
+		unref[total++] = f;
+		KASSERT(total <= unp_unreachable,
+		    ("%s: incorrect unreachable count.", __func__));
+	}
 	UNP_LINK_RUNLOCK();
 
 	/*
diff --git a/sys/kern/vfs_mount.c b/sys/kern/vfs_mount.c
index 55e46c085d462..47a5193eebf55 100644
--- a/sys/kern/vfs_mount.c
+++ b/sys/kern/vfs_mount.c
@@ -1294,12 +1294,19 @@ struct unmount_args {
 int
 sys_unmount(struct thread *td, struct unmount_args *uap)
 {
+
+	return (kern_unmount(td, uap->path, uap->flags));
+}
+
+int
+kern_unmount(struct thread *td, const char *path, int flags)
+{
 	struct nameidata nd;
 	struct mount *mp;
 	char *pathbuf;
 	int error, id0, id1;
 
-	AUDIT_ARG_VALUE(uap->flags);
+	AUDIT_ARG_VALUE(flags);
 	if (jailed(td->td_ucred) || usermount == 0) {
 		error = priv_check(td, PRIV_VFS_UNMOUNT);
 		if (error)
@@ -1307,12 +1314,12 @@ sys_unmount(struct thread *td, struct unmount_args *uap)
 	}
 
 	pathbuf = malloc(MNAMELEN, M_TEMP, M_WAITOK);
-	error = copyinstr(uap->path, pathbuf, MNAMELEN, NULL);
+	error = copyinstr(path, pathbuf, MNAMELEN, NULL);
 	if (error) {
 		free(pathbuf, M_TEMP);
 		return (error);
 	}
-	if (uap->flags & MNT_BYFSID) {
+	if (flags & MNT_BYFSID) {
 		AUDIT_ARG_TEXT(pathbuf);
 		/* Decode the filesystem ID. */
 		if (sscanf(pathbuf, "FSID:%d:%d", &id0, &id1) != 2) {
@@ -1359,7 +1366,7 @@ sys_unmount(struct thread *td, struct unmount_args *uap)
 		 * now, so in the !MNT_BYFSID case return the more likely
 		 * EINVAL for compatibility.
 		 */
-		return ((uap->flags & MNT_BYFSID) ? ENOENT : EINVAL);
+		return ((flags & MNT_BYFSID) ? ENOENT : EINVAL);
 	}
 
 	/*
@@ -1369,7 +1376,7 @@ sys_unmount(struct thread *td, struct unmount_args *uap)
 		vfs_rel(mp);
 		return (EINVAL);
 	}
-	error = dounmount(mp, uap->flags, td);
+	error = dounmount(mp, flags, td);
 	return (error);
 }
 
diff --git a/sys/kern/vfs_subr.c b/sys/kern/vfs_subr.c
index d8d4d074b88b1..b8278f1cd5213 100644
--- a/sys/kern/vfs_subr.c
+++ b/sys/kern/vfs_subr.c
@@ -2826,11 +2826,10 @@ v_decr_devcount(struct vnode *vp)
  * see doomed vnodes.  If inactive processing was delayed in
  * vput try to do it here.
  *
- * usecount is manipulated using atomics without holding any locks,
- * except when transitioning 0->1 in which case the interlock is held.
-
- * holdcnt is manipulated using atomics without holding any locks,
- * except when transitioning 1->0 in which case the interlock is held.
+ * usecount is manipulated using atomics without holding any locks.
+ *
+ * holdcnt can be manipulated using atomics without holding any locks,
+ * except when transitioning 1<->0, in which case the interlock is held.
  */
 enum vgetstate
 vget_prep(struct vnode *vp)
@@ -2857,10 +2856,46 @@ vget(struct vnode *vp, int flags, struct thread *td)
 	return (vget_finish(vp, flags, vs));
 }
 
+static int __noinline
+vget_finish_vchr(struct vnode *vp)
+{
+
+	VNASSERT(vp->v_type == VCHR, vp, ("type != VCHR)"));
+
+	/*
+	 * See the comment in vget_finish before usecount bump.
+	 */
+	if (refcount_acquire_if_not_zero(&vp->v_usecount)) {
+#ifdef INVARIANTS
+		int old = atomic_fetchadd_int(&vp->v_holdcnt, -1);
+		VNASSERT(old > 0, vp, ("%s: wrong hold count %d", __func__, old));
+#else
+		refcount_release(&vp->v_holdcnt);
+#endif
+		return (0);
+	}
+
+	VI_LOCK(vp);
+	if (refcount_acquire_if_not_zero(&vp->v_usecount)) {
+#ifdef INVARIANTS
+		int old = atomic_fetchadd_int(&vp->v_holdcnt, -1);
+		VNASSERT(old > 1, vp, ("%s: wrong hold count %d", __func__, old));
+#else
+		refcount_release(&vp->v_holdcnt);
+#endif
+		VI_UNLOCK(vp);
+		return (0);
+	}
+	v_incr_devcount(vp);
+	refcount_acquire(&vp->v_usecount);
+	VI_UNLOCK(vp);
+	return (0);
+}
+
 int
 vget_finish(struct vnode *vp, int flags, enum vgetstate vs)
 {
-	int error, oweinact;
+	int error, old;
 
 	VNASSERT((flags & LK_TYPE_MASK) != 0, vp,
 	    ("%s: invalid lock operation", __func__));
@@ -2887,93 +2922,109 @@ vget_finish(struct vnode *vp, int flags, enum vgetstate vs)
 	}
 
 	if (vs == VGET_USECOUNT) {
-		VNASSERT((vp->v_iflag & VI_OWEINACT) == 0, vp,
-		    ("%s: vnode with usecount and VI_OWEINACT set", __func__));
 		return (0);
 	}
 
+	if (__predict_false(vp->v_type == VCHR))
+		return (vget_finish_vchr(vp));
+
 	/*
 	 * We hold the vnode. If the usecount is 0 it will be utilized to keep
 	 * the vnode around. Otherwise someone else lended their hold count and
 	 * we have to drop ours.
 	 */
-	if (refcount_acquire_if_not_zero(&vp->v_usecount)) {
+	old = atomic_fetchadd_int(&vp->v_usecount, 1);
+	VNASSERT(old >= 0, vp, ("%s: wrong use count %d", __func__, old));
+	if (old != 0) {
 #ifdef INVARIANTS
-		int old = atomic_fetchadd_int(&vp->v_holdcnt, -1);
+		old = atomic_fetchadd_int(&vp->v_holdcnt, -1);
 		VNASSERT(old > 1, vp, ("%s: wrong hold count %d", __func__, old));
 #else
 		refcount_release(&vp->v_holdcnt);
 #endif
-		VNODE_REFCOUNT_FENCE_ACQ();
-		VNASSERT((vp->v_iflag & VI_OWEINACT) == 0, vp,
-		    ("%s: vnode with usecount and VI_OWEINACT set", __func__));
-		return (0);
 	}
+	return (0);
+}
+
+/*
+ * Increase the reference (use) and hold count of a vnode.
+ * This will also remove the vnode from the free list if it is presently free.
+ */
+static void __noinline
+vref_vchr(struct vnode *vp, bool interlock)
+{
 
 	/*
-	 * We don't guarantee that any particular close will
-	 * trigger inactive processing so just make a best effort
-	 * here at preventing a reference to a removed file.  If
-	 * we don't succeed no harm is done.
-	 *
-	 * Upgrade our holdcnt to a usecount.
-	 */
-	VI_LOCK(vp);
-	/*
-	 * See the previous section. By the time we get here we may find
-	 * ourselves in the same spot.
+	 * See the comment in vget_finish before usecount bump.
 	 */
+	if (!interlock) {
+		if (refcount_acquire_if_not_zero(&vp->v_usecount)) {
+			VNODE_REFCOUNT_FENCE_ACQ();
+			VNASSERT(vp->v_holdcnt > 0, vp,
+			    ("%s: active vnode not held", __func__));
+			return;
+		}
+		VI_LOCK(vp);
+		/*
+		 * By the time we get here the vnode might have been doomed, at
+		 * which point the 0->1 use count transition is no longer
+		 * protected by the interlock. Since it can't bounce back to
+		 * VCHR and requires vref semantics, punt it back
+		 */
+		if (__predict_false(vp->v_type == VBAD)) {
+			VI_UNLOCK(vp);
+			vref(vp);
+			return;
+		}
+	}
+	VNASSERT(vp->v_type == VCHR, vp, ("type != VCHR)"));
 	if (refcount_acquire_if_not_zero(&vp->v_usecount)) {
-#ifdef INVARIANTS
-		int old = atomic_fetchadd_int(&vp->v_holdcnt, -1);
-		VNASSERT(old > 1, vp, ("%s: wrong hold count %d", __func__, old));
-#else
-		refcount_release(&vp->v_holdcnt);
-#endif
 		VNODE_REFCOUNT_FENCE_ACQ();
-		VNASSERT((vp->v_iflag & VI_OWEINACT) == 0, vp,
-		    ("%s: vnode with usecount and VI_OWEINACT set",
-		    __func__));
-		VI_UNLOCK(vp);
-		return (0);
-	}
-	if ((vp->v_iflag & VI_OWEINACT) == 0) {
-		oweinact = 0;
-	} else {
-		oweinact = 1;
-		vp->v_iflag &= ~VI_OWEINACT;
-		VNODE_REFCOUNT_FENCE_REL();
+		VNASSERT(vp->v_holdcnt > 0, vp,
+		    ("%s: active vnode not held", __func__));
+		if (!interlock)
+			VI_UNLOCK(vp);
+		return;
 	}
+	vhold(vp);
 	v_incr_devcount(vp);
 	refcount_acquire(&vp->v_usecount);
-	if (oweinact && VOP_ISLOCKED(vp) == LK_EXCLUSIVE &&
-	    (flags & LK_NOWAIT) == 0)
-		vinactive(vp);
-	VI_UNLOCK(vp);
-	return (0);
+	if (!interlock)
+		VI_UNLOCK(vp);
+	return;
 }
 
-/*
- * Increase the reference (use) and hold count of a vnode.
- * This will also remove the vnode from the free list if it is presently free.
- */
 void
 vref(struct vnode *vp)
 {
+	int old;
 
-	ASSERT_VI_UNLOCKED(vp, __func__);
 	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
+	if (__predict_false(vp->v_type == VCHR)) {
+		 vref_vchr(vp, false);
+		 return;
+	}
+
 	if (refcount_acquire_if_not_zero(&vp->v_usecount)) {
 		VNODE_REFCOUNT_FENCE_ACQ();
 		VNASSERT(vp->v_holdcnt > 0, vp,
 		    ("%s: active vnode not held", __func__));
-		VNASSERT((vp->v_iflag & VI_OWEINACT) == 0, vp,
-		    ("%s: vnode with usecount and VI_OWEINACT set", __func__));
 		return;
 	}
-	VI_LOCK(vp);
-	vrefl(vp);
-	VI_UNLOCK(vp);
+	vhold(vp);
+	/*
+	 * See the comment in vget_finish.
+	 */
+	old = atomic_fetchadd_int(&vp->v_usecount, 1);
+	VNASSERT(old >= 0, vp, ("%s: wrong use count %d", __func__, old));
+	if (old != 0) {
+#ifdef INVARIANTS
+		old = atomic_fetchadd_int(&vp->v_holdcnt, -1);
+		VNASSERT(old > 1, vp, ("%s: wrong hold count %d", __func__, old));
+#else
+		refcount_release(&vp->v_holdcnt);
+#endif
+	}
 }
 
 void
@@ -2982,21 +3033,11 @@ vrefl(struct vnode *vp)
 
 	ASSERT_VI_LOCKED(vp, __func__);
 	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
-	if (refcount_acquire_if_not_zero(&vp->v_usecount)) {
-		VNODE_REFCOUNT_FENCE_ACQ();
-		VNASSERT(vp->v_holdcnt > 0, vp,
-		    ("%s: active vnode not held", __func__));
-		VNASSERT((vp->v_iflag & VI_OWEINACT) == 0, vp,
-		    ("%s: vnode with usecount and VI_OWEINACT set", __func__));
+	if (__predict_false(vp->v_type == VCHR)) {
+		vref_vchr(vp, true);
 		return;
 	}
-	vholdl(vp);
-	if ((vp->v_iflag & VI_OWEINACT) != 0) {
-		vp->v_iflag &= ~VI_OWEINACT;
-		VNODE_REFCOUNT_FENCE_REL();
-	}
-	v_incr_devcount(vp);
-	refcount_acquire(&vp->v_usecount);
+	vref(vp);
 }
 
 void
@@ -3052,8 +3093,8 @@ vdefer_inactive(struct vnode *vp)
 {
 
 	ASSERT_VI_LOCKED(vp, __func__);
-	VNASSERT(vp->v_iflag & VI_OWEINACT, vp,
-	    ("%s: vnode without VI_OWEINACT", __func__));
+	VNASSERT(vp->v_holdcnt > 0, vp,
+	    ("%s: vnode without hold count", __func__));
 	if (VN_IS_DOOMED(vp)) {
 		vdropl(vp);
 		return;
@@ -3063,6 +3104,11 @@ vdefer_inactive(struct vnode *vp)
 		vdropl(vp);
 		return;
 	}
+	if (vp->v_usecount > 0) {
+		vp->v_iflag &= ~VI_OWEINACT;
+		vdropl(vp);
+		return;
+	}
 	vlazy(vp);
 	vp->v_iflag |= VI_DEFINACT;
 	VI_UNLOCK(vp);
@@ -3070,11 +3116,10 @@ vdefer_inactive(struct vnode *vp)
 }
 
 static void
-vdefer_inactive_cond(struct vnode *vp)
+vdefer_inactive_unlocked(struct vnode *vp)
 {
 
 	VI_LOCK(vp);
-	VNASSERT(vp->v_holdcnt > 0, vp, ("vnode without hold count"));
 	if ((vp->v_iflag & VI_OWEINACT) == 0) {
 		vdropl(vp);
 		return;
@@ -3088,6 +3133,11 @@ enum vputx_op { VPUTX_VRELE, VPUTX_VPUT, VPUTX_VUNREF };
  * Decrement the use and hold counts for a vnode.
  *
  * See an explanation near vget() as to why atomic operation is safe.
+ *
+ * XXX Some filesystems pass in an exclusively locked vnode and strongly depend
+ * on the lock being held all the way until VOP_INACTIVE. This in particular
+ * happens with UFS which adds half-constructed vnodes to the hash, where they
+ * can be found by other code.
  */
 static void
 vputx(struct vnode *vp, enum vputx_op func)
@@ -3097,6 +3147,8 @@ vputx(struct vnode *vp, enum vputx_op func)
 	KASSERT(vp != NULL, ("vputx: null vp"));
 	if (func == VPUTX_VUNREF)
 		ASSERT_VOP_LOCKED(vp, "vunref");
+	else if (func == VPUTX_VPUT)
+		ASSERT_VOP_LOCKED(vp, "vput");
 	ASSERT_VI_UNLOCKED(vp, __func__);
 	VNASSERT(vp->v_holdcnt > 0 && vp->v_usecount > 0, vp,
 	    ("%s: wrong ref counts", __func__));
@@ -3112,22 +3164,19 @@ vputx(struct vnode *vp, enum vputx_op func)
 	 * count which provides liveness of the vnode, in which case we
 	 * have to vdrop.
 	 */
-	if (!refcount_release(&vp->v_usecount))
+	if (!refcount_release(&vp->v_usecount)) {
+		if (func == VPUTX_VPUT)
+			VOP_UNLOCK(vp);
 		return;
+	}
 	VI_LOCK(vp);
 	v_decr_devcount(vp);
 	/*
 	 * By the time we got here someone else might have transitioned
 	 * the count back to > 0.
 	 */
-	if (vp->v_usecount > 0) {
-		vdropl(vp);
-		return;
-	}
-	if (vp->v_iflag & VI_DOINGINACT) {
-		vdropl(vp);
-		return;
-	}
+	if (vp->v_usecount > 0 || vp->v_iflag & VI_DOINGINACT)
+		goto out;
 
 	/*
 	 * Check if the fs wants to perform inactive processing. Note we
@@ -3137,10 +3186,8 @@ vputx(struct vnode *vp, enum vputx_op func)
 	 * here but to drop our hold count.
 	 */
 	if (__predict_false(VN_IS_DOOMED(vp)) ||
-	    VOP_NEED_INACTIVE(vp) == 0) {
-		vdropl(vp);
-		return;
-	}
+	    VOP_NEED_INACTIVE(vp) == 0)
+		goto out;
 
 	/*
 	 * We must call VOP_INACTIVE with the node locked. Mark
@@ -3153,8 +3200,12 @@ vputx(struct vnode *vp, enum vputx_op func)
 		VI_LOCK(vp);
 		break;
 	case VPUTX_VPUT:
-		error = VOP_LOCK(vp, LK_EXCLUSIVE | LK_INTERLOCK | LK_NOWAIT);
-		VI_LOCK(vp);
+		error = 0;
+		if (VOP_ISLOCKED(vp) != LK_EXCLUSIVE) {
+			error = VOP_LOCK(vp, LK_UPGRADE | LK_INTERLOCK |
+			    LK_NOWAIT);
+			VI_LOCK(vp);
+		}
 		break;
 	case VPUTX_VUNREF:
 		error = 0;
@@ -3164,19 +3215,19 @@ vputx(struct vnode *vp, enum vputx_op func)
 		}
 		break;
 	}
-	VNASSERT(vp->v_usecount == 0 || (vp->v_iflag & VI_OWEINACT) == 0, vp,
-	    ("vnode with usecount and VI_OWEINACT set"));
 	if (error == 0) {
-		if (vp->v_iflag & VI_OWEINACT)
-			vinactive(vp);
+		vinactive(vp);
 		if (func != VPUTX_VUNREF)
 			VOP_UNLOCK(vp);
 		vdropl(vp);
-	} else if (vp->v_iflag & VI_OWEINACT) {
-		vdefer_inactive(vp);
 	} else {
-		vdropl(vp);
+		vdefer_inactive(vp);
 	}
+	return;
+out:
+	if (func == VPUTX_VPUT)
+		VOP_UNLOCK(vp);
+	vdropl(vp);
 }
 
 /*
@@ -3194,21 +3245,11 @@ vrele(struct vnode *vp)
  * Release an already locked vnode.  This give the same effects as
  * unlock+vrele(), but takes less time and avoids releasing and
  * re-aquiring the lock (as vrele() acquires the lock internally.)
- *
- * It is an invariant that all VOP_* calls operate on a held vnode.
- * We may be only having an implicit hold stemming from our usecount,
- * which we are about to release. If we unlock the vnode afterwards we
- * open a time window where someone else dropped the last usecount and
- * proceeded to free the vnode before our unlock finished. For this
- * reason we unlock the vnode early. This is a little bit wasteful as
- * it may be the vnode is exclusively locked and inactive processing is
- * needed, in which case we are adding work.
  */
 void
 vput(struct vnode *vp)
 {
 
-	VOP_UNLOCK(vp);
 	vputx(vp, VPUTX_VPUT);
 }
 
@@ -3441,11 +3482,9 @@ vdropl(struct vnode *vp)
 /*
  * Call VOP_INACTIVE on the vnode and manage the DOINGINACT and OWEINACT
  * flags.  DOINGINACT prevents us from recursing in calls to vinactive.
- * OWEINACT tracks whether a vnode missed a call to inactive due to a
- * failed lock upgrade.
  */
-void
-vinactive(struct vnode *vp)
+static void
+vinactivef(struct vnode *vp)
 {
 	struct vm_object *obj;
 
@@ -3480,6 +3519,25 @@ vinactive(struct vnode *vp)
 	vp->v_iflag &= ~VI_DOINGINACT;
 }
 
+void
+vinactive(struct vnode *vp)
+{
+
+	ASSERT_VOP_ELOCKED(vp, "vinactive");
+	ASSERT_VI_LOCKED(vp, "vinactive");
+	CTR2(KTR_VFS, "%s: vp %p", __func__, vp);
+
+	if ((vp->v_iflag & VI_OWEINACT) == 0)
+		return;
+	if (vp->v_iflag & VI_DOINGINACT)
+		return;
+	if (vp->v_usecount > 0) {
+		vp->v_iflag &= ~VI_OWEINACT;
+		return;
+	}
+	vinactivef(vp);
+}
+
 /*
  * Remove any vnodes in the vnode table belonging to mount point mp.
  *
@@ -3778,8 +3836,7 @@ vgonel(struct vnode *vp)
 		VOP_CLOSE(vp, FNONBLOCK, NOCRED, td);
 	if (oweinact || active) {
 		VI_LOCK(vp);
-		if ((vp->v_iflag & VI_DOINGINACT) == 0)
-			vinactive(vp);
+		vinactivef(vp);
 		VI_UNLOCK(vp);
 	}
 	if (vp->v_type == VSOCK)
@@ -4529,13 +4586,12 @@ vfs_deferred_inactive(struct vnode *vp, int lkflags)
 	}
 	if (vn_lock(vp, lkflags) == 0) {
 		VI_LOCK(vp);
-		if ((vp->v_iflag & (VI_OWEINACT | VI_DOINGINACT)) == VI_OWEINACT)
-			vinactive(vp);
+		vinactive(vp);
 		VOP_UNLOCK(vp);
 		vdropl(vp);
 		return;
 	}
-	vdefer_inactive_cond(vp);
+	vdefer_inactive_unlocked(vp);
 }
 
 static int
@@ -4635,7 +4691,7 @@ vfs_periodic_msync_inactive(struct mount *mp, int flags)
 				vdrop(vp);
 		} else {
 			if (seen_defer)
-				vdefer_inactive_cond(vp);
+				vdefer_inactive_unlocked(vp);
 		}
 	}
 }