330 files changed, 12423 insertions, 6301 deletions

diff --git a/sys/amd64/amd64/elf_machdep.c b/sys/amd64/amd64/elf_machdep.c
index d1cffd9a38cc9..d961f092cdf31 100644
--- a/sys/amd64/amd64/elf_machdep.c
+++ b/sys/amd64/amd64/elf_machdep.c
@@ -168,6 +168,7 @@ elf_reloc_internal(linker_file_t lf, Elf_Addr relocbase, const void *data,
 	Elf_Size rtype, symidx;
 	const Elf_Rel *rel;
 	const Elf_Rela *rela;
+	int error;
 
 	switch (type) {
 	case ELF_RELOC_REL:
@@ -203,29 +204,29 @@ elf_reloc_internal(linker_file_t lf, Elf_Addr relocbase, const void *data,
 			break;
 
 		case R_X86_64_64:		/* S + A */
-			addr = lookup(lf, symidx, 1);
+			error = lookup(lf, symidx, 1, &addr);
 			val = addr + addend;
-			if (addr == 0)
+			if (error != 0)
 				return -1;
 			if (*where != val)
 				*where = val;
 			break;
 
 		case R_X86_64_PC32:	/* S + A - P */
-			addr = lookup(lf, symidx, 1);
+			error = lookup(lf, symidx, 1, &addr);
 			where32 = (Elf32_Addr *)where;
 			val32 = (Elf32_Addr)(addr + addend - (Elf_Addr)where);
-			if (addr == 0)
+			if (error != 0)
 				return -1;
 			if (*where32 != val32)
 				*where32 = val32;
 			break;
 
 		case R_X86_64_32S:	/* S + A sign extend */
-			addr = lookup(lf, symidx, 1);
+			error = lookup(lf, symidx, 1, &addr);
 			val32 = (Elf32_Addr)(addr + addend);
 			where32 = (Elf32_Addr *)where;
-			if (addr == 0)
+			if (error != 0)
 				return -1;
 			if (*where32 != val32)
 				*where32 = val32;
@@ -242,8 +243,8 @@ elf_reloc_internal(linker_file_t lf, Elf_Addr relocbase, const void *data,
 
 		case R_X86_64_GLOB_DAT:	/* S */
 		case R_X86_64_JMP_SLOT:	/* XXX need addend + offset */
-			addr = lookup(lf, symidx, 1);
-			if (addr == 0)
+			error = lookup(lf, symidx, 1, &addr);
+			if (error != 0)
 				return -1;
 			if (*where != addr)
 				*where = addr;
diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c
index f3e54805154e7..cc1b73efdae53 100644
--- a/sys/amd64/amd64/pmap.c
+++ b/sys/amd64/amd64/pmap.c
@@ -2860,7 +2860,7 @@ free_pv_chunk(struct pv_chunk *pc)
 	/* entire chunk is free, return it */
 	m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pc));
 	dump_drop_page(m->phys_addr);
-	vm_page_unwire(m, PQ_INACTIVE);
+	vm_page_unwire(m, PQ_NONE);
 	vm_page_free(m);
 }
 
diff --git a/sys/amd64/include/smp.h b/sys/amd64/include/smp.h
index 4fd6aac856a29..850289a701f62 100644
--- a/sys/amd64/include/smp.h
+++ b/sys/amd64/include/smp.h
@@ -35,7 +35,6 @@ extern int			mp_naps;
 extern int			boot_cpu_id;
 extern struct pcb		stoppcbs[];
 extern int			cpu_apic_ids[];
-extern int bootAP;
 extern void *dpcpu;
 extern char *bootSTK;
 extern int bootAP;
diff --git a/sys/arm/allwinner/a20/a20_if_dwc.c b/sys/arm/allwinner/a20/a20_if_dwc.c
new file mode 100644
index 0000000000000..88981efb5dc29
--- /dev/null
+++ b/sys/arm/allwinner/a20/a20_if_dwc.c
@@ -0,0 +1,107 @@
+/*-
+ * Copyright (c) 2015 Luiz Otavio O Souza <loos@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/module.h>
+
+#include <machine/bus.h>
+
+#include <dev/dwc/if_dwc.h>
+#include <dev/dwc/if_dwcvar.h>
+#include <dev/ofw/ofw_bus.h>
+#include <dev/ofw/ofw_bus_subr.h>
+
+#include <arm/allwinner/a10_clk.h>
+#include <arm/allwinner/a10_gpio.h>
+
+#include "if_dwc_if.h"
+
+static int
+a20_if_dwc_probe(device_t dev)
+{
+
+	if (!ofw_bus_status_okay(dev))
+		return (ENXIO);
+	if (!ofw_bus_is_compatible(dev, "allwinner,sun7i-a20-gmac"))
+		return (ENXIO);
+	device_set_desc(dev, "A20 Gigabit Ethernet Controller");
+
+	return (BUS_PROBE_DEFAULT);
+}
+
+static int
+a20_if_dwc_init(device_t dev)
+{
+
+	/* Activate GMAC clock and set the pin mux to rgmii. */
+	if (a10_clk_gmac_activate(ofw_bus_get_node(dev)) != 0 ||
+	    a10_gpio_ethernet_activate(A10_GPIO_FUNC_RGMII)) {
+		device_printf(dev, "could not activate gmac module\n");
+		return (ENXIO);
+	}
+
+	return (0);
+}
+
+static int
+a20_if_dwc_mac_type(device_t dev)
+{
+
+	return (DWC_GMAC_ALT_DESC);
+}
+
+static int
+a20_if_dwc_mii_clk(device_t dev)
+{
+
+	return (GMAC_MII_CLK_150_250M_DIV102);
+}
+
+static device_method_t a20_dwc_methods[] = {
+	DEVMETHOD(device_probe,		a20_if_dwc_probe),
+
+	DEVMETHOD(if_dwc_init,		a20_if_dwc_init),
+	DEVMETHOD(if_dwc_mac_type,	a20_if_dwc_mac_type),
+	DEVMETHOD(if_dwc_mii_clk,	a20_if_dwc_mii_clk),
+
+	DEVMETHOD_END
+};
+
+static devclass_t a20_dwc_devclass;
+
+extern driver_t dwc_driver;
+
+DEFINE_CLASS_1(dwc, a20_dwc_driver, a20_dwc_methods, sizeof(struct dwc_softc),
+    dwc_driver);
+DRIVER_MODULE(a20_dwc, simplebus, a20_dwc_driver, a20_dwc_devclass, 0, 0);
+
+MODULE_DEPEND(a20_dwc, dwc, 1, 1, 1);
diff --git a/sys/arm/allwinner/a20/files.a20 b/sys/arm/allwinner/a20/files.a20
index 25fd880103246..fe475b69a2e8c 100644
--- a/sys/arm/allwinner/a20/files.a20
+++ b/sys/arm/allwinner/a20/files.a20
@@ -1,3 +1,4 @@
 # $FreeBSD$
 
 arm/allwinner/a20/a20_mp.c		optional	smp
+arm/allwinner/a20/a20_if_dwc.c		optional	dwc
diff --git a/sys/arm/altera/socfpga/files.socfpga b/sys/arm/altera/socfpga/files.socfpga
index 901069b79a76e..8c930df746a83 100644
--- a/sys/arm/altera/socfpga/files.socfpga
+++ b/sys/arm/altera/socfpga/files.socfpga
@@ -15,7 +15,6 @@ arm/altera/socfpga/socfpga_rstmgr.c		standard
 arm/altera/socfpga/socfpga_mp.c			optional smp
 arm/altera/socfpga/socfpga_gpio.c		optional gpio
 
-dev/dwc/if_dwc.c				optional dwc
 dev/mii/micphy.c				optional micphy
 dev/mmc/host/dwmmc.c				optional dwmmc
 
diff --git a/sys/arm/amlogic/aml8726/files.aml8726 b/sys/arm/amlogic/aml8726/files.aml8726
index e4bc31e73a4a6..0f198aaa5ada9 100644
--- a/sys/arm/amlogic/aml8726/files.aml8726
+++ b/sys/arm/amlogic/aml8726/files.aml8726
@@ -32,5 +32,3 @@ arm/amlogic/aml8726/aml8726_pinctrl.c		optional	fdt_pinctrl
 arm/amlogic/aml8726/uart_dev_aml8726.c		optional	uart
 arm/amlogic/aml8726/aml8726_usb_phy-m3.c	optional	dwcotg usb gpio
 arm/amlogic/aml8726/aml8726_usb_phy-m6.c	optional	dwcotg usb gpio
-
-dev/dwc/if_dwc.c				optional	dwc
diff --git a/sys/arm/arm/elf_machdep.c b/sys/arm/arm/elf_machdep.c
index e5cc6a023240f..34761ff0e93f2 100644
--- a/sys/arm/arm/elf_machdep.c
+++ b/sys/arm/arm/elf_machdep.c
@@ -164,6 +164,7 @@ elf_reloc_internal(linker_file_t lf, Elf_Addr relocbase, const void *data,
 	Elf_Word rtype, symidx;
 	const Elf_Rel *rel;
 	const Elf_Rela *rela;
+	int error;
 
 	switch (type) {
 	case ELF_RELOC_REL:
@@ -199,8 +200,8 @@ elf_reloc_internal(linker_file_t lf, Elf_Addr relocbase, const void *data,
 			break;
 
 		case R_ARM_ABS32:
-			addr = lookup(lf, symidx, 1);
-			if (addr == 0)
+			error = lookup(lf, symidx, 1, &addr);
+			if (error != 0)
 				return -1;
 			store_ptr(where, addr + load_ptr(where));
 			break;
@@ -215,8 +216,8 @@ elf_reloc_internal(linker_file_t lf, Elf_Addr relocbase, const void *data,
 			break;
 
 		case R_ARM_JUMP_SLOT:
-			addr = lookup(lf, symidx, 1);
-			if (addr) {
+			error = lookup(lf, symidx, 1, &addr);
+			if (error == 0) {
 				store_ptr(where, addr);
 				return (0);
 			}
diff --git a/sys/arm/arm/pmap-v6-new.c b/sys/arm/arm/pmap-v6-new.c
index 864e05cae8ec2..d0a3f0f5421d5 100644
--- a/sys/arm/arm/pmap-v6-new.c
+++ b/sys/arm/arm/pmap-v6-new.c
@@ -2866,7 +2866,7 @@ free_pv_chunk(struct pv_chunk *pc)
 	/* entire chunk is free, return it */
 	m = PHYS_TO_VM_PAGE(pmap_kextract((vm_offset_t)pc));
 	pmap_qremove((vm_offset_t)pc, 1);
-	vm_page_unwire(m, PQ_INACTIVE);
+	vm_page_unwire(m, PQ_NONE);
 	vm_page_free(m);
 	pmap_pte2list_free(&pv_vafree, (vm_offset_t)pc);
 }
diff --git a/sys/arm/arm/pmap-v6.c b/sys/arm/arm/pmap-v6.c
index 50172f253f545..437891e5d0f8b 100644
--- a/sys/arm/arm/pmap-v6.c
+++ b/sys/arm/arm/pmap-v6.c
@@ -4292,7 +4292,7 @@ pmap_free_pv_chunk(struct pv_chunk *pc)
 	/* entire chunk is free, return it */
 	m = PHYS_TO_VM_PAGE(pmap_kextract((vm_offset_t)pc));
 	pmap_qremove((vm_offset_t)pc, 1);
-	vm_page_unwire(m, PQ_INACTIVE);
+	vm_page_unwire(m, PQ_NONE);
 	vm_page_free(m);
 	pmap_ptelist_free(&pv_vafree, (vm_offset_t)pc);
 
diff --git a/sys/arm/arm/stdatomic.c b/sys/arm/arm/stdatomic.c
index 3c0b99729f6c0..b12a82e2288bf 100644
--- a/sys/arm/arm/stdatomic.c
+++ b/sys/arm/arm/stdatomic.c
@@ -270,11 +270,11 @@ __atomic_compare_exchange_##N(uintN_t *mem, uintN_t *pexpected,		\
 	}								\
 }
 
-#define	EMIT_FETCH_OP_N(N, uintN_t, ldr, str, name, op)			\
+#define	EMIT_FETCH_OP_N(N, uintN_t, ldr, str, name, op, ret)		\
 uintN_t									\
 __atomic_##name##_##N(uintN_t *mem, uintN_t val, int model __unused)	\
 {									\
-	uint32_t old, temp, ras_start;					\
+	uint32_t old, new, ras_start;					\
 									\
 	ras_start = ARM_RAS_START;					\
 	__asm volatile (						\
@@ -295,9 +295,9 @@ __atomic_##name##_##N(uintN_t *mem, uintN_t val, int model __unused)	\
 		"\tstr   %2, [%5]\n"					\
 		"\tmov   %2, #0xffffffff\n"				\
 		"\tstr   %2, [%5, #4]\n"				\
-		: "=&r" (old), "=m" (*mem), "=&r" (temp)		\
+		: "=&r" (old), "=m" (*mem), "=&r" (new)			\
 		: "r" (val), "m" (*mem), "r" (ras_start));		\
-	return (old);							\
+	return (ret);							\
 }
 
 #define	EMIT_ALL_OPS_N(N, uintN_t, ldr, str, streq)			\
@@ -305,11 +305,16 @@ EMIT_LOAD_N(N, uintN_t)							\
 EMIT_STORE_N(N, uintN_t)						\
 EMIT_EXCHANGE_N(N, uintN_t, ldr, str)					\
 EMIT_COMPARE_EXCHANGE_N(N, uintN_t, ldr, streq)				\
-EMIT_FETCH_OP_N(N, uintN_t, ldr, str, fetch_add, "add")			\
-EMIT_FETCH_OP_N(N, uintN_t, ldr, str, fetch_and, "and")			\
-EMIT_FETCH_OP_N(N, uintN_t, ldr, str, fetch_or, "orr")			\
-EMIT_FETCH_OP_N(N, uintN_t, ldr, str, fetch_sub, "sub")			\
-EMIT_FETCH_OP_N(N, uintN_t, ldr, str, fetch_xor, "eor")
+EMIT_FETCH_OP_N(N, uintN_t, ldr, str, fetch_add, "add", old)		\
+EMIT_FETCH_OP_N(N, uintN_t, ldr, str, fetch_and, "and", old)		\
+EMIT_FETCH_OP_N(N, uintN_t, ldr, str, fetch_or,  "orr", old)		\
+EMIT_FETCH_OP_N(N, uintN_t, ldr, str, fetch_sub, "sub", old)		\
+EMIT_FETCH_OP_N(N, uintN_t, ldr, str, fetch_xor, "eor", old)		\
+EMIT_FETCH_OP_N(N, uintN_t, ldr, str, add_fetch, "add", new)		\
+EMIT_FETCH_OP_N(N, uintN_t, ldr, str, and_fetch, "and", new)		\
+EMIT_FETCH_OP_N(N, uintN_t, ldr, str, or_fetch,  "orr", new)		\
+EMIT_FETCH_OP_N(N, uintN_t, ldr, str, sub_fetch, "sub", new)		\
+EMIT_FETCH_OP_N(N, uintN_t, ldr, str, xor_fetch, "eor", new)
 
 EMIT_ALL_OPS_N(1, uint8_t, "ldrb", "strb", "strbeq")
 EMIT_ALL_OPS_N(2, uint16_t, "ldrh", "strh", "strheq")
diff --git a/sys/arm/conf/A20 b/sys/arm/conf/A20
index 00cdb8f41cdf4..3bac455185850 100644
--- a/sys/arm/conf/A20
+++ b/sys/arm/conf/A20
@@ -47,7 +47,7 @@ options 	WITNESS_SKIPSPIN	# Don't run witness on spinlocks for speed
 #options 	BOOTP_NFSROOT
 #options 	BOOTP_COMPAT
 #options 	BOOTP_NFSV3
-#options 	BOOTP_WIRED_TO=cpsw0
+#options 	BOOTP_WIRED_TO=dwc0
 
 # Boot device is 2nd slice on MMC/SD card
 options 	ROOTDEVNAME=\"ufs:/dev/da0s2\"
@@ -102,7 +102,8 @@ device		ether
 device		mii
 device		bpf
 
-device		emac
+#device		emac			# 10/100 integrated EMAC controller
+device		dwc			# 10/100/1000 integrated GMAC controller
 
 # USB ethernet support, requires miibus
 device		miibus
diff --git a/sys/arm/versatile/versatile_pci.c b/sys/arm/versatile/versatile_pci.c
index 10f31190cefb0..f5ef6e94ec2c6 100644
--- a/sys/arm/versatile/versatile_pci.c
+++ b/sys/arm/versatile/versatile_pci.c
@@ -266,7 +266,7 @@ versatile_pci_attach(device_t dev)
 		versatile_pci_conf_write_4((slot << 11) + PCIR_COMMAND, val);
 	}
 
-	device_add_child(dev, "pci", 0);
+	device_add_child(dev, "pci", -1);
 	return (bus_generic_attach(dev));
 }
 
diff --git a/sys/arm/xscale/i80321/i80321_pci.c b/sys/arm/xscale/i80321/i80321_pci.c
index 4e2814830004e..5f78577e8704e 100644
--- a/sys/arm/xscale/i80321/i80321_pci.c
+++ b/sys/arm/xscale/i80321/i80321_pci.c
@@ -117,7 +117,7 @@ i80321_pci_attach(device_t dev)
 	if (rman_init(&sc->sc_irq_rman) != 0 ||
 	    rman_manage_region(&sc->sc_irq_rman, 26, 32) != 0)
 		panic("i80321_pci_probe: failed to set up IRQ rman");
-	device_add_child(dev, "pci",busno);
+	device_add_child(dev, "pci", -1);
 	return (bus_generic_attach(dev));
 }
 
diff --git a/sys/arm/xscale/i8134x/i81342_pci.c b/sys/arm/xscale/i8134x/i81342_pci.c
index 68d2fbf4ea14a..d9d978c8a4bca 100644
--- a/sys/arm/xscale/i8134x/i81342_pci.c
+++ b/sys/arm/xscale/i8134x/i81342_pci.c
@@ -209,7 +209,7 @@ i81342_pci_attach(device_t dev)
 	}
 	bus_space_write_4(sc->sc_st, sc->sc_atu_sh, ATU_ISR,
 	    bus_space_read_4(sc->sc_st, sc->sc_atu_sh, ATU_ISR) & ATUX_ISR_ERRMSK);
-	device_add_child(dev, "pci", busno);
+	device_add_child(dev, "pci", -1);
 	return (bus_generic_attach(dev));
 }
 
diff --git a/sys/arm64/arm64/exception.S b/sys/arm64/arm64/exception.S
index b05941f5eac21..2e6fee87e0823 100644
--- a/sys/arm64/arm64/exception.S
+++ b/sys/arm64/arm64/exception.S
@@ -63,8 +63,13 @@ __FBSDID("$FreeBSD$");
 .endm
 
 .macro	restore_registers el
-	msr	daifset, #2		/* Disable interrupts, x18 may change
-					 * in the interrupt exception handler */
+.if \el == 1
+	msr	daifset, #2
+	/*
+	 * Disable interrupts, x18 may change in the interrupt exception
+	 * handler.  For EL0 exceptions, do_ast already did this.
+	 */
+.endif
 	ldp	x18, lr, [sp], #16
 	ldp	x10, x11, [sp], #16
 .if \el == 0
@@ -100,10 +105,11 @@ __FBSDID("$FreeBSD$");
 .macro	do_ast
 	/* Disable interrupts */
 	mrs	x19, daif
+1:
 	msr	daifset, #2
 
 	/* Read the current thread flags */
-1:	ldr	x1, [x18, #PC_CURTHREAD]	/* Load curthread */
+	ldr	x1, [x18, #PC_CURTHREAD]	/* Load curthread */
 	ldr	x2, [x1, #TD_FLAGS]
 
 	/* Check if we have either bits set */
@@ -120,13 +126,9 @@ __FBSDID("$FreeBSD$");
 	mov	x0, sp
 	bl	_C_LABEL(ast)
 
-	/* Disable interrupts */
-	mrs	x19, daif
-	msr	daifset, #2
-
+	/* Re-check for new ast scheduled */
+	b	1b
 2:
-	/* Restore interrupts */
-	msr	daif, x19
 .endm
 
 handle_el1h_sync:
@@ -158,6 +160,7 @@ handle_el0_irq:
 	save_registers 0
 	mov	x0, sp
 	bl	arm_cpu_intr
+	do_ast
 	restore_registers 0
 	eret
 
diff --git a/sys/arm64/arm64/gic_v3_its.c b/sys/arm64/arm64/gic_v3_its.c
index d3e9ab720f664..cb8a4887e90b1 100644
--- a/sys/arm64/arm64/gic_v3_its.c
+++ b/sys/arm64/arm64/gic_v3_its.c
@@ -180,6 +180,19 @@ gic_v3_its_attach(device_t dev)
 	sc = device_get_softc(dev);
 
 	/*
+	 * XXX ARM64TODO: Avoid configuration of more than one ITS
+	 * device. To be removed when multi-PIC support is added
+	 * to FreeBSD (or at least multi-ITS is implemented). Limit
+	 * supported ITS sockets to '0' only.
+	 */
+	if (device_get_unit(dev) != 0) {
+		device_printf(dev,
+		    "Only single instance of ITS is supported, exitting...\n");
+		return (ENXIO);
+	}
+	sc->its_socket = 0;
+
+	/*
 	 * Initialize sleep & spin mutex for ITS
 	 */
 	/* Protects ITS device list and assigned LPIs bitmaps. */
@@ -558,6 +571,10 @@ its_init_cpu(struct gic_v3_its_softc *sc)
 			sc = its_sc;
 		} else
 			return (ENXIO);
+
+		/* Skip if running secondary init on a wrong socket */
+		if (sc->its_socket != CPU_CURRENT_SOCKET)
+			return (ENXIO);
 	}
 
 	/*
diff --git a/sys/arm64/arm64/gic_v3_var.h b/sys/arm64/arm64/gic_v3_var.h
index c8519b0ac856d..7bcd068a7fdc1 100644
--- a/sys/arm64/arm64/gic_v3_var.h
+++ b/sys/arm64/arm64/gic_v3_var.h
@@ -232,6 +232,8 @@ struct gic_v3_its_softc {
 
 	struct mtx		its_mtx;
 	struct mtx		its_spin_mtx;
+
+	uint32_t		its_socket;	/* Socket number ITS is attached to */
 };
 
 /* Stuff that is specific to the vendor's implementation */
diff --git a/sys/arm64/arm64/machdep.c b/sys/arm64/arm64/machdep.c
index bfdbb27ca6f16..4bb3f674b9f42 100644
--- a/sys/arm64/arm64/machdep.c
+++ b/sys/arm64/arm64/machdep.c
@@ -894,8 +894,11 @@ DB_SHOW_COMMAND(specialregs, db_show_spregs)
 	PRINT_REG(elr_el1);
 	PRINT_REG(esr_el1);
 	PRINT_REG(far_el1);
+#if 0
+	/* ARM64TODO: Enable VFP before reading floating-point registers */
 	PRINT_REG(fpcr);
 	PRINT_REG(fpsr);
+#endif
 	PRINT_REG(id_aa64afr0_el1);
 	PRINT_REG(id_aa64afr1_el1);
 	PRINT_REG(id_aa64dfr0_el1);
diff --git a/sys/arm64/arm64/mp_machdep.c b/sys/arm64/arm64/mp_machdep.c
index b95b2ed0789c2..688cc7913d9c4 100644
--- a/sys/arm64/arm64/mp_machdep.c
+++ b/sys/arm64/arm64/mp_machdep.c
@@ -190,7 +190,7 @@ release_aps(void *dummy __unused)
 		DELAY(1000);
 	}
 
-	printf("AP's not started\n");
+	printf("APs not started\n");
 }
 SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL);
 
@@ -352,7 +352,6 @@ cpu_init_fdt(u_int id, phandle_t node, u_int addr_size, pcell_t *reg)
 	if (id == 0)
 		return (1);
 
-	CPU_SET(id, &all_cpus);
 
 	pcpup = &__pcpu[id];
 	pcpu_init(pcpup, id, sizeof(struct pcpu));
@@ -371,8 +370,17 @@ cpu_init_fdt(u_int id, phandle_t node, u_int addr_size, pcell_t *reg)
 	pa = pmap_extract(kernel_pmap, (vm_offset_t)mpentry);
 
 	err = psci_cpu_on(target_cpu, pa, id);
-	if (err != PSCI_RETVAL_SUCCESS)
-		printf("Failed to start CPU %u\n", id);
+	if (err != PSCI_RETVAL_SUCCESS) {
+		/* Panic here if INVARIANTS are enabled */
+		KASSERT(0, ("Failed to start CPU %u (%lx)\n", id, target_cpu));
+
+		pcpu_destroy(pcpup);
+		kmem_free(kernel_arena, (vm_offset_t)dpcpu[id - 1], DPCPU_SIZE);
+		dpcpu[id - 1] = NULL;
+		/* Notify the user that the CPU failed to start */
+		printf("Failed to start CPU %u (%lx)\n", id, target_cpu);
+	} else
+		CPU_SET(id, &all_cpus);
 
 	return (1);
 }
diff --git a/sys/arm64/arm64/pmap.c b/sys/arm64/arm64/pmap.c
index 62fe394bd393b..b558149b69173 100644
--- a/sys/arm64/arm64/pmap.c
+++ b/sys/arm64/arm64/pmap.c
@@ -1517,7 +1517,7 @@ free_pv_chunk(struct pv_chunk *pc)
 	/* entire chunk is free, return it */
 	m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pc));
 	dump_drop_page(m->phys_addr);
-	vm_page_unwire(m, PQ_INACTIVE);
+	vm_page_unwire(m, PQ_NONE);
 	vm_page_free(m);
 }
 
diff --git a/sys/arm64/arm64/trap.c b/sys/arm64/arm64/trap.c
index 2a3cfb9662f39..e54c20e54ce90 100644
--- a/sys/arm64/arm64/trap.c
+++ b/sys/arm64/arm64/trap.c
@@ -76,6 +76,7 @@ extern register_t fsu_intr_fault;
 void do_el1h_sync(struct trapframe *);
 void do_el0_sync(struct trapframe *);
 void do_el0_error(struct trapframe *);
+static void print_registers(struct trapframe *frame);
 
 int (*dtrace_invop_jump_addr)(struct trapframe *);
 
@@ -144,7 +145,7 @@ svc_handler(struct trapframe *frame)
 }
 
 static void
-data_abort(struct trapframe *frame, uint64_t esr, int lower)
+data_abort(struct trapframe *frame, uint64_t esr, uint64_t far, int lower)
 {
 	struct vm_map *map;
 	struct thread *td;
@@ -152,9 +153,15 @@ data_abort(struct trapframe *frame, uint64_t esr, int lower)
 	struct pcb *pcb;
 	vm_prot_t ftype;
 	vm_offset_t va;
-	uint64_t far;
 	int error, sig, ucode;
 
+	/*
+	 * According to the ARMv8-A rev. A.g, B2.10.5 "Load-Exclusive
+	 * and Store-Exclusive instruction usage restrictions", state
+	 * of the exclusive monitors after data abort exception is unknown.
+	 */
+	clrex();
+
 #ifdef KDB
 	if (kdb_active) {
 		kdb_reenter();
@@ -174,17 +181,23 @@ data_abort(struct trapframe *frame, uint64_t esr, int lower)
 		return;
 	}
 
-	far = READ_SPECIALREG(far_el1);
-	p = td->td_proc;
+	KASSERT(td->td_md.md_spinlock_count == 0,
+	    ("data abort with spinlock held"));
+	if (td->td_critnest != 0 || WITNESS_CHECK(WARN_SLEEPOK |
+	    WARN_GIANTOK, NULL, "Kernel page fault") != 0) {
+		print_registers(frame);
+		panic("data abort in critical section or under mutex");
+	}
 
+	p = td->td_proc;
 	if (lower)
-		map = &td->td_proc->p_vmspace->vm_map;
+		map = &p->p_vmspace->vm_map;
 	else {
 		/* The top bit tells us which range to use */
 		if ((far >> 63) == 1)
 			map = kernel_map;
 		else
-			map = &td->td_proc->p_vmspace->vm_map;
+			map = &p->p_vmspace->vm_map;
 	}
 
 	va = trunc_page(far);
@@ -239,7 +252,7 @@ void
 do_el1h_sync(struct trapframe *frame)
 {
 	uint32_t exception;
-	uint64_t esr;
+	uint64_t esr, far;
 
 	/* Read the esr register to get the exception details */
 	esr = READ_SPECIALREG(esr_el1);
@@ -274,7 +287,9 @@ do_el1h_sync(struct trapframe *frame)
 		print_registers(frame);
 		panic("VFP exception in the kernel");
 	case EXCP_DATA_ABORT:
-		data_abort(frame, esr, 0);
+		far = READ_SPECIALREG(far_el1);
+		intr_enable();
+		data_abort(frame, esr, far, 0);
 		break;
 	case EXCP_BRK:
 #ifdef KDTRACE_HOOKS
@@ -321,7 +336,7 @@ do_el0_sync(struct trapframe *frame)
 {
 	struct thread *td;
 	uint32_t exception;
-	uint64_t esr;
+	uint64_t esr, far;
 
 	/* Check we have a sane environment when entering from userland */
 	KASSERT((uintptr_t)get_pcpu() >= VM_MIN_KERNEL_ADDRESS,
@@ -330,6 +345,13 @@ do_el0_sync(struct trapframe *frame)
 
 	esr = READ_SPECIALREG(esr_el1);
 	exception = ESR_ELx_EXCEPTION(esr);
+	switch (exception) {
+	case EXCP_INSN_ABORT_L:
+	case EXCP_DATA_ABORT_L:
+	case EXCP_DATA_ABORT:
+		far = READ_SPECIALREG(far_el1);
+	}
+	intr_enable();
 
 	CTR4(KTR_TRAP,
 	    "do_el0_sync: curthread: %p, esr %lx, elr: %lx, frame: %p",
@@ -345,18 +367,12 @@ do_el0_sync(struct trapframe *frame)
 #endif
 		break;
 	case EXCP_SVC:
-		/*
-		 * Ensure the svc_handler is being run with interrupts enabled.
-		 * They will be automatically restored when returning from
-		 * exception handler.
-		 */
-		intr_enable();
 		svc_handler(frame);
 		break;
 	case EXCP_INSN_ABORT_L:
 	case EXCP_DATA_ABORT_L:
 	case EXCP_DATA_ABORT:
-		data_abort(frame, esr, 1);
+		data_abort(frame, esr, far, 1);
 		break;
 	case EXCP_UNKNOWN:
 		el0_excp_unknown(frame);
diff --git a/sys/arm64/cavium/thunder_pcie_pem.c b/sys/arm64/cavium/thunder_pcie_pem.c
index 23f1446b8d229..f36057b93ff49 100644
--- a/sys/arm64/cavium/thunder_pcie_pem.c
+++ b/sys/arm64/cavium/thunder_pcie_pem.c
@@ -183,8 +183,16 @@ static int
 thunder_pem_maxslots(device_t dev)
 {
 
+#if 0
 	/* max slots per bus acc. to standard */
 	return (PCI_SLOTMAX);
+#else
+	/*
+	 * ARM64TODO Workaround - otherwise an em(4) interface appears to be
+	 * present on every PCI function on the bus to which it is connected
+	 */
+	return (0);
+#endif
 }
 
 static int
diff --git a/sys/arm64/conf/GENERIC b/sys/arm64/conf/GENERIC
index eb46c1168f8d6..26ca51df86ac4 100644
--- a/sys/arm64/conf/GENERIC
+++ b/sys/arm64/conf/GENERIC
@@ -107,6 +107,9 @@ device		ahci
 device		scbus
 device		da
 
+# ATA/SCSI peripherals
+device		pass		# Passthrough device (direct ATA/SCSI access)
+
 # MMC/SD/SDIO Card slot support
 device		mmc			# mmc/sd bus
 device		mmcsd			# mmc/sd flash cards
@@ -119,7 +122,10 @@ device		pl011
 # USB support
 options 	USB_DEBUG		# enable debug msgs
 device		dwcotg			# DWC OTG controller
+device		xhci			# XHCI PCI->USB interface (USB 3.0)
 device		usb			# USB Bus (required)
+device		ukbd			# Keyboard
+device		umass			# Disks/Mass storage - Requires scbus and da
 
 # Pseudo devices.
 device		loop		# Network loopback
diff --git a/sys/arm64/include/cpu.h b/sys/arm64/include/cpu.h
index f979af5c27412..14bffbc1f5369 100644
--- a/sys/arm64/include/cpu.h
+++ b/sys/arm64/include/cpu.h
@@ -148,6 +148,8 @@ void	identify_cpu(void);
 void	swi_vm(void *v);
 
 #define	CPU_AFFINITY(cpu)	__cpu_affinity[(cpu)]
+#define	CPU_CURRENT_SOCKET				\
+    (CPU_AFF2(CPU_AFFINITY(PCPU_GET(cpuid))))
 
 static __inline uint64_t
 get_cyclecount(void)
diff --git a/sys/arm64/include/cpufunc.h b/sys/arm64/include/cpufunc.h
index 5a7f08b61895f..56f1acff7264c 100644
--- a/sys/arm64/include/cpufunc.h
+++ b/sys/arm64/include/cpufunc.h
@@ -108,6 +108,17 @@ get_mpidr(void)
 	return (mpidr);
 }
 
+static __inline void
+clrex(void)
+{
+
+	/*
+	 * Ensure compiler barrier, otherwise the monitor clear might
+	 * occur too late for us ?
+	 */
+	__asm __volatile("clrex" : : : "memory");
+}
+
 #define	cpu_nullop()			arm64_nullop()
 #define	cpufunc_nullop()		arm64_nullop()
 #define	cpu_setttb(a)			arm64_setttb(a)
diff --git a/sys/boot/efi/boot1/Makefile b/sys/boot/efi/boot1/Makefile
index c6a6d59e50d18..c1b3714a5271e 100644
--- a/sys/boot/efi/boot1/Makefile
+++ b/sys/boot/efi/boot1/Makefile
@@ -9,7 +9,7 @@ MAN=
 
 MK_SSP=		no
 
-PROG=		loader.sym
+PROG=		boot1.sym
 INTERNALPROG=
 
 # architecture-specific loader code
@@ -50,7 +50,7 @@ DPADD+=		${LIBSTAND}
 LDADD+=		-lstand
 .endif
 
-${PROG}:	${LDSCRIPT}
+DPADD+=		${LDSCRIPT}
 
 OBJCOPY?=	objcopy
 OBJDUMP?=	objdump
@@ -63,19 +63,19 @@ EFI_TARGET=	efi-app-ia32
 EFI_TARGET=	binary
 .endif
 
-boot1.efi: loader.sym
+boot1.efi: ${PROG}
 	if [ `${OBJDUMP} -t ${.ALLSRC} | fgrep '*UND*' | wc -l` != 0 ]; then \
 		${OBJDUMP} -t ${.ALLSRC} | fgrep '*UND*'; \
 		exit 1; \
 	fi
 	${OBJCOPY} -j .peheader -j .text -j .sdata -j .data \
 		-j .dynamic -j .dynsym -j .rel.dyn \
-		-j .rela.dyn -j .reloc -j .eh_frame -j set_Xcommand_set \
+		-j .rela.dyn -j .reloc -j .eh_frame \
 		--output-target=${EFI_TARGET} ${.ALLSRC} ${.TARGET}
 
 boot1.o: ${.CURDIR}/../../common/ufsread.c
 
-# The following inserts out objects into a template FAT file system
+# The following inserts our objects into a template FAT file system
 # created by generate-fat.sh
 
 .include "${.CURDIR}/Makefile.fat"
diff --git a/sys/boot/efi/libefi/Makefile.depend b/sys/boot/efi/libefi/Makefile.depend
index c77da59895bc4..18be76b0cb6f4 100644
--- a/sys/boot/efi/libefi/Makefile.depend
+++ b/sys/boot/efi/libefi/Makefile.depend
@@ -1,10 +1,6 @@
 # $FreeBSD$
 # Autogenerated - do NOT edit!
 
-DEP_RELDIR := ${_PARSEDIR:S,${SRCTOP}/,,}
-
-DEP_MACHINE := ${.PARSEFILE:E}
-
 DIRDEPS = \
 	include \
 	include/xlocale \
diff --git a/sys/boot/efi/loader/Makefile b/sys/boot/efi/loader/Makefile
index b341704675b4f..c607f46a065dd 100644
--- a/sys/boot/efi/loader/Makefile
+++ b/sys/boot/efi/loader/Makefile
@@ -12,7 +12,6 @@ MK_SSP=		no
 PROG=		loader.sym
 INTERNALPROG=
 
-.PATH: ${.CURDIR}/../../efi/loader
 # architecture-specific loader code
 SRCS=	autoload.c \
 	bootinfo.c \
@@ -97,7 +96,7 @@ EFI_TARGET=	efi-app-ia32
 EFI_TARGET=	binary
 .endif
 
-loader.efi: loader.sym
+loader.efi: ${PROG}
 	if [ `${OBJDUMP} -t ${.ALLSRC} | fgrep '*UND*' | wc -l` != 0 ]; then \
 		${OBJDUMP} -t ${.ALLSRC} | fgrep '*UND*'; \
 		exit 1; \
diff --git a/sys/boot/efi/loader/copy.c b/sys/boot/efi/loader/copy.c
index 10f4cb2e73c6b..716e9ea328a2b 100644
--- a/sys/boot/efi/loader/copy.c
+++ b/sys/boot/efi/loader/copy.c
@@ -38,10 +38,10 @@ __FBSDID("$FreeBSD$");
 #include <efilib.h>
 
 #ifndef EFI_STAGING_SIZE
-#define	EFI_STAGING_SIZE	32
+#define	EFI_STAGING_SIZE	48
 #endif
 
-#define	STAGE_PAGES	((EFI_STAGING_SIZE) * 1024 * 1024 / 4096)
+#define	STAGE_PAGES	EFI_SIZE_TO_PAGES((EFI_STAGING_SIZE) * 1024 * 1024)
 
 EFI_PHYSICAL_ADDRESS	staging, staging_end;
 int			stage_offset_set = 0;
@@ -59,7 +59,7 @@ efi_copy_init(void)
 		    (unsigned long)(status & EFI_ERROR_MASK));
 		return (status);
 	}
-	staging_end = staging + STAGE_PAGES * 4096;
+	staging_end = staging + STAGE_PAGES * EFI_PAGE_SIZE;
 
 #if defined(__aarch64__) || defined(__arm__)
 	/*
@@ -132,7 +132,7 @@ efi_copy_finish(void)
 
 	src = (uint64_t *)staging;
 	dst = (uint64_t *)(staging - stage_offset);
-	last = (uint64_t *)(staging + STAGE_PAGES * EFI_PAGE_SIZE);
+	last = (uint64_t *)staging_end;
 
 	while (src < last)
 		*dst++ = *src++;
diff --git a/sys/boot/fdt/dts/arm/bcm2836.dtsi b/sys/boot/fdt/dts/arm/bcm2836.dtsi
index bd75c843ca481..c1caac34c7021 100644
--- a/sys/boot/fdt/dts/arm/bcm2836.dtsi
+++ b/sys/boot/fdt/dts/arm/bcm2836.dtsi
@@ -379,6 +379,8 @@
 		};
 
 		bsc0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
 			compatible = "broadcom,bcm2835-bsc",
 				     "broadcom,bcm2708-bsc";
 			reg = <0x205000 0x20>;
@@ -387,6 +389,8 @@
 		};
 
 		bsc1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
 			compatible = "broadcom,bcm2835-bsc",
 				     "broadcom,bcm2708-bsc";
 			reg = <0x804000 0x20>;
diff --git a/sys/boot/fdt/dts/arm/cubieboard2.dts b/sys/boot/fdt/dts/arm/cubieboard2.dts
index 700952a81ea2c..480df660e7330 100644
--- a/sys/boot/fdt/dts/arm/cubieboard2.dts
+++ b/sys/boot/fdt/dts/arm/cubieboard2.dts
@@ -67,6 +67,10 @@
 			status = "okay";
 		};
 
+		gmac@01c50000 {
+			status = "okay";
+		};
+
 		ahci: sata@01c18000 {
 			status = "okay";
 		};
diff --git a/sys/boot/ficl/Makefile.depend b/sys/boot/ficl/Makefile.depend
index ab9c549baedf6..729ef23e98b4e 100644
--- a/sys/boot/ficl/Makefile.depend
+++ b/sys/boot/ficl/Makefile.depend
@@ -1,10 +1,6 @@
 # $FreeBSD$
 # Autogenerated - do NOT edit!
 
-DEP_RELDIR := ${_PARSEDIR:S,${SRCTOP}/,,}
-
-DEP_MACHINE := ${.PARSEFILE:E}
-
 DIRDEPS = \
 	include \
 	include/xlocale \
diff --git a/sys/boot/i386/boot0/Makefile.depend b/sys/boot/i386/boot0/Makefile.depend
index 3af2d7f1f4d20..f80275d86ab17 100644
--- a/sys/boot/i386/boot0/Makefile.depend
+++ b/sys/boot/i386/boot0/Makefile.depend
@@ -1,10 +1,6 @@
 # $FreeBSD$
 # Autogenerated - do NOT edit!
 
-DEP_RELDIR := ${_PARSEDIR:S,${SRCTOP}/,,}
-
-DEP_MACHINE := ${.PARSEFILE:E}
-
 DIRDEPS = \
 
 
diff --git a/sys/boot/i386/boot0sio/Makefile.depend b/sys/boot/i386/boot0sio/Makefile.depend
index 3af2d7f1f4d20..f80275d86ab17 100644
--- a/sys/boot/i386/boot0sio/Makefile.depend
+++ b/sys/boot/i386/boot0sio/Makefile.depend
@@ -1,10 +1,6 @@
 # $FreeBSD$
 # Autogenerated - do NOT edit!
 
-DEP_RELDIR := ${_PARSEDIR:S,${SRCTOP}/,,}
-
-DEP_MACHINE := ${.PARSEFILE:E}
-
 DIRDEPS = \
 
 
diff --git a/sys/boot/i386/boot2/Makefile.depend b/sys/boot/i386/boot2/Makefile.depend
index 3af2d7f1f4d20..f80275d86ab17 100644
--- a/sys/boot/i386/boot2/Makefile.depend
+++ b/sys/boot/i386/boot2/Makefile.depend
@@ -1,10 +1,6 @@
 # $FreeBSD$
 # Autogenerated - do NOT edit!
 
-DEP_RELDIR := ${_PARSEDIR:S,${SRCTOP}/,,}
-
-DEP_MACHINE := ${.PARSEFILE:E}
-
 DIRDEPS = \
 
 
diff --git a/sys/boot/i386/loader/Makefile.depend b/sys/boot/i386/loader/Makefile.depend
index 3af2d7f1f4d20..f80275d86ab17 100644
--- a/sys/boot/i386/loader/Makefile.depend
+++ b/sys/boot/i386/loader/Makefile.depend
@@ -1,10 +1,6 @@
 # $FreeBSD$
 # Autogenerated - do NOT edit!
 
-DEP_RELDIR := ${_PARSEDIR:S,${SRCTOP}/,,}
-
-DEP_MACHINE := ${.PARSEFILE:E}
-
 DIRDEPS = \
 
 
diff --git a/sys/boot/i386/zfsloader/Makefile.depend b/sys/boot/i386/zfsloader/Makefile.depend
index 3af2d7f1f4d20..f80275d86ab17 100644
--- a/sys/boot/i386/zfsloader/Makefile.depend
+++ b/sys/boot/i386/zfsloader/Makefile.depend
@@ -1,10 +1,6 @@
 # $FreeBSD$
 # Autogenerated - do NOT edit!
 
-DEP_RELDIR := ${_PARSEDIR:S,${SRCTOP}/,,}
-
-DEP_MACHINE := ${.PARSEFILE:E}
-
 DIRDEPS = \
 
 
diff --git a/sys/boot/usb/usbcore.mk b/sys/boot/usb/usbcore.mk
index e99fe877c6634..121e036d5b9e5 100644
--- a/sys/boot/usb/usbcore.mk
+++ b/sys/boot/usb/usbcore.mk
@@ -143,6 +143,7 @@ KSRCS+=	usb_template_kbd.c
 KSRCS+=	usb_template_audio.c
 KSRCS+=	usb_template_phone.c
 KSRCS+=	usb_template_serialnet.c
+KSRCS+=	usb_template_midi.c
 
 #
 # USB mass storage support
diff --git a/sys/cam/cam_compat.c b/sys/cam/cam_compat.c
index bcce814ffdf12..b8e57bb7e6b89 100644
--- a/sys/cam/cam_compat.c
+++ b/sys/cam/cam_compat.c
@@ -300,7 +300,7 @@ cam_compat_translate_dev_match_0x18(union ccb *ccb)
 
 	/* Remap the CCB into kernel address space */
 	bzero(&mapinfo, sizeof(mapinfo));
-	cam_periph_mapmem(ccb, &mapinfo);
+	cam_periph_mapmem(ccb, &mapinfo, MAXPHYS);
 
 	dm = ccb->cdm.matches;
 	/* Translate in-place: old fields are smaller */
diff --git a/sys/cam/cam_periph.c b/sys/cam/cam_periph.c
index 956e5d0a7a251..91cb45d371d74 100644
--- a/sys/cam/cam_periph.c
+++ b/sys/cam/cam_periph.c
@@ -716,16 +716,19 @@ camperiphfree(struct cam_periph *periph)
  * buffers to map stuff in and out, we're limited to the buffer size.
  */
 int
-cam_periph_mapmem(union ccb *ccb, struct cam_periph_map_info *mapinfo)
+cam_periph_mapmem(union ccb *ccb, struct cam_periph_map_info *mapinfo,
+    u_int maxmap)
 {
 	int numbufs, i, j;
 	int flags[CAM_PERIPH_MAXMAPS];
 	u_int8_t **data_ptrs[CAM_PERIPH_MAXMAPS];
 	u_int32_t lengths[CAM_PERIPH_MAXMAPS];
 	u_int32_t dirs[CAM_PERIPH_MAXMAPS];
-	/* Some controllers may not be able to handle more data. */
-	size_t maxmap = DFLTPHYS;
 
+	if (maxmap == 0)
+		maxmap = DFLTPHYS;	/* traditional default */
+	else if (maxmap > MAXPHYS)
+		maxmap = MAXPHYS;	/* for safety */
 	switch(ccb->ccb_h.func_code) {
 	case XPT_DEV_MATCH:
 		if (ccb->cdm.match_buf_len == 0) {
diff --git a/sys/cam/cam_periph.h b/sys/cam/cam_periph.h
index ebcf1a42bdafa..e28d5b11214d1 100644
--- a/sys/cam/cam_periph.h
+++ b/sys/cam/cam_periph.h
@@ -160,7 +160,8 @@ int		cam_periph_hold(struct cam_periph *periph, int priority);
 void		cam_periph_unhold(struct cam_periph *periph);
 void		cam_periph_invalidate(struct cam_periph *periph);
 int		cam_periph_mapmem(union ccb *ccb,
-				  struct cam_periph_map_info *mapinfo);
+				  struct cam_periph_map_info *mapinfo,
+				  u_int maxmap);
 void		cam_periph_unmapmem(union ccb *ccb,
 				    struct cam_periph_map_info *mapinfo);
 union ccb	*cam_periph_getccb(struct cam_periph *periph,
diff --git a/sys/cam/cam_xpt.c b/sys/cam/cam_xpt.c
index ad55373c25a63..4ce48a386fe46 100644
--- a/sys/cam/cam_xpt.c
+++ b/sys/cam/cam_xpt.c
@@ -536,7 +536,7 @@ xptdoioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *
 			 * Map the pattern and match buffers into kernel
 			 * virtual address space.
 			 */
-			error = cam_periph_mapmem(inccb, &mapinfo);
+			error = cam_periph_mapmem(inccb, &mapinfo, MAXPHYS);
 
 			if (error) {
 				inccb->ccb_h.path = old_path;
diff --git a/sys/cam/ctl/README.ctl.txt b/sys/cam/ctl/README.ctl.txt
index c37527bcc0cb2..89030e20298ae 100644
--- a/sys/cam/ctl/README.ctl.txt
+++ b/sys/cam/ctl/README.ctl.txt
@@ -19,9 +19,9 @@ Userland Commands
 Introduction:
 ============
 
-CTL is a disk and processor device emulation subsystem originally written
-for Copan Systems under Linux starting in 2003.  It has been shipping in
-Copan (now SGI) products since 2005.
+CTL is a disk, processor and cdrom device emulation subsystem originally
+written for Copan Systems under Linux starting in 2003.  It has been
+shipping in Copan (now SGI) products since 2005.
 
 It was ported to FreeBSD in 2008, and thanks to an agreement between SGI
 (who acquired Copan's assets in 2010) and Spectra Logic in 2010, CTL is
@@ -31,7 +31,7 @@ that Spectra would work to get CTL into the FreeBSD tree.
 Features:
 ========
 
- - Disk and processor device emulation.
+ - Disk, processor and cdrom device emulation.
  - Tagged queueing
  - SCSI task attribute support (ordered, head of queue, simple tags)
  - SCSI implicit command ordering support.  (e.g. if a read follows a mode
diff --git a/sys/cam/ctl/ctl.c b/sys/cam/ctl/ctl.c
index ddac04fc8562e..1cbf8209b68aa 100644
--- a/sys/cam/ctl/ctl.c
+++ b/sys/cam/ctl/ctl.c
@@ -69,6 +69,7 @@ __FBSDID("$FreeBSD$");
 
 #include <cam/cam.h>
 #include <cam/scsi/scsi_all.h>
+#include <cam/scsi/scsi_cd.h>
 #include <cam/scsi/scsi_da.h>
 #include <cam/ctl/ctl_io.h>
 #include <cam/ctl/ctl.h>
@@ -261,6 +262,26 @@ const static struct scsi_control_page control_page_changeable = {
 	/*extended_selftest_completion_time*/{0, 0}
 };
 
+#define CTL_CEM_LEN	(sizeof(struct scsi_control_ext_page) - 4)
+
+const static struct scsi_control_ext_page control_ext_page_default = {
+	/*page_code*/SMS_CONTROL_MODE_PAGE | SMPH_SPF,
+	/*subpage_code*/0x01,
+	/*page_length*/{CTL_CEM_LEN >> 8, CTL_CEM_LEN},
+	/*flags*/0,
+	/*prio*/0,
+	/*max_sense*/0
+};
+
+const static struct scsi_control_ext_page control_ext_page_changeable = {
+	/*page_code*/SMS_CONTROL_MODE_PAGE | SMPH_SPF,
+	/*subpage_code*/0x01,
+	/*page_length*/{CTL_CEM_LEN >> 8, CTL_CEM_LEN},
+	/*flags*/0,
+	/*prio*/0,
+	/*max_sense*/0
+};
+
 const static struct scsi_info_exceptions_page ie_page_default = {
 	/*page_code*/SMS_INFO_EXCEPTIONS_PAGE,
 	/*page_length*/sizeof(struct scsi_info_exceptions_page) - 2,
@@ -333,6 +354,52 @@ const static struct ctl_logical_block_provisioning_page lbp_page_changeable = {{
 	}
 };
 
+const static struct scsi_cddvd_capabilities_page cddvd_page_default = {
+	/*page_code*/SMS_CDDVD_CAPS_PAGE,
+	/*page_length*/sizeof(struct scsi_cddvd_capabilities_page) - 2,
+	/*caps1*/0x3f,
+	/*caps2*/0x00,
+	/*caps3*/0xf0,
+	/*caps4*/0x00,
+	/*caps5*/0x29,
+	/*caps6*/0x00,
+	/*obsolete*/{0, 0},
+	/*nvol_levels*/{0, 0},
+	/*buffer_size*/{8, 0},
+	/*obsolete2*/{0, 0},
+	/*reserved*/0,
+	/*digital*/0,
+	/*obsolete3*/0,
+	/*copy_management*/0,
+	/*reserved2*/0,
+	/*rotation_control*/0,
+	/*cur_write_speed*/0,
+	/*num_speed_descr*/0,
+};
+
+const static struct scsi_cddvd_capabilities_page cddvd_page_changeable = {
+	/*page_code*/SMS_CDDVD_CAPS_PAGE,
+	/*page_length*/sizeof(struct scsi_cddvd_capabilities_page) - 2,
+	/*caps1*/0,
+	/*caps2*/0,
+	/*caps3*/0,
+	/*caps4*/0,
+	/*caps5*/0,
+	/*caps6*/0,
+	/*obsolete*/{0, 0},
+	/*nvol_levels*/{0, 0},
+	/*buffer_size*/{0, 0},
+	/*obsolete2*/{0, 0},
+	/*reserved*/0,
+	/*digital*/0,
+	/*obsolete3*/0,
+	/*copy_management*/0,
+	/*reserved2*/0,
+	/*rotation_control*/0,
+	/*cur_write_speed*/0,
+	/*num_speed_descr*/0,
+};
+
 SYSCTL_NODE(_kern_cam, OID_AUTO, ctl, CTLFLAG_RD, 0, "CAM Target Layer");
 static int worker_threads = -1;
 SYSCTL_INT(_kern_cam_ctl, OID_AUTO, worker_threads, CTLFLAG_RDTUN,
@@ -358,7 +425,7 @@ void ctl_shutdown(void);
 static int ctl_open(struct cdev *dev, int flags, int fmt, struct thread *td);
 static int ctl_close(struct cdev *dev, int flags, int fmt, struct thread *td);
 static int ctl_serialize_other_sc_cmd(struct ctl_scsiio *ctsio);
-static int ctl_ioctl_fill_ooa(struct ctl_lun *lun, uint32_t *cur_fill_num,
+static void ctl_ioctl_fill_ooa(struct ctl_lun *lun, uint32_t *cur_fill_num,
 			      struct ctl_ooa *ooa_hdr,
 			      struct ctl_ooa_entry *kern_entries);
 static int ctl_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
@@ -404,13 +471,7 @@ static int ctl_check_blocked(struct ctl_lun *lun);
 static int ctl_scsiio_lun_check(struct ctl_lun *lun,
 				const struct ctl_cmd_entry *entry,
 				struct ctl_scsiio *ctsio);
-static void ctl_failover_lun(struct ctl_lun *lun);
-static void ctl_est_ua(struct ctl_lun *lun, uint32_t initidx, ctl_ua_type ua);
-static void ctl_est_ua_all(struct ctl_lun *lun, uint32_t except, ctl_ua_type ua);
-static void ctl_clr_ua(struct ctl_lun *lun, uint32_t initidx, ctl_ua_type ua);
-static void ctl_clr_ua_all(struct ctl_lun *lun, uint32_t except, ctl_ua_type ua);
-static void ctl_clr_ua_allluns(struct ctl_softc *ctl_softc, uint32_t initidx,
-			 ctl_ua_type ua_type);
+static void ctl_failover_lun(union ctl_io *io);
 static int ctl_scsiio_precheck(struct ctl_softc *ctl_softc,
 			       struct ctl_scsiio *ctsio);
 static int ctl_scsiio(struct ctl_scsiio *ctsio);
@@ -418,11 +479,14 @@ static int ctl_scsiio(struct ctl_scsiio *ctsio);
 static int ctl_bus_reset(struct ctl_softc *ctl_softc, union ctl_io *io);
 static int ctl_target_reset(struct ctl_softc *ctl_softc, union ctl_io *io,
 			    ctl_ua_type ua_type);
-static int ctl_lun_reset(struct ctl_lun *lun, union ctl_io *io,
+static int ctl_do_lun_reset(struct ctl_lun *lun, union ctl_io *io,
 			 ctl_ua_type ua_type);
+static int ctl_lun_reset(struct ctl_softc *ctl_softc, union ctl_io *io);
 static int ctl_abort_task(union ctl_io *io);
 static int ctl_abort_task_set(union ctl_io *io);
+static int ctl_query_task(union ctl_io *io, int task_set);
 static int ctl_i_t_nexus_reset(union ctl_io *io);
+static int ctl_query_async_event(union ctl_io *io);
 static void ctl_run_task(union ctl_io *io);
 #ifdef CTL_IO_DELAY
 static void ctl_datamove_timer_wakeup(void *arg);
@@ -440,7 +504,7 @@ static int ctl_datamove_remote_xfer(union ctl_io *io, unsigned command,
 				    ctl_ha_dt_cb callback);
 static void ctl_datamove_remote_read(union ctl_io *io);
 static void ctl_datamove_remote(union ctl_io *io);
-static int ctl_process_done(union ctl_io *io);
+static void ctl_process_done(union ctl_io *io);
 static void ctl_lun_thread(void *arg);
 static void ctl_thresh_thread(void *arg);
 static void ctl_work_thread(void *arg);
@@ -498,6 +562,151 @@ static struct ctl_frontend ha_frontend =
 };
 
 static void
+ctl_ha_datamove(union ctl_io *io)
+{
+	struct ctl_lun *lun;
+	struct ctl_sg_entry *sgl;
+	union ctl_ha_msg msg;
+	uint32_t sg_entries_sent;
+	int do_sg_copy, i, j;
+
+	lun = (struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
+	memset(&msg.dt, 0, sizeof(msg.dt));
+	msg.hdr.msg_type = CTL_MSG_DATAMOVE;
+	msg.hdr.original_sc = io->io_hdr.original_sc;
+	msg.hdr.serializing_sc = io;
+	msg.hdr.nexus = io->io_hdr.nexus;
+	msg.hdr.status = io->io_hdr.status;
+	msg.dt.flags = io->io_hdr.flags;
+
+	/*
+	 * We convert everything into a S/G list here.  We can't
+	 * pass by reference, only by value between controllers.
+	 * So we can't pass a pointer to the S/G list, only as many
+	 * S/G entries as we can fit in here.  If it's possible for
+	 * us to get more than CTL_HA_MAX_SG_ENTRIES S/G entries,
+	 * then we need to break this up into multiple transfers.
+	 */
+	if (io->scsiio.kern_sg_entries == 0) {
+		msg.dt.kern_sg_entries = 1;
+#if 0
+		if (io->io_hdr.flags & CTL_FLAG_BUS_ADDR) {
+			msg.dt.sg_list[0].addr = io->scsiio.kern_data_ptr;
+		} else {
+			/* XXX KDM use busdma here! */
+			msg.dt.sg_list[0].addr =
+			    (void *)vtophys(io->scsiio.kern_data_ptr);
+		}
+#else
+		KASSERT((io->io_hdr.flags & CTL_FLAG_BUS_ADDR) == 0,
+		    ("HA does not support BUS_ADDR"));
+		msg.dt.sg_list[0].addr = io->scsiio.kern_data_ptr;
+#endif
+		msg.dt.sg_list[0].len = io->scsiio.kern_data_len;
+		do_sg_copy = 0;
+	} else {
+		msg.dt.kern_sg_entries = io->scsiio.kern_sg_entries;
+		do_sg_copy = 1;
+	}
+
+	msg.dt.kern_data_len = io->scsiio.kern_data_len;
+	msg.dt.kern_total_len = io->scsiio.kern_total_len;
+	msg.dt.kern_data_resid = io->scsiio.kern_data_resid;
+	msg.dt.kern_rel_offset = io->scsiio.kern_rel_offset;
+	msg.dt.sg_sequence = 0;
+
+	/*
+	 * Loop until we've sent all of the S/G entries.  On the
+	 * other end, we'll recompose these S/G entries into one
+	 * contiguous list before processing.
+	 */
+	for (sg_entries_sent = 0; sg_entries_sent < msg.dt.kern_sg_entries;
+	    msg.dt.sg_sequence++) {
+		msg.dt.cur_sg_entries = MIN((sizeof(msg.dt.sg_list) /
+		    sizeof(msg.dt.sg_list[0])),
+		    msg.dt.kern_sg_entries - sg_entries_sent);
+		if (do_sg_copy != 0) {
+			sgl = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
+			for (i = sg_entries_sent, j = 0;
+			     i < msg.dt.cur_sg_entries; i++, j++) {
+#if 0
+				if (io->io_hdr.flags & CTL_FLAG_BUS_ADDR) {
+					msg.dt.sg_list[j].addr = sgl[i].addr;
+				} else {
+					/* XXX KDM use busdma here! */
+					msg.dt.sg_list[j].addr =
+					    (void *)vtophys(sgl[i].addr);
+				}
+#else
+				KASSERT((io->io_hdr.flags &
+				    CTL_FLAG_BUS_ADDR) == 0,
+				    ("HA does not support BUS_ADDR"));
+				msg.dt.sg_list[j].addr = sgl[i].addr;
+#endif
+				msg.dt.sg_list[j].len = sgl[i].len;
+			}
+		}
+
+		sg_entries_sent += msg.dt.cur_sg_entries;
+		msg.dt.sg_last = (sg_entries_sent >= msg.dt.kern_sg_entries);
+		if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg,
+		    sizeof(msg.dt) - sizeof(msg.dt.sg_list) +
+		    sizeof(struct ctl_sg_entry) * msg.dt.cur_sg_entries,
+		    M_WAITOK) > CTL_HA_STATUS_SUCCESS) {
+			io->io_hdr.port_status = 31341;
+			io->scsiio.be_move_done(io);
+			return;
+		}
+		msg.dt.sent_sg_entries = sg_entries_sent;
+	}
+
+	/*
+	 * Officially handover the request from us to peer.
+	 * If failover has just happened, then we must return error.
+	 * If failover happen just after, then it is not our problem.
+	 */
+	if (lun)
+		mtx_lock(&lun->lun_lock);
+	if (io->io_hdr.flags & CTL_FLAG_FAILOVER) {
+		if (lun)
+			mtx_unlock(&lun->lun_lock);
+		io->io_hdr.port_status = 31342;
+		io->scsiio.be_move_done(io);
+		return;
+	}
+	io->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE;
+	io->io_hdr.flags |= CTL_FLAG_DMA_INPROG;
+	if (lun)
+		mtx_unlock(&lun->lun_lock);
+}
+
+static void
+ctl_ha_done(union ctl_io *io)
+{
+	union ctl_ha_msg msg;
+
+	if (io->io_hdr.io_type == CTL_IO_SCSI) {
+		memset(&msg, 0, sizeof(msg));
+		msg.hdr.msg_type = CTL_MSG_FINISH_IO;
+		msg.hdr.original_sc = io->io_hdr.original_sc;
+		msg.hdr.nexus = io->io_hdr.nexus;
+		msg.hdr.status = io->io_hdr.status;
+		msg.scsi.scsi_status = io->scsiio.scsi_status;
+		msg.scsi.tag_num = io->scsiio.tag_num;
+		msg.scsi.tag_type = io->scsiio.tag_type;
+		msg.scsi.sense_len = io->scsiio.sense_len;
+		msg.scsi.sense_residual = io->scsiio.sense_residual;
+		msg.scsi.residual = io->scsiio.residual;
+		memcpy(&msg.scsi.sense_data, &io->scsiio.sense_data,
+		    io->scsiio.sense_len);
+		ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg,
+		    sizeof(msg.scsi) - sizeof(msg.scsi.sense_data) +
+		    msg.scsi.sense_len, M_WAITOK);
+	}
+	ctl_free_io(io);
+}
+
+static void
 ctl_isc_handler_finish_xfer(struct ctl_softc *ctl_softc,
 			    union ctl_ha_msg *msg_info)
 {
@@ -519,8 +728,6 @@ ctl_isc_handler_finish_xfer(struct ctl_softc *ctl_softc,
 	ctsio->residual = msg_info->scsi.residual;
 	memcpy(&ctsio->sense_data, &msg_info->scsi.sense_data,
 	       msg_info->scsi.sense_len);
-	memcpy(&ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN].bytes,
-	       &msg_info->scsi.lbalen, sizeof(msg_info->scsi.lbalen));
 	ctl_enqueue_isc((union ctl_io *)ctsio);
 }
 
@@ -596,12 +803,20 @@ alloc:
 	ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg->port, sizeof(msg->port) + i,
 	    M_WAITOK);
 	free(msg, M_CTL);
+
+	if (lun->flags & CTL_LUN_PRIMARY_SC) {
+		for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
+			ctl_isc_announce_mode(lun, -1,
+			    lun->mode_pages.index[i].page_code & SMPH_PC_MASK,
+			    lun->mode_pages.index[i].subpage);
+		}
+	}
 }
 
 void
 ctl_isc_announce_port(struct ctl_port *port)
 {
-	struct ctl_softc *softc = control_softc;
+	struct ctl_softc *softc = port->ctl_softc;
 	union ctl_ha_msg *msg;
 	int i;
 
@@ -616,6 +831,8 @@ ctl_isc_announce_port(struct ctl_port *port)
 		i += port->port_devid->len;
 	if (port->target_devid)
 		i += port->target_devid->len;
+	if (port->init_devid)
+		i += port->init_devid->len;
 	msg = malloc(i, M_CTL, M_WAITOK);
 	bzero(&msg->port, sizeof(msg->port));
 	msg->hdr.msg_type = CTL_MSG_PORT_SYNC;
@@ -646,19 +863,111 @@ ctl_isc_announce_port(struct ctl_port *port)
 		    msg->port.target_devid_len);
 		i += msg->port.target_devid_len;
 	}
+	if (port->init_devid) {
+		msg->port.init_devid_len = port->init_devid->len;
+		memcpy(&msg->port.data[i], port->init_devid->data,
+		    msg->port.init_devid_len);
+		i += msg->port.init_devid_len;
+	}
 	ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg->port, sizeof(msg->port) + i,
 	    M_WAITOK);
 	free(msg, M_CTL);
 }
 
+void
+ctl_isc_announce_iid(struct ctl_port *port, int iid)
+{
+	struct ctl_softc *softc = port->ctl_softc;
+	union ctl_ha_msg *msg;
+	int i, l;
+
+	if (port->targ_port < softc->port_min ||
+	    port->targ_port >= softc->port_max ||
+	    softc->ha_link != CTL_HA_LINK_ONLINE)
+		return;
+	mtx_lock(&softc->ctl_lock);
+	i = sizeof(msg->iid);
+	l = 0;
+	if (port->wwpn_iid[iid].name)
+		l = strlen(port->wwpn_iid[iid].name) + 1;
+	i += l;
+	msg = malloc(i, M_CTL, M_NOWAIT);
+	if (msg == NULL) {
+		mtx_unlock(&softc->ctl_lock);
+		return;
+	}
+	bzero(&msg->iid, sizeof(msg->iid));
+	msg->hdr.msg_type = CTL_MSG_IID_SYNC;
+	msg->hdr.nexus.targ_port = port->targ_port;
+	msg->hdr.nexus.initid = iid;
+	msg->iid.in_use = port->wwpn_iid[iid].in_use;
+	msg->iid.name_len = l;
+	msg->iid.wwpn = port->wwpn_iid[iid].wwpn;
+	if (port->wwpn_iid[iid].name)
+		strlcpy(msg->iid.data, port->wwpn_iid[iid].name, l);
+	mtx_unlock(&softc->ctl_lock);
+	ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg->iid, i, M_NOWAIT);
+	free(msg, M_CTL);
+}
+
+void
+ctl_isc_announce_mode(struct ctl_lun *lun, uint32_t initidx,
+    uint8_t page, uint8_t subpage)
+{
+	struct ctl_softc *softc = lun->ctl_softc;
+	union ctl_ha_msg msg;
+	int i;
+
+	if (softc->ha_link != CTL_HA_LINK_ONLINE)
+		return;
+	for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
+		if ((lun->mode_pages.index[i].page_code & SMPH_PC_MASK) ==
+		    page && lun->mode_pages.index[i].subpage == subpage)
+			break;
+	}
+	if (i == CTL_NUM_MODE_PAGES)
+		return;
+	bzero(&msg.mode, sizeof(msg.mode));
+	msg.hdr.msg_type = CTL_MSG_MODE_SYNC;
+	msg.hdr.nexus.targ_port = initidx / CTL_MAX_INIT_PER_PORT;
+	msg.hdr.nexus.initid = initidx % CTL_MAX_INIT_PER_PORT;
+	msg.hdr.nexus.targ_lun = lun->lun;
+	msg.hdr.nexus.targ_mapped_lun = lun->lun;
+	msg.mode.page_code = page;
+	msg.mode.subpage = subpage;
+	msg.mode.page_len = lun->mode_pages.index[i].page_len;
+	memcpy(msg.mode.data, lun->mode_pages.index[i].page_data,
+	    msg.mode.page_len);
+	ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg.mode, sizeof(msg.mode),
+	    M_WAITOK);
+}
+
 static void
 ctl_isc_ha_link_up(struct ctl_softc *softc)
 {
 	struct ctl_port *port;
 	struct ctl_lun *lun;
+	union ctl_ha_msg msg;
+	int i;
 
-	STAILQ_FOREACH(port, &softc->port_list, links)
+	/* Announce this node parameters to peer for validation. */
+	msg.login.msg_type = CTL_MSG_LOGIN;
+	msg.login.version = CTL_HA_VERSION;
+	msg.login.ha_mode = softc->ha_mode;
+	msg.login.ha_id = softc->ha_id;
+	msg.login.max_luns = CTL_MAX_LUNS;
+	msg.login.max_ports = CTL_MAX_PORTS;
+	msg.login.max_init_per_port = CTL_MAX_INIT_PER_PORT;
+	ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg.login, sizeof(msg.login),
+	    M_WAITOK);
+
+	STAILQ_FOREACH(port, &softc->port_list, links) {
 		ctl_isc_announce_port(port);
+		for (i = 0; i < CTL_MAX_INIT_PER_PORT; i++) {
+			if (port->wwpn_iid[i].in_use)
+				ctl_isc_announce_iid(port, i);
+		}
+	}
 	STAILQ_FOREACH(lun, &softc->lun_list, links)
 		ctl_isc_announce_lun(lun);
 }
@@ -669,11 +978,15 @@ ctl_isc_ha_link_down(struct ctl_softc *softc)
 	struct ctl_port *port;
 	struct ctl_lun *lun;
 	union ctl_io *io;
+	int i;
 
 	mtx_lock(&softc->ctl_lock);
 	STAILQ_FOREACH(lun, &softc->lun_list, links) {
 		mtx_lock(&lun->lun_lock);
-		lun->flags &= ~CTL_LUN_PEER_SC_PRIMARY;
+		if (lun->flags & CTL_LUN_PEER_SC_PRIMARY) {
+			lun->flags &= ~CTL_LUN_PEER_SC_PRIMARY;
+			ctl_est_ua_all(lun, -1, CTL_UA_ASYM_ACC_CHANGE);
+		}
 		mtx_unlock(&lun->lun_lock);
 
 		mtx_unlock(&softc->ctl_lock);
@@ -690,6 +1003,11 @@ ctl_isc_ha_link_down(struct ctl_softc *softc)
 		    port->targ_port < softc->port_max)
 			continue;
 		port->status &= ~CTL_PORT_STATUS_ONLINE;
+		for (i = 0; i < CTL_MAX_INIT_PER_PORT; i++) {
+			port->wwpn_iid[i].in_use = 0;
+			free(port->wwpn_iid[i].name, M_CTL);
+			port->wwpn_iid[i].name = NULL;
+		}
 	}
 	mtx_unlock(&softc->ctl_lock);
 }
@@ -700,8 +1018,14 @@ ctl_isc_ua(struct ctl_softc *softc, union ctl_ha_msg *msg, int len)
 	struct ctl_lun *lun;
 	uint32_t iid = ctl_get_initindex(&msg->hdr.nexus);
 
+	mtx_lock(&softc->ctl_lock);
 	if (msg->hdr.nexus.targ_lun < CTL_MAX_LUNS &&
-	    (lun = softc->ctl_luns[msg->hdr.nexus.targ_lun]) != NULL) {
+	    (lun = softc->ctl_luns[msg->hdr.nexus.targ_mapped_lun]) != NULL) {
+		mtx_lock(&lun->lun_lock);
+		mtx_unlock(&softc->ctl_lock);
+		if (msg->ua.ua_type == CTL_UA_THIN_PROV_THRES &&
+		    msg->ua.ua_set)
+			memcpy(lun->ua_tpt_info, msg->ua.ua_info, 8);
 		if (msg->ua.ua_all) {
 			if (msg->ua.ua_set)
 				ctl_est_ua_all(lun, iid, msg->ua.ua_type);
@@ -713,7 +1037,9 @@ ctl_isc_ua(struct ctl_softc *softc, union ctl_ha_msg *msg, int len)
 			else
 				ctl_clr_ua(lun, iid, msg->ua.ua_type);
 		}
-	}
+		mtx_unlock(&lun->lun_lock);
+	} else
+		mtx_unlock(&softc->ctl_lock);
 }
 
 static void
@@ -722,58 +1048,69 @@ ctl_isc_lun_sync(struct ctl_softc *softc, union ctl_ha_msg *msg, int len)
 	struct ctl_lun *lun;
 	struct ctl_ha_msg_lun_pr_key pr_key;
 	int i, k;
+	ctl_lun_flags oflags;
+	uint32_t targ_lun;
 
-	lun = softc->ctl_luns[msg->hdr.nexus.targ_lun];
-	if (lun == NULL) {
-		CTL_DEBUG_PRINT(("%s: Unknown LUN %d\n", __func__,
-		    msg->hdr.nexus.targ_lun));
+	targ_lun = msg->hdr.nexus.targ_mapped_lun;
+	mtx_lock(&softc->ctl_lock);
+	if ((targ_lun >= CTL_MAX_LUNS) ||
+	    ((lun = softc->ctl_luns[targ_lun]) == NULL)) {
+		mtx_unlock(&softc->ctl_lock);
+		return;
+	}
+	mtx_lock(&lun->lun_lock);
+	mtx_unlock(&softc->ctl_lock);
+	if (lun->flags & CTL_LUN_DISABLED) {
+		mtx_unlock(&lun->lun_lock);
+		return;
+	}
+	i = (lun->lun_devid != NULL) ? lun->lun_devid->len : 0;
+	if (msg->lun.lun_devid_len != i || (i > 0 &&
+	    memcmp(&msg->lun.data[0], lun->lun_devid->data, i) != 0)) {
+		mtx_unlock(&lun->lun_lock);
+		printf("%s: Received conflicting HA LUN %d\n",
+		    __func__, msg->hdr.nexus.targ_lun);
+		return;
 	} else {
-		mtx_lock(&lun->lun_lock);
-		i = (lun->lun_devid != NULL) ? lun->lun_devid->len : 0;
-		if (msg->lun.lun_devid_len != i || (i > 0 &&
-		    memcmp(&msg->lun.data[0], lun->lun_devid->data, i) != 0)) {
-			mtx_unlock(&lun->lun_lock);
-			printf("%s: Received conflicting HA LUN %d\n",
-			    __func__, msg->hdr.nexus.targ_lun);
-			return;
-		} else {
-			/* Record whether peer is primary. */
-			if ((msg->lun.flags & CTL_LUN_PRIMARY_SC) &&
-			    (msg->lun.flags & CTL_LUN_DISABLED) == 0)
-				lun->flags |= CTL_LUN_PEER_SC_PRIMARY;
-			else
-				lun->flags &= ~CTL_LUN_PEER_SC_PRIMARY;
+		/* Record whether peer is primary. */
+		oflags = lun->flags;
+		if ((msg->lun.flags & CTL_LUN_PRIMARY_SC) &&
+		    (msg->lun.flags & CTL_LUN_DISABLED) == 0)
+			lun->flags |= CTL_LUN_PEER_SC_PRIMARY;
+		else
+			lun->flags &= ~CTL_LUN_PEER_SC_PRIMARY;
+		if (oflags != lun->flags)
+			ctl_est_ua_all(lun, -1, CTL_UA_ASYM_ACC_CHANGE);
 
-			/* If peer is primary and we are not -- use data */
-			if ((lun->flags & CTL_LUN_PRIMARY_SC) == 0 &&
-			    (lun->flags & CTL_LUN_PEER_SC_PRIMARY)) {
-				lun->PRGeneration = msg->lun.pr_generation;
-				lun->pr_res_idx = msg->lun.pr_res_idx;
-				lun->res_type = msg->lun.pr_res_type;
-				lun->pr_key_count = msg->lun.pr_key_count;
-				for (k = 0; k < CTL_MAX_INITIATORS; k++)
-					ctl_clr_prkey(lun, k);
-				for (k = 0; k < msg->lun.pr_key_count; k++) {
-					memcpy(&pr_key, &msg->lun.data[i],
-					    sizeof(pr_key));
-					ctl_alloc_prkey(lun, pr_key.pr_iid);
-					ctl_set_prkey(lun, pr_key.pr_iid,
-					    pr_key.pr_key);
-					i += sizeof(pr_key);
-				}
+		/* If peer is primary and we are not -- use data */
+		if ((lun->flags & CTL_LUN_PRIMARY_SC) == 0 &&
+		    (lun->flags & CTL_LUN_PEER_SC_PRIMARY)) {
+			lun->PRGeneration = msg->lun.pr_generation;
+			lun->pr_res_idx = msg->lun.pr_res_idx;
+			lun->res_type = msg->lun.pr_res_type;
+			lun->pr_key_count = msg->lun.pr_key_count;
+			for (k = 0; k < CTL_MAX_INITIATORS; k++)
+				ctl_clr_prkey(lun, k);
+			for (k = 0; k < msg->lun.pr_key_count; k++) {
+				memcpy(&pr_key, &msg->lun.data[i],
+				    sizeof(pr_key));
+				ctl_alloc_prkey(lun, pr_key.pr_iid);
+				ctl_set_prkey(lun, pr_key.pr_iid,
+				    pr_key.pr_key);
+				i += sizeof(pr_key);
 			}
+		}
 
-			mtx_unlock(&lun->lun_lock);
-			CTL_DEBUG_PRINT(("%s: Known LUN %d, peer is %s\n",
-			    __func__, msg->hdr.nexus.targ_lun,
-			    (msg->lun.flags & CTL_LUN_PRIMARY_SC) ?
-			    "primary" : "secondary"));
+		mtx_unlock(&lun->lun_lock);
+		CTL_DEBUG_PRINT(("%s: Known LUN %d, peer is %s\n",
+		    __func__, msg->hdr.nexus.targ_lun,
+		    (msg->lun.flags & CTL_LUN_PRIMARY_SC) ?
+		    "primary" : "secondary"));
 
-			/* If we are primary but peer doesn't know -- notify */
-			if ((lun->flags & CTL_LUN_PRIMARY_SC) &&
-			    (msg->lun.flags & CTL_LUN_PEER_SC_PRIMARY) == 0)
-				ctl_isc_announce_lun(lun);
-		}
+		/* If we are primary but peer doesn't know -- notify */
+		if ((lun->flags & CTL_LUN_PRIMARY_SC) &&
+		    (msg->lun.flags & CTL_LUN_PEER_SC_PRIMARY) == 0)
+			ctl_isc_announce_lun(lun);
 	}
 }
 
@@ -781,6 +1118,7 @@ static void
 ctl_isc_port_sync(struct ctl_softc *softc, union ctl_ha_msg *msg, int len)
 {
 	struct ctl_port *port;
+	struct ctl_lun *lun;
 	int i, new;
 
 	port = softc->ctl_ports[msg->hdr.nexus.targ_port];
@@ -791,6 +1129,8 @@ ctl_isc_port_sync(struct ctl_softc *softc, union ctl_ha_msg *msg, int len)
 		port = malloc(sizeof(*port), M_CTL, M_WAITOK | M_ZERO);
 		port->frontend = &ha_frontend;
 		port->targ_port = msg->hdr.nexus.targ_port;
+		port->fe_datamove = ctl_ha_datamove;
+		port->fe_done = ctl_ha_done;
 	} else if (port->frontend == &ha_frontend) {
 		CTL_DEBUG_PRINT(("%s: Updated port %d\n", __func__,
 		    msg->hdr.nexus.targ_port));
@@ -847,8 +1187,23 @@ ctl_isc_port_sync(struct ctl_softc *softc, union ctl_ha_msg *msg, int len)
 		port->target_devid->len = msg->port.target_devid_len;
 		i += msg->port.target_devid_len;
 	} else {
-		free(port->port_devid, M_CTL);
-		port->port_devid = NULL;
+		free(port->target_devid, M_CTL);
+		port->target_devid = NULL;
+	}
+	if (msg->port.init_devid_len != 0) {
+		if (port->init_devid == NULL ||
+		    port->init_devid->len != msg->port.init_devid_len) {
+			free(port->init_devid, M_CTL);
+			port->init_devid = malloc(sizeof(struct ctl_devid) +
+			    msg->port.init_devid_len, M_CTL, M_WAITOK);
+		}
+		memcpy(port->init_devid->data, &msg->port.data[i],
+		    msg->port.init_devid_len);
+		port->init_devid->len = msg->port.init_devid_len;
+		i += msg->port.init_devid_len;
+	} else {
+		free(port->init_devid, M_CTL);
+		port->init_devid = NULL;
 	}
 	if (new) {
 		if (ctl_port_register(port) != 0) {
@@ -856,6 +1211,106 @@ ctl_isc_port_sync(struct ctl_softc *softc, union ctl_ha_msg *msg, int len)
 			    __func__);
 		}
 	}
+	mtx_lock(&softc->ctl_lock);
+	STAILQ_FOREACH(lun, &softc->lun_list, links) {
+		if (ctl_lun_map_to_port(port, lun->lun) >= CTL_MAX_LUNS)
+			continue;
+		mtx_lock(&lun->lun_lock);
+		ctl_est_ua_all(lun, -1, CTL_UA_INQ_CHANGE);
+		mtx_unlock(&lun->lun_lock);
+	}
+	mtx_unlock(&softc->ctl_lock);
+}
+
+static void
+ctl_isc_iid_sync(struct ctl_softc *softc, union ctl_ha_msg *msg, int len)
+{
+	struct ctl_port *port;
+	int iid;
+
+	port = softc->ctl_ports[msg->hdr.nexus.targ_port];
+	if (port == NULL) {
+		printf("%s: Received IID for unknown port %d\n",
+		    __func__, msg->hdr.nexus.targ_port);
+		return;
+	}
+	iid = msg->hdr.nexus.initid;
+	port->wwpn_iid[iid].in_use = msg->iid.in_use;
+	port->wwpn_iid[iid].wwpn = msg->iid.wwpn;
+	free(port->wwpn_iid[iid].name, M_CTL);
+	if (msg->iid.name_len) {
+		port->wwpn_iid[iid].name = strndup(&msg->iid.data[0],
+		    msg->iid.name_len, M_CTL);
+	} else
+		port->wwpn_iid[iid].name = NULL;
+}
+
+static void
+ctl_isc_login(struct ctl_softc *softc, union ctl_ha_msg *msg, int len)
+{
+
+	if (msg->login.version != CTL_HA_VERSION) {
+		printf("CTL HA peers have different versions %d != %d\n",
+		    msg->login.version, CTL_HA_VERSION);
+		ctl_ha_msg_abort(CTL_HA_CHAN_CTL);
+		return;
+	}
+	if (msg->login.ha_mode != softc->ha_mode) {
+		printf("CTL HA peers have different ha_mode %d != %d\n",
+		    msg->login.ha_mode, softc->ha_mode);
+		ctl_ha_msg_abort(CTL_HA_CHAN_CTL);
+		return;
+	}
+	if (msg->login.ha_id == softc->ha_id) {
+		printf("CTL HA peers have same ha_id %d\n", msg->login.ha_id);
+		ctl_ha_msg_abort(CTL_HA_CHAN_CTL);
+		return;
+	}
+	if (msg->login.max_luns != CTL_MAX_LUNS ||
+	    msg->login.max_ports != CTL_MAX_PORTS ||
+	    msg->login.max_init_per_port != CTL_MAX_INIT_PER_PORT) {
+		printf("CTL HA peers have different limits\n");
+		ctl_ha_msg_abort(CTL_HA_CHAN_CTL);
+		return;
+	}
+}
+
+static void
+ctl_isc_mode_sync(struct ctl_softc *softc, union ctl_ha_msg *msg, int len)
+{
+	struct ctl_lun *lun;
+	int i;
+	uint32_t initidx, targ_lun;
+
+	targ_lun = msg->hdr.nexus.targ_mapped_lun;
+	mtx_lock(&softc->ctl_lock);
+	if ((targ_lun >= CTL_MAX_LUNS) ||
+	    ((lun = softc->ctl_luns[targ_lun]) == NULL)) {
+		mtx_unlock(&softc->ctl_lock);
+		return;
+	}
+	mtx_lock(&lun->lun_lock);
+	mtx_unlock(&softc->ctl_lock);
+	if (lun->flags & CTL_LUN_DISABLED) {
+		mtx_unlock(&lun->lun_lock);
+		return;
+	}
+	for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
+		if ((lun->mode_pages.index[i].page_code & SMPH_PC_MASK) ==
+		    msg->mode.page_code &&
+		    lun->mode_pages.index[i].subpage == msg->mode.subpage)
+			break;
+	}
+	if (i == CTL_NUM_MODE_PAGES) {
+		mtx_unlock(&lun->lun_lock);
+		return;
+	}
+	memcpy(lun->mode_pages.index[i].page_data, msg->mode.data,
+	    lun->mode_pages.index[i].page_len);
+	initidx = ctl_get_initindex(&msg->hdr.nexus);
+	if (initidx != -1)
+		ctl_est_ua_all(lun, initidx, CTL_UA_MODE_CHANGE);
+	mtx_unlock(&lun->lun_lock);
 }
 
 /*
@@ -865,12 +1320,11 @@ ctl_isc_port_sync(struct ctl_softc *softc, union ctl_ha_msg *msg, int len)
 static void
 ctl_isc_event_handler(ctl_ha_channel channel, ctl_ha_event event, int param)
 {
-	struct ctl_softc *softc;
+	struct ctl_softc *softc = control_softc;
 	union ctl_io *io;
 	struct ctl_prio *presio;
 	ctl_ha_status isc_status;
 
-	softc = control_softc;
 	CTL_DEBUG_PRINT(("CTL: Isc Msg event %d\n", event));
 	if (event == CTL_HA_EVT_MSG_RECV) {
 		union ctl_ha_msg *msg, msgbuf;
@@ -920,7 +1374,7 @@ ctl_isc_event_handler(ctl_ha_channel channel, ctl_ha_event event, int param)
 			io->scsiio.tag_type = msg->scsi.tag_type;
 #ifdef CTL_TIME_IO
 			io->io_hdr.start_time = time_uptime;
-			getbintime(&io->io_hdr.start_bt);
+			getbinuptime(&io->io_hdr.start_bt);
 #endif /* CTL_TIME_IO */
 			io->scsiio.cdb_len = msg->scsi.cdb_len;
 			memcpy(io->scsiio.cdb, msg->scsi.cdb,
@@ -954,10 +1408,15 @@ ctl_isc_event_handler(ctl_ha_channel channel, ctl_ha_event event, int param)
 			 * when the datamove is complete.
 			 */
 			io->io_hdr.serializing_sc = msg->hdr.serializing_sc;
+			if (msg->hdr.status == CTL_SUCCESS)
+				io->io_hdr.status = msg->hdr.status;
 
 			if (msg->dt.sg_sequence == 0) {
+#ifdef CTL_TIME_IO
+				getbinuptime(&io->io_hdr.dma_start_bt);
+#endif
 				i = msg->dt.kern_sg_entries +
-				    io->scsiio.kern_data_len /
+				    msg->dt.kern_data_len /
 				    CTL_HA_DATAMOVE_SEGMENT + 1;
 				sgl = malloc(sizeof(*sgl) * i, M_CTL,
 				    M_WAITOK | M_ZERO);
@@ -993,11 +1452,8 @@ ctl_isc_event_handler(ctl_ha_channel channel, ctl_ha_event event, int param)
 				sgl[i].len = msg->dt.sg_list[j].len;
 
 #if 0
-				printf("%s: L: %p,%d -> %p,%d j=%d, i=%d\n",
-				       __func__,
-				       msg->dt.sg_list[j].addr,
-				       msg->dt.sg_list[j].len,
-				       sgl[i].addr, sgl[i].len, j, i);
+				printf("%s: DATAMOVE: %p,%lu j=%d, i=%d\n",
+				    __func__, sgl[i].addr, sgl[i].len, j, i);
 #endif
 			}
 
@@ -1025,6 +1481,7 @@ ctl_isc_event_handler(ctl_ha_channel channel, ctl_ha_event event, int param)
 			 */
 			io = msg->hdr.serializing_sc;
 			io->io_hdr.msg_type = CTL_MSG_DATAMOVE_DONE;
+			io->io_hdr.flags &= ~CTL_FLAG_DMA_INPROG;
 			io->io_hdr.flags |= CTL_FLAG_IO_ACTIVE;
 			io->io_hdr.port_status = msg->scsi.fetd_status;
 			io->scsiio.residual = msg->scsi.residual;
@@ -1036,6 +1493,8 @@ ctl_isc_event_handler(ctl_ha_channel channel, ctl_ha_event event, int param)
 				memcpy(&io->scsiio.sense_data,
 				    &msg->scsi.sense_data,
 				    msg->scsi.sense_len);
+				if (msg->hdr.status == CTL_SUCCESS)
+					io->io_hdr.flags |= CTL_FLAG_STATUS_SENT;
 			}
 			ctl_enqueue_isc(io);
 			break;
@@ -1104,7 +1563,7 @@ ctl_isc_event_handler(ctl_ha_channel channel, ctl_ha_event event, int param)
 			taskio->tag_type = msg->task.tag_type;
 #ifdef CTL_TIME_IO
 			taskio->io_hdr.start_time = time_uptime;
-			getbintime(&taskio->io_hdr.start_bt);
+			getbinuptime(&taskio->io_hdr.start_bt);
 #endif /* CTL_TIME_IO */
 			ctl_run_task((union ctl_io *)taskio);
 			break;
@@ -1129,9 +1588,19 @@ ctl_isc_event_handler(ctl_ha_channel channel, ctl_ha_event event, int param)
 		case CTL_MSG_LUN_SYNC:
 			ctl_isc_lun_sync(softc, msg, param);
 			break;
+		case CTL_MSG_IID_SYNC:
+			ctl_isc_iid_sync(softc, msg, param);
+			break;
+		case CTL_MSG_LOGIN:
+			ctl_isc_login(softc, msg, param);
+			break;
+		case CTL_MSG_MODE_SYNC:
+			ctl_isc_mode_sync(softc, msg, param);
+			break;
 		default:
 			printf("Received HA message of unknown type %d\n",
 			    msg->hdr.msg_type);
+			ctl_ha_msg_abort(CTL_HA_CHAN_CTL);
 			break;
 		}
 		if (msg != &msgbuf)
@@ -1178,7 +1647,7 @@ ctl_copy_sense_data_back(union ctl_io *src, union ctl_ha_msg *dest)
 	dest->hdr.status = src->io_hdr.status;
 }
 
-static void
+void
 ctl_est_ua(struct ctl_lun *lun, uint32_t initidx, ctl_ua_type ua)
 {
 	struct ctl_softc *softc = lun->ctl_softc;
@@ -1193,25 +1662,33 @@ ctl_est_ua(struct ctl_lun *lun, uint32_t initidx, ctl_ua_type ua)
 	pu[initidx % CTL_MAX_INIT_PER_PORT] |= ua;
 }
 
-static void
-ctl_est_ua_all(struct ctl_lun *lun, uint32_t except, ctl_ua_type ua)
+void
+ctl_est_ua_port(struct ctl_lun *lun, int port, uint32_t except, ctl_ua_type ua)
 {
-	struct ctl_softc *softc = lun->ctl_softc;
-	int i, j;
+	int i;
 
 	mtx_assert(&lun->lun_lock, MA_OWNED);
-	for (i = softc->port_min; i < softc->port_max; i++) {
-		if (lun->pending_ua[i] == NULL)
+	if (lun->pending_ua[port] == NULL)
+		return;
+	for (i = 0; i < CTL_MAX_INIT_PER_PORT; i++) {
+		if (port * CTL_MAX_INIT_PER_PORT + i == except)
 			continue;
-		for (j = 0; j < CTL_MAX_INIT_PER_PORT; j++) {
-			if (i * CTL_MAX_INIT_PER_PORT + j == except)
-				continue;
-			lun->pending_ua[i][j] |= ua;
-		}
+		lun->pending_ua[port][i] |= ua;
 	}
 }
 
-static void
+void
+ctl_est_ua_all(struct ctl_lun *lun, uint32_t except, ctl_ua_type ua)
+{
+	struct ctl_softc *softc = lun->ctl_softc;
+	int i;
+
+	mtx_assert(&lun->lun_lock, MA_OWNED);
+	for (i = softc->port_min; i < softc->port_max; i++)
+		ctl_est_ua_port(lun, i, except, ua);
+}
+
+void
 ctl_clr_ua(struct ctl_lun *lun, uint32_t initidx, ctl_ua_type ua)
 {
 	struct ctl_softc *softc = lun->ctl_softc;
@@ -1226,7 +1703,7 @@ ctl_clr_ua(struct ctl_lun *lun, uint32_t initidx, ctl_ua_type ua)
 	pu[initidx % CTL_MAX_INIT_PER_PORT] &= ~ua;
 }
 
-static void
+void
 ctl_clr_ua_all(struct ctl_lun *lun, uint32_t except, ctl_ua_type ua)
 {
 	struct ctl_softc *softc = lun->ctl_softc;
@@ -1244,7 +1721,7 @@ ctl_clr_ua_all(struct ctl_lun *lun, uint32_t except, ctl_ua_type ua)
 	}
 }
 
-static void
+void
 ctl_clr_ua_allluns(struct ctl_softc *ctl_softc, uint32_t initidx,
     ctl_ua_type ua_type)
 {
@@ -1298,16 +1775,13 @@ ctl_init(void)
 {
 	struct ctl_softc *softc;
 	void *other_pool;
-	int i, error, retval;
+	int i, error;
 
-	retval = 0;
-	control_softc = malloc(sizeof(*control_softc), M_DEVBUF,
+	softc = control_softc = malloc(sizeof(*control_softc), M_DEVBUF,
 			       M_WAITOK | M_ZERO);
-	softc = control_softc;
 
 	softc->dev = make_dev(&ctl_cdevsw, 0, UID_ROOT, GID_OPERATOR, 0600,
 			      "cam/ctl");
-
 	softc->dev->si_drv1 = softc;
 
 	sysctl_ctx_init(&softc->sysctl_ctx);
@@ -1326,13 +1800,7 @@ ctl_init(void)
 	mtx_init(&softc->ctl_lock, "CTL mutex", NULL, MTX_DEF);
 	softc->io_zone = uma_zcreate("CTL IO", sizeof(union ctl_io),
 	    NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
-	softc->open_count = 0;
-
-	/*
-	 * Default to actually sending a SYNCHRONIZE CACHE command down to
-	 * the drive.
-	 */
-	softc->flags = CTL_FLAG_REAL_SYNC;
+	softc->flags = 0;
 
 	SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree),
 	    OID_AUTO, "ha_mode", CTLFLAG_RDTUN, (int *)&softc->ha_mode, 0,
@@ -1439,31 +1907,23 @@ ctl_init(void)
 void
 ctl_shutdown(void)
 {
-	struct ctl_softc *softc;
+	struct ctl_softc *softc = control_softc;
 	struct ctl_lun *lun, *next_lun;
 
-	softc = (struct ctl_softc *)control_softc;
-
 	if (softc->is_single == 0) {
+		ctl_ha_msg_shutdown(softc);
 		if (ctl_ha_msg_deregister(CTL_HA_CHAN_CTL)
-		    != CTL_HA_STATUS_SUCCESS) {
-			printf("ctl_shutdown: ctl_ha_msg_deregister failed.\n");
-		}
-		if (ctl_ha_msg_shutdown(softc) != CTL_HA_STATUS_SUCCESS) {
-			printf("ctl_shutdown: ctl_ha_msg_shutdown failed.\n");
-		}
+		    != CTL_HA_STATUS_SUCCESS)
+			printf("%s: ctl_ha_msg_deregister failed.\n", __func__);
+		if (ctl_ha_msg_destroy(softc) != CTL_HA_STATUS_SUCCESS)
+			printf("%s: ctl_ha_msg_destroy failed.\n", __func__);
 		ctl_frontend_deregister(&ha_frontend);
 	}
 
 	mtx_lock(&softc->ctl_lock);
 
-	/*
-	 * Free up each LUN.
-	 */
-	for (lun = STAILQ_FIRST(&softc->lun_list); lun != NULL; lun = next_lun){
-		next_lun = STAILQ_NEXT(lun, links);
+	STAILQ_FOREACH_SAFE(lun, &softc->lun_list, links, next_lun)
 		ctl_free_lun(lun);
-	}
 
 	mtx_unlock(&softc->ctl_lock);
 
@@ -1521,7 +1981,7 @@ ctl_close(struct cdev *dev, int flags, int fmt, struct thread *td)
 int
 ctl_remove_initiator(struct ctl_port *port, int iid)
 {
-	struct ctl_softc *softc = control_softc;
+	struct ctl_softc *softc = port->ctl_softc;
 
 	mtx_assert(&softc->ctl_lock, MA_NOTOWNED);
 
@@ -1535,6 +1995,7 @@ ctl_remove_initiator(struct ctl_port *port, int iid)
 	port->wwpn_iid[iid].in_use--;
 	port->wwpn_iid[iid].last_use = time_uptime;
 	mtx_unlock(&softc->ctl_lock);
+	ctl_isc_announce_iid(port, iid);
 
 	return (0);
 }
@@ -1546,7 +2007,7 @@ ctl_remove_initiator(struct ctl_port *port, int iid)
 int
 ctl_add_initiator(struct ctl_port *port, int iid, uint64_t wwpn, char *name)
 {
-	struct ctl_softc *softc = control_softc;
+	struct ctl_softc *softc = port->ctl_softc;
 	time_t best_time;
 	int i, best;
 
@@ -1650,6 +2111,7 @@ take:
 	port->wwpn_iid[iid].wwpn = wwpn;
 	port->wwpn_iid[iid].in_use++;
 	mtx_unlock(&softc->ctl_lock);
+	ctl_isc_announce_iid(port, iid);
 
 	return (iid);
 }
@@ -1720,30 +2182,43 @@ ctl_create_iid(struct ctl_port *port, int iid, uint8_t *buf)
 static int
 ctl_serialize_other_sc_cmd(struct ctl_scsiio *ctsio)
 {
-	struct ctl_softc *softc;
+	struct ctl_softc *softc = control_softc;
 	union ctl_ha_msg msg_info;
+	struct ctl_port *port;
 	struct ctl_lun *lun;
 	const struct ctl_cmd_entry *entry;
 	int retval = 0;
 	uint32_t targ_lun;
 
-	softc = control_softc;
-
 	targ_lun = ctsio->io_hdr.nexus.targ_mapped_lun;
+	mtx_lock(&softc->ctl_lock);
+
+	/* Make sure that we know about this port. */
+	port = ctl_io_port(&ctsio->io_hdr);
+	if (port == NULL || (port->status & CTL_PORT_STATUS_ONLINE) == 0) {
+		ctl_set_internal_failure(ctsio, /*sks_valid*/ 0,
+					 /*retry_count*/ 1);
+		goto badjuju;
+	}
+
+	/* Make sure that we know about this LUN. */
 	if ((targ_lun < CTL_MAX_LUNS) &&
 	    ((lun = softc->ctl_luns[targ_lun]) != NULL)) {
+		mtx_lock(&lun->lun_lock);
+		mtx_unlock(&softc->ctl_lock);
 		/*
 		 * If the LUN is invalid, pretend that it doesn't exist.
 		 * It will go away as soon as all pending I/O has been
 		 * completed.
 		 */
-		mtx_lock(&lun->lun_lock);
 		if (lun->flags & CTL_LUN_DISABLED) {
 			mtx_unlock(&lun->lun_lock);
 			lun = NULL;
 		}
-	} else
+	} else {
+		mtx_unlock(&softc->ctl_lock);
 		lun = NULL;
+	}
 	if (lun == NULL) {
 		/*
 		 * The other node would not send this request to us unless
@@ -1752,25 +2227,13 @@ ctl_serialize_other_sc_cmd(struct ctl_scsiio *ctsio)
 		 * a race, so respond to initiator in the most opaque way.
 		 */
 		ctl_set_busy(ctsio);
-		ctl_copy_sense_data_back((union ctl_io *)ctsio, &msg_info);
-		msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
-		msg_info.hdr.serializing_sc = NULL;
-		msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
-		ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
-		    sizeof(msg_info.scsi), M_WAITOK);
-		return(1);
+		goto badjuju;
 	}
 
 	entry = ctl_get_cmd_entry(ctsio, NULL);
 	if (ctl_scsiio_lun_check(lun, entry, ctsio) != 0) {
 		mtx_unlock(&lun->lun_lock);
-		ctl_copy_sense_data_back((union ctl_io *)ctsio, &msg_info);
-		msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
-		msg_info.hdr.serializing_sc = NULL;
-		msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
-		ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
-		    sizeof(msg_info.scsi), M_WAITOK);
-		return(1);
+		goto badjuju;
 	}
 
 	ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr = lun;
@@ -1816,42 +2279,28 @@ ctl_serialize_other_sc_cmd(struct ctl_scsiio *ctsio)
 	case CTL_ACTION_OVERLAP:
 		TAILQ_REMOVE(&lun->ooa_queue, &ctsio->io_hdr, ooa_links);
 		mtx_unlock(&lun->lun_lock);
-		retval = 1;
-
 		ctl_set_overlapped_cmd(ctsio);
-		ctl_copy_sense_data_back((union ctl_io *)ctsio, &msg_info);
-		msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
-		msg_info.hdr.serializing_sc = NULL;
-		msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
-		ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
-		    sizeof(msg_info.scsi), M_WAITOK);
-		break;
+		goto badjuju;
 	case CTL_ACTION_OVERLAP_TAG:
 		TAILQ_REMOVE(&lun->ooa_queue, &ctsio->io_hdr, ooa_links);
 		mtx_unlock(&lun->lun_lock);
-		retval = 1;
 		ctl_set_overlapped_tag(ctsio, ctsio->tag_num);
-		ctl_copy_sense_data_back((union ctl_io *)ctsio, &msg_info);
-		msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
-		msg_info.hdr.serializing_sc = NULL;
-		msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
-		ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
-		    sizeof(msg_info.scsi), M_WAITOK);
-		break;
+		goto badjuju;
 	case CTL_ACTION_ERROR:
 	default:
 		TAILQ_REMOVE(&lun->ooa_queue, &ctsio->io_hdr, ooa_links);
 		mtx_unlock(&lun->lun_lock);
-		retval = 1;
 
 		ctl_set_internal_failure(ctsio, /*sks_valid*/ 0,
 					 /*retry_count*/ 0);
+badjuju:
 		ctl_copy_sense_data_back((union ctl_io *)ctsio, &msg_info);
 		msg_info.hdr.original_sc = ctsio->io_hdr.original_sc;
 		msg_info.hdr.serializing_sc = NULL;
 		msg_info.hdr.msg_type = CTL_MSG_BAD_JUJU;
 		ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
 		    sizeof(msg_info.scsi), M_WAITOK);
+		retval = 1;
 		break;
 	}
 	return (retval);
@@ -1860,14 +2309,11 @@ ctl_serialize_other_sc_cmd(struct ctl_scsiio *ctsio)
 /*
  * Returns 0 for success, errno for failure.
  */
-static int
+static void
 ctl_ioctl_fill_ooa(struct ctl_lun *lun, uint32_t *cur_fill_num,
 		   struct ctl_ooa *ooa_hdr, struct ctl_ooa_entry *kern_entries)
 {
 	union ctl_io *io;
-	int retval;
-
-	retval = 0;
 
 	mtx_lock(&lun->lun_lock);
 	for (io = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue); (io != NULL);
@@ -1907,8 +2353,6 @@ ctl_ioctl_fill_ooa(struct ctl_lun *lun, uint32_t *cur_fill_num,
 			entry->cmd_flags |= CTL_OOACMD_FLAG_DMA_QUEUED;
 	}
 	mtx_unlock(&lun->lun_lock);
-
-	return (retval);
 }
 
 static void *
@@ -2091,11 +2535,10 @@ static int
 ctl_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
 	  struct thread *td)
 {
-	struct ctl_softc *softc;
+	struct ctl_softc *softc = dev->si_drv1;
+	struct ctl_lun *lun;
 	int retval;
 
-	softc = control_softc;
-
 	retval = 0;
 
 	switch (cmd) {
@@ -2179,115 +2622,7 @@ ctl_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
 		mtx_unlock(&softc->ctl_lock);
 		break;
 	}
-	case CTL_GET_PORT_LIST: {
-		struct ctl_port *port;
-		struct ctl_port_list *list;
-		int i;
-
-		list = (struct ctl_port_list *)addr;
-
-		if (list->alloc_len != (list->alloc_num *
-		    sizeof(struct ctl_port_entry))) {
-			printf("%s: CTL_GET_PORT_LIST: alloc_len %u != "
-			       "alloc_num %u * sizeof(struct ctl_port_entry) "
-			       "%zu\n", __func__, list->alloc_len,
-			       list->alloc_num, sizeof(struct ctl_port_entry));
-			retval = EINVAL;
-			break;
-		}
-		list->fill_len = 0;
-		list->fill_num = 0;
-		list->dropped_num = 0;
-		i = 0;
-		mtx_lock(&softc->ctl_lock);
-		STAILQ_FOREACH(port, &softc->port_list, links) {
-			struct ctl_port_entry entry, *list_entry;
-
-			if (list->fill_num >= list->alloc_num) {
-				list->dropped_num++;
-				continue;
-			}
-
-			entry.port_type = port->port_type;
-			strlcpy(entry.port_name, port->port_name,
-				sizeof(entry.port_name));
-			entry.targ_port = port->targ_port;
-			entry.physical_port = port->physical_port;
-			entry.virtual_port = port->virtual_port;
-			entry.wwnn = port->wwnn;
-			entry.wwpn = port->wwpn;
-			if (port->status & CTL_PORT_STATUS_ONLINE)
-				entry.online = 1;
-			else
-				entry.online = 0;
-
-			list_entry = &list->entries[i];
-
-			retval = copyout(&entry, list_entry, sizeof(entry));
-			if (retval != 0) {
-				printf("%s: CTL_GET_PORT_LIST: copyout "
-				       "returned %d\n", __func__, retval);
-				break;
-			}
-			i++;
-			list->fill_num++;
-			list->fill_len += sizeof(entry);
-		}
-		mtx_unlock(&softc->ctl_lock);
-
-		/*
-		 * If this is non-zero, we had a copyout fault, so there's
-		 * probably no point in attempting to set the status inside
-		 * the structure.
-		 */
-		if (retval != 0)
-			break;
-
-		if (list->dropped_num > 0)
-			list->status = CTL_PORT_LIST_NEED_MORE_SPACE;
-		else
-			list->status = CTL_PORT_LIST_OK;
-		break;
-	}
-	case CTL_DUMP_OOA: {
-		struct ctl_lun *lun;
-		union ctl_io *io;
-		char printbuf[128];
-		struct sbuf sb;
-
-		mtx_lock(&softc->ctl_lock);
-		printf("Dumping OOA queues:\n");
-		STAILQ_FOREACH(lun, &softc->lun_list, links) {
-			mtx_lock(&lun->lun_lock);
-			for (io = (union ctl_io *)TAILQ_FIRST(
-			     &lun->ooa_queue); io != NULL;
-			     io = (union ctl_io *)TAILQ_NEXT(&io->io_hdr,
-			     ooa_links)) {
-				sbuf_new(&sb, printbuf, sizeof(printbuf),
-					 SBUF_FIXEDLEN);
-				sbuf_printf(&sb, "LUN %jd tag 0x%04x%s%s%s%s: ",
-					    (intmax_t)lun->lun,
-					    io->scsiio.tag_num,
-					    (io->io_hdr.flags &
-					    CTL_FLAG_BLOCKED) ? "" : " BLOCKED",
-					    (io->io_hdr.flags &
-					    CTL_FLAG_DMA_INPROG) ? " DMA" : "",
-					    (io->io_hdr.flags &
-					    CTL_FLAG_ABORT) ? " ABORT" : "",
-			                    (io->io_hdr.flags &
-		                        CTL_FLAG_IS_WAS_ON_RTR) ? " RTR" : "");
-				ctl_scsi_command_string(&io->scsiio, NULL, &sb);
-				sbuf_finish(&sb);
-				printf("%s\n", sbuf_data(&sb));
-			}
-			mtx_unlock(&lun->lun_lock);
-		}
-		printf("OOA queues dump done\n");
-		mtx_unlock(&softc->ctl_lock);
-		break;
-	}
 	case CTL_GET_OOA: {
-		struct ctl_lun *lun;
 		struct ctl_ooa *ooa_hdr;
 		struct ctl_ooa_entry *entries;
 		uint32_t cur_fill_num;
@@ -2337,21 +2672,13 @@ ctl_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
 
 		if (ooa_hdr->flags & CTL_OOA_FLAG_ALL_LUNS) {
 			STAILQ_FOREACH(lun, &softc->lun_list, links) {
-				retval = ctl_ioctl_fill_ooa(lun, &cur_fill_num,
-					ooa_hdr, entries);
-				if (retval != 0)
-					break;
-			}
-			if (retval != 0) {
-				mtx_unlock(&softc->ctl_lock);
-				free(entries, M_CTL);
-				break;
+				ctl_ioctl_fill_ooa(lun, &cur_fill_num,
+				    ooa_hdr, entries);
 			}
 		} else {
 			lun = softc->ctl_luns[ooa_hdr->lun_num];
-
-			retval = ctl_ioctl_fill_ooa(lun, &cur_fill_num,ooa_hdr,
-						    entries);
+			ctl_ioctl_fill_ooa(lun, &cur_fill_num, ooa_hdr,
+			    entries);
 		}
 		mtx_unlock(&softc->ctl_lock);
 
@@ -2364,7 +2691,7 @@ ctl_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
 			       __func__, ooa_hdr->fill_len);
 		}
 
-		getbintime(&ooa_hdr->cur_bt);
+		getbinuptime(&ooa_hdr->cur_bt);
 
 		if (cur_fill_num > ooa_hdr->alloc_num) {
 			ooa_hdr->dropped_num = cur_fill_num -ooa_hdr->alloc_num;
@@ -2377,44 +2704,8 @@ ctl_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
 		free(entries, M_CTL);
 		break;
 	}
-	case CTL_CHECK_OOA: {
-		union ctl_io *io;
-		struct ctl_lun *lun;
-		struct ctl_ooa_info *ooa_info;
-
-
-		ooa_info = (struct ctl_ooa_info *)addr;
-
-		if (ooa_info->lun_id >= CTL_MAX_LUNS) {
-			ooa_info->status = CTL_OOA_INVALID_LUN;
-			break;
-		}
-		mtx_lock(&softc->ctl_lock);
-		lun = softc->ctl_luns[ooa_info->lun_id];
-		if (lun == NULL) {
-			mtx_unlock(&softc->ctl_lock);
-			ooa_info->status = CTL_OOA_INVALID_LUN;
-			break;
-		}
-		mtx_lock(&lun->lun_lock);
-		mtx_unlock(&softc->ctl_lock);
-		ooa_info->num_entries = 0;
-		for (io = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue);
-		     io != NULL; io = (union ctl_io *)TAILQ_NEXT(
-		     &io->io_hdr, ooa_links)) {
-			ooa_info->num_entries++;
-		}
-		mtx_unlock(&lun->lun_lock);
-
-		ooa_info->status = CTL_OOA_SUCCESS;
-
-		break;
-	}
 	case CTL_DELAY_IO: {
 		struct ctl_io_delay_info *delay_info;
-#ifdef CTL_IO_DELAY
-		struct ctl_lun *lun;
-#endif /* CTL_IO_DELAY */
 
 		delay_info = (struct ctl_io_delay_info *)addr;
 
@@ -2468,73 +2759,8 @@ ctl_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
 #endif /* CTL_IO_DELAY */
 		break;
 	}
-	case CTL_REALSYNC_SET: {
-		int *syncstate;
-
-		syncstate = (int *)addr;
-
-		mtx_lock(&softc->ctl_lock);
-		switch (*syncstate) {
-		case 0:
-			softc->flags &= ~CTL_FLAG_REAL_SYNC;
-			break;
-		case 1:
-			softc->flags |= CTL_FLAG_REAL_SYNC;
-			break;
-		default:
-			retval = EINVAL;
-			break;
-		}
-		mtx_unlock(&softc->ctl_lock);
-		break;
-	}
-	case CTL_REALSYNC_GET: {
-		int *syncstate;
-
-		syncstate = (int*)addr;
-
-		mtx_lock(&softc->ctl_lock);
-		if (softc->flags & CTL_FLAG_REAL_SYNC)
-			*syncstate = 1;
-		else
-			*syncstate = 0;
-		mtx_unlock(&softc->ctl_lock);
-
-		break;
-	}
-	case CTL_SETSYNC:
-	case CTL_GETSYNC: {
-		struct ctl_sync_info *sync_info;
-		struct ctl_lun *lun;
-
-		sync_info = (struct ctl_sync_info *)addr;
-
-		mtx_lock(&softc->ctl_lock);
-		lun = softc->ctl_luns[sync_info->lun_id];
-		if (lun == NULL) {
-			mtx_unlock(&softc->ctl_lock);
-			sync_info->status = CTL_GS_SYNC_NO_LUN;
-		}
-		/*
-		 * Get or set the sync interval.  We're not bounds checking
-		 * in the set case, hopefully the user won't do something
-		 * silly.
-		 */
-		mtx_lock(&lun->lun_lock);
-		mtx_unlock(&softc->ctl_lock);
-		if (cmd == CTL_GETSYNC)
-			sync_info->sync_interval = lun->sync_interval;
-		else
-			lun->sync_interval = sync_info->sync_interval;
-		mtx_unlock(&lun->lun_lock);
-
-		sync_info->status = CTL_GS_SYNC_OK;
-
-		break;
-	}
 	case CTL_GETSTATS: {
 		struct ctl_stats *stats;
-		struct ctl_lun *lun;
 		int i;
 
 		stats = (struct ctl_stats *)addr;
@@ -2549,9 +2775,9 @@ ctl_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
 		 * XXX KDM no locking here.  If the LUN list changes,
 		 * things can blow up.
 		 */
-		for (i = 0, lun = STAILQ_FIRST(&softc->lun_list); lun != NULL;
-		     i++, lun = STAILQ_NEXT(lun, links)) {
-			retval = copyout(&lun->stats, &stats->lun_stats[i],
+		i = 0;
+		STAILQ_FOREACH(lun, &softc->lun_list, links) {
+			retval = copyout(&lun->stats, &stats->lun_stats[i++],
 					 sizeof(lun->stats));
 			if (retval != 0)
 				break;
@@ -2570,7 +2796,6 @@ ctl_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
 	}
 	case CTL_ERROR_INJECT: {
 		struct ctl_error_desc *err_desc, *new_err_desc;
-		struct ctl_lun *lun;
 
 		err_desc = (struct ctl_error_desc *)addr;
 
@@ -2617,7 +2842,6 @@ ctl_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
 	}
 	case CTL_ERROR_INJECT_DELETE: {
 		struct ctl_error_desc *delete_desc, *desc, *desc2;
-		struct ctl_lun *lun;
 		int delete_done;
 
 		delete_desc = (struct ctl_error_desc *)addr;
@@ -2661,8 +2885,6 @@ ctl_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
 		mtx_lock(&softc->ctl_lock);
 		printf("CTL Persistent Reservation information start:\n");
 		for (i = 0; i < CTL_MAX_LUNS; i++) {
-			struct ctl_lun *lun;
-
 			lun = softc->ctl_luns[i];
 
 			if ((lun == NULL)
@@ -2756,7 +2978,6 @@ ctl_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
 	}
 	case CTL_LUN_LIST: {
 		struct sbuf *sb;
-		struct ctl_lun *lun;
 		struct ctl_lun_list *list;
 		struct ctl_option *opt;
 
@@ -3129,6 +3350,17 @@ ctl_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
 			mtx_unlock(&softc->ctl_lock);
 			return (ENXIO);
 		}
+		if (port->status & CTL_PORT_STATUS_ONLINE) {
+			STAILQ_FOREACH(lun, &softc->lun_list, links) {
+				if (ctl_lun_map_to_port(port, lun->lun) >=
+				    CTL_MAX_LUNS)
+					continue;
+				mtx_lock(&lun->lun_lock);
+				ctl_est_ua_port(lun, lm->port, -1,
+				    CTL_UA_LUN_CHANGE);
+				mtx_unlock(&lun->lun_lock);
+			}
+		}
 		mtx_unlock(&softc->ctl_lock); // XXX: port_enable sleeps
 		if (lm->plun < CTL_MAX_LUNS) {
 			if (lm->lun == UINT32_MAX)
@@ -3145,6 +3377,8 @@ ctl_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
 				retval = ctl_lun_map_init(port);
 		} else
 			return (ENXIO);
+		if (port->status & CTL_PORT_STATUS_ONLINE)
+			ctl_isc_announce_port(port);
 		break;
 	}
 	default: {
@@ -3193,7 +3427,7 @@ ctl_get_initindex(struct ctl_nexus *nexus)
 int
 ctl_lun_map_init(struct ctl_port *port)
 {
-	struct ctl_softc *softc = control_softc;
+	struct ctl_softc *softc = port->ctl_softc;
 	struct ctl_lun *lun;
 	uint32_t i;
 
@@ -3217,7 +3451,7 @@ ctl_lun_map_init(struct ctl_port *port)
 int
 ctl_lun_map_deinit(struct ctl_port *port)
 {
-	struct ctl_softc *softc = control_softc;
+	struct ctl_softc *softc = port->ctl_softc;
 	struct ctl_lun *lun;
 
 	if (port->lun_map == NULL)
@@ -3523,46 +3757,6 @@ ctl_zero_io(union ctl_io *io)
 	io->io_hdr.pool = pool_ref;
 }
 
-/*
- * This routine is currently used for internal copies of ctl_ios that need
- * to persist for some reason after we've already returned status to the
- * FETD.  (Thus the flag set.)
- *
- * XXX XXX
- * Note that this makes a blind copy of all fields in the ctl_io, except
- * for the pool reference.  This includes any memory that has been
- * allocated!  That memory will no longer be valid after done has been
- * called, so this would be VERY DANGEROUS for command that actually does
- * any reads or writes.  Right now (11/7/2005), this is only used for immediate
- * start and stop commands, which don't transfer any data, so this is not a
- * problem.  If it is used for anything else, the caller would also need to
- * allocate data buffer space and this routine would need to be modified to
- * copy the data buffer(s) as well.
- */
-void
-ctl_copy_io(union ctl_io *src, union ctl_io *dest)
-{
-	void *pool_ref;
-
-	if ((src == NULL)
-	 || (dest == NULL))
-		return;
-
-	/*
-	 * May need to preserve linked list pointers at some point too.
-	 */
-	pool_ref = dest->io_hdr.pool;
-
-	memcpy(dest, src, MIN(sizeof(*src), sizeof(*dest)));
-
-	dest->io_hdr.pool = pool_ref;
-	/*
-	 * We need to know that this is an internal copy, and doesn't need
-	 * to get passed back to the FETD that allocated it.
-	 */
-	dest->io_hdr.flags |= CTL_FLAG_INT_COPY;
-}
-
 int
 ctl_expand_number(const char *buf, uint64_t *num)
 {
@@ -3627,14 +3821,14 @@ ctl_init_page_index(struct ctl_lun *lun)
 	for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
 
 		page_index = &lun->mode_pages.index[i];
-		/*
-		 * If this is a disk-only mode page, there's no point in
-		 * setting it up.  For some pages, we have to have some
-		 * basic information about the disk in order to calculate the
-		 * mode page data.
-		 */
-		if ((lun->be_lun->lun_type != T_DIRECT)
-		 && (page_index->page_flags & CTL_PAGE_FLAG_DISK_ONLY))
+		if (lun->be_lun->lun_type == T_DIRECT &&
+		    (page_index->page_flags & CTL_PAGE_FLAG_DIRECT) == 0)
+			continue;
+		if (lun->be_lun->lun_type == T_PROCESSOR &&
+		    (page_index->page_flags & CTL_PAGE_FLAG_PROC) == 0)
+			continue;
+		if (lun->be_lun->lun_type == T_CDROM &&
+		    (page_index->page_flags & CTL_PAGE_FLAG_CDROM) == 0)
 			continue;
 
 		switch (page_index->page_code & SMPH_PC_MASK) {
@@ -3815,35 +4009,65 @@ ctl_init_page_index(struct ctl_lun *lun)
 			break;
 		}
 		case SMS_CONTROL_MODE_PAGE: {
-			struct scsi_control_page *control_page;
-
-			if (page_index->subpage != SMS_SUBPAGE_PAGE_0)
-				panic("invalid subpage value %d",
-				      page_index->subpage);
+			switch (page_index->subpage) {
+			case SMS_SUBPAGE_PAGE_0: {
+				struct scsi_control_page *control_page;
 
-			memcpy(&lun->mode_pages.control_page[CTL_PAGE_DEFAULT],
-			       &control_page_default,
-			       sizeof(control_page_default));
-			memcpy(&lun->mode_pages.control_page[
-			       CTL_PAGE_CHANGEABLE], &control_page_changeable,
-			       sizeof(control_page_changeable));
-			memcpy(&lun->mode_pages.control_page[CTL_PAGE_SAVED],
-			       &control_page_default,
-			       sizeof(control_page_default));
-			control_page = &lun->mode_pages.control_page[
-			    CTL_PAGE_SAVED];
-			value = ctl_get_opt(&lun->be_lun->options, "reordering");
-			if (value != NULL && strcmp(value, "unrestricted") == 0) {
-				control_page->queue_flags &= ~SCP_QUEUE_ALG_MASK;
-				control_page->queue_flags |= SCP_QUEUE_ALG_UNRESTRICTED;
+				memcpy(&lun->mode_pages.control_page[
+				    CTL_PAGE_DEFAULT],
+				       &control_page_default,
+				       sizeof(control_page_default));
+				memcpy(&lun->mode_pages.control_page[
+				    CTL_PAGE_CHANGEABLE],
+				       &control_page_changeable,
+				       sizeof(control_page_changeable));
+				memcpy(&lun->mode_pages.control_page[
+				    CTL_PAGE_SAVED],
+				       &control_page_default,
+				       sizeof(control_page_default));
+				control_page = &lun->mode_pages.control_page[
+				    CTL_PAGE_SAVED];
+				value = ctl_get_opt(&lun->be_lun->options,
+				    "reordering");
+				if (value != NULL &&
+				    strcmp(value, "unrestricted") == 0) {
+					control_page->queue_flags &=
+					    ~SCP_QUEUE_ALG_MASK;
+					control_page->queue_flags |=
+					    SCP_QUEUE_ALG_UNRESTRICTED;
+				}
+				memcpy(&lun->mode_pages.control_page[
+				    CTL_PAGE_CURRENT],
+				       &lun->mode_pages.control_page[
+				    CTL_PAGE_SAVED],
+				       sizeof(control_page_default));
+				page_index->page_data =
+				    (uint8_t *)lun->mode_pages.control_page;
+				break;
+			}
+			case 0x01:
+				memcpy(&lun->mode_pages.control_ext_page[
+				    CTL_PAGE_DEFAULT],
+				       &control_ext_page_default,
+				       sizeof(control_ext_page_default));
+				memcpy(&lun->mode_pages.control_ext_page[
+				    CTL_PAGE_CHANGEABLE],
+				       &control_ext_page_changeable,
+				       sizeof(control_ext_page_changeable));
+				memcpy(&lun->mode_pages.control_ext_page[
+				    CTL_PAGE_SAVED],
+				       &control_ext_page_default,
+				       sizeof(control_ext_page_default));
+				memcpy(&lun->mode_pages.control_ext_page[
+				    CTL_PAGE_CURRENT],
+				       &lun->mode_pages.control_ext_page[
+				    CTL_PAGE_SAVED],
+				       sizeof(control_ext_page_default));
+				page_index->page_data =
+				    (uint8_t *)lun->mode_pages.control_ext_page;
+				break;
 			}
-			memcpy(&lun->mode_pages.control_page[CTL_PAGE_CURRENT],
-			       &lun->mode_pages.control_page[CTL_PAGE_SAVED],
-			       sizeof(control_page_default));
-			page_index->page_data =
-				(uint8_t *)lun->mode_pages.control_page;
 			break;
-
 		}
 		case SMS_INFO_EXCEPTIONS_PAGE: {
 			switch (page_index->subpage) {
@@ -3936,12 +4160,26 @@ ctl_init_page_index(struct ctl_lun *lun)
 			}}
 			break;
 		}
+		case SMS_CDDVD_CAPS_PAGE:{
+			memcpy(&lun->mode_pages.cddvd_page[CTL_PAGE_DEFAULT],
+			       &cddvd_page_default,
+			       sizeof(cddvd_page_default));
+			memcpy(&lun->mode_pages.cddvd_page[
+			       CTL_PAGE_CHANGEABLE], &cddvd_page_changeable,
+			       sizeof(cddvd_page_changeable));
+			memcpy(&lun->mode_pages.cddvd_page[CTL_PAGE_SAVED],
+			       &cddvd_page_default,
+			       sizeof(cddvd_page_default));
+			memcpy(&lun->mode_pages.cddvd_page[CTL_PAGE_CURRENT],
+			       &lun->mode_pages.cddvd_page[CTL_PAGE_SAVED],
+			       sizeof(cddvd_page_default));
+			page_index->page_data =
+				(uint8_t *)lun->mode_pages.cddvd_page;
+			break;
+		}
 		case SMS_VENDOR_SPECIFIC_PAGE:{
 			switch (page_index->subpage) {
 			case DBGCNF_SUBPAGE_CODE: {
-				struct copan_debugconf_subpage *current_page,
-							       *saved_page;
-
 				memcpy(&lun->mode_pages.debugconf_subpage[
 				       CTL_PAGE_CURRENT],
 				       &debugconf_page_default,
@@ -3959,16 +4197,7 @@ ctl_init_page_index(struct ctl_lun *lun)
 				       &debugconf_page_default,
 				       sizeof(debugconf_page_default));
 				page_index->page_data =
-					(uint8_t *)lun->mode_pages.debugconf_subpage;
-
-				current_page = (struct copan_debugconf_subpage *)
-					(page_index->page_data +
-					 (page_index->page_len *
-					  CTL_PAGE_CURRENT));
-				saved_page = (struct copan_debugconf_subpage *)
-					(page_index->page_data +
-					 (page_index->page_len *
-					  CTL_PAGE_SAVED));
+				    (uint8_t *)lun->mode_pages.debugconf_subpage;
 				break;
 			}
 			default:
@@ -4001,18 +4230,18 @@ ctl_init_log_page_index(struct ctl_lun *lun)
 	for (i = 0, j = 0, k = 0; i < CTL_NUM_LOG_PAGES; i++) {
 
 		page_index = &lun->log_pages.index[i];
-		/*
-		 * If this is a disk-only mode page, there's no point in
-		 * setting it up.  For some pages, we have to have some
-		 * basic information about the disk in order to calculate the
-		 * mode page data.
-		 */
-		if ((lun->be_lun->lun_type != T_DIRECT)
-		 && (page_index->page_flags & CTL_PAGE_FLAG_DISK_ONLY))
+		if (lun->be_lun->lun_type == T_DIRECT &&
+		    (page_index->page_flags & CTL_PAGE_FLAG_DIRECT) == 0)
+			continue;
+		if (lun->be_lun->lun_type == T_PROCESSOR &&
+		    (page_index->page_flags & CTL_PAGE_FLAG_PROC) == 0)
+			continue;
+		if (lun->be_lun->lun_type == T_CDROM &&
+		    (page_index->page_flags & CTL_PAGE_FLAG_CDROM) == 0)
 			continue;
 
 		if (page_index->page_code == SLS_LOGICAL_BLOCK_PROVISIONING &&
-		     lun->backend->lun_attr == NULL)
+		    lun->backend->lun_attr == NULL)
 			continue;
 
 		if (page_index->page_code != prev) {
@@ -4084,7 +4313,7 @@ ctl_alloc_lun(struct ctl_softc *ctl_softc, struct ctl_lun *ctl_lun,
 	struct ctl_lun *nlun, *lun;
 	struct scsi_vpd_id_descriptor *desc;
 	struct scsi_vpd_id_t10 *t10id;
-	const char *eui, *naa, *scsiname, *vendor;
+	const char *eui, *naa, *scsiname, *vendor, *value;
 	int lun_number, i, lun_malloced;
 	int devidlen, idlen1, idlen2 = 0, len;
 
@@ -4096,8 +4325,8 @@ ctl_alloc_lun(struct ctl_softc *ctl_softc, struct ctl_lun *ctl_lun,
 	 */
 	switch (be_lun->lun_type) {
 	case T_DIRECT:
-		break;
 	case T_PROCESSOR:
+	case T_CDROM:
 		break;
 	case T_SEQUENTIAL:
 	case T_CHANGER:
@@ -4238,18 +4467,23 @@ ctl_alloc_lun(struct ctl_softc *ctl_softc, struct ctl_lun *ctl_lun,
 	be_lun->ctl_lun = lun;
 	be_lun->lun_id = lun_number;
 	atomic_add_int(&be_lun->be->num_luns, 1);
-	if (be_lun->flags & CTL_LUN_FLAG_OFFLINE)
-		lun->flags |= CTL_LUN_OFFLINE;
-
-	if (be_lun->flags & CTL_LUN_FLAG_POWERED_OFF)
+	if (be_lun->flags & CTL_LUN_FLAG_EJECTED)
+		lun->flags |= CTL_LUN_EJECTED;
+	if (be_lun->flags & CTL_LUN_FLAG_NO_MEDIA)
+		lun->flags |= CTL_LUN_NO_MEDIA;
+	if (be_lun->flags & CTL_LUN_FLAG_STOPPED)
 		lun->flags |= CTL_LUN_STOPPED;
 
-	if (be_lun->flags & CTL_LUN_FLAG_INOPERABLE)
-		lun->flags |= CTL_LUN_INOPERABLE;
-
 	if (be_lun->flags & CTL_LUN_FLAG_PRIMARY)
 		lun->flags |= CTL_LUN_PRIMARY_SC;
 
+	value = ctl_get_opt(&be_lun->options, "removable");
+	if (value != NULL) {
+		if (strcmp(value, "on") == 0)
+			lun->flags |= CTL_LUN_REMOVABLE;
+	} else if (be_lun->lun_type == T_CDROM)
+		lun->flags |= CTL_LUN_REMOVABLE;
+
 	lun->ctl_softc = ctl_softc;
 #ifdef CTL_TIME_IO
 	lun->last_busy = getsbinuptime();
@@ -4284,9 +4518,8 @@ ctl_alloc_lun(struct ctl_softc *ctl_softc, struct ctl_lun *ctl_lun,
 	/* Setup statistics gathering */
 	lun->stats.device_type = be_lun->lun_type;
 	lun->stats.lun_number = lun_number;
-	if (lun->stats.device_type == T_DIRECT)
-		lun->stats.blocksize = be_lun->blocksize;
-	else
+	lun->stats.blocksize = be_lun->blocksize;
+	if (be_lun->blocksize == 0)
 		lun->stats.flags = CTL_LUN_STATS_NO_BLOCKSIZE;
 	for (i = 0;i < CTL_MAX_PORTS;i++)
 		lun->stats.ports[i].targ_port = i;
@@ -4354,14 +4587,11 @@ ctl_free_lun(struct ctl_lun *lun)
 static void
 ctl_create_lun(struct ctl_be_lun *be_lun)
 {
-	struct ctl_softc *softc;
-
-	softc = control_softc;
 
 	/*
 	 * ctl_alloc_lun() should handle all potential failure cases.
 	 */
-	ctl_alloc_lun(softc, NULL, be_lun);
+	ctl_alloc_lun(control_softc, NULL, be_lun);
 }
 
 int
@@ -4402,8 +4632,7 @@ ctl_enable_lun(struct ctl_be_lun *be_lun)
 	lun->flags &= ~CTL_LUN_DISABLED;
 	mtx_unlock(&lun->lun_lock);
 
-	for (port = STAILQ_FIRST(&softc->port_list); port != NULL; port = nport) {
-		nport = STAILQ_NEXT(port, links);
+	STAILQ_FOREACH_SAFE(port, &softc->port_list, links, nport) {
 		if ((port->status & CTL_PORT_STATUS_ONLINE) == 0 ||
 		    port->lun_map != NULL || port->lun_enable == NULL)
 			continue;
@@ -4503,23 +4732,51 @@ ctl_stop_lun(struct ctl_be_lun *be_lun)
 }
 
 int
-ctl_lun_offline(struct ctl_be_lun *be_lun)
+ctl_lun_no_media(struct ctl_be_lun *be_lun)
 {
 	struct ctl_lun *lun = (struct ctl_lun *)be_lun->ctl_lun;
 
 	mtx_lock(&lun->lun_lock);
-	lun->flags |= CTL_LUN_OFFLINE;
+	lun->flags |= CTL_LUN_NO_MEDIA;
 	mtx_unlock(&lun->lun_lock);
 	return (0);
 }
 
 int
-ctl_lun_online(struct ctl_be_lun *be_lun)
+ctl_lun_has_media(struct ctl_be_lun *be_lun)
+{
+	struct ctl_lun *lun = (struct ctl_lun *)be_lun->ctl_lun;
+	union ctl_ha_msg msg;
+
+	mtx_lock(&lun->lun_lock);
+	lun->flags &= ~(CTL_LUN_NO_MEDIA | CTL_LUN_EJECTED);
+	if (lun->flags & CTL_LUN_REMOVABLE)
+		ctl_est_ua_all(lun, -1, CTL_UA_MEDIUM_CHANGE);
+	mtx_unlock(&lun->lun_lock);
+	if ((lun->flags & CTL_LUN_REMOVABLE) &&
+	    lun->ctl_softc->ha_mode == CTL_HA_MODE_XFER) {
+		bzero(&msg.ua, sizeof(msg.ua));
+		msg.hdr.msg_type = CTL_MSG_UA;
+		msg.hdr.nexus.initid = -1;
+		msg.hdr.nexus.targ_port = -1;
+		msg.hdr.nexus.targ_lun = lun->lun;
+		msg.hdr.nexus.targ_mapped_lun = lun->lun;
+		msg.ua.ua_all = 1;
+		msg.ua.ua_set = 1;
+		msg.ua.ua_type = CTL_UA_MEDIUM_CHANGE;
+		ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg, sizeof(msg.ua),
+		    M_WAITOK);
+	}
+	return (0);
+}
+
+int
+ctl_lun_ejected(struct ctl_be_lun *be_lun)
 {
 	struct ctl_lun *lun = (struct ctl_lun *)be_lun->ctl_lun;
 
 	mtx_lock(&lun->lun_lock);
-	lun->flags &= ~CTL_LUN_OFFLINE;
+	lun->flags |= CTL_LUN_EJECTED;
 	mtx_unlock(&lun->lun_lock);
 	return (0);
 }
@@ -4531,8 +4788,8 @@ ctl_lun_primary(struct ctl_be_lun *be_lun)
 
 	mtx_lock(&lun->lun_lock);
 	lun->flags |= CTL_LUN_PRIMARY_SC;
-	mtx_unlock(&lun->lun_lock);
 	ctl_est_ua_all(lun, -1, CTL_UA_ASYM_ACC_CHANGE);
+	mtx_unlock(&lun->lun_lock);
 	ctl_isc_announce_lun(lun);
 	return (0);
 }
@@ -4544,8 +4801,8 @@ ctl_lun_secondary(struct ctl_be_lun *be_lun)
 
 	mtx_lock(&lun->lun_lock);
 	lun->flags &= ~CTL_LUN_PRIMARY_SC;
-	mtx_unlock(&lun->lun_lock);
 	ctl_est_ua_all(lun, -1, CTL_UA_ASYM_ACC_CHANGE);
+	mtx_unlock(&lun->lun_lock);
 	ctl_isc_announce_lun(lun);
 	return (0);
 }
@@ -4589,28 +4846,6 @@ ctl_invalidate_lun(struct ctl_be_lun *be_lun)
 	return (0);
 }
 
-int
-ctl_lun_inoperable(struct ctl_be_lun *be_lun)
-{
-	struct ctl_lun *lun = (struct ctl_lun *)be_lun->ctl_lun;
-
-	mtx_lock(&lun->lun_lock);
-	lun->flags |= CTL_LUN_INOPERABLE;
-	mtx_unlock(&lun->lun_lock);
-	return (0);
-}
-
-int
-ctl_lun_operable(struct ctl_be_lun *be_lun)
-{
-	struct ctl_lun *lun = (struct ctl_lun *)be_lun->ctl_lun;
-
-	mtx_lock(&lun->lun_lock);
-	lun->flags &= ~CTL_LUN_INOPERABLE;
-	mtx_unlock(&lun->lun_lock);
-	return (0);
-}
-
 void
 ctl_lun_capacity_changed(struct ctl_be_lun *be_lun)
 {
@@ -4618,7 +4853,7 @@ ctl_lun_capacity_changed(struct ctl_be_lun *be_lun)
 	union ctl_ha_msg msg;
 
 	mtx_lock(&lun->lun_lock);
-	ctl_est_ua_all(lun, -1, CTL_UA_CAPACITY_CHANGED);
+	ctl_est_ua_all(lun, -1, CTL_UA_CAPACITY_CHANGE);
 	mtx_unlock(&lun->lun_lock);
 	if (lun->ctl_softc->ha_mode == CTL_HA_MODE_XFER) {
 		/* Send msg to other side. */
@@ -4630,7 +4865,7 @@ ctl_lun_capacity_changed(struct ctl_be_lun *be_lun)
 		msg.hdr.nexus.targ_mapped_lun = lun->lun;
 		msg.ua.ua_all = 1;
 		msg.ua.ua_set = 1;
-		msg.ua.ua_type = CTL_UA_CAPACITY_CHANGED;
+		msg.ua.ua_type = CTL_UA_CAPACITY_CHANGE;
 		ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg, sizeof(msg.ua),
 		    M_WAITOK);
 	}
@@ -4807,36 +5042,14 @@ ctl_config_read_done(union ctl_io *io)
 int
 ctl_scsi_release(struct ctl_scsiio *ctsio)
 {
-	int length, longid, thirdparty_id, resv_id;
 	struct ctl_lun *lun;
 	uint32_t residx;
 
-	length = 0;
-	resv_id = 0;
-
 	CTL_DEBUG_PRINT(("ctl_scsi_release\n"));
 
 	residx = ctl_get_initindex(&ctsio->io_hdr.nexus);
 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
 
-	switch (ctsio->cdb[0]) {
-	case RELEASE_10: {
-		struct scsi_release_10 *cdb;
-
-		cdb = (struct scsi_release_10 *)ctsio->cdb;
-
-		if (cdb->byte2 & SR10_LONGID)
-			longid = 1;
-		else
-			thirdparty_id = cdb->thirdparty_id;
-
-		resv_id = cdb->resv_id;
-		length = scsi_2btoul(cdb->length);
-		break;
-	}
-	}
-
-
 	/*
 	 * XXX KDM right now, we only support LUN reservation.  We don't
 	 * support 3rd party reservations, or extent reservations, which
@@ -4844,25 +5057,6 @@ ctl_scsi_release(struct ctl_scsiio *ctsio)
 	 * far, we've got a LUN reservation.  Anything else got kicked out
 	 * above.  So, according to SPC, ignore the length.
 	 */
-	length = 0;
-
-	if (((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0)
-	 && (length > 0)) {
-		ctsio->kern_data_ptr = malloc(length, M_CTL, M_WAITOK);
-		ctsio->kern_data_len = length;
-		ctsio->kern_total_len = length;
-		ctsio->kern_data_resid = 0;
-		ctsio->kern_rel_offset = 0;
-		ctsio->kern_sg_entries = 0;
-		ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
-		ctsio->be_move_done = ctl_config_move_done;
-		ctl_datamove((union ctl_io *)ctsio);
-
-		return (CTL_RETVAL_COMPLETE);
-	}
-
-	if (length > 0)
-		thirdparty_id = scsi_8btou64(ctsio->kern_data_ptr);
 
 	mtx_lock(&lun->lun_lock);
 
@@ -4878,11 +5072,6 @@ ctl_scsi_release(struct ctl_scsiio *ctsio)
 
 	mtx_unlock(&lun->lun_lock);
 
-	if (ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) {
-		free(ctsio->kern_data_ptr, M_CTL);
-		ctsio->io_hdr.flags &= ~CTL_FLAG_ALLOCATED;
-	}
-
 	ctl_set_success(ctsio);
 	ctl_done((union ctl_io *)ctsio);
 	return (CTL_RETVAL_COMPLETE);
@@ -4891,41 +5080,14 @@ ctl_scsi_release(struct ctl_scsiio *ctsio)
 int
 ctl_scsi_reserve(struct ctl_scsiio *ctsio)
 {
-	int extent, thirdparty, longid;
-	int resv_id, length;
-	uint64_t thirdparty_id;
 	struct ctl_lun *lun;
 	uint32_t residx;
 
-	extent = 0;
-	thirdparty = 0;
-	longid = 0;
-	resv_id = 0;
-	length = 0;
-	thirdparty_id = 0;
-
 	CTL_DEBUG_PRINT(("ctl_reserve\n"));
 
 	residx = ctl_get_initindex(&ctsio->io_hdr.nexus);
 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
 
-	switch (ctsio->cdb[0]) {
-	case RESERVE_10: {
-		struct scsi_reserve_10 *cdb;
-
-		cdb = (struct scsi_reserve_10 *)ctsio->cdb;
-
-		if (cdb->byte2 & SR10_LONGID)
-			longid = 1;
-		else
-			thirdparty_id = cdb->thirdparty_id;
-
-		resv_id = cdb->resv_id;
-		length = scsi_2btoul(cdb->length);
-		break;
-	}
-	}
-
 	/*
 	 * XXX KDM right now, we only support LUN reservation.  We don't
 	 * support 3rd party reservations, or extent reservations, which
@@ -4933,45 +5095,18 @@ ctl_scsi_reserve(struct ctl_scsiio *ctsio)
 	 * far, we've got a LUN reservation.  Anything else got kicked out
 	 * above.  So, according to SPC, ignore the length.
 	 */
-	length = 0;
-
-	if (((ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) == 0)
-	 && (length > 0)) {
-		ctsio->kern_data_ptr = malloc(length, M_CTL, M_WAITOK);
-		ctsio->kern_data_len = length;
-		ctsio->kern_total_len = length;
-		ctsio->kern_data_resid = 0;
-		ctsio->kern_rel_offset = 0;
-		ctsio->kern_sg_entries = 0;
-		ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
-		ctsio->be_move_done = ctl_config_move_done;
-		ctl_datamove((union ctl_io *)ctsio);
-
-		return (CTL_RETVAL_COMPLETE);
-	}
-
-	if (length > 0)
-		thirdparty_id = scsi_8btou64(ctsio->kern_data_ptr);
 
 	mtx_lock(&lun->lun_lock);
 	if ((lun->flags & CTL_LUN_RESERVED) && (lun->res_idx != residx)) {
 		ctl_set_reservation_conflict(ctsio);
 		goto bailout;
 	}
-
 	lun->flags |= CTL_LUN_RESERVED;
 	lun->res_idx = residx;
-
 	ctl_set_success(ctsio);
 
 bailout:
 	mtx_unlock(&lun->lun_lock);
-
-	if (ctsio->io_hdr.flags & CTL_FLAG_ALLOCATED) {
-		free(ctsio->kern_data_ptr, M_CTL);
-		ctsio->io_hdr.flags &= ~CTL_FLAG_ALLOCATED;
-	}
-
 	ctl_done((union ctl_io *)ctsio);
 	return (CTL_RETVAL_COMPLETE);
 }
@@ -4986,113 +5121,83 @@ ctl_start_stop(struct ctl_scsiio *ctsio)
 	CTL_DEBUG_PRINT(("ctl_start_stop\n"));
 
 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
-	retval = 0;
-
 	cdb = (struct scsi_start_stop_unit *)ctsio->cdb;
 
-	/*
-	 * XXX KDM
-	 * We don't support the immediate bit on a stop unit.  In order to
-	 * do that, we would need to code up a way to know that a stop is
-	 * pending, and hold off any new commands until it completes, one
-	 * way or another.  Then we could accept or reject those commands
-	 * depending on its status.  We would almost need to do the reverse
-	 * of what we do below for an immediate start -- return the copy of
-	 * the ctl_io to the FETD with status to send to the host (and to
-	 * free the copy!) and then free the original I/O once the stop
-	 * actually completes.  That way, the OOA queue mechanism can work
-	 * to block commands that shouldn't proceed.  Another alternative
-	 * would be to put the copy in the queue in place of the original,
-	 * and return the original back to the caller.  That could be
-	 * slightly safer..
-	 */
-	if ((cdb->byte2 & SSS_IMMED)
-	 && ((cdb->how & SSS_START) == 0)) {
-		ctl_set_invalid_field(ctsio,
-				      /*sks_valid*/ 1,
-				      /*command*/ 1,
-				      /*field*/ 1,
-				      /*bit_valid*/ 1,
-				      /*bit*/ 0);
-		ctl_done((union ctl_io *)ctsio);
-		return (CTL_RETVAL_COMPLETE);
-	}
+	if ((cdb->how & SSS_PC_MASK) == 0) {
+		if ((lun->flags & CTL_LUN_PR_RESERVED) &&
+		    (cdb->how & SSS_START) == 0) {
+			uint32_t residx;
 
-	if ((lun->flags & CTL_LUN_PR_RESERVED)
-	 && ((cdb->how & SSS_START)==0)) {
-		uint32_t residx;
+			residx = ctl_get_initindex(&ctsio->io_hdr.nexus);
+			if (ctl_get_prkey(lun, residx) == 0 ||
+			    (lun->pr_res_idx != residx && lun->res_type < 4)) {
 
-		residx = ctl_get_initindex(&ctsio->io_hdr.nexus);
-		if (ctl_get_prkey(lun, residx) == 0
-		 || (lun->pr_res_idx!=residx && lun->res_type < 4)) {
+				ctl_set_reservation_conflict(ctsio);
+				ctl_done((union ctl_io *)ctsio);
+				return (CTL_RETVAL_COMPLETE);
+			}
+		}
 
-			ctl_set_reservation_conflict(ctsio);
+		if ((cdb->how & SSS_LOEJ) &&
+		    (lun->flags & CTL_LUN_REMOVABLE) == 0) {
+			ctl_set_invalid_field(ctsio,
+					      /*sks_valid*/ 1,
+					      /*command*/ 1,
+					      /*field*/ 4,
+					      /*bit_valid*/ 1,
+					      /*bit*/ 1);
+			ctl_done((union ctl_io *)ctsio);
+			return (CTL_RETVAL_COMPLETE);
+		}
+
+		if ((cdb->how & SSS_START) == 0 && (cdb->how & SSS_LOEJ) &&
+		    lun->prevent_count > 0) {
+			/* "Medium removal prevented" */
+			ctl_set_sense(ctsio, /*current_error*/ 1,
+			    /*sense_key*/(lun->flags & CTL_LUN_NO_MEDIA) ?
+			     SSD_KEY_NOT_READY : SSD_KEY_ILLEGAL_REQUEST,
+			    /*asc*/ 0x53, /*ascq*/ 0x02, SSD_ELEM_NONE);
 			ctl_done((union ctl_io *)ctsio);
 			return (CTL_RETVAL_COMPLETE);
 		}
 	}
 
-	/*
-	 * If there is no backend on this device, we can't start or stop
-	 * it.  In theory we shouldn't get any start/stop commands in the
-	 * first place at this level if the LUN doesn't have a backend.
-	 * That should get stopped by the command decode code.
-	 */
-	if (lun->backend == NULL) {
+	retval = lun->backend->config_write((union ctl_io *)ctsio);
+	return (retval);
+}
+
+int
+ctl_prevent_allow(struct ctl_scsiio *ctsio)
+{
+	struct ctl_lun *lun;
+	struct scsi_prevent *cdb;
+	int retval;
+	uint32_t initidx;
+
+	CTL_DEBUG_PRINT(("ctl_prevent_allow\n"));
+
+	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
+	cdb = (struct scsi_prevent *)ctsio->cdb;
+
+	if ((lun->flags & CTL_LUN_REMOVABLE) == 0) {
 		ctl_set_invalid_opcode(ctsio);
 		ctl_done((union ctl_io *)ctsio);
 		return (CTL_RETVAL_COMPLETE);
 	}
 
-	/*
-	 * XXX KDM Copan-specific offline behavior.
-	 * Figure out a reasonable way to port this?
-	 */
-#ifdef NEEDTOPORT
+	initidx = ctl_get_initindex(&ctsio->io_hdr.nexus);
 	mtx_lock(&lun->lun_lock);
-
-	if (((cdb->byte2 & SSS_ONOFFLINE) == 0)
-	 && (lun->flags & CTL_LUN_OFFLINE)) {
-		/*
-		 * If the LUN is offline, and the on/offline bit isn't set,
-		 * reject the start or stop.  Otherwise, let it through.
-		 */
-		mtx_unlock(&lun->lun_lock);
-		ctl_set_lun_not_ready(ctsio);
-		ctl_done((union ctl_io *)ctsio);
-	} else {
-		mtx_unlock(&lun->lun_lock);
-#endif /* NEEDTOPORT */
-		/*
-		 * This could be a start or a stop when we're online,
-		 * or a stop/offline or start/online.  A start or stop when
-		 * we're offline is covered in the case above.
-		 */
-		/*
-		 * In the non-immediate case, we send the request to
-		 * the backend and return status to the user when
-		 * it is done.
-		 *
-		 * In the immediate case, we allocate a new ctl_io
-		 * to hold a copy of the request, and send that to
-		 * the backend.  We then set good status on the
-		 * user's request and return it immediately.
-		 */
-		if (cdb->byte2 & SSS_IMMED) {
-			union ctl_io *new_io;
-
-			new_io = ctl_alloc_io(ctsio->io_hdr.pool);
-			ctl_copy_io((union ctl_io *)ctsio, new_io);
-			retval = lun->backend->config_write(new_io);
-			ctl_set_success(ctsio);
-			ctl_done((union ctl_io *)ctsio);
-		} else {
-			retval = lun->backend->config_write(
-				(union ctl_io *)ctsio);
-		}
-#ifdef NEEDTOPORT
+	if ((cdb->how & PR_PREVENT) &&
+	    ctl_is_set(lun->prevent, initidx) == 0) {
+		ctl_set_mask(lun->prevent, initidx);
+		lun->prevent_count++;
+	} else if ((cdb->how & PR_PREVENT) == 0 &&
+	    ctl_is_set(lun->prevent, initidx)) {
+		ctl_clear_mask(lun->prevent, initidx);
+		lun->prevent_count--;
 	}
-#endif
+	mtx_unlock(&lun->lun_lock);
+	retval = lun->backend->config_write((union ctl_io *)ctsio);
 	return (retval);
 }
 
@@ -5157,38 +5262,13 @@ ctl_sync_cache(struct ctl_scsiio *ctsio)
 		goto bailout;
 	}
 
-	/*
-	 * If this LUN has no backend, we can't flush the cache anyway.
-	 */
-	if (lun->backend == NULL) {
-		ctl_set_invalid_opcode(ctsio);
-		ctl_done((union ctl_io *)ctsio);
-		goto bailout;
-	}
-
 	lbalen = (struct ctl_lba_len_flags *)&ctsio->io_hdr.ctl_private[CTL_PRIV_LBA_LEN];
 	lbalen->lba = starting_lba;
 	lbalen->len = block_count;
 	lbalen->flags = byte2;
-
-	/*
-	 * Check to see whether we're configured to send the SYNCHRONIZE
-	 * CACHE command directly to the back end.
-	 */
-	mtx_lock(&lun->lun_lock);
-	if ((softc->flags & CTL_FLAG_REAL_SYNC)
-	 && (++(lun->sync_count) >= lun->sync_interval)) {
-		lun->sync_count = 0;
-		mtx_unlock(&lun->lun_lock);
-		retval = lun->backend->config_write((union ctl_io *)ctsio);
-	} else {
-		mtx_unlock(&lun->lun_lock);
-		ctl_set_success(ctsio);
-		ctl_done((union ctl_io *)ctsio);
-	}
+	retval = lun->backend->config_write((union ctl_io *)ctsio);
 
 bailout:
-
 	return (retval);
 }
 
@@ -5266,17 +5346,6 @@ ctl_format(struct ctl_scsiio *ctsio)
 		}
 	}
 
-	/*
-	 * The format command will clear out the "Medium format corrupted"
-	 * status if set by the configuration code.  That status is really
-	 * just a way to notify the host that we have lost the media, and
-	 * get them to issue a command that will basically make them think
-	 * they're blowing away the media.
-	 */
-	mtx_lock(&lun->lun_lock);
-	lun->flags &= ~CTL_LUN_INOPERABLE;
-	mtx_unlock(&lun->lun_lock);
-
 	ctl_set_success(ctsio);
 bailout:
 
@@ -5292,20 +5361,43 @@ bailout:
 int
 ctl_read_buffer(struct ctl_scsiio *ctsio)
 {
-	struct scsi_read_buffer *cdb;
 	struct ctl_lun *lun;
-	int buffer_offset, len;
+	uint64_t buffer_offset;
+	uint32_t len;
+	uint8_t byte2;
 	static uint8_t descr[4];
 	static uint8_t echo_descr[4] = { 0 };
 
 	CTL_DEBUG_PRINT(("ctl_read_buffer\n"));
-
 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
-	cdb = (struct scsi_read_buffer *)ctsio->cdb;
+	switch (ctsio->cdb[0]) {
+	case READ_BUFFER: {
+		struct scsi_read_buffer *cdb;
+
+		cdb = (struct scsi_read_buffer *)ctsio->cdb;
+		buffer_offset = scsi_3btoul(cdb->offset);
+		len = scsi_3btoul(cdb->length);
+		byte2 = cdb->byte2;
+		break;
+	}
+	case READ_BUFFER_16: {
+		struct scsi_read_buffer_16 *cdb;
+
+		cdb = (struct scsi_read_buffer_16 *)ctsio->cdb;
+		buffer_offset = scsi_8btou64(cdb->offset);
+		len = scsi_4btoul(cdb->length);
+		byte2 = cdb->byte2;
+		break;
+	}
+	default: /* This shouldn't happen. */
+		ctl_set_invalid_opcode(ctsio);
+		ctl_done((union ctl_io *)ctsio);
+		return (CTL_RETVAL_COMPLETE);
+	}
 
-	if ((cdb->byte2 & RWB_MODE) != RWB_MODE_DATA &&
-	    (cdb->byte2 & RWB_MODE) != RWB_MODE_ECHO_DESCR &&
-	    (cdb->byte2 & RWB_MODE) != RWB_MODE_DESCR) {
+	if ((byte2 & RWB_MODE) != RWB_MODE_DATA &&
+	    (byte2 & RWB_MODE) != RWB_MODE_ECHO_DESCR &&
+	    (byte2 & RWB_MODE) != RWB_MODE_DESCR) {
 		ctl_set_invalid_field(ctsio,
 				      /*sks_valid*/ 1,
 				      /*command*/ 1,
@@ -5316,10 +5408,8 @@ ctl_read_buffer(struct ctl_scsiio *ctsio)
 		return (CTL_RETVAL_COMPLETE);
 	}
 
-	len = scsi_3btoul(cdb->length);
-	buffer_offset = scsi_3btoul(cdb->offset);
-
-	if (buffer_offset + len > CTL_WRITE_BUFFER_SIZE) {
+	if (buffer_offset > CTL_WRITE_BUFFER_SIZE ||
+	    buffer_offset + len > CTL_WRITE_BUFFER_SIZE) {
 		ctl_set_invalid_field(ctsio,
 				      /*sks_valid*/ 1,
 				      /*command*/ 1,
@@ -5330,12 +5420,12 @@ ctl_read_buffer(struct ctl_scsiio *ctsio)
 		return (CTL_RETVAL_COMPLETE);
 	}
 
-	if ((cdb->byte2 & RWB_MODE) == RWB_MODE_DESCR) {
+	if ((byte2 & RWB_MODE) == RWB_MODE_DESCR) {
 		descr[0] = 0;
 		scsi_ulto3b(CTL_WRITE_BUFFER_SIZE, &descr[1]);
 		ctsio->kern_data_ptr = descr;
 		len = min(len, sizeof(descr));
-	} else if ((cdb->byte2 & RWB_MODE) == RWB_MODE_ECHO_DESCR) {
+	} else if ((byte2 & RWB_MODE) == RWB_MODE_ECHO_DESCR) {
 		ctsio->kern_data_ptr = echo_descr;
 		len = min(len, sizeof(echo_descr));
 	} else {
@@ -5430,8 +5520,6 @@ ctl_write_same(struct ctl_scsiio *ctsio)
 	int len, retval;
 	uint8_t byte2;
 
-	retval = CTL_RETVAL_COMPLETE;
-
 	CTL_DEBUG_PRINT(("ctl_write_same\n"));
 
 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
@@ -5469,9 +5557,8 @@ ctl_write_same(struct ctl_scsiio *ctsio)
 		break; /* NOTREACHED */
 	}
 
-	/* NDOB and ANCHOR flags can be used only together with UNMAP */
-	if ((byte2 & SWS_UNMAP) == 0 &&
-	    (byte2 & (SWS_NDOB | SWS_ANCHOR)) != 0) {
+	/* ANCHOR flag can be used only together with UNMAP */
+	if ((byte2 & SWS_UNMAP) == 0 && (byte2 & SWS_ANCHOR) != 0) {
 		ctl_set_invalid_field(ctsio, /*sks_valid*/ 1,
 		    /*command*/ 1, /*field*/ 1, /*bit_valid*/ 1, /*bit*/ 0);
 		ctl_done((union ctl_io *)ctsio);
@@ -5549,8 +5636,6 @@ ctl_unmap(struct ctl_scsiio *ctsio)
 	int len, retval;
 	uint8_t byte2;
 
-	retval = CTL_RETVAL_COMPLETE;
-
 	CTL_DEBUG_PRINT(("ctl_unmap\n"));
 
 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
@@ -5715,7 +5800,11 @@ ctl_control_page_handler(struct ctl_scsiio *ctsio,
 	if (set_ua != 0)
 		ctl_est_ua_all(lun, initidx, CTL_UA_MODE_CHANGE);
 	mtx_unlock(&lun->lun_lock);
-
+	if (set_ua) {
+		ctl_isc_announce_mode(lun,
+		    ctl_get_initindex(&ctsio->io_hdr.nexus),
+		    page_index->page_code, page_index->subpage);
+	}
 	return (0);
 }
 
@@ -5752,7 +5841,11 @@ ctl_caching_sp_handler(struct ctl_scsiio *ctsio,
 	if (set_ua != 0)
 		ctl_est_ua_all(lun, initidx, CTL_UA_MODE_CHANGE);
 	mtx_unlock(&lun->lun_lock);
-
+	if (set_ua) {
+		ctl_isc_announce_mode(lun,
+		    ctl_get_initindex(&ctsio->io_hdr.nexus),
+		    page_index->page_code, page_index->subpage);
+	}
 	return (0);
 }
 
@@ -5801,9 +5894,6 @@ ctl_debugconf_sp_sense_handler(struct ctl_scsiio *ctsio,
 		page->ctl_time_io_secs[1] = ctl_time_io_secs >> 0;
 		break;
 	default:
-#ifdef NEEDTOPORT
-		EPRINT(0, "Invalid PC %d!!", pc);
-#endif /* NEEDTOPORT */
 		break;
 	}
 	return (0);
@@ -5816,8 +5906,7 @@ ctl_do_mode_select(union ctl_io *io)
 	struct scsi_mode_page_header *page_header;
 	struct ctl_page_index *page_index;
 	struct ctl_scsiio *ctsio;
-	int control_dev, page_len;
-	int page_len_offset, page_len_size;
+	int page_len, page_len_offset, page_len_size;
 	union ctl_modepage_info *modepage_info;
 	struct ctl_lun *lun;
 	int *len_left, *len_used;
@@ -5826,15 +5915,8 @@ ctl_do_mode_select(union ctl_io *io)
 	ctsio = &io->scsiio;
 	page_index = NULL;
 	page_len = 0;
-	retval = CTL_RETVAL_COMPLETE;
-
 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
 
-	if (lun->be_lun->lun_type != T_DIRECT)
-		control_dev = 1;
-	else
-		control_dev = 0;
-
 	modepage_info = (union ctl_modepage_info *)
 		ctsio->io_hdr.ctl_private[CTL_PRIV_MODEPAGE].bytes;
 	len_left = &modepage_info->header.len_left;
@@ -5871,13 +5953,18 @@ do_next_page:
 	 * XXX KDM should we do something with the block descriptor?
 	 */
 	for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
-
-		if ((control_dev != 0)
-		 && (lun->mode_pages.index[i].page_flags &
-		     CTL_PAGE_FLAG_DISK_ONLY))
+		page_index = &lun->mode_pages.index[i];
+		if (lun->be_lun->lun_type == T_DIRECT &&
+		    (page_index->page_flags & CTL_PAGE_FLAG_DIRECT) == 0)
+			continue;
+		if (lun->be_lun->lun_type == T_PROCESSOR &&
+		    (page_index->page_flags & CTL_PAGE_FLAG_PROC) == 0)
+			continue;
+		if (lun->be_lun->lun_type == T_CDROM &&
+		    (page_index->page_flags & CTL_PAGE_FLAG_CDROM) == 0)
 			continue;
 
-		if ((lun->mode_pages.index[i].page_code & SMPH_PC_MASK) !=
+		if ((page_index->page_code & SMPH_PC_MASK) !=
 		    (page_header->page_code & SMPH_PC_MASK))
 			continue;
 
@@ -5885,9 +5972,8 @@ do_next_page:
 		 * If neither page has a subpage code, then we've got a
 		 * match.
 		 */
-		if (((lun->mode_pages.index[i].page_code & SMPH_SPF) == 0)
+		if (((page_index->page_code & SMPH_SPF) == 0)
 		 && ((page_header->page_code & SMPH_SPF) == 0)) {
-			page_index = &lun->mode_pages.index[i];
 			page_len = page_header->page_length;
 			break;
 		}
@@ -5896,15 +5982,12 @@ do_next_page:
 		 * If both pages have subpages, then the subpage numbers
 		 * have to match.
 		 */
-		if ((lun->mode_pages.index[i].page_code & SMPH_SPF)
+		if ((page_index->page_code & SMPH_SPF)
 		  && (page_header->page_code & SMPH_SPF)) {
 			struct scsi_mode_page_header_sp *sph;
 
 			sph = (struct scsi_mode_page_header_sp *)page_header;
-
-			if (lun->mode_pages.index[i].subpage ==
-			    sph->subpage) {
-				page_index = &lun->mode_pages.index[i];
+			if (page_index->subpage == sph->subpage) {
 				page_len = scsi_2btoul(sph->page_length);
 				break;
 			}
@@ -5915,7 +5998,7 @@ do_next_page:
 	 * If we couldn't find the page, or if we don't have a mode select
 	 * handler for it, send back an error to the user.
 	 */
-	if ((page_index == NULL)
+	if ((i >= CTL_NUM_MODE_PAGES)
 	 || (page_index->select_handler == NULL)) {
 		ctl_set_invalid_field(ctsio,
 				      /*sks_valid*/ 1,
@@ -6045,28 +6128,7 @@ ctl_mode_select(struct ctl_scsiio *ctsio)
 {
 	int param_len, pf, sp;
 	int header_size, bd_len;
-	int len_left, len_used;
-	struct ctl_page_index *page_index;
-	struct ctl_lun *lun;
-	int control_dev, page_len;
 	union ctl_modepage_info *modepage_info;
-	int retval;
-
-	pf = 0;
-	sp = 0;
-	page_len = 0;
-	len_used = 0;
-	len_left = 0;
-	retval = 0;
-	bd_len = 0;
-	page_index = NULL;
-
-	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
-
-	if (lun->be_lun->lun_type != T_DIRECT)
-		control_dev = 1;
-	else
-		control_dev = 0;
 
 	switch (ctsio->cdb[0]) {
 	case MODE_SELECT_6: {
@@ -6076,7 +6138,6 @@ ctl_mode_select(struct ctl_scsiio *ctsio)
 
 		pf = (cdb->byte2 & SMS_PF) ? 1 : 0;
 		sp = (cdb->byte2 & SMS_SP) ? 1 : 0;
-
 		param_len = cdb->length;
 		header_size = sizeof(struct scsi_mode_header_6);
 		break;
@@ -6088,7 +6149,6 @@ ctl_mode_select(struct ctl_scsiio *ctsio)
 
 		pf = (cdb->byte2 & SMS_PF) ? 1 : 0;
 		sp = (cdb->byte2 & SMS_SP) ? 1 : 0;
-
 		param_len = scsi_2btoul(cdb->length);
 		header_size = sizeof(struct scsi_mode_header_10);
 		break;
@@ -6097,7 +6157,6 @@ ctl_mode_select(struct ctl_scsiio *ctsio)
 		ctl_set_invalid_opcode(ctsio);
 		ctl_done((union ctl_io *)ctsio);
 		return (CTL_RETVAL_COMPLETE);
-		break; /* NOTREACHED */
 	}
 
 	/*
@@ -6179,14 +6238,9 @@ ctl_mode_select(struct ctl_scsiio *ctsio)
 
 	modepage_info = (union ctl_modepage_info *)
 		ctsio->io_hdr.ctl_private[CTL_PRIV_MODEPAGE].bytes;
-
 	memset(modepage_info, 0, sizeof(*modepage_info));
-
-	len_left = param_len - header_size - bd_len;
-	len_used = header_size + bd_len;
-
-	modepage_info->header.len_left = len_left;
-	modepage_info->header.len_used = len_used;
+	modepage_info->header.len_left = param_len - header_size - bd_len;
+	modepage_info->header.len_used = header_size + bd_len;
 
 	return (ctl_do_mode_select((union ctl_io *)ctsio));
 }
@@ -6199,22 +6253,14 @@ ctl_mode_sense(struct ctl_scsiio *ctsio)
 	int alloc_len, page_len, header_len, total_len;
 	struct scsi_mode_block_descr *block_desc;
 	struct ctl_page_index *page_index;
-	int control_dev;
 
 	dbd = 0;
 	llba = 0;
 	block_desc = NULL;
-	page_index = NULL;
 
 	CTL_DEBUG_PRINT(("ctl_mode_sense\n"));
 
 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
-
-	if (lun->be_lun->lun_type != T_DIRECT)
-		control_dev = 1;
-	else
-		control_dev = 0;
-
 	switch (ctsio->cdb[0]) {
 	case MODE_SENSE_6: {
 		struct scsi_mode_sense_6 *cdb;
@@ -6287,26 +6333,33 @@ ctl_mode_sense(struct ctl_scsiio *ctsio)
 		}
 
 		for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
-			if ((control_dev != 0)
-			 && (lun->mode_pages.index[i].page_flags &
-			     CTL_PAGE_FLAG_DISK_ONLY))
+			page_index = &lun->mode_pages.index[i];
+
+			/* Make sure the page is supported for this dev type */
+			if (lun->be_lun->lun_type == T_DIRECT &&
+			    (page_index->page_flags & CTL_PAGE_FLAG_DIRECT) == 0)
+				continue;
+			if (lun->be_lun->lun_type == T_PROCESSOR &&
+			    (page_index->page_flags & CTL_PAGE_FLAG_PROC) == 0)
+				continue;
+			if (lun->be_lun->lun_type == T_CDROM &&
+			    (page_index->page_flags & CTL_PAGE_FLAG_CDROM) == 0)
 				continue;
 
 			/*
 			 * We don't use this subpage if the user didn't
 			 * request all subpages.
 			 */
-			if ((lun->mode_pages.index[i].subpage != 0)
+			if ((page_index->subpage != 0)
 			 && (subpage == SMS_SUBPAGE_PAGE_0))
 				continue;
 
 #if 0
 			printf("found page %#x len %d\n",
-			       lun->mode_pages.index[i].page_code &
-			       SMPH_PC_MASK,
-			       lun->mode_pages.index[i].page_len);
+			       page_index->page_code & SMPH_PC_MASK,
+			       page_index->page_len);
 #endif
-			page_len += lun->mode_pages.index[i].page_len;
+			page_len += page_index->page_len;
 		}
 		break;
 	}
@@ -6316,30 +6369,35 @@ ctl_mode_sense(struct ctl_scsiio *ctsio)
 		page_len = 0;
 
 		for (i = 0; i < CTL_NUM_MODE_PAGES; i++) {
+			page_index = &lun->mode_pages.index[i];
+
+			/* Make sure the page is supported for this dev type */
+			if (lun->be_lun->lun_type == T_DIRECT &&
+			    (page_index->page_flags & CTL_PAGE_FLAG_DIRECT) == 0)
+				continue;
+			if (lun->be_lun->lun_type == T_PROCESSOR &&
+			    (page_index->page_flags & CTL_PAGE_FLAG_PROC) == 0)
+				continue;
+			if (lun->be_lun->lun_type == T_CDROM &&
+			    (page_index->page_flags & CTL_PAGE_FLAG_CDROM) == 0)
+				continue;
+
 			/* Look for the right page code */
-			if ((lun->mode_pages.index[i].page_code &
-			     SMPH_PC_MASK) != page_code)
+			if ((page_index->page_code & SMPH_PC_MASK) != page_code)
 				continue;
 
 			/* Look for the right subpage or the subpage wildcard*/
-			if ((lun->mode_pages.index[i].subpage != subpage)
+			if ((page_index->subpage != subpage)
 			 && (subpage != SMS_SUBPAGE_ALL))
 				continue;
 
-			/* Make sure the page is supported for this dev type */
-			if ((control_dev != 0)
-			 && (lun->mode_pages.index[i].page_flags &
-			     CTL_PAGE_FLAG_DISK_ONLY))
-				continue;
-
 #if 0
 			printf("found page %#x len %d\n",
-			       lun->mode_pages.index[i].page_code &
-			       SMPH_PC_MASK,
-			       lun->mode_pages.index[i].page_len);
+			       page_index->page_code & SMPH_PC_MASK,
+			       page_index->page_len);
 #endif
 
-			page_len += lun->mode_pages.index[i].page_len;
+			page_len += page_index->page_len;
 		}
 
 		if (page_len == 0) {
@@ -6383,7 +6441,7 @@ ctl_mode_sense(struct ctl_scsiio *ctsio)
 		header = (struct scsi_mode_hdr_6 *)ctsio->kern_data_ptr;
 
 		header->datalen = MIN(total_len - 1, 254);
-		if (control_dev == 0) {
+		if (lun->be_lun->lun_type == T_DIRECT) {
 			header->dev_specific = 0x10; /* DPOFUA */
 			if ((lun->be_lun->flags & CTL_LUN_FLAG_READONLY) ||
 			    (lun->mode_pages.control_page[CTL_PAGE_CURRENT]
@@ -6406,7 +6464,7 @@ ctl_mode_sense(struct ctl_scsiio *ctsio)
 
 		datalen = MIN(total_len - 2, 65533);
 		scsi_ulto2b(datalen, header->datalen);
-		if (control_dev == 0) {
+		if (lun->be_lun->lun_type == T_DIRECT) {
 			header->dev_specific = 0x10; /* DPOFUA */
 			if ((lun->be_lun->flags & CTL_LUN_FLAG_READONLY) ||
 			    (lun->mode_pages.control_page[CTL_PAGE_CURRENT]
@@ -6431,7 +6489,7 @@ ctl_mode_sense(struct ctl_scsiio *ctsio)
 	 * descriptor.  Otherwise, just set it to 0.
 	 */
 	if (dbd == 0) {
-		if (control_dev == 0)
+		if (lun->be_lun->lun_type == T_DIRECT)
 			scsi_ulto3b(lun->be_lun->blocksize,
 				    block_desc->block_len);
 		else
@@ -6447,10 +6505,14 @@ ctl_mode_sense(struct ctl_scsiio *ctsio)
 			struct ctl_page_index *page_index;
 
 			page_index = &lun->mode_pages.index[i];
-
-			if ((control_dev != 0)
-			 && (page_index->page_flags &
-			    CTL_PAGE_FLAG_DISK_ONLY))
+			if (lun->be_lun->lun_type == T_DIRECT &&
+			    (page_index->page_flags & CTL_PAGE_FLAG_DIRECT) == 0)
+				continue;
+			if (lun->be_lun->lun_type == T_PROCESSOR &&
+			    (page_index->page_flags & CTL_PAGE_FLAG_PROC) == 0)
+				continue;
+			if (lun->be_lun->lun_type == T_CDROM &&
+			    (page_index->page_flags & CTL_PAGE_FLAG_CDROM) == 0)
 				continue;
 
 			/*
@@ -6498,9 +6560,14 @@ ctl_mode_sense(struct ctl_scsiio *ctsio)
 				continue;
 
 			/* Make sure the page is supported for this dev type */
-			if ((control_dev != 0)
-			 && (page_index->page_flags &
-			     CTL_PAGE_FLAG_DISK_ONLY))
+			if (lun->be_lun->lun_type == T_DIRECT &&
+			    (page_index->page_flags & CTL_PAGE_FLAG_DIRECT) == 0)
+				continue;
+			if (lun->be_lun->lun_type == T_PROCESSOR &&
+			    (page_index->page_flags & CTL_PAGE_FLAG_PROC) == 0)
+				continue;
+			if (lun->be_lun->lun_type == T_CDROM &&
+			    (page_index->page_flags & CTL_PAGE_FLAG_CDROM) == 0)
 				continue;
 
 			/*
@@ -7325,8 +7392,9 @@ ctl_report_supported_tmf(struct ctl_scsiio *ctsio)
 	ctsio->kern_rel_offset = 0;
 
 	data = (struct scsi_report_supported_tmf_data *)ctsio->kern_data_ptr;
-	data->byte1 |= RST_ATS | RST_ATSS | RST_CTSS | RST_LURS | RST_TRS;
-	data->byte2 |= RST_ITNRS;
+	data->byte1 |= RST_ATS | RST_ATSS | RST_CTSS | RST_LURS | RST_QTS |
+	    RST_TRS;
+	data->byte2 |= RST_QAES | RST_QTSS | RST_ITNRS;
 
 	ctl_set_success(ctsio);
 	ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
@@ -7489,18 +7557,6 @@ retry:
 			 * sync), we've got a problem.
 			 */
 			if (key_count >= lun->pr_key_count) {
-#ifdef NEEDTOPORT
-				csevent_log(CSC_CTL | CSC_SHELF_SW |
-					    CTL_PR_ERROR,
-					    csevent_LogType_Fault,
-					    csevent_AlertLevel_Yellow,
-					    csevent_FRU_ShelfController,
-					    csevent_FRU_Firmware,
-				        csevent_FRU_Unknown,
-					    "registered keys %d >= key "
-					    "count %d", key_count,
-					    lun->pr_key_count);
-#endif
 				key_count++;
 				continue;
 			}
@@ -8377,16 +8433,25 @@ done:
 static void
 ctl_hndl_per_res_out_on_other_sc(union ctl_ha_msg *msg)
 {
+	struct ctl_softc *softc = control_softc;
 	struct ctl_lun *lun;
-	struct ctl_softc *softc;
 	int i;
-	uint32_t targ_lun;
-
-	softc = control_softc;
+	uint32_t residx, targ_lun;
 
 	targ_lun = msg->hdr.nexus.targ_mapped_lun;
-	lun = softc->ctl_luns[targ_lun];
+	mtx_lock(&softc->ctl_lock);
+	if ((targ_lun >= CTL_MAX_LUNS) ||
+	    ((lun = softc->ctl_luns[targ_lun]) == NULL)) {
+		mtx_unlock(&softc->ctl_lock);
+		return;
+	}
 	mtx_lock(&lun->lun_lock);
+	mtx_unlock(&softc->ctl_lock);
+	if (lun->flags & CTL_LUN_DISABLED) {
+		mtx_unlock(&lun->lun_lock);
+		return;
+	}
+	residx = ctl_get_initindex(&msg->hdr.nexus);
 	switch(msg->pr.pr_info.action) {
 	case CTL_PR_REG_KEY:
 		ctl_alloc_prkey(lun, msg->pr.pr_info.residx);
@@ -8451,8 +8516,9 @@ ctl_hndl_per_res_out_on_other_sc(union ctl_ha_msg *msg)
 		if (lun->res_type != SPR_TYPE_EX_AC
 		 && lun->res_type != SPR_TYPE_WR_EX) {
 			for (i = softc->init_min; i < softc->init_max; i++)
-				if (ctl_get_prkey(lun, i) != 0)
-					ctl_est_ua(lun, i, CTL_UA_RES_RELEASE);
+				if (i == residx || ctl_get_prkey(lun, i) == 0)
+					continue;
+				ctl_est_ua(lun, i, CTL_UA_RES_RELEASE);
 		}
 
 		lun->flags &= ~CTL_LUN_PR_RESERVED;
@@ -8497,8 +8563,6 @@ ctl_read_write(struct ctl_scsiio *ctsio)
 	CTL_DEBUG_PRINT(("ctl_read_write: command: %#x\n", ctsio->cdb[0]));
 
 	flags = 0;
-	retval = CTL_RETVAL_COMPLETE;
-
 	isread = ctsio->cdb[0] == READ_6  || ctsio->cdb[0] == READ_10
 	      || ctsio->cdb[0] == READ_12 || ctsio->cdb[0] == READ_16;
 	switch (ctsio->cdb[0]) {
@@ -8581,7 +8645,7 @@ ctl_read_write(struct ctl_scsiio *ctsio)
 		break;
 	}
 	case WRITE_ATOMIC_16: {
-		struct scsi_rw_16 *cdb;
+		struct scsi_write_atomic_16 *cdb;
 
 		if (lun->be_lun->atomicblock == 0) {
 			ctl_set_invalid_opcode(ctsio);
@@ -8589,13 +8653,13 @@ ctl_read_write(struct ctl_scsiio *ctsio)
 			return (CTL_RETVAL_COMPLETE);
 		}
 
-		cdb = (struct scsi_rw_16 *)ctsio->cdb;
+		cdb = (struct scsi_write_atomic_16 *)ctsio->cdb;
 		if (cdb->byte2 & SRW12_FUA)
 			flags |= CTL_LLF_FUA;
 		if (cdb->byte2 & SRW12_DPO)
 			flags |= CTL_LLF_DPO;
 		lba = scsi_8btou64(cdb->addr);
-		num_blocks = scsi_4btoul(cdb->length);
+		num_blocks = scsi_2btoul(cdb->length);
 		if (num_blocks > lun->be_lun->atomicblock) {
 			ctl_set_invalid_field(ctsio, /*sks_valid*/ 1,
 			    /*command*/ 1, /*field*/ 12, /*bit_valid*/ 0,
@@ -8675,7 +8739,6 @@ ctl_read_write(struct ctl_scsiio *ctsio)
 	CTL_DEBUG_PRINT(("ctl_read_write: calling data_submit()\n"));
 
 	retval = lun->backend->data_submit((union ctl_io *)ctsio);
-
 	return (retval);
 }
 
@@ -8715,8 +8778,6 @@ ctl_cnw(struct ctl_scsiio *ctsio)
 	CTL_DEBUG_PRINT(("ctl_cnw: command: %#x\n", ctsio->cdb[0]));
 
 	flags = 0;
-	retval = CTL_RETVAL_COMPLETE;
-
 	switch (ctsio->cdb[0]) {
 	case COMPARE_AND_WRITE: {
 		struct scsi_compare_and_write *cdb;
@@ -8807,8 +8868,6 @@ ctl_verify(struct ctl_scsiio *ctsio)
 
 	bytchk = 0;
 	flags = CTL_LLF_FUA;
-	retval = CTL_RETVAL_COMPLETE;
-
 	switch (ctsio->cdb[0]) {
 	case VERIFY_10: {
 		struct scsi_verify_10 *cdb;
@@ -8899,21 +8958,20 @@ ctl_verify(struct ctl_scsiio *ctsio)
 int
 ctl_report_luns(struct ctl_scsiio *ctsio)
 {
-	struct ctl_softc *softc = control_softc;
+	struct ctl_softc *softc;
 	struct scsi_report_luns *cdb;
 	struct scsi_report_luns_data *lun_data;
 	struct ctl_lun *lun, *request_lun;
 	struct ctl_port *port;
 	int num_luns, retval;
 	uint32_t alloc_len, lun_datalen;
-	int num_filled, well_known;
+	int num_filled;
 	uint32_t initidx, targ_lun_id, lun_id;
 
 	retval = CTL_RETVAL_COMPLETE;
-	well_known = 0;
-
 	cdb = (struct scsi_report_luns *)ctsio->cdb;
 	port = ctl_io_port(&ctsio->io_hdr);
+	softc = port->ctl_softc;
 
 	CTL_DEBUG_PRINT(("ctl_report_luns\n"));
 
@@ -8928,9 +8986,11 @@ ctl_report_luns(struct ctl_scsiio *ctsio)
 	switch (cdb->select_report) {
 	case RPL_REPORT_DEFAULT:
 	case RPL_REPORT_ALL:
+	case RPL_REPORT_NONSUBSID:
 		break;
 	case RPL_REPORT_WELLKNOWN:
-		well_known = 1;
+	case RPL_REPORT_ADMIN:
+	case RPL_REPORT_CONGLOM:
 		num_luns = 0;
 		break;
 	default:
@@ -9736,7 +9796,6 @@ ctl_inquiry_evpd_block_limits(struct ctl_scsiio *ctsio, int alloc_len)
 {
 	struct scsi_vpd_block_limits *bl_ptr;
 	struct ctl_lun *lun;
-	int bs;
 
 	lun = (struct ctl_lun *)ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
 
@@ -9773,7 +9832,6 @@ ctl_inquiry_evpd_block_limits(struct ctl_scsiio *ctsio, int alloc_len)
 	bl_ptr->max_cmp_write_len = 0xff;
 	scsi_ulto4b(0xffffffff, bl_ptr->max_txfer_len);
 	if (lun != NULL) {
-		bs = lun->be_lun->blocksize;
 		scsi_ulto4b(lun->be_lun->opttxferlen, bl_ptr->opt_txfer_len);
 		if (lun->be_lun->flags & CTL_LUN_FLAG_UNMAP) {
 			scsi_ulto4b(0xffffffff, bl_ptr->max_unmap_lba_cnt);
@@ -9789,6 +9847,8 @@ ctl_inquiry_evpd_block_limits(struct ctl_scsiio *ctsio, int alloc_len)
 		    bl_ptr->max_atomic_transfer_length);
 		scsi_ulto4b(0, bl_ptr->atomic_alignment);
 		scsi_ulto4b(0, bl_ptr->atomic_transfer_length_granularity);
+		scsi_ulto4b(0, bl_ptr->max_atomic_transfer_length_with_atomic_boundary);
+		scsi_ulto4b(0, bl_ptr->max_atomic_boundary_size);
 	}
 	scsi_u64to8b(UINT64_MAX, bl_ptr->max_write_same_length);
 
@@ -9986,25 +10046,15 @@ ctl_inquiry_std(struct ctl_scsiio *ctsio)
 {
 	struct scsi_inquiry_data *inq_ptr;
 	struct scsi_inquiry *cdb;
-	struct ctl_softc *softc;
+	struct ctl_softc *softc = control_softc;
 	struct ctl_port *port;
 	struct ctl_lun *lun;
 	char *val;
 	uint32_t alloc_len, data_len;
 	ctl_port_type port_type;
 
-	softc = control_softc;
-
-	/*
-	 * Figure out whether we're talking to a Fibre Channel port or not.
-	 * We treat the ioctl front end, and any SCSI adapters, as packetized
-	 * SCSI front ends.
-	 */
 	port = ctl_io_port(&ctsio->io_hdr);
-	if (port != NULL)
-		port_type = port->port_type;
-	else
-		port_type = CTL_PORT_SCSI;
+	port_type = port->port_type;
 	if (port_type == CTL_PORT_IOCTL || port_type == CTL_PORT_INTERNAL)
 		port_type = CTL_PORT_SCSI;
 
@@ -10043,6 +10093,8 @@ ctl_inquiry_std(struct ctl_scsiio *ctsio)
 			inq_ptr->device = (SID_QUAL_LU_OFFLINE << 5) |
 			    lun->be_lun->lun_type;
 		}
+		if (lun->flags & CTL_LUN_REMOVABLE)
+			inq_ptr->dev_qual2 |= SID_RMB;
 	} else
 		inq_ptr->device = (SID_QUAL_BAD_LU << 5) | T_NODEVICE;
 
@@ -10074,23 +10126,13 @@ ctl_inquiry_std(struct ctl_scsiio *ctsio)
 			 inq_ptr->additional_length));
 
 	inq_ptr->spc3_flags = SPC3_SID_3PC | SPC3_SID_TPGS_IMPLICIT;
-	/* 16 bit addressing */
 	if (port_type == CTL_PORT_SCSI)
 		inq_ptr->spc2_flags = SPC2_SID_ADDR16;
-	/* XXX set the SID_MultiP bit here if we're actually going to
-	   respond on multiple ports */
 	inq_ptr->spc2_flags |= SPC2_SID_MultiP;
-
-	/* 16 bit data bus, synchronous transfers */
+	inq_ptr->flags = SID_CmdQue;
 	if (port_type == CTL_PORT_SCSI)
-		inq_ptr->flags = SID_WBus16 | SID_Sync;
-	/*
-	 * XXX KDM do we want to support tagged queueing on the control
-	 * device at all?
-	 */
-	if ((lun == NULL)
-	 || (lun->be_lun->lun_type != T_PROCESSOR))
-		inq_ptr->flags |= SID_CmdQue;
+		inq_ptr->flags |= SID_WBus16 | SID_Sync;
+
 	/*
 	 * Per SPC-3, unused bytes in ASCII strings are filled with spaces.
 	 * We have 8 bytes for the vendor name, and 16 bytes for the device
@@ -10117,6 +10159,10 @@ ctl_inquiry_std(struct ctl_scsiio *ctsio)
 			strncpy(inq_ptr->product, CTL_PROCESSOR_PRODUCT,
 			    sizeof(inq_ptr->product));
 			break;
+		case T_CDROM:
+			strncpy(inq_ptr->product, CTL_CDROM_PRODUCT,
+			    sizeof(inq_ptr->product));
+			break;
 		default:
 			strncpy(inq_ptr->product, CTL_UNKNOWN_PRODUCT,
 			    sizeof(inq_ptr->product));
@@ -10179,6 +10225,11 @@ ctl_inquiry_std(struct ctl_scsiio *ctsio)
 			scsi_ulto2b(0x0600, inq_ptr->version4);
 			break;
 		case T_PROCESSOR:
+			break;
+		case T_CDROM:
+			/* MMC-6 (no version claimed) */
+			scsi_ulto2b(0x04E0, inq_ptr->version4);
+			break;
 		default:
 			break;
 		}
@@ -10218,6 +10269,398 @@ ctl_inquiry(struct ctl_scsiio *ctsio)
 	return (retval);
 }
 
+int
+ctl_get_config(struct ctl_scsiio *ctsio)
+{
+	struct scsi_get_config_header *hdr;
+	struct scsi_get_config_feature *feature;
+	struct scsi_get_config *cdb;
+	struct ctl_lun *lun;
+	uint32_t alloc_len, data_len;
+	int rt, starting;
+
+	lun = ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
+	cdb = (struct scsi_get_config *)ctsio->cdb;
+	rt = (cdb->rt & SGC_RT_MASK);
+	starting = scsi_2btoul(cdb->starting_feature);
+	alloc_len = scsi_2btoul(cdb->length);
+
+	data_len = sizeof(struct scsi_get_config_header) +
+	    sizeof(struct scsi_get_config_feature) + 8 +
+	    sizeof(struct scsi_get_config_feature) + 8 +
+	    sizeof(struct scsi_get_config_feature) + 4 +
+	    sizeof(struct scsi_get_config_feature) + 4 +
+	    sizeof(struct scsi_get_config_feature) + 8 +
+	    sizeof(struct scsi_get_config_feature) +
+	    sizeof(struct scsi_get_config_feature) + 4 +
+	    sizeof(struct scsi_get_config_feature) + 4 +
+	    sizeof(struct scsi_get_config_feature) + 4 +
+	    sizeof(struct scsi_get_config_feature) + 4 +
+	    sizeof(struct scsi_get_config_feature) + 4 +
+	    sizeof(struct scsi_get_config_feature) + 4;
+	ctsio->kern_data_ptr = malloc(data_len, M_CTL, M_WAITOK | M_ZERO);
+	ctsio->kern_sg_entries = 0;
+	ctsio->kern_data_resid = 0;
+	ctsio->kern_rel_offset = 0;
+
+	hdr = (struct scsi_get_config_header *)ctsio->kern_data_ptr;
+	if (lun->flags & CTL_LUN_NO_MEDIA)
+		scsi_ulto2b(0x0000, hdr->current_profile);
+	else
+		scsi_ulto2b(0x0010, hdr->current_profile);
+	feature = (struct scsi_get_config_feature *)(hdr + 1);
+
+	if (starting > 0x003b)
+		goto done;
+	if (starting > 0x003a)
+		goto f3b;
+	if (starting > 0x002b)
+		goto f3a;
+	if (starting > 0x002a)
+		goto f2b;
+	if (starting > 0x001f)
+		goto f2a;
+	if (starting > 0x001e)
+		goto f1f;
+	if (starting > 0x001d)
+		goto f1e;
+	if (starting > 0x0010)
+		goto f1d;
+	if (starting > 0x0003)
+		goto f10;
+	if (starting > 0x0002)
+		goto f3;
+	if (starting > 0x0001)
+		goto f2;
+	if (starting > 0x0000)
+		goto f1;
+
+	/* Profile List */
+	scsi_ulto2b(0x0000, feature->feature_code);
+	feature->flags = SGC_F_PERSISTENT | SGC_F_CURRENT;
+	feature->add_length = 8;
+	scsi_ulto2b(0x0008, &feature->feature_data[0]);	/* CD-ROM */
+	feature->feature_data[2] = 0x00;
+	scsi_ulto2b(0x0010, &feature->feature_data[4]);	/* DVD-ROM */
+	feature->feature_data[6] = 0x01;
+	feature = (struct scsi_get_config_feature *)
+	    &feature->feature_data[feature->add_length];
+
+f1:	/* Core */
+	scsi_ulto2b(0x0001, feature->feature_code);
+	feature->flags = 0x08 | SGC_F_PERSISTENT | SGC_F_CURRENT;
+	feature->add_length = 8;
+	scsi_ulto4b(0x00000000, &feature->feature_data[0]);
+	feature->feature_data[4] = 0x03;
+	feature = (struct scsi_get_config_feature *)
+	    &feature->feature_data[feature->add_length];
+
+f2:	/* Morphing */
+	scsi_ulto2b(0x0002, feature->feature_code);
+	feature->flags = 0x04 | SGC_F_PERSISTENT | SGC_F_CURRENT;
+	feature->add_length = 4;
+	feature->feature_data[0] = 0x02;
+	feature = (struct scsi_get_config_feature *)
+	    &feature->feature_data[feature->add_length];
+
+f3:	/* Removable Medium */
+	scsi_ulto2b(0x0003, feature->feature_code);
+	feature->flags = 0x04 | SGC_F_PERSISTENT | SGC_F_CURRENT;
+	feature->add_length = 4;
+	feature->feature_data[0] = 0x39;
+	feature = (struct scsi_get_config_feature *)
+	    &feature->feature_data[feature->add_length];
+
+	if (rt == SGC_RT_CURRENT && (lun->flags & CTL_LUN_NO_MEDIA))
+		goto done;
+
+f10:	/* Random Read */
+	scsi_ulto2b(0x0010, feature->feature_code);
+	feature->flags = 0x00;
+	if ((lun->flags & CTL_LUN_NO_MEDIA) == 0)
+		feature->flags |= SGC_F_CURRENT;
+	feature->add_length = 8;
+	scsi_ulto4b(lun->be_lun->blocksize, &feature->feature_data[0]);
+	scsi_ulto2b(1, &feature->feature_data[4]);
+	feature->feature_data[6] = 0x00;
+	feature = (struct scsi_get_config_feature *)
+	    &feature->feature_data[feature->add_length];
+
+f1d:	/* Multi-Read */
+	scsi_ulto2b(0x001D, feature->feature_code);
+	feature->flags = 0x00;
+	if ((lun->flags & CTL_LUN_NO_MEDIA) == 0)
+		feature->flags |= SGC_F_CURRENT;
+	feature->add_length = 0;
+	feature = (struct scsi_get_config_feature *)
+	    &feature->feature_data[feature->add_length];
+
+f1e:	/* CD Read */
+	scsi_ulto2b(0x001E, feature->feature_code);
+	feature->flags = 0x00;
+	if ((lun->flags & CTL_LUN_NO_MEDIA) == 0)
+		feature->flags |= SGC_F_CURRENT;
+	feature->add_length = 4;
+	feature->feature_data[0] = 0x00;
+	feature = (struct scsi_get_config_feature *)
+	    &feature->feature_data[feature->add_length];
+
+f1f:	/* DVD Read */
+	scsi_ulto2b(0x001F, feature->feature_code);
+	feature->flags = 0x08;
+	if ((lun->flags & CTL_LUN_NO_MEDIA) == 0)
+		feature->flags |= SGC_F_CURRENT;
+	feature->add_length = 4;
+	feature->feature_data[0] = 0x01;
+	feature->feature_data[2] = 0x03;
+	feature = (struct scsi_get_config_feature *)
+	    &feature->feature_data[feature->add_length];
+
+f2a:	/* DVD+RW */
+	scsi_ulto2b(0x002A, feature->feature_code);
+	feature->flags = 0x04;
+	if ((lun->flags & CTL_LUN_NO_MEDIA) == 0)
+		feature->flags |= SGC_F_CURRENT;
+	feature->add_length = 4;
+	feature->feature_data[0] = 0x00;
+	feature->feature_data[1] = 0x00;
+	feature = (struct scsi_get_config_feature *)
+	    &feature->feature_data[feature->add_length];
+
+f2b:	/* DVD+R */
+	scsi_ulto2b(0x002B, feature->feature_code);
+	feature->flags = 0x00;
+	if ((lun->flags & CTL_LUN_NO_MEDIA) == 0)
+		feature->flags |= SGC_F_CURRENT;
+	feature->add_length = 4;
+	feature->feature_data[0] = 0x00;
+	feature = (struct scsi_get_config_feature *)
+	    &feature->feature_data[feature->add_length];
+
+f3a:	/* DVD+RW Dual Layer */
+	scsi_ulto2b(0x003A, feature->feature_code);
+	feature->flags = 0x00;
+	if ((lun->flags & CTL_LUN_NO_MEDIA) == 0)
+		feature->flags |= SGC_F_CURRENT;
+	feature->add_length = 4;
+	feature->feature_data[0] = 0x00;
+	feature->feature_data[1] = 0x00;
+	feature = (struct scsi_get_config_feature *)
+	    &feature->feature_data[feature->add_length];
+
+f3b:	/* DVD+R Dual Layer */
+	scsi_ulto2b(0x003B, feature->feature_code);
+	feature->flags = 0x00;
+	if ((lun->flags & CTL_LUN_NO_MEDIA) == 0)
+		feature->flags |= SGC_F_CURRENT;
+	feature->add_length = 4;
+	feature->feature_data[0] = 0x00;
+	feature = (struct scsi_get_config_feature *)
+	    &feature->feature_data[feature->add_length];
+
+done:
+	data_len = (uint8_t *)feature - (uint8_t *)hdr;
+	if (rt == SGC_RT_SPECIFIC && data_len > 4) {
+		feature = (struct scsi_get_config_feature *)(hdr + 1);
+		if (scsi_2btoul(feature->feature_code) == starting)
+			feature = (struct scsi_get_config_feature *)
+			    &feature->feature_data[feature->add_length];
+		data_len = (uint8_t *)feature - (uint8_t *)hdr;
+	}
+	scsi_ulto4b(data_len - 4, hdr->data_length);
+	if (data_len < alloc_len) {
+		ctsio->residual = alloc_len - data_len;
+		ctsio->kern_data_len = data_len;
+		ctsio->kern_total_len = data_len;
+	} else {
+		ctsio->residual = 0;
+		ctsio->kern_data_len = alloc_len;
+		ctsio->kern_total_len = alloc_len;
+	}
+
+	ctl_set_success(ctsio);
+	ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
+	ctsio->be_move_done = ctl_config_move_done;
+	ctl_datamove((union ctl_io *)ctsio);
+	return (CTL_RETVAL_COMPLETE);
+}
+
+int
+ctl_get_event_status(struct ctl_scsiio *ctsio)
+{
+	struct scsi_get_event_status_header *hdr;
+	struct scsi_get_event_status *cdb;
+	struct ctl_lun *lun;
+	uint32_t alloc_len, data_len;
+	int notif_class;
+
+	lun = ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
+	cdb = (struct scsi_get_event_status *)ctsio->cdb;
+	if ((cdb->byte2 & SGESN_POLLED) == 0) {
+		ctl_set_invalid_field(ctsio, /*sks_valid*/ 1, /*command*/ 1,
+		    /*field*/ 1, /*bit_valid*/ 1, /*bit*/ 0);
+		ctl_done((union ctl_io *)ctsio);
+		return (CTL_RETVAL_COMPLETE);
+	}
+	notif_class = cdb->notif_class;
+	alloc_len = scsi_2btoul(cdb->length);
+
+	data_len = sizeof(struct scsi_get_event_status_header);
+	ctsio->kern_data_ptr = malloc(data_len, M_CTL, M_WAITOK | M_ZERO);
+	ctsio->kern_sg_entries = 0;
+	ctsio->kern_data_resid = 0;
+	ctsio->kern_rel_offset = 0;
+
+	if (data_len < alloc_len) {
+		ctsio->residual = alloc_len - data_len;
+		ctsio->kern_data_len = data_len;
+		ctsio->kern_total_len = data_len;
+	} else {
+		ctsio->residual = 0;
+		ctsio->kern_data_len = alloc_len;
+		ctsio->kern_total_len = alloc_len;
+	}
+
+	hdr = (struct scsi_get_event_status_header *)ctsio->kern_data_ptr;
+	scsi_ulto2b(0, hdr->descr_length);
+	hdr->nea_class = SGESN_NEA;
+	hdr->supported_class = 0;
+
+	ctl_set_success(ctsio);
+	ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
+	ctsio->be_move_done = ctl_config_move_done;
+	ctl_datamove((union ctl_io *)ctsio);
+	return (CTL_RETVAL_COMPLETE);
+}
+
+int
+ctl_mechanism_status(struct ctl_scsiio *ctsio)
+{
+	struct scsi_mechanism_status_header *hdr;
+	struct scsi_mechanism_status *cdb;
+	struct ctl_lun *lun;
+	uint32_t alloc_len, data_len;
+
+	lun = ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
+	cdb = (struct scsi_mechanism_status *)ctsio->cdb;
+	alloc_len = scsi_2btoul(cdb->length);
+
+	data_len = sizeof(struct scsi_mechanism_status_header);
+	ctsio->kern_data_ptr = malloc(data_len, M_CTL, M_WAITOK | M_ZERO);
+	ctsio->kern_sg_entries = 0;
+	ctsio->kern_data_resid = 0;
+	ctsio->kern_rel_offset = 0;
+
+	if (data_len < alloc_len) {
+		ctsio->residual = alloc_len - data_len;
+		ctsio->kern_data_len = data_len;
+		ctsio->kern_total_len = data_len;
+	} else {
+		ctsio->residual = 0;
+		ctsio->kern_data_len = alloc_len;
+		ctsio->kern_total_len = alloc_len;
+	}
+
+	hdr = (struct scsi_mechanism_status_header *)ctsio->kern_data_ptr;
+	hdr->state1 = 0x00;
+	hdr->state2 = 0xe0;
+	scsi_ulto3b(0, hdr->lba);
+	hdr->slots_num = 0;
+	scsi_ulto2b(0, hdr->slots_length);
+
+	ctl_set_success(ctsio);
+	ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
+	ctsio->be_move_done = ctl_config_move_done;
+	ctl_datamove((union ctl_io *)ctsio);
+	return (CTL_RETVAL_COMPLETE);
+}
+
+static void
+ctl_ultomsf(uint32_t lba, uint8_t *buf)
+{
+
+	lba += 150;
+	buf[0] = 0;
+	buf[1] = bin2bcd((lba / 75) / 60);
+	buf[2] = bin2bcd((lba / 75) % 60);
+	buf[3] = bin2bcd(lba % 75);
+}
+
+int
+ctl_read_toc(struct ctl_scsiio *ctsio)
+{
+	struct scsi_read_toc_hdr *hdr;
+	struct scsi_read_toc_type01_descr *descr;
+	struct scsi_read_toc *cdb;
+	struct ctl_lun *lun;
+	uint32_t alloc_len, data_len;
+	int format, msf;
+
+	lun = ctsio->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
+	cdb = (struct scsi_read_toc *)ctsio->cdb;
+	msf = (cdb->byte2 & CD_MSF) != 0;
+	format = cdb->format;
+	alloc_len = scsi_2btoul(cdb->data_len);
+
+	data_len = sizeof(struct scsi_read_toc_hdr);
+	if (format == 0)
+		data_len += 2 * sizeof(struct scsi_read_toc_type01_descr);
+	else
+		data_len += sizeof(struct scsi_read_toc_type01_descr);
+	ctsio->kern_data_ptr = malloc(data_len, M_CTL, M_WAITOK | M_ZERO);
+	ctsio->kern_sg_entries = 0;
+	ctsio->kern_data_resid = 0;
+	ctsio->kern_rel_offset = 0;
+
+	if (data_len < alloc_len) {
+		ctsio->residual = alloc_len - data_len;
+		ctsio->kern_data_len = data_len;
+		ctsio->kern_total_len = data_len;
+	} else {
+		ctsio->residual = 0;
+		ctsio->kern_data_len = alloc_len;
+		ctsio->kern_total_len = alloc_len;
+	}
+
+	hdr = (struct scsi_read_toc_hdr *)ctsio->kern_data_ptr;
+	if (format == 0) {
+		scsi_ulto2b(0x12, hdr->data_length);
+		hdr->first = 1;
+		hdr->last = 1;
+		descr = (struct scsi_read_toc_type01_descr *)(hdr + 1);
+		descr->addr_ctl = 0x14;
+		descr->track_number = 1;
+		if (msf)
+			ctl_ultomsf(0, descr->track_start);
+		else
+			scsi_ulto4b(0, descr->track_start);
+		descr++;
+		descr->addr_ctl = 0x14;
+		descr->track_number = 0xaa;
+		if (msf)
+			ctl_ultomsf(lun->be_lun->maxlba+1, descr->track_start);
+		else
+			scsi_ulto4b(lun->be_lun->maxlba+1, descr->track_start);
+	} else {
+		scsi_ulto2b(0x0a, hdr->data_length);
+		hdr->first = 1;
+		hdr->last = 1;
+		descr = (struct scsi_read_toc_type01_descr *)(hdr + 1);
+		descr->addr_ctl = 0x14;
+		descr->track_number = 1;
+		if (msf)
+			ctl_ultomsf(0, descr->track_start);
+		else
+			scsi_ulto4b(0, descr->track_start);
+	}
+
+	ctl_set_success(ctsio);
+	ctsio->io_hdr.flags |= CTL_FLAG_ALLOCATED;
+	ctsio->be_move_done = ctl_config_move_done;
+	ctl_datamove((union ctl_io *)ctsio);
+	return (CTL_RETVAL_COMPLETE);
+}
+
 /*
  * For known CDB types, parse the LBA and length.
  */
@@ -10288,8 +10731,7 @@ ctl_get_lba_len(union ctl_io *io, uint64_t *lba, uint64_t *len)
 		break;
 	}
 	case READ_16:
-	case WRITE_16:
-	case WRITE_ATOMIC_16: {
+	case WRITE_16: {
 		struct scsi_rw_16 *cdb;
 
 		cdb = (struct scsi_rw_16 *)io->scsiio.cdb;
@@ -10298,6 +10740,15 @@ ctl_get_lba_len(union ctl_io *io, uint64_t *lba, uint64_t *len)
 		*len = scsi_4btoul(cdb->length);
 		break;
 	}
+	case WRITE_ATOMIC_16: {
+		struct scsi_write_atomic_16 *cdb;
+
+		cdb = (struct scsi_write_atomic_16 *)io->scsiio.cdb;
+
+		*lba = scsi_8btou64(cdb->addr);
+		*len = scsi_2btoul(cdb->length);
+		break;
+	}
 	case WRITE_VERIFY_16: {
 		struct scsi_write_verify_16 *cdb;
 
@@ -10437,6 +10888,8 @@ ctl_extent_check(union ctl_io *io1, union ctl_io *io2, bool seq)
 	if (ctl_get_lba_len(io1, &lba1, &len1) != 0)
 		return (CTL_ACTION_ERROR);
 
+	if (io1->io_hdr.flags & CTL_FLAG_SERSEQ_DONE)
+		seq = FALSE;
 	return (ctl_extent_check_lba(lba1, len1, lba2, len2, seq));
 }
 
@@ -10446,6 +10899,8 @@ ctl_extent_check_seq(union ctl_io *io1, union ctl_io *io2)
 	uint64_t lba1, lba2;
 	uint64_t len1, len2;
 
+	if (io1->io_hdr.flags & CTL_FLAG_SERSEQ_DONE)
+		return (CTL_ACTION_PASS);
 	if (ctl_get_lba_len(io1, &lba1, &len1) != 0)
 		return (CTL_ACTION_ERROR);
 	if (ctl_get_lba_len(io2, &lba2, &len2) != 0)
@@ -10461,7 +10916,7 @@ ctl_check_for_blockage(struct ctl_lun *lun, union ctl_io *pending_io,
     union ctl_io *ooa_io)
 {
 	const struct ctl_cmd_entry *pending_entry, *ooa_entry;
-	ctl_serialize_action *serialize_row;
+	const ctl_serialize_action *serialize_row;
 
 	/*
 	 * The initiator attempted multiple untagged commands at the same
@@ -10798,9 +11253,7 @@ ctl_scsiio_lun_check(struct ctl_lun *lun,
 	if (entry->pattern & CTL_LUN_PAT_WRITE) {
 		if (lun->be_lun &&
 		    lun->be_lun->flags & CTL_LUN_FLAG_READONLY) {
-			ctl_set_sense(ctsio, /*current_error*/ 1,
-			    /*sense_key*/ SSD_KEY_DATA_PROTECT,
-			    /*asc*/ 0x27, /*ascq*/ 0x01, SSD_ELEM_NONE);
+			ctl_set_hw_write_protected(ctsio);
 			retval = 1;
 			goto bailout;
 		}
@@ -10851,25 +11304,18 @@ ctl_scsiio_lun_check(struct ctl_lun *lun,
 		}
 	}
 
-	if ((lun->flags & CTL_LUN_OFFLINE)
-	 && ((entry->flags & CTL_CMD_FLAG_OK_ON_STANDBY) == 0)) {
-		ctl_set_lun_not_ready(ctsio);
-		retval = 1;
-		goto bailout;
-	}
-
-	if ((lun->flags & CTL_LUN_STOPPED)
-	 && ((entry->flags & CTL_CMD_FLAG_OK_ON_STOPPED) == 0)) {
-		/* "Logical unit not ready, initializing cmd. required" */
-		ctl_set_lun_stopped(ctsio);
-		retval = 1;
-		goto bailout;
-	}
-
-	if ((lun->flags & CTL_LUN_INOPERABLE)
-	 && ((entry->flags & CTL_CMD_FLAG_OK_ON_INOPERABLE) == 0)) {
-		/* "Medium format corrupted" */
-		ctl_set_medium_format_corrupted(ctsio);
+	if ((entry->flags & CTL_CMD_FLAG_OK_ON_NO_MEDIA) == 0) {
+		if (lun->flags & CTL_LUN_EJECTED)
+			ctl_set_lun_ejected(ctsio);
+		else if (lun->flags & CTL_LUN_NO_MEDIA) {
+			if (lun->flags & CTL_LUN_REMOVABLE)
+				ctl_set_lun_no_media(ctsio);
+			else
+				ctl_set_lun_int_reqd(ctsio);
+		} else if (lun->flags & CTL_LUN_STOPPED)
+			ctl_set_lun_stopped(ctsio);
+		else
+			goto bailout;
 		retval = 1;
 		goto bailout;
 	}
@@ -10886,20 +11332,41 @@ ctl_failover_io(union ctl_io *io, int have_lock)
 }
 
 static void
-ctl_failover_lun(struct ctl_lun *lun)
+ctl_failover_lun(union ctl_io *rio)
 {
-	struct ctl_softc *softc = lun->ctl_softc;
+	struct ctl_softc *softc = control_softc;
+	struct ctl_lun *lun;
 	struct ctl_io_hdr *io, *next_io;
+	uint32_t targ_lun;
+
+	targ_lun = rio->io_hdr.nexus.targ_mapped_lun;
+	CTL_DEBUG_PRINT(("FAILOVER for lun %ju\n", targ_lun));
+
+	/* Find and lock the LUN. */
+	mtx_lock(&softc->ctl_lock);
+	if ((targ_lun < CTL_MAX_LUNS) &&
+	    ((lun = softc->ctl_luns[targ_lun]) != NULL)) {
+		mtx_lock(&lun->lun_lock);
+		mtx_unlock(&softc->ctl_lock);
+		if (lun->flags & CTL_LUN_DISABLED) {
+			mtx_unlock(&lun->lun_lock);
+			return;
+		}
+	} else {
+		mtx_unlock(&softc->ctl_lock);
+		return;
+	}
 
-	CTL_DEBUG_PRINT(("FAILOVER for lun %ju\n", lun->lun));
 	if (softc->ha_mode == CTL_HA_MODE_XFER) {
 		TAILQ_FOREACH_SAFE(io, &lun->ooa_queue, ooa_links, next_io) {
 			/* We are master */
 			if (io->flags & CTL_FLAG_FROM_OTHER_SC) {
 				if (io->flags & CTL_FLAG_IO_ACTIVE) {
 					io->flags |= CTL_FLAG_ABORT;
+					io->flags |= CTL_FLAG_FAILOVER;
 				} else { /* This can be only due to DATAMOVE */
 					io->msg_type = CTL_MSG_DATAMOVE_DONE;
+					io->flags &= ~CTL_FLAG_DMA_INPROG;
 					io->flags |= CTL_FLAG_IO_ACTIVE;
 					io->port_status = 31340;
 					ctl_enqueue_isc((union ctl_io *)io);
@@ -10947,6 +11414,7 @@ ctl_failover_lun(struct ctl_lun *lun)
 		}
 		ctl_check_blocked(lun);
 	}
+	mtx_unlock(&lun->lun_lock);
 }
 
 static int
@@ -11017,7 +11485,7 @@ ctl_scsiio_precheck(struct ctl_softc *softc, struct ctl_scsiio *ctsio)
 	 * it on the rtr queue.
 	 */
 	if (lun == NULL) {
-		if (entry->flags & CTL_CMD_FLAG_OK_ON_ALL_LUNS) {
+		if (entry->flags & CTL_CMD_FLAG_OK_ON_NO_LUN) {
 			ctsio->io_hdr.flags |= CTL_FLAG_IS_WAS_ON_RTR;
 			ctl_enqueue_rtr((union ctl_io *)ctsio);
 			return (retval);
@@ -11078,15 +11546,9 @@ ctl_scsiio_precheck(struct ctl_softc *softc, struct ctl_scsiio *ctsio)
 	 */
 	if ((entry->flags & CTL_CMD_FLAG_NO_SENSE) == 0) {
 		ctl_ua_type ua_type;
-		scsi_sense_data_type sense_format;
-
-		if (lun->flags & CTL_LUN_SENSE_DESC)
-			sense_format = SSD_TYPE_DESC;
-		else
-			sense_format = SSD_TYPE_FIXED;
 
 		ua_type = ctl_build_ua(lun, initidx, &ctsio->sense_data,
-		    sense_format);
+		    SSD_TYPE_NONE);
 		if (ua_type != CTL_UA_NONE) {
 			mtx_unlock(&lun->lun_lock);
 			ctsio->scsi_status = SCSI_STATUS_CHECK_COND;
@@ -11114,7 +11576,8 @@ ctl_scsiio_precheck(struct ctl_softc *softc, struct ctl_scsiio *ctsio)
 	 * side so when we are done we can find the copy.
 	 */
 	if ((lun->flags & CTL_LUN_PRIMARY_SC) == 0 &&
-	    (lun->flags & CTL_LUN_PEER_SC_PRIMARY) != 0) {
+	    (lun->flags & CTL_LUN_PEER_SC_PRIMARY) != 0 &&
+	    (entry->flags & CTL_CMD_FLAG_RUN_HERE) == 0) {
 		union ctl_ha_msg msg_info;
 		int isc_retval;
 
@@ -11240,14 +11703,16 @@ ctl_cmd_applicable(uint8_t lun_type, const struct ctl_cmd_entry *entry)
 {
 
 	switch (lun_type) {
+	case T_DIRECT:
+		if ((entry->flags & CTL_CMD_FLAG_OK_ON_DIRECT) == 0)
+			return (0);
+		break;
 	case T_PROCESSOR:
-		if (((entry->flags & CTL_CMD_FLAG_OK_ON_PROC) == 0) &&
-		    ((entry->flags & CTL_CMD_FLAG_OK_ON_ALL_LUNS) == 0))
+		if ((entry->flags & CTL_CMD_FLAG_OK_ON_PROC) == 0)
 			return (0);
 		break;
-	case T_DIRECT:
-		if (((entry->flags & CTL_CMD_FLAG_OK_ON_SLUN) == 0) &&
-		    ((entry->flags & CTL_CMD_FLAG_OK_ON_ALL_LUNS) == 0))
+	case T_CDROM:
+		if ((entry->flags & CTL_CMD_FLAG_OK_ON_CDROM) == 0)
 			return (0);
 		break;
 	default:
@@ -11301,6 +11766,7 @@ static int
 ctl_target_reset(struct ctl_softc *softc, union ctl_io *io,
 		 ctl_ua_type ua_type)
 {
+	struct ctl_port *port;
 	struct ctl_lun *lun;
 	int retval;
 
@@ -11321,10 +11787,15 @@ ctl_target_reset(struct ctl_softc *softc, union ctl_io *io,
 	retval = 0;
 
 	mtx_lock(&softc->ctl_lock);
-	STAILQ_FOREACH(lun, &softc->lun_list, links)
-		retval += ctl_lun_reset(lun, io, ua_type);
+	port = ctl_io_port(&io->io_hdr);
+	STAILQ_FOREACH(lun, &softc->lun_list, links) {
+		if (port != NULL &&
+		    ctl_lun_map_to_port(port, lun->lun) >= CTL_MAX_LUNS)
+			continue;
+		retval += ctl_do_lun_reset(lun, io, ua_type);
+	}
 	mtx_unlock(&softc->ctl_lock);
-
+	io->taskio.task_status = CTL_TASK_FUNCTION_COMPLETE;
 	return (retval);
 }
 
@@ -11350,15 +11821,13 @@ ctl_target_reset(struct ctl_softc *softc, union ctl_io *io,
  * XXX KDM for now, we're setting unit attention for all initiators.
  */
 static int
-ctl_lun_reset(struct ctl_lun *lun, union ctl_io *io, ctl_ua_type ua_type)
+ctl_do_lun_reset(struct ctl_lun *lun, union ctl_io *io, ctl_ua_type ua_type)
 {
 	union ctl_io *xio;
 #if 0
 	uint32_t initidx;
 #endif
-#ifdef CTL_WITH_CA
 	int i;
-#endif
 
 	mtx_lock(&lun->lun_lock);
 	/*
@@ -11393,11 +11862,47 @@ ctl_lun_reset(struct ctl_lun *lun, union ctl_io *io, ctl_ua_type ua_type)
 	for (i = 0; i < CTL_MAX_INITIATORS; i++)
 		ctl_clear_mask(lun->have_ca, i);
 #endif
+	lun->prevent_count = 0;
+	for (i = 0; i < CTL_MAX_INITIATORS; i++)
+		ctl_clear_mask(lun->prevent, i);
 	mtx_unlock(&lun->lun_lock);
 
 	return (0);
 }
 
+static int
+ctl_lun_reset(struct ctl_softc *softc, union ctl_io *io)
+{
+	struct ctl_lun *lun;
+	uint32_t targ_lun;
+	int retval;
+
+	targ_lun = io->io_hdr.nexus.targ_mapped_lun;
+	mtx_lock(&softc->ctl_lock);
+	if ((targ_lun >= CTL_MAX_LUNS) ||
+	    (lun = softc->ctl_luns[targ_lun]) == NULL) {
+		mtx_unlock(&softc->ctl_lock);
+		io->taskio.task_status = CTL_TASK_LUN_DOES_NOT_EXIST;
+		return (1);
+	}
+	retval = ctl_do_lun_reset(lun, io, CTL_UA_LUN_RESET);
+	mtx_unlock(&softc->ctl_lock);
+	io->taskio.task_status = CTL_TASK_FUNCTION_COMPLETE;
+
+	if ((io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC) == 0) {
+		union ctl_ha_msg msg_info;
+
+		msg_info.hdr.msg_type = CTL_MSG_MANAGE_TASKS;
+		msg_info.hdr.nexus = io->io_hdr.nexus;
+		msg_info.task.task_action = CTL_TASK_LUN_RESET;
+		msg_info.hdr.original_sc = NULL;
+		msg_info.hdr.serializing_sc = NULL;
+		ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
+		    sizeof(msg_info.task), M_WAITOK);
+	}
+	return (retval);
+}
+
 static void
 ctl_abort_tasks_lun(struct ctl_lun *lun, uint32_t targ_port, uint32_t init_id,
     int other_sc)
@@ -11453,10 +11958,10 @@ ctl_abort_task_set(union ctl_io *io)
 	 */
 	targ_lun = io->io_hdr.nexus.targ_mapped_lun;
 	mtx_lock(&softc->ctl_lock);
-	if ((targ_lun < CTL_MAX_LUNS) && (softc->ctl_luns[targ_lun] != NULL))
-		lun = softc->ctl_luns[targ_lun];
-	else {
+	if ((targ_lun >= CTL_MAX_LUNS) ||
+	    (lun = softc->ctl_luns[targ_lun]) == NULL) {
 		mtx_unlock(&softc->ctl_lock);
+		io->taskio.task_status = CTL_TASK_LUN_DOES_NOT_EXIST;
 		return (1);
 	}
 
@@ -11471,6 +11976,7 @@ ctl_abort_task_set(union ctl_io *io)
 		    (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC) != 0);
 	}
 	mtx_unlock(&lun->lun_lock);
+	io->taskio.task_status = CTL_TASK_FUNCTION_COMPLETE;
 	return (0);
 }
 
@@ -11481,22 +11987,38 @@ ctl_i_t_nexus_reset(union ctl_io *io)
 	struct ctl_lun *lun;
 	uint32_t initidx;
 
+	if (!(io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)) {
+		union ctl_ha_msg msg_info;
+
+		msg_info.hdr.nexus = io->io_hdr.nexus;
+		msg_info.task.task_action = CTL_TASK_I_T_NEXUS_RESET;
+		msg_info.hdr.msg_type = CTL_MSG_MANAGE_TASKS;
+		msg_info.hdr.original_sc = NULL;
+		msg_info.hdr.serializing_sc = NULL;
+		ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
+		    sizeof(msg_info.task), M_WAITOK);
+	}
+
 	initidx = ctl_get_initindex(&io->io_hdr.nexus);
 	mtx_lock(&softc->ctl_lock);
 	STAILQ_FOREACH(lun, &softc->lun_list, links) {
 		mtx_lock(&lun->lun_lock);
 		ctl_abort_tasks_lun(lun, io->io_hdr.nexus.targ_port,
-		    io->io_hdr.nexus.initid,
-		    (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC) != 0);
+		    io->io_hdr.nexus.initid, 1);
 #ifdef CTL_WITH_CA
 		ctl_clear_mask(lun->have_ca, initidx);
 #endif
 		if ((lun->flags & CTL_LUN_RESERVED) && (lun->res_idx == initidx))
 			lun->flags &= ~CTL_LUN_RESERVED;
+		if (ctl_is_set(lun->prevent, initidx)) {
+			ctl_clear_mask(lun->prevent, initidx);
+			lun->prevent_count--;
+		}
 		ctl_est_ua(lun, initidx, CTL_UA_I_T_NEXUS_LOSS);
 		mtx_unlock(&lun->lun_lock);
 	}
 	mtx_unlock(&softc->ctl_lock);
+	io->taskio.task_status = CTL_TASK_FUNCTION_COMPLETE;
 	return (0);
 }
 
@@ -11521,11 +12043,10 @@ ctl_abort_task(union ctl_io *io)
 	 */
 	targ_lun = io->io_hdr.nexus.targ_mapped_lun;
 	mtx_lock(&softc->ctl_lock);
-	if ((targ_lun < CTL_MAX_LUNS)
-	 && (softc->ctl_luns[targ_lun] != NULL))
-		lun = softc->ctl_luns[targ_lun];
-	else {
+	if ((targ_lun >= CTL_MAX_LUNS) ||
+	    (lun = softc->ctl_luns[targ_lun]) == NULL) {
 		mtx_unlock(&softc->ctl_lock);
+		io->taskio.task_status = CTL_TASK_LUN_DOES_NOT_EXIST;
 		return (1);
 	}
 
@@ -11631,6 +12152,77 @@ ctl_abort_task(union ctl_io *io)
 		       io->taskio.tag_type);
 #endif
 	}
+	io->taskio.task_status = CTL_TASK_FUNCTION_COMPLETE;
+	return (0);
+}
+
+static int
+ctl_query_task(union ctl_io *io, int task_set)
+{
+	union ctl_io *xio;
+	struct ctl_lun *lun;
+	struct ctl_softc *softc;
+	int found = 0;
+	uint32_t targ_lun;
+
+	softc = control_softc;
+	targ_lun = io->io_hdr.nexus.targ_mapped_lun;
+	mtx_lock(&softc->ctl_lock);
+	if ((targ_lun >= CTL_MAX_LUNS) ||
+	    (lun = softc->ctl_luns[targ_lun]) == NULL) {
+		mtx_unlock(&softc->ctl_lock);
+		io->taskio.task_status = CTL_TASK_LUN_DOES_NOT_EXIST;
+		return (1);
+	}
+	mtx_lock(&lun->lun_lock);
+	mtx_unlock(&softc->ctl_lock);
+	for (xio = (union ctl_io *)TAILQ_FIRST(&lun->ooa_queue); xio != NULL;
+	     xio = (union ctl_io *)TAILQ_NEXT(&xio->io_hdr, ooa_links)) {
+
+		if ((xio->io_hdr.nexus.targ_port != io->io_hdr.nexus.targ_port)
+		 || (xio->io_hdr.nexus.initid != io->io_hdr.nexus.initid)
+		 || (xio->io_hdr.flags & CTL_FLAG_ABORT))
+			continue;
+
+		if (task_set || xio->scsiio.tag_num == io->taskio.tag_num) {
+			found = 1;
+			break;
+		}
+	}
+	mtx_unlock(&lun->lun_lock);
+	if (found)
+		io->taskio.task_status = CTL_TASK_FUNCTION_SUCCEEDED;
+	else
+		io->taskio.task_status = CTL_TASK_FUNCTION_COMPLETE;
+	return (0);
+}
+
+static int
+ctl_query_async_event(union ctl_io *io)
+{
+	struct ctl_lun *lun;
+	struct ctl_softc *softc;
+	ctl_ua_type ua;
+	uint32_t targ_lun, initidx;
+
+	softc = control_softc;
+	targ_lun = io->io_hdr.nexus.targ_mapped_lun;
+	mtx_lock(&softc->ctl_lock);
+	if ((targ_lun >= CTL_MAX_LUNS) ||
+	    (lun = softc->ctl_luns[targ_lun]) == NULL) {
+		mtx_unlock(&softc->ctl_lock);
+		io->taskio.task_status = CTL_TASK_LUN_DOES_NOT_EXIST;
+		return (1);
+	}
+	mtx_lock(&lun->lun_lock);
+	mtx_unlock(&softc->ctl_lock);
+	initidx = ctl_get_initindex(&io->io_hdr.nexus);
+	ua = ctl_build_qae(lun, initidx, io->taskio.task_resp);
+	mtx_unlock(&lun->lun_lock);
+	if (ua != CTL_UA_NONE)
+		io->taskio.task_status = CTL_TASK_FUNCTION_SUCCEEDED;
+	else
+		io->taskio.task_status = CTL_TASK_FUNCTION_COMPLETE;
 	return (0);
 }
 
@@ -11639,41 +12231,12 @@ ctl_run_task(union ctl_io *io)
 {
 	struct ctl_softc *softc = control_softc;
 	int retval = 1;
-	const char *task_desc;
 
 	CTL_DEBUG_PRINT(("ctl_run_task\n"));
-
 	KASSERT(io->io_hdr.io_type == CTL_IO_TASK,
-	    ("ctl_run_task: Unextected io_type %d\n",
-	     io->io_hdr.io_type));
-
-	task_desc = ctl_scsi_task_string(&io->taskio);
-	if (task_desc != NULL) {
-#ifdef NEEDTOPORT
-		csevent_log(CSC_CTL | CSC_SHELF_SW |
-			    CTL_TASK_REPORT,
-			    csevent_LogType_Trace,
-			    csevent_Severity_Information,
-			    csevent_AlertLevel_Green,
-			    csevent_FRU_Firmware,
-			    csevent_FRU_Unknown,
-			    "CTL: received task: %s",task_desc);
-#endif
-	} else {
-#ifdef NEEDTOPORT
-		csevent_log(CSC_CTL | CSC_SHELF_SW |
-			    CTL_TASK_REPORT,
-			    csevent_LogType_Trace,
-			    csevent_Severity_Information,
-			    csevent_AlertLevel_Green,
-			    csevent_FRU_Firmware,
-			    csevent_FRU_Unknown,
-			    "CTL: received unknown task "
-			    "type: %d (%#x)",
-			    io->taskio.task_action,
-			    io->taskio.task_action);
-#endif
-	}
+	    ("ctl_run_task: Unextected io_type %d\n", io->io_hdr.io_type));
+	io->taskio.task_status = CTL_TASK_FUNCTION_NOT_SUPPORTED;
+	bzero(io->taskio.task_resp, sizeof(io->taskio.task_resp));
 	switch (io->taskio.task_action) {
 	case CTL_TASK_ABORT_TASK:
 		retval = ctl_abort_task(io);
@@ -11687,36 +12250,9 @@ ctl_run_task(union ctl_io *io)
 	case CTL_TASK_I_T_NEXUS_RESET:
 		retval = ctl_i_t_nexus_reset(io);
 		break;
-	case CTL_TASK_LUN_RESET: {
-		struct ctl_lun *lun;
-		uint32_t targ_lun;
-
-		targ_lun = io->io_hdr.nexus.targ_mapped_lun;
-		mtx_lock(&softc->ctl_lock);
-		if ((targ_lun < CTL_MAX_LUNS)
-		 && (softc->ctl_luns[targ_lun] != NULL))
-			lun = softc->ctl_luns[targ_lun];
-		else {
-			mtx_unlock(&softc->ctl_lock);
-			retval = 1;
-			break;
-		}
-		retval = ctl_lun_reset(lun, io, CTL_UA_LUN_RESET);
-		mtx_unlock(&softc->ctl_lock);
-
-		if ((io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC) == 0) {
-			union ctl_ha_msg msg_info;
-
-			msg_info.hdr.msg_type = CTL_MSG_MANAGE_TASKS;
-			msg_info.hdr.nexus = io->io_hdr.nexus;
-			msg_info.task.task_action = CTL_TASK_LUN_RESET;
-			msg_info.hdr.original_sc = NULL;
-			msg_info.hdr.serializing_sc = NULL;
-			ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg_info,
-			    sizeof(msg_info.task), M_WAITOK);
-		}
+	case CTL_TASK_LUN_RESET:
+		retval = ctl_lun_reset(softc, io);
 		break;
-	}
 	case CTL_TASK_TARGET_RESET:
 		retval = ctl_target_reset(softc, io, CTL_UA_TARG_RESET);
 		break;
@@ -11727,9 +12263,18 @@ ctl_run_task(union ctl_io *io)
 		break;
 	case CTL_TASK_PORT_LOGOUT:
 		break;
+	case CTL_TASK_QUERY_TASK:
+		retval = ctl_query_task(io, 0);
+		break;
+	case CTL_TASK_QUERY_TASK_SET:
+		retval = ctl_query_task(io, 1);
+		break;
+	case CTL_TASK_QUERY_ASYNC_EVENT:
+		retval = ctl_query_async_event(io);
+		break;
 	default:
-		printf("ctl_run_task: got unknown task management event %d\n",
-		       io->taskio.task_action);
+		printf("%s: got unknown task management event %d\n",
+		       __func__, io->taskio.task_action);
 		break;
 	}
 	if (retval == 0)
@@ -11748,11 +12293,9 @@ ctl_handle_isc(union ctl_io *io)
 {
 	int free_io;
 	struct ctl_lun *lun;
-	struct ctl_softc *softc;
+	struct ctl_softc *softc = control_softc;
 	uint32_t targ_lun;
 
-	softc = control_softc;
-
 	targ_lun = io->io_hdr.nexus.targ_mapped_lun;
 	lun = softc->ctl_luns[targ_lun];
 
@@ -11813,9 +12356,7 @@ ctl_handle_isc(union ctl_io *io)
 		io->scsiio.be_move_done(io);
 		break;
 	case CTL_MSG_FAILOVER:
-		mtx_lock(&lun->lun_lock);
-		ctl_failover_lun(lun);
-		mtx_unlock(&lun->lun_lock);
+		ctl_failover_lun(io);
 		free_io = 1;
 		break;
 	default:
@@ -11918,7 +12459,9 @@ ctl_inject_error(struct ctl_lun *lun, union ctl_io *io)
 			ctl_set_aborted(&io->scsiio);
 			break;
 		case CTL_LUN_INJ_MEDIUM_ERR:
-			ctl_set_medium_error(&io->scsiio);
+			ctl_set_medium_error(&io->scsiio,
+			    (io->io_hdr.flags & CTL_FLAG_DATA_MASK) !=
+			     CTL_FLAG_DATA_OUT);
 			break;
 		case CTL_LUN_INJ_UA:
 			/* 29h/00h  POWER ON, RESET, OR BUS DEVICE RESET
@@ -11975,12 +12518,14 @@ ctl_datamove_timer_wakeup(void *arg)
 void
 ctl_datamove(union ctl_io *io)
 {
+	struct ctl_lun *lun;
 	void (*fe_datamove)(union ctl_io *io);
 
 	mtx_assert(&control_softc->ctl_lock, MA_NOTOWNED);
 
 	CTL_DEBUG_PRINT(("ctl_datamove\n"));
 
+	lun = (struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
 #ifdef CTL_TIME_IO
 	if ((time_uptime - io->io_hdr.start_time) > ctl_time_io_secs) {
 		char str[256];
@@ -12021,9 +12566,6 @@ ctl_datamove(union ctl_io *io)
 	if (io->io_hdr.flags & CTL_FLAG_DELAY_DONE) {
 		io->io_hdr.flags &= ~CTL_FLAG_DELAY_DONE;
 	} else {
-		struct ctl_lun *lun;
-
-		lun =(struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
 		if ((lun != NULL)
 		 && (lun->delay_info.datamove_delay > 0)) {
 
@@ -12065,159 +12607,19 @@ ctl_datamove(union ctl_io *io)
 		return;
 	}
 
-	/*
-	 * If we're in XFER mode and this I/O is from the other shelf
-	 * controller, we need to send the DMA to the other side to
-	 * actually transfer the data to/from the host.  In serialize only
-	 * mode the transfer happens below CTL and ctl_datamove() is only
-	 * called on the machine that originally received the I/O.
-	 */
-	if ((control_softc->ha_mode == CTL_HA_MODE_XFER)
-	 && (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)) {
-		union ctl_ha_msg msg;
-		uint32_t sg_entries_sent;
-		int do_sg_copy;
-		int i;
-
-		memset(&msg, 0, sizeof(msg));
-		msg.hdr.msg_type = CTL_MSG_DATAMOVE;
-		msg.hdr.original_sc = io->io_hdr.original_sc;
-		msg.hdr.serializing_sc = io;
-		msg.hdr.nexus = io->io_hdr.nexus;
-		msg.dt.flags = io->io_hdr.flags;
-		/*
-		 * We convert everything into a S/G list here.  We can't
-		 * pass by reference, only by value between controllers.
-		 * So we can't pass a pointer to the S/G list, only as many
-		 * S/G entries as we can fit in here.  If it's possible for
-		 * us to get more than CTL_HA_MAX_SG_ENTRIES S/G entries,
-		 * then we need to break this up into multiple transfers.
-		 */
-		if (io->scsiio.kern_sg_entries == 0) {
-			msg.dt.kern_sg_entries = 1;
-#if 0
-			/*
-			 * Convert to a physical address if this is a
-			 * virtual address.
-			 */
-			if (io->io_hdr.flags & CTL_FLAG_BUS_ADDR) {
-				msg.dt.sg_list[0].addr =
-					io->scsiio.kern_data_ptr;
-			} else {
-				/*
-				 * XXX KDM use busdma here!
-				 */
-				msg.dt.sg_list[0].addr = (void *)
-					vtophys(io->scsiio.kern_data_ptr);
-			}
-#else
-			KASSERT((io->io_hdr.flags & CTL_FLAG_BUS_ADDR) == 0,
-			    ("HA does not support BUS_ADDR"));
-			msg.dt.sg_list[0].addr = io->scsiio.kern_data_ptr;
-#endif
-
-			msg.dt.sg_list[0].len = io->scsiio.kern_data_len;
-			do_sg_copy = 0;
-		} else {
-			msg.dt.kern_sg_entries = io->scsiio.kern_sg_entries;
-			do_sg_copy = 1;
-		}
-
-		msg.dt.kern_data_len = io->scsiio.kern_data_len;
-		msg.dt.kern_total_len = io->scsiio.kern_total_len;
-		msg.dt.kern_data_resid = io->scsiio.kern_data_resid;
-		msg.dt.kern_rel_offset = io->scsiio.kern_rel_offset;
-		msg.dt.sg_sequence = 0;
-
-		/*
-		 * Loop until we've sent all of the S/G entries.  On the
-		 * other end, we'll recompose these S/G entries into one
-		 * contiguous list before passing it to the
-		 */
-		for (sg_entries_sent = 0; sg_entries_sent <
-		     msg.dt.kern_sg_entries; msg.dt.sg_sequence++) {
-			msg.dt.cur_sg_entries = MIN((sizeof(msg.dt.sg_list)/
-				sizeof(msg.dt.sg_list[0])),
-				msg.dt.kern_sg_entries - sg_entries_sent);
-
-			if (do_sg_copy != 0) {
-				struct ctl_sg_entry *sgl;
-				int j;
-
-				sgl = (struct ctl_sg_entry *)
-					io->scsiio.kern_data_ptr;
-				/*
-				 * If this is in cached memory, flush the cache
-				 * before we send the DMA request to the other
-				 * controller.  We want to do this in either
-				 * the * read or the write case.  The read
-				 * case is straightforward.  In the write
-				 * case, we want to make sure nothing is
-				 * in the local cache that could overwrite
-				 * the DMAed data.
-				 */
-
-				for (i = sg_entries_sent, j = 0;
-				     i < msg.dt.cur_sg_entries; i++, j++) {
-#if 0
-					if ((io->io_hdr.flags &
-					     CTL_FLAG_BUS_ADDR) == 0) {
-						/*
-						 * XXX KDM use busdma.
-						 */
-						msg.dt.sg_list[j].addr =(void *)
-						       vtophys(sgl[i].addr);
-					} else {
-						msg.dt.sg_list[j].addr =
-							sgl[i].addr;
-					}
-#else
-					KASSERT((io->io_hdr.flags &
-					    CTL_FLAG_BUS_ADDR) == 0,
-					    ("HA does not support BUS_ADDR"));
-					msg.dt.sg_list[j].addr = sgl[i].addr;
-#endif
-					msg.dt.sg_list[j].len = sgl[i].len;
-				}
-			}
-
-			sg_entries_sent += msg.dt.cur_sg_entries;
-			if (sg_entries_sent >= msg.dt.kern_sg_entries)
-				msg.dt.sg_last = 1;
-			else
-				msg.dt.sg_last = 0;
-
-			if (ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg,
-			    sizeof(msg.dt) - sizeof(msg.dt.sg_list) +
-			    sizeof(struct ctl_sg_entry)*msg.dt.cur_sg_entries,
-			    M_WAITOK) > CTL_HA_STATUS_SUCCESS) {
-				io->io_hdr.port_status = 31341;
-				io->scsiio.be_move_done(io);
-				return;
-			}
-
-			msg.dt.sent_sg_entries = sg_entries_sent;
-		}
-		io->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE;
-	} else {
-
-		/*
-		 * Lookup the fe_datamove() function for this particular
-		 * front end.
-		 */
-		fe_datamove = ctl_io_port(&io->io_hdr)->fe_datamove;
-
-		fe_datamove(io);
-	}
+	fe_datamove = ctl_io_port(&io->io_hdr)->fe_datamove;
+	fe_datamove(io);
 }
 
 static void
 ctl_send_datamove_done(union ctl_io *io, int have_lock)
 {
 	union ctl_ha_msg msg;
+#ifdef CTL_TIME_IO
+	struct bintime cur_bt;
+#endif
 
 	memset(&msg, 0, sizeof(msg));
-
 	msg.hdr.msg_type = CTL_MSG_DATAMOVE_DONE;
 	msg.hdr.original_sc = io;
 	msg.hdr.serializing_sc = io->io_hdr.serializing_sc;
@@ -12233,15 +12635,20 @@ ctl_send_datamove_done(union ctl_io *io, int have_lock)
 	msg.scsi.fetd_status = io->io_hdr.port_status;
 	msg.scsi.residual = io->scsiio.residual;
 	io->io_hdr.flags &= ~CTL_FLAG_IO_ACTIVE;
-
 	if (io->io_hdr.flags & CTL_FLAG_FAILOVER) {
 		ctl_failover_io(io, /*have_lock*/ have_lock);
 		return;
 	}
-
 	ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg,
 	    sizeof(msg.scsi) - sizeof(msg.scsi.sense_data) +
 	    msg.scsi.sense_len, M_WAITOK);
+
+#ifdef CTL_TIME_IO
+	getbinuptime(&cur_bt);
+	bintime_sub(&cur_bt, &io->io_hdr.dma_start_bt);
+	bintime_add(&io->io_hdr.dma_bt, &cur_bt);
+#endif
+	io->io_hdr.num_dmas++;
 }
 
 /*
@@ -12288,11 +12695,8 @@ ctl_datamove_remote_dm_write_cb(union ctl_io *io)
 {
 	int retval;
 
-	retval = 0;
-
 	retval = ctl_datamove_remote_xfer(io, CTL_HA_DT_CMD_WRITE,
 					  ctl_datamove_remote_write_cb);
-
 	return (retval);
 }
 
@@ -12322,11 +12726,7 @@ ctl_datamove_remote_write(union ctl_io *io)
 	io->scsiio.be_move_done = ctl_datamove_remote_dm_write_cb;
 
 	fe_datamove = ctl_io_port(&io->io_hdr)->fe_datamove;
-
 	fe_datamove(io);
-
-	return;
-
 }
 
 static int
@@ -12401,23 +12801,19 @@ ctl_datamove_remote_read_cb(struct ctl_ha_dt_req *rq)
 	/* XXX KDM add checks like the ones in ctl_datamove? */
 
 	fe_datamove = ctl_io_port(&io->io_hdr)->fe_datamove;
-
 	fe_datamove(io);
 }
 
 static int
 ctl_datamove_remote_sgl_setup(union ctl_io *io)
 {
-	struct ctl_sg_entry *local_sglist, *remote_sglist;
-	struct ctl_softc *softc;
+	struct ctl_sg_entry *local_sglist;
 	uint32_t len_to_go;
 	int retval;
 	int i;
 
 	retval = 0;
-	softc = control_softc;
 	local_sglist = io->io_hdr.local_sglist;
-	remote_sglist = io->io_hdr.remote_sglist;
 	len_to_go = io->scsiio.kern_data_len;
 
 	/*
@@ -12443,7 +12839,7 @@ ctl_datamove_remote_sgl_setup(union ctl_io *io)
 	printf("%s: kern_sg_entries = %d\n", __func__,
 	       io->scsiio.kern_sg_entries);
 	for (i = 0; i < io->scsiio.kern_sg_entries; i++)
-		printf("%s: sg[%d] = %p, %d\n", __func__, i,
+		printf("%s: sg[%d] = %p, %lu\n", __func__, i,
 		       local_sglist[i].addr, local_sglist[i].len);
 #endif
 
@@ -12467,10 +12863,12 @@ ctl_datamove_remote_xfer(union ctl_io *io, unsigned command,
 	 * failure.
 	 */
 	if ((rq == NULL)
-	 && ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE))
+	 && ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE &&
+	     (io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS))
 		ctl_set_busy(&io->scsiio);
 
-	if ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE) {
+	if ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_STATUS_NONE &&
+	    (io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS) {
 
 		if (rq != NULL)
 			ctl_dt_req_free(rq);
@@ -12559,7 +12957,7 @@ ctl_datamove_remote_xfer(union ctl_io *io, unsigned command,
 			rq->callback = callback;
 
 #if 0
-		printf("%s: %s: local %#x remote %#x size %d\n", __func__,
+		printf("%s: %s: local %p remote %p size %d\n", __func__,
 		       (command == CTL_HA_DT_CMD_WRITE) ? "WRITE" : "READ",
 		       rq->local, rq->remote, rq->size);
 #endif
@@ -12605,8 +13003,6 @@ ctl_datamove_remote_read(union ctl_io *io)
 		io->io_hdr.remote_sglist = NULL;
 		io->io_hdr.local_sglist = NULL;
 	}
-
-	return;
 }
 
 /*
@@ -12653,7 +13049,7 @@ ctl_datamove_remote(union ctl_io *io)
 	}
 }
 
-static int
+static void
 ctl_process_done(union ctl_io *io)
 {
 	struct ctl_lun *lun;
@@ -12663,11 +13059,7 @@ ctl_process_done(union ctl_io *io)
 	uint32_t targ_port = io->io_hdr.nexus.targ_port;
 
 	CTL_DEBUG_PRINT(("ctl_process_done\n"));
-
-	if ((io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC) == 0)
-		fe_done = softc->ctl_ports[targ_port]->fe_done;
-	else
-		fe_done = NULL;
+	fe_done = softc->ctl_ports[targ_port]->fe_done;
 
 #ifdef CTL_TIME_IO
 	if ((time_uptime - io->io_hdr.start_time) > ctl_time_io_secs) {
@@ -12711,11 +13103,8 @@ ctl_process_done(union ctl_io *io)
 	case CTL_IO_TASK:
 		if (ctl_debug & CTL_DEBUG_INFO)
 			ctl_io_error_print(io, NULL);
-		if (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)
-			ctl_free_io(io);
-		else
-			fe_done(io);
-		return (CTL_RETVAL_COMPLETE);
+		fe_done(io);
+		return;
 	default:
 		panic("ctl_process_done: invalid io type %d\n",
 		      io->io_hdr.io_type);
@@ -12735,7 +13124,7 @@ ctl_process_done(union ctl_io *io)
 	 * Check to see if we have any errors to inject here.  We only
 	 * inject errors for commands that don't already have errors set.
 	 */
-	if ((STAILQ_FIRST(&lun->error_list) != NULL) &&
+	if (!STAILQ_EMPTY(&lun->error_list) &&
 	    ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS) &&
 	    ((io->io_hdr.flags & CTL_FLAG_STATUS_SENT) == 0))
 		ctl_inject_error(lun, io);
@@ -12768,12 +13157,12 @@ ctl_process_done(union ctl_io *io)
 #ifdef CTL_TIME_IO
 		bintime_add(&lun->stats.ports[targ_port].dma_time[type],
 		   &io->io_hdr.dma_bt);
-		lun->stats.ports[targ_port].num_dmas[type] +=
-		    io->io_hdr.num_dmas;
-		getbintime(&cur_bt);
+		getbinuptime(&cur_bt);
 		bintime_sub(&cur_bt, &io->io_hdr.start_bt);
 		bintime_add(&lun->stats.ports[targ_port].time[type], &cur_bt);
 #endif
+		lun->stats.ports[targ_port].num_dmas[type] +=
+		    io->io_hdr.num_dmas;
 	}
 
 	/*
@@ -12836,39 +13225,8 @@ bailout:
 		    sizeof(msg.scsi) - sizeof(msg.scsi.sense_data),
 		    M_WAITOK);
 	}
-	if ((softc->ha_mode == CTL_HA_MODE_XFER)
-	 && (io->io_hdr.flags & CTL_FLAG_FROM_OTHER_SC)) {
-		memset(&msg, 0, sizeof(msg));
-		msg.hdr.msg_type = CTL_MSG_FINISH_IO;
-		msg.hdr.original_sc = io->io_hdr.original_sc;
-		msg.hdr.nexus = io->io_hdr.nexus;
-		msg.hdr.status = io->io_hdr.status;
-		msg.scsi.scsi_status = io->scsiio.scsi_status;
-		msg.scsi.tag_num = io->scsiio.tag_num;
-		msg.scsi.tag_type = io->scsiio.tag_type;
-		msg.scsi.sense_len = io->scsiio.sense_len;
-		msg.scsi.sense_residual = io->scsiio.sense_residual;
-		msg.scsi.residual = io->scsiio.residual;
-		memcpy(&msg.scsi.sense_data, &io->scsiio.sense_data,
-		       io->scsiio.sense_len);
-		/*
-		 * We copy this whether or not this is an I/O-related
-		 * command.  Otherwise, we'd have to go and check to see
-		 * whether it's a read/write command, and it really isn't
-		 * worth it.
-		 */
-		memcpy(&msg.scsi.lbalen,
-		       &io->io_hdr.ctl_private[CTL_PRIV_LBA_LEN].bytes,
-		       sizeof(msg.scsi.lbalen));
 
-		ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg,
-		    sizeof(msg.scsi) - sizeof(msg.scsi.sense_data) +
-		    msg.scsi.sense_len, M_WAITOK);
-		ctl_free_io(io);
-	} else 
-		fe_done(io);
-
-	return (CTL_RETVAL_COMPLETE);
+	fe_done(io);
 }
 
 #ifdef CTL_WITH_CA
@@ -12948,7 +13306,7 @@ ctl_queue(union ctl_io *io)
 
 #ifdef CTL_TIME_IO
 	io->io_hdr.start_time = time_uptime;
-	getbintime(&io->io_hdr.start_bt);
+	getbinuptime(&io->io_hdr.start_bt);
 #endif /* CTL_TIME_IO */
 
 	/* Map FE-specific LUN ID into global one. */
@@ -12983,6 +13341,21 @@ ctl_done_timer_wakeup(void *arg)
 #endif /* CTL_IO_DELAY */
 
 void
+ctl_serseq_done(union ctl_io *io)
+{
+	struct ctl_lun *lun;
+
+	lun = (struct ctl_lun *)io->io_hdr.ctl_private[CTL_PRIV_LUN].ptr;
+	if (lun->be_lun == NULL ||
+	    lun->be_lun->serseq == CTL_LUN_SERSEQ_OFF)
+		return;
+	mtx_lock(&lun->lun_lock);
+	io->io_hdr.flags |= CTL_FLAG_SERSEQ_DONE;
+	ctl_check_blocked(lun);
+	mtx_unlock(&lun->lun_lock);
+}
+
+void
 ctl_done(union ctl_io *io)
 {
 
@@ -13059,8 +13432,6 @@ ctl_work_thread(void *arg)
 	CTL_DEBUG_PRINT(("ctl_work_thread starting\n"));
 
 	for (;;) {
-		retval = 0;
-
 		/*
 		 * We handle the queues in this order:
 		 * - ISC
@@ -13084,7 +13455,7 @@ ctl_work_thread(void *arg)
 			STAILQ_REMOVE_HEAD(&thr->done_queue, links);
 			/* clear any blocked commands, call fe_done */
 			mtx_unlock(&thr->queue_lock);
-			retval = ctl_process_done(io);
+			ctl_process_done(io);
 			continue;
 		}
 		io = (union ctl_io *)STAILQ_FIRST(&thr->incoming_queue);
@@ -13117,12 +13488,10 @@ ctl_lun_thread(void *arg)
 {
 	struct ctl_softc *softc = (struct ctl_softc *)arg;
 	struct ctl_be_lun *be_lun;
-	int retval;
 
 	CTL_DEBUG_PRINT(("ctl_lun_thread starting\n"));
 
 	for (;;) {
-		retval = 0;
 		mtx_lock(&softc->ctl_lock);
 		be_lun = STAILQ_FIRST(&softc->pending_lun_queue);
 		if (be_lun != NULL) {
@@ -13143,7 +13512,6 @@ ctl_thresh_thread(void *arg)
 {
 	struct ctl_softc *softc = (struct ctl_softc *)arg;
 	struct ctl_lun *lun;
-	struct ctl_be_lun *be_lun;
 	struct scsi_da_rw_recovery_page *rwpage;
 	struct ctl_logical_block_provisioning_page *page;
 	const char *attr;
@@ -13156,9 +13524,8 @@ ctl_thresh_thread(void *arg)
 	for (;;) {
 		mtx_lock(&softc->ctl_lock);
 		STAILQ_FOREACH(lun, &softc->lun_list, links) {
-			be_lun = lun->be_lun;
 			if ((lun->flags & CTL_LUN_DISABLED) ||
-			    (lun->flags & CTL_LUN_OFFLINE) ||
+			    (lun->flags & CTL_LUN_NO_MEDIA) ||
 			    lun->backend->lun_attr == NULL)
 				continue;
 			if ((lun->flags & CTL_LUN_PRIMARY_SC) == 0 &&
@@ -13198,12 +13565,16 @@ ctl_thresh_thread(void *arg)
 					continue;
 				if ((page->descr[i].flags & SLBPPD_ARMING_MASK)
 				    == SLBPPD_ARMING_INC)
-					e |= (val >= thres);
+					e = (val >= thres);
 				else
-					e |= (val <= thres);
+					e = (val <= thres);
+				if (e)
+					break;
 			}
 			mtx_lock(&lun->lun_lock);
 			if (e) {
+				scsi_u64to8b((uint8_t *)&page->descr[i] -
+				    (uint8_t *)page, lun->ua_tpt_info);
 				if (lun->lasttpt == 0 ||
 				    time_uptime - lun->lasttpt >= CTL_LBP_UA_PERIOD) {
 					lun->lasttpt = time_uptime;
@@ -13229,6 +13600,7 @@ ctl_thresh_thread(void *arg)
 				msg.ua.ua_all = 1;
 				msg.ua.ua_set = (set > 0);
 				msg.ua.ua_type = CTL_UA_THIN_PROV_THRES;
+				memcpy(msg.ua.ua_info, lun->ua_tpt_info, 8);
 				mtx_unlock(&softc->ctl_lock); // XXX
 				ctl_ha_msg_send(CTL_HA_CHAN_CTL, &msg,
 				    sizeof(msg.ua), M_WAITOK);
diff --git a/sys/cam/ctl/ctl.h b/sys/cam/ctl/ctl.h
index 630e3bbb49838..35fc4c39e7f0b 100644
--- a/sys/cam/ctl/ctl.h
+++ b/sys/cam/ctl/ctl.h
@@ -120,12 +120,14 @@ typedef enum {
 	CTL_UA_LUN_CHANGE	= 0x0020,
 	CTL_UA_MODE_CHANGE	= 0x0040,
 	CTL_UA_LOG_CHANGE	= 0x0080,
+	CTL_UA_INQ_CHANGE	= 0x0100,
 	CTL_UA_RES_PREEMPT	= 0x0400,
 	CTL_UA_RES_RELEASE	= 0x0800,
-	CTL_UA_REG_PREEMPT  	= 0x1000,
-	CTL_UA_ASYM_ACC_CHANGE  = 0x2000,
-	CTL_UA_CAPACITY_CHANGED = 0x4000,
-	CTL_UA_THIN_PROV_THRES	= 0x8000
+	CTL_UA_REG_PREEMPT	= 0x1000,
+	CTL_UA_ASYM_ACC_CHANGE	= 0x2000,
+	CTL_UA_CAPACITY_CHANGE	= 0x4000,
+	CTL_UA_THIN_PROV_THRES	= 0x8000,
+	CTL_UA_MEDIUM_CHANGE	= 0x10000
 } ctl_ua_type;
 
 #ifdef	_KERNEL
@@ -138,6 +140,10 @@ struct ctl_page_index;
 SYSCTL_DECL(_kern_cam_ctl);
 #endif
 
+struct ctl_lun;
+struct ctl_port;
+struct ctl_softc;
+
 /*
  * Put a string into an sbuf, escaping characters that are illegal or not
  * recommended in XML.  Note this doesn't escape everything, just > < and &.
@@ -167,6 +173,7 @@ int ctl_sap_log_sense_handler(struct ctl_scsiio *ctsio,
 				   int pc);
 int ctl_config_move_done(union ctl_io *io);
 void ctl_datamove(union ctl_io *io);
+void ctl_serseq_done(union ctl_io *io);
 void ctl_done(union ctl_io *io);
 void ctl_data_submit_done(union ctl_io *io);
 void ctl_config_read_done(union ctl_io *io);
@@ -174,10 +181,21 @@ void ctl_config_write_done(union ctl_io *io);
 void ctl_portDB_changed(int portnum);
 int ctl_ioctl_io(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
 		 struct thread *td);
-struct ctl_lun;
+
+void ctl_est_ua(struct ctl_lun *lun, uint32_t initidx, ctl_ua_type ua);
+void ctl_est_ua_port(struct ctl_lun *lun, int port, uint32_t except,
+    ctl_ua_type ua);
+void ctl_est_ua_all(struct ctl_lun *lun, uint32_t except, ctl_ua_type ua);
+void ctl_clr_ua(struct ctl_lun *lun, uint32_t initidx, ctl_ua_type ua);
+void ctl_clr_ua_all(struct ctl_lun *lun, uint32_t except, ctl_ua_type ua);
+void ctl_clr_ua_allluns(struct ctl_softc *ctl_softc, uint32_t initidx,
+    ctl_ua_type ua_type);
+
 void ctl_isc_announce_lun(struct ctl_lun *lun);
-struct ctl_port;
 void ctl_isc_announce_port(struct ctl_port *port);
+void ctl_isc_announce_iid(struct ctl_port *port, int iid);
+void ctl_isc_announce_mode(struct ctl_lun *lun, uint32_t initidx,
+    uint8_t page, uint8_t subpage);
 
 /*
  * KPI to manipulate LUN/port options
diff --git a/sys/cam/ctl/ctl_backend.c b/sys/cam/ctl/ctl_backend.c
index f243d0bfa31c0..91576c485cb47 100644
--- a/sys/cam/ctl/ctl_backend.c
+++ b/sys/cam/ctl/ctl_backend.c
@@ -65,11 +65,9 @@ extern struct ctl_softc *control_softc;
 int
 ctl_backend_register(struct ctl_backend_driver *be)
 {
-	struct ctl_softc *softc;
+	struct ctl_softc *softc = control_softc;
 	struct ctl_backend_driver *be_tmp;
 
-	softc = control_softc;
-
 	mtx_lock(&softc->ctl_lock);
 	/*
 	 * Sanity check, make sure this isn't a duplicate registration.
@@ -120,9 +118,7 @@ ctl_backend_register(struct ctl_backend_driver *be)
 int
 ctl_backend_deregister(struct ctl_backend_driver *be)
 {
-	struct ctl_softc *softc;
-
-	softc = control_softc;
+	struct ctl_softc *softc = control_softc;
 
 	mtx_lock(&softc->ctl_lock);
 
@@ -153,20 +149,16 @@ ctl_backend_deregister(struct ctl_backend_driver *be)
 struct ctl_backend_driver *
 ctl_backend_find(char *backend_name)
 {
-	struct ctl_softc *softc;
+	struct ctl_softc *softc = control_softc;
 	struct ctl_backend_driver *be_tmp;
 
-	softc = control_softc;
-
 	mtx_lock(&softc->ctl_lock);
-
 	STAILQ_FOREACH(be_tmp, &softc->be_list, links) {
 		if (strcmp(be_tmp->name, backend_name) == 0) {
 			mtx_unlock(&softc->ctl_lock);
 			return (be_tmp);
 		}
 	}
-
 	mtx_unlock(&softc->ctl_lock);
 
 	return (NULL);
diff --git a/sys/cam/ctl/ctl_backend.h b/sys/cam/ctl/ctl_backend.h
index f5bfd83666c28..4177e2d0eba0a 100644
--- a/sys/cam/ctl/ctl_backend.h
+++ b/sys/cam/ctl/ctl_backend.h
@@ -1,5 +1,6 @@
 /*-
  * Copyright (c) 2003 Silicon Graphics International Corp.
+ * Copyright (c) 2014-2015 Alexander Motin <mav@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -49,14 +50,11 @@
  * particular LUN ID in the req_lun_id field.  If we cannot allocate that
  * LUN ID, the ctl_add_lun() call will fail.
  *
- * The POWERED_OFF flag tells us that the LUN should default to the powered
+ * The STOPPED flag tells us that the LUN should default to the powered
  * off state.  It will return 0x04,0x02 until it is powered up.  ("Logical
  * unit not ready, initializing command required.")
  *
- * The INOPERABLE flag tells us that this LUN is not operable for whatever
- * reason.  This means that user data may have been (or has been?) lost.
- * We will return 0x31,0x00 ("Medium format corrupted") until the host
- * issues a FORMAT UNIT command to clear the error.
+ * The NO_MEDIA flag tells us that the LUN has no media inserted.
  *
  * The PRIMARY flag tells us that this LUN is registered as a Primary LUN
  * which is accessible via the Master shelf controller in an HA. This flag
@@ -72,20 +70,22 @@
  *
  * The DEV_TYPE flag tells us that the device_type field is filled in.
  *
+ * The EJECTED flag tells us that the removable LUN has tray open.
+ *
  * The UNMAP flag tells us that this LUN supports UNMAP.
  *
  * The OFFLINE flag tells us that this LUN can not access backing store.
  */
 typedef enum {
 	CTL_LUN_FLAG_ID_REQ		= 0x01,
-	CTL_LUN_FLAG_POWERED_OFF	= 0x02,
-	CTL_LUN_FLAG_INOPERABLE		= 0x04,
+	CTL_LUN_FLAG_STOPPED		= 0x02,
+	CTL_LUN_FLAG_NO_MEDIA		= 0x04,
 	CTL_LUN_FLAG_PRIMARY		= 0x08,
 	CTL_LUN_FLAG_SERIAL_NUM		= 0x10,
 	CTL_LUN_FLAG_DEVID		= 0x20,
 	CTL_LUN_FLAG_DEV_TYPE		= 0x40,
 	CTL_LUN_FLAG_UNMAP		= 0x80,
-	CTL_LUN_FLAG_OFFLINE		= 0x100,
+	CTL_LUN_FLAG_EJECTED		= 0x100,
 	CTL_LUN_FLAG_READONLY		= 0x200
 } ctl_backend_lun_flags;
 
@@ -289,23 +289,11 @@ int ctl_start_lun(struct ctl_be_lun *be_lun);
 int ctl_stop_lun(struct ctl_be_lun *be_lun);
 
 /*
- * If a LUN is inoperable, call ctl_lun_inoperable().  Generally the LUN
- * will become operable once again when the user issues the SCSI FORMAT UNIT
- * command.  (CTL will automatically clear the inoperable flag.)  If we
- * need to re-enable the LUN, we can call ctl_lun_operable() to enable it
- * without a SCSI command.
- */
-int ctl_lun_inoperable(struct ctl_be_lun *be_lun);
-int ctl_lun_operable(struct ctl_be_lun *be_lun);
-
-/*
- * To take a LUN offline, call ctl_lun_offline().  Generally the LUN will
- * be online again once the user sends a SCSI START STOP UNIT command with
- * the start and on/offline bits set.  The backend can bring the LUN back
- * online via the ctl_lun_online() function, if necessary.
+ * Methods to notify about media and tray status changes.
  */
-int ctl_lun_offline(struct ctl_be_lun *be_lun);
-int ctl_lun_online(struct ctl_be_lun *be_lun);
+int ctl_lun_no_media(struct ctl_be_lun *be_lun);
+int ctl_lun_has_media(struct ctl_be_lun *be_lun);
+int ctl_lun_ejected(struct ctl_be_lun *be_lun);
 
 /*
  * Called on LUN HA role change.
@@ -314,7 +302,7 @@ int ctl_lun_primary(struct ctl_be_lun *be_lun);
 int ctl_lun_secondary(struct ctl_be_lun *be_lun);
 
 /*
- * Let the backend notify the initiator about changed capacity.
+ * Let the backend notify the initiators about changes.
  */
 void ctl_lun_capacity_changed(struct ctl_be_lun *be_lun);
 
diff --git a/sys/cam/ctl/ctl_backend_block.c b/sys/cam/ctl/ctl_backend_block.c
index c9d2a570a2bc2..40a0c796927c3 100644
--- a/sys/cam/ctl/ctl_backend_block.c
+++ b/sys/cam/ctl/ctl_backend_block.c
@@ -2,6 +2,7 @@
  * Copyright (c) 2003 Silicon Graphics International Corp.
  * Copyright (c) 2009-2011 Spectra Logic Corporation
  * Copyright (c) 2012 The FreeBSD Foundation
+ * Copyright (c) 2014-2015 Alexander Motin <mav@FreeBSD.org>
  * All rights reserved.
  *
  * Portions of this software were developed by Edward Tomasz Napierala
@@ -256,17 +257,12 @@ static int ctl_be_block_open_file(struct ctl_be_block_lun *be_lun,
 static int ctl_be_block_open_dev(struct ctl_be_block_lun *be_lun,
 				 struct ctl_lun_req *req);
 static int ctl_be_block_close(struct ctl_be_block_lun *be_lun);
-static int ctl_be_block_open(struct ctl_be_block_softc *softc,
-			     struct ctl_be_block_lun *be_lun,
+static int ctl_be_block_open(struct ctl_be_block_lun *be_lun,
 			     struct ctl_lun_req *req);
 static int ctl_be_block_create(struct ctl_be_block_softc *softc,
 			       struct ctl_lun_req *req);
 static int ctl_be_block_rm(struct ctl_be_block_softc *softc,
 			   struct ctl_lun_req *req);
-static int ctl_be_block_modify_file(struct ctl_be_block_lun *be_lun,
-				  struct ctl_lun_req *req);
-static int ctl_be_block_modify_dev(struct ctl_be_block_lun *be_lun,
-				 struct ctl_lun_req *req);
 static int ctl_be_block_modify(struct ctl_be_block_softc *softc,
 			   struct ctl_lun_req *req);
 static void ctl_be_block_lun_shutdown(void *be_lun);
@@ -351,6 +347,48 @@ ctl_complete_beio(struct ctl_be_block_io *beio)
 	}
 }
 
+static size_t
+cmp(uint8_t *a, uint8_t *b, size_t size)
+{
+	size_t i;
+
+	for (i = 0; i < size; i++) {
+		if (a[i] != b[i])
+			break;
+	}
+	return (i);
+}
+
+static void
+ctl_be_block_compare(union ctl_io *io)
+{
+	struct ctl_be_block_io *beio;
+	uint64_t off, res;
+	int i;
+	uint8_t info[8];
+
+	beio = (struct ctl_be_block_io *)PRIV(io)->ptr;
+	off = 0;
+	for (i = 0; i < beio->num_segs; i++) {
+		res = cmp(beio->sg_segs[i].addr,
+		    beio->sg_segs[i + CTLBLK_HALF_SEGS].addr,
+		    beio->sg_segs[i].len);
+		off += res;
+		if (res < beio->sg_segs[i].len)
+			break;
+	}
+	if (i < beio->num_segs) {
+		scsi_u64to8b(off, info);
+		ctl_set_sense(&io->scsiio, /*current_error*/ 1,
+		    /*sense_key*/ SSD_KEY_MISCOMPARE,
+		    /*asc*/ 0x1D, /*ascq*/ 0x00,
+		    /*type*/ SSD_ELEM_INFO,
+		    /*size*/ sizeof(info), /*data*/ &info,
+		    /*type*/ SSD_ELEM_NONE);
+	} else
+		ctl_set_success(&io->scsiio);
+}
+
 static int
 ctl_be_block_move_done(union ctl_io *io)
 {
@@ -360,7 +398,6 @@ ctl_be_block_move_done(union ctl_io *io)
 #ifdef CTL_TIME_IO
 	struct bintime cur_bt;
 #endif
-	int i;
 
 	beio = (struct ctl_be_block_io *)PRIV(io)->ptr;
 	be_lun = beio->lun;
@@ -368,11 +405,11 @@ ctl_be_block_move_done(union ctl_io *io)
 	DPRINTF("entered\n");
 
 #ifdef CTL_TIME_IO
-	getbintime(&cur_bt);
+	getbinuptime(&cur_bt);
 	bintime_sub(&cur_bt, &io->io_hdr.dma_start_bt);
 	bintime_add(&io->io_hdr.dma_bt, &cur_bt);
+#endif
 	io->io_hdr.num_dmas++;
-#endif  
 	io->scsiio.kern_rel_offset += io->scsiio.kern_data_len;
 
 	/*
@@ -388,21 +425,7 @@ ctl_be_block_move_done(union ctl_io *io)
 			ctl_set_success(&io->scsiio);
 		} else if (lbalen->flags & CTL_LLF_COMPARE) {
 			/* We have two data blocks ready for comparison. */
-			for (i = 0; i < beio->num_segs; i++) {
-				if (memcmp(beio->sg_segs[i].addr,
-				    beio->sg_segs[i + CTLBLK_HALF_SEGS].addr,
-				    beio->sg_segs[i].len) != 0)
-					break;
-			}
-			if (i < beio->num_segs)
-				ctl_set_sense(&io->scsiio,
-				    /*current_error*/ 1,
-				    /*sense_key*/ SSD_KEY_MISCOMPARE,
-				    /*asc*/ 0x1D,
-				    /*ascq*/ 0x00,
-				    SSD_ELEM_NONE);
-			else
-				ctl_set_success(&io->scsiio);
+			ctl_be_block_compare(io);
 		}
 	} else if ((io->io_hdr.port_status != 0) &&
 	    ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_STATUS_NONE ||
@@ -440,14 +463,8 @@ ctl_be_block_move_done(union ctl_io *io)
 	 * interrupt context, and therefore we cannot block.
 	 */
 	mtx_lock(&be_lun->queue_lock);
-	/*
-	 * XXX KDM make sure that links is okay to use at this point.
-	 * Otherwise, we either need to add another field to ctl_io_hdr,
-	 * or deal with resource allocation here.
-	 */
 	STAILQ_INSERT_TAIL(&be_lun->datamove_queue, &io->io_hdr, links);
 	mtx_unlock(&be_lun->queue_lock);
-
 	taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task);
 
 	return (0);
@@ -508,13 +525,17 @@ ctl_be_block_biodone(struct bio *bio)
 			ctl_set_invalid_opcode(&io->scsiio);
 		} else if (error == ENOSPC || error == EDQUOT) {
 			ctl_set_space_alloc_fail(&io->scsiio);
+		} else if (error == EROFS || error == EACCES) {
+			ctl_set_hw_write_protected(&io->scsiio);
 		} else if (beio->bio_cmd == BIO_FLUSH) {
 			/* XXX KDM is there is a better error here? */
 			ctl_set_internal_failure(&io->scsiio,
 						 /*sks_valid*/ 1,
 						 /*retry_count*/ 0xbad2);
-		} else
-			ctl_set_medium_error(&io->scsiio);
+		} else {
+			ctl_set_medium_error(&io->scsiio,
+			    beio->bio_cmd == BIO_READ);
+		}
 		ctl_complete_beio(beio);
 		return;
 	}
@@ -531,11 +552,13 @@ ctl_be_block_biodone(struct bio *bio)
 		ctl_complete_beio(beio);
 	} else {
 		if ((ARGS(io)->flags & CTL_LLF_READ) &&
-		    beio->beio_cont == NULL)
+		    beio->beio_cont == NULL) {
 			ctl_set_success(&io->scsiio);
+			ctl_serseq_done(io);
+		}
 #ifdef CTL_TIME_IO
-        	getbintime(&io->io_hdr.dma_start_bt);
-#endif  
+		getbinuptime(&io->io_hdr.dma_start_bt);
+#endif
 		ctl_datamove(io);
 	}
 }
@@ -557,15 +580,12 @@ ctl_be_block_flush_file(struct ctl_be_block_lun *be_lun,
 
 	(void) vn_start_write(be_lun->vn, &mountpoint, V_WAIT);
 
-	if (MNT_SHARED_WRITES(mountpoint)
-	 || ((mountpoint == NULL)
-	  && MNT_SHARED_WRITES(be_lun->vn->v_mount)))
+	if (MNT_SHARED_WRITES(mountpoint) ||
+	    ((mountpoint == NULL) && MNT_SHARED_WRITES(be_lun->vn->v_mount)))
 		lock_flags = LK_SHARED;
 	else
 		lock_flags = LK_EXCLUSIVE;
-
 	vn_lock(be_lun->vn, lock_flags | LK_RETRY);
-
 	error = VOP_FSYNC(be_lun->vn, beio->io_arg ? MNT_NOWAIT : MNT_WAIT,
 	    curthread);
 	VOP_UNLOCK(be_lun->vn, 0);
@@ -603,8 +623,8 @@ ctl_be_block_dispatch_file(struct ctl_be_block_lun *be_lun,
 	union ctl_io *io;
 	struct uio xuio;
 	struct iovec *xiovec;
-	int flags;
-	int error, i;
+	size_t s;
+	int error, flags, i;
 
 	DPRINTF("entered\n");
 
@@ -665,19 +685,33 @@ ctl_be_block_dispatch_file(struct ctl_be_block_lun *be_lun,
 
 		VOP_UNLOCK(be_lun->vn, 0);
 		SDT_PROBE(cbb, kernel, read, file_done, 0, 0, 0, 0, 0);
+		if (error == 0 && xuio.uio_resid > 0) {
+			/*
+			 * If we red less then requested (EOF), then
+			 * we should clean the rest of the buffer.
+			 */
+			s = beio->io_len - xuio.uio_resid;
+			for (i = 0; i < beio->num_segs; i++) {
+				if (s >= beio->sg_segs[i].len) {
+					s -= beio->sg_segs[i].len;
+					continue;
+				}
+				bzero((uint8_t *)beio->sg_segs[i].addr + s,
+				    beio->sg_segs[i].len - s);
+				s = 0;
+			}
+		}
 	} else {
 		struct mount *mountpoint;
 		int lock_flags;
 
 		(void)vn_start_write(be_lun->vn, &mountpoint, V_WAIT);
 
-		if (MNT_SHARED_WRITES(mountpoint)
-		 || ((mountpoint == NULL)
+		if (MNT_SHARED_WRITES(mountpoint) || ((mountpoint == NULL)
 		  && MNT_SHARED_WRITES(be_lun->vn->v_mount)))
 			lock_flags = LK_SHARED;
 		else
 			lock_flags = LK_EXCLUSIVE;
-
 		vn_lock(be_lun->vn, lock_flags | LK_RETRY);
 
 		/*
@@ -713,15 +747,14 @@ ctl_be_block_dispatch_file(struct ctl_be_block_lun *be_lun,
 	 * return the I/O to the user.
 	 */
 	if (error != 0) {
-		char path_str[32];
-
-		ctl_scsi_path_string(io, path_str, sizeof(path_str));
-		printf("%s%s command returned errno %d\n", path_str,
-		       (beio->bio_cmd == BIO_READ) ? "READ" : "WRITE", error);
 		if (error == ENOSPC || error == EDQUOT) {
 			ctl_set_space_alloc_fail(&io->scsiio);
-		} else
-			ctl_set_medium_error(&io->scsiio);
+		} else if (error == EROFS || error == EACCES) {
+			ctl_set_hw_write_protected(&io->scsiio);
+		} else {
+			ctl_set_medium_error(&io->scsiio,
+			    beio->bio_cmd == BIO_READ);
+		}
 		ctl_complete_beio(beio);
 		return;
 	}
@@ -736,11 +769,13 @@ ctl_be_block_dispatch_file(struct ctl_be_block_lun *be_lun,
 		ctl_complete_beio(beio);
 	} else {
 		if ((ARGS(io)->flags & CTL_LLF_READ) &&
-		    beio->beio_cont == NULL)
+		    beio->beio_cont == NULL) {
 			ctl_set_success(&io->scsiio);
+			ctl_serseq_done(io);
+		}
 #ifdef CTL_TIME_IO
-        	getbintime(&io->io_hdr.dma_start_bt);
-#endif  
+		getbinuptime(&io->io_hdr.dma_start_bt);
+#endif
 		ctl_datamove(io);
 	}
 }
@@ -885,8 +920,12 @@ ctl_be_block_dispatch_zvol(struct ctl_be_block_lun *be_lun,
 	if (error != 0) {
 		if (error == ENOSPC || error == EDQUOT) {
 			ctl_set_space_alloc_fail(&io->scsiio);
-		} else
-			ctl_set_medium_error(&io->scsiio);
+		} else if (error == EROFS || error == EACCES) {
+			ctl_set_hw_write_protected(&io->scsiio);
+		} else {
+			ctl_set_medium_error(&io->scsiio,
+			    beio->bio_cmd == BIO_READ);
+		}
 		ctl_complete_beio(beio);
 		return;
 	}
@@ -901,11 +940,13 @@ ctl_be_block_dispatch_zvol(struct ctl_be_block_lun *be_lun,
 		ctl_complete_beio(beio);
 	} else {
 		if ((ARGS(io)->flags & CTL_LLF_READ) &&
-		    beio->beio_cont == NULL)
+		    beio->beio_cont == NULL) {
 			ctl_set_success(&io->scsiio);
+			ctl_serseq_done(io);
+		}
 #ifdef CTL_TIME_IO
-        	getbintime(&io->io_hdr.dma_start_bt);
-#endif  
+		getbinuptime(&io->io_hdr.dma_start_bt);
+#endif
 		ctl_datamove(io);
 	}
 }
@@ -962,13 +1003,10 @@ ctl_be_block_flush_dev(struct ctl_be_block_lun *be_lun,
 		       struct ctl_be_block_io *beio)
 {
 	struct bio *bio;
-	union ctl_io *io;
 	struct cdevsw *csw;
 	struct cdev *dev;
 	int ref;
 
-	io = beio->io;
-
 	DPRINTF("entered\n");
 
 	/* This can't fail, it's a blocking allocation. */
@@ -1305,7 +1343,12 @@ ctl_be_block_cw_dispatch_ws(struct ctl_be_block_lun *be_lun,
 		buf = beio->sg_segs[i].addr;
 		end = buf + seglen;
 		for (; buf < end; buf += cbe_lun->blocksize) {
-			memcpy(buf, io->scsiio.kern_data_ptr, cbe_lun->blocksize);
+			if (lbalen->flags & SWS_NDOB) {
+				memset(buf, 0, cbe_lun->blocksize);
+			} else {
+				memcpy(buf, io->scsiio.kern_data_ptr,
+				    cbe_lun->blocksize);
+			}
 			if (lbalen->flags & SWS_LBDATA)
 				scsi_ulto4b(lbalen->lba + lba, buf);
 			lba++;
@@ -1483,14 +1526,8 @@ ctl_be_block_next(struct ctl_be_block_io *beio)
 	io->io_hdr.status |= CTL_STATUS_NONE;
 
 	mtx_lock(&be_lun->queue_lock);
-	/*
-	 * XXX KDM make sure that links is okay to use at this point.
-	 * Otherwise, we either need to add another field to ctl_io_hdr,
-	 * or deal with resource allocation here.
-	 */
 	STAILQ_INSERT_TAIL(&be_lun->input_queue, &io->io_hdr, links);
 	mtx_unlock(&be_lun->queue_lock);
-
 	taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task);
 }
 
@@ -1593,7 +1630,7 @@ ctl_be_block_dispatch(struct ctl_be_block_lun *be_lun,
 	io->scsiio.kern_data_len = beio->io_len;
 	io->scsiio.kern_data_resid = 0;
 	io->scsiio.kern_sg_entries = beio->num_segs;
-	io->io_hdr.flags |= CTL_FLAG_ALLOCATED | CTL_FLAG_KDPTR_SGLIST;
+	io->io_hdr.flags |= CTL_FLAG_ALLOCATED;
 
 	/*
 	 * For the read case, we need to read the data into our buffers and
@@ -1606,8 +1643,8 @@ ctl_be_block_dispatch(struct ctl_be_block_lun *be_lun,
 	} else {
 		SDT_PROBE(cbb, kernel, write, alloc_done, 0, 0, 0, 0, 0);
 #ifdef CTL_TIME_IO
-        	getbintime(&io->io_hdr.dma_start_bt);
-#endif  
+		getbinuptime(&io->io_hdr.dma_start_bt);
+#endif
 		ctl_datamove(io);
 	}
 }
@@ -1623,7 +1660,7 @@ ctl_be_block_worker(void *context, int pending)
 	DPRINTF("entered\n");
 	/*
 	 * Fetch and process I/Os from all queues.  If we detect LUN
-	 * CTL_LUN_FLAG_OFFLINE status here -- it is result of a race,
+	 * CTL_LUN_FLAG_NO_MEDIA status here -- it is result of a race,
 	 * so make response maximally opaque to not confuse initiator.
 	 */
 	for (;;) {
@@ -1635,7 +1672,7 @@ ctl_be_block_worker(void *context, int pending)
 				      ctl_io_hdr, links);
 			mtx_unlock(&be_lun->queue_lock);
 			beio = (struct ctl_be_block_io *)PRIV(io)->ptr;
-			if (cbe_lun->flags & CTL_LUN_FLAG_OFFLINE) {
+			if (cbe_lun->flags & CTL_LUN_FLAG_NO_MEDIA) {
 				ctl_set_busy(&io->scsiio);
 				ctl_complete_beio(beio);
 				return;
@@ -1649,7 +1686,7 @@ ctl_be_block_worker(void *context, int pending)
 			STAILQ_REMOVE(&be_lun->config_write_queue, &io->io_hdr,
 				      ctl_io_hdr, links);
 			mtx_unlock(&be_lun->queue_lock);
-			if (cbe_lun->flags & CTL_LUN_FLAG_OFFLINE) {
+			if (cbe_lun->flags & CTL_LUN_FLAG_NO_MEDIA) {
 				ctl_set_busy(&io->scsiio);
 				ctl_config_write_done(io);
 				return;
@@ -1663,7 +1700,7 @@ ctl_be_block_worker(void *context, int pending)
 			STAILQ_REMOVE(&be_lun->config_read_queue, &io->io_hdr,
 				      ctl_io_hdr, links);
 			mtx_unlock(&be_lun->queue_lock);
-			if (cbe_lun->flags & CTL_LUN_FLAG_OFFLINE) {
+			if (cbe_lun->flags & CTL_LUN_FLAG_NO_MEDIA) {
 				ctl_set_busy(&io->scsiio);
 				ctl_config_read_done(io);
 				return;
@@ -1677,7 +1714,7 @@ ctl_be_block_worker(void *context, int pending)
 			STAILQ_REMOVE(&be_lun->input_queue, &io->io_hdr,
 				      ctl_io_hdr, links);
 			mtx_unlock(&be_lun->queue_lock);
-			if (cbe_lun->flags & CTL_LUN_FLAG_OFFLINE) {
+			if (cbe_lun->flags & CTL_LUN_FLAG_NO_MEDIA) {
 				ctl_set_busy(&io->scsiio);
 				ctl_data_submit_done(io);
 				return;
@@ -1721,11 +1758,6 @@ ctl_be_block_submit(union ctl_io *io)
 	PRIV(io)->len = 0;
 
 	mtx_lock(&be_lun->queue_lock);
-	/*
-	 * XXX KDM make sure that links is okay to use at this point.
-	 * Otherwise, we either need to add another field to ctl_io_hdr,
-	 * or deal with resource allocation here.
-	 */
 	STAILQ_INSERT_TAIL(&be_lun->input_queue, &io->io_hdr, links);
 	mtx_unlock(&be_lun->queue_lock);
 	taskqueue_enqueue(be_lun->io_taskqueue, &be_lun->io_task);
@@ -1788,7 +1820,6 @@ ctl_be_block_open_file(struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req)
 	off_t			      ps, pss, po, pos, us, uss, uo, uos;
 	int			      error;
 
-	error = 0;
 	cbe_lun = &be_lun->cbe_lun;
 	file_data = &be_lun->backend.file;
 	params = &be_lun->params;
@@ -1838,6 +1869,8 @@ ctl_be_block_open_file(struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req)
 	 */
 	if (params->blocksize_bytes != 0)
 		cbe_lun->blocksize = params->blocksize_bytes;
+	else if (cbe_lun->lun_type == T_CDROM)
+		cbe_lun->blocksize = 2048;
 	else
 		cbe_lun->blocksize = 512;
 	be_lun->size_blocks = be_lun->size_bytes / cbe_lun->blocksize;
@@ -1966,7 +1999,9 @@ ctl_be_block_open_dev(struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req)
 			 "requested blocksize %u < backing device "
 			 "blocksize %u", params->blocksize_bytes, tmp);
 		return (EINVAL);
-	} else
+	} else if (cbe_lun->lun_type == T_CDROM)
+		cbe_lun->blocksize = MAX(tmp, 2048);
+	else
 		cbe_lun->blocksize = tmp;
 
 	error = csw->d_ioctl(dev, DIOCGMEDIASIZE, (caddr_t)&otmp, FREAD,
@@ -2099,8 +2134,7 @@ ctl_be_block_close(struct ctl_be_block_lun *be_lun)
 }
 
 static int
-ctl_be_block_open(struct ctl_be_block_softc *softc,
-		  struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req)
+ctl_be_block_open(struct ctl_be_block_lun *be_lun, struct ctl_lun_req *req)
 {
 	struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;
 	struct nameidata nd;
@@ -2126,7 +2160,10 @@ ctl_be_block_open(struct ctl_be_block_softc *softc,
 
 	flags = FREAD;
 	value = ctl_get_opt(&cbe_lun->options, "readonly");
-	if (value == NULL || strcmp(value, "on") != 0)
+	if (value != NULL) {
+		if (strcmp(value, "on") != 0)
+			flags |= FWRITE;
+	} else if (cbe_lun->lun_type == T_DIRECT)
 		flags |= FWRITE;
 
 again:
@@ -2242,10 +2279,13 @@ ctl_be_block_create(struct ctl_be_block_softc *softc, struct ctl_lun_req *req)
 	} else if (control_softc->flags & CTL_FLAG_ACTIVE_SHELF)
 		cbe_lun->flags |= CTL_LUN_FLAG_PRIMARY;
 
-	if (cbe_lun->lun_type == T_DIRECT) {
+	if (cbe_lun->lun_type == T_DIRECT ||
+	    cbe_lun->lun_type == T_CDROM) {
 		be_lun->size_bytes = params->lun_size_bytes;
 		if (params->blocksize_bytes != 0)
 			cbe_lun->blocksize = params->blocksize_bytes;
+		else if (cbe_lun->lun_type == T_CDROM)
+			cbe_lun->blocksize = 2048;
 		else
 			cbe_lun->blocksize = 512;
 		be_lun->size_blocks = be_lun->size_bytes / cbe_lun->blocksize;
@@ -2254,7 +2294,7 @@ ctl_be_block_create(struct ctl_be_block_softc *softc, struct ctl_lun_req *req)
 
 		if ((cbe_lun->flags & CTL_LUN_FLAG_PRIMARY) ||
 		    control_softc->ha_mode == CTL_HA_MODE_SER_ONLY) {
-			retval = ctl_be_block_open(softc, be_lun, req);
+			retval = ctl_be_block_open(be_lun, req);
 			if (retval != 0) {
 				retval = 0;
 				req->status = CTL_LUN_WARNING;
@@ -2265,10 +2305,6 @@ ctl_be_block_create(struct ctl_be_block_softc *softc, struct ctl_lun_req *req)
 		num_threads = 1;
 	}
 
-	/*
-	 * XXX This searching loop might be refactored to be combined with
-	 * the loop above,
-	 */
 	value = ctl_get_opt(&cbe_lun->options, "num_threads");
 	if (value != NULL) {
 		tmp_num_threads = strtol(value, NULL, 0);
@@ -2288,7 +2324,7 @@ ctl_be_block_create(struct ctl_be_block_softc *softc, struct ctl_lun_req *req)
 	}
 
 	if (be_lun->vn == NULL)
-		cbe_lun->flags |= CTL_LUN_FLAG_OFFLINE;
+		cbe_lun->flags |= CTL_LUN_FLAG_NO_MEDIA;
 	/* Tell the user the blocksize we ended up using */
 	params->lun_size_bytes = be_lun->size_bytes;
 	params->blocksize_bytes = cbe_lun->blocksize;
@@ -2458,7 +2494,6 @@ ctl_be_block_rm(struct ctl_be_block_softc *softc, struct ctl_lun_req *req)
 			break;
 	}
 	mtx_unlock(&softc->lock);
-
 	if (be_lun == NULL) {
 		snprintf(req->error_str, sizeof(req->error_str),
 			 "LUN %u is not managed by the block backend",
@@ -2476,8 +2511,8 @@ ctl_be_block_rm(struct ctl_be_block_softc *softc, struct ctl_lun_req *req)
 	}
 
 	if (be_lun->vn != NULL) {
-		cbe_lun->flags |= CTL_LUN_FLAG_OFFLINE;
-		ctl_lun_offline(cbe_lun);
+		cbe_lun->flags |= CTL_LUN_FLAG_NO_MEDIA;
+		ctl_lun_no_media(cbe_lun);
 		taskqueue_drain_all(be_lun->io_taskqueue);
 		ctl_be_block_close(be_lun);
 	}
@@ -2526,92 +2561,10 @@ ctl_be_block_rm(struct ctl_be_block_softc *softc, struct ctl_lun_req *req)
 	free(be_lun, M_CTLBLK);
 
 	req->status = CTL_LUN_OK;
-
 	return (0);
 
 bailout_error:
-
 	req->status = CTL_LUN_ERROR;
-
-	return (0);
-}
-
-static int
-ctl_be_block_modify_file(struct ctl_be_block_lun *be_lun,
-			 struct ctl_lun_req *req)
-{
-	struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;
-	struct vattr vattr;
-	int error;
-	struct ctl_lun_create_params *params = &be_lun->params;
-
-	if (params->lun_size_bytes != 0) {
-		be_lun->size_bytes = params->lun_size_bytes;
-	} else  {
-		vn_lock(be_lun->vn, LK_SHARED | LK_RETRY);
-		error = VOP_GETATTR(be_lun->vn, &vattr, curthread->td_ucred);
-		VOP_UNLOCK(be_lun->vn, 0);
-		if (error != 0) {
-			snprintf(req->error_str, sizeof(req->error_str),
-				 "error calling VOP_GETATTR() for file %s",
-				 be_lun->dev_path);
-			return (error);
-		}
-		be_lun->size_bytes = vattr.va_size;
-	}
-	be_lun->size_blocks = be_lun->size_bytes / cbe_lun->blocksize;
-	cbe_lun->maxlba = (be_lun->size_blocks == 0) ?
-	    0 : (be_lun->size_blocks - 1);
-	return (0);
-}
-
-static int
-ctl_be_block_modify_dev(struct ctl_be_block_lun *be_lun,
-			struct ctl_lun_req *req)
-{
-	struct ctl_be_lun *cbe_lun = &be_lun->cbe_lun;
-	struct ctl_lun_create_params *params = &be_lun->params;
-	struct cdevsw *csw;
-	struct cdev *dev;
-	uint64_t size_bytes;
-	int error, ref;
-
-	csw = devvn_refthread(be_lun->vn, &dev, &ref);
-	if (csw == NULL)
-		return (ENXIO);
-	if (csw->d_ioctl == NULL) {
-		dev_relthread(dev, ref);
-		snprintf(req->error_str, sizeof(req->error_str),
-			 "no d_ioctl for device %s!", be_lun->dev_path);
-		return (ENODEV);
-	}
-
-	error = csw->d_ioctl(dev, DIOCGMEDIASIZE, (caddr_t)&size_bytes, FREAD,
-	    curthread);
-	dev_relthread(dev, ref);
-	if (error) {
-		snprintf(req->error_str, sizeof(req->error_str),
-			 "error %d returned for DIOCGMEDIASIZE ioctl "
-			 "on %s!", error, be_lun->dev_path);
-		return (error);
-	}
-
-	if (params->lun_size_bytes != 0) {
-		if (params->lun_size_bytes > size_bytes) {
-			snprintf(req->error_str, sizeof(req->error_str),
-				 "requested LUN size %ju > backing device "
-				 "size %ju",
-				 (uintmax_t)params->lun_size_bytes,
-				 (uintmax_t)size_bytes);
-			return (EINVAL);
-		}
-		be_lun->size_bytes = params->lun_size_bytes;
-	} else {
-		be_lun->size_bytes = size_bytes;
-	}
-	be_lun->size_blocks = be_lun->size_bytes / cbe_lun->blocksize;
-	cbe_lun->maxlba = (be_lun->size_blocks == 0) ?
-	    0 : (be_lun->size_blocks - 1);
 	return (0);
 }
 
@@ -2633,7 +2586,6 @@ ctl_be_block_modify(struct ctl_be_block_softc *softc, struct ctl_lun_req *req)
 			break;
 	}
 	mtx_unlock(&softc->lock);
-
 	if (be_lun == NULL) {
 		snprintf(req->error_str, sizeof(req->error_str),
 			 "LUN %u is not managed by the block backend",
@@ -2668,22 +2620,27 @@ ctl_be_block_modify(struct ctl_be_block_softc *softc, struct ctl_lun_req *req)
 	if ((cbe_lun->flags & CTL_LUN_FLAG_PRIMARY) ||
 	    control_softc->ha_mode == CTL_HA_MODE_SER_ONLY) {
 		if (be_lun->vn == NULL)
-			error = ctl_be_block_open(softc, be_lun, req);
+			error = ctl_be_block_open(be_lun, req);
 		else if (vn_isdisk(be_lun->vn, &error))
-			error = ctl_be_block_modify_dev(be_lun, req);
+			error = ctl_be_block_open_dev(be_lun, req);
 		else if (be_lun->vn->v_type == VREG)
-			error = ctl_be_block_modify_file(be_lun, req);
+			error = ctl_be_block_open_file(be_lun, req);
 		else
 			error = EINVAL;
-		if ((cbe_lun->flags & CTL_LUN_FLAG_OFFLINE) &&
+		if ((cbe_lun->flags & CTL_LUN_FLAG_NO_MEDIA) &&
 		    be_lun->vn != NULL) {
-			cbe_lun->flags &= ~CTL_LUN_FLAG_OFFLINE;
-			ctl_lun_online(cbe_lun);
+			cbe_lun->flags &= ~CTL_LUN_FLAG_NO_MEDIA;
+			ctl_lun_has_media(cbe_lun);
+		} else if ((cbe_lun->flags & CTL_LUN_FLAG_NO_MEDIA) == 0 &&
+		    be_lun->vn == NULL) {
+			cbe_lun->flags |= CTL_LUN_FLAG_NO_MEDIA;
+			ctl_lun_no_media(cbe_lun);
 		}
+		cbe_lun->flags &= ~CTL_LUN_FLAG_EJECTED;
 	} else {
 		if (be_lun->vn != NULL) {
-			cbe_lun->flags |= CTL_LUN_FLAG_OFFLINE;
-			ctl_lun_offline(cbe_lun);
+			cbe_lun->flags |= CTL_LUN_FLAG_NO_MEDIA;
+			ctl_lun_no_media(cbe_lun);
 			taskqueue_drain_all(be_lun->io_taskqueue);
 			error = ctl_be_block_close(be_lun);
 		} else
@@ -2710,7 +2667,6 @@ ctl_be_block_lun_shutdown(void *be_lun)
 	struct ctl_be_block_softc *softc;
 
 	lun = (struct ctl_be_block_lun *)be_lun;
-
 	softc = lun->softc;
 
 	mtx_lock(&softc->lock);
@@ -2718,7 +2674,6 @@ ctl_be_block_lun_shutdown(void *be_lun)
 	if (lun->flags & CTL_BE_BLOCK_LUN_WAITING)
 		wakeup(lun);
 	mtx_unlock(&softc->lock);
-
 }
 
 static void
@@ -2767,14 +2722,13 @@ ctl_be_block_config_write(union ctl_io *io)
 	struct ctl_be_lun *cbe_lun;
 	int retval;
 
-	retval = 0;
-
 	DPRINTF("entered\n");
 
 	cbe_lun = (struct ctl_be_lun *)io->io_hdr.ctl_private[
 		CTL_PRIV_BACKEND_LUN].ptr;
 	be_lun = (struct ctl_be_block_lun *)cbe_lun->be_lun;
 
+	retval = 0;
 	switch (io->scsiio.cdb[0]) {
 	case SYNCHRONIZE_CACHE:
 	case SYNCHRONIZE_CACHE_16:
@@ -2797,40 +2751,46 @@ ctl_be_block_config_write(union ctl_io *io)
 		break;
 	case START_STOP_UNIT: {
 		struct scsi_start_stop_unit *cdb;
+		struct ctl_lun_req req;
 
 		cdb = (struct scsi_start_stop_unit *)io->scsiio.cdb;
-
-		if (cdb->how & SSS_START)
-			retval = ctl_start_lun(cbe_lun);
-		else {
-			retval = ctl_stop_lun(cbe_lun);
-			/*
-			 * XXX KDM Copan-specific offline behavior.
-			 * Figure out a reasonable way to port this?
-			 */
-#ifdef NEEDTOPORT
-			if ((retval == 0)
-			 && (cdb->byte2 & SSS_ONOFFLINE))
-				retval = ctl_lun_offline(cbe_lun);
-#endif
+		if ((cdb->how & SSS_PC_MASK) != 0) {
+			ctl_set_success(&io->scsiio);
+			ctl_config_write_done(io);
+			break;
 		}
-
-		/*
-		 * In general, the above routines should not fail.  They
-		 * just set state for the LUN.  So we've got something
-		 * pretty wrong here if we can't start or stop the LUN.
-		 */
-		if (retval != 0) {
-			ctl_set_internal_failure(&io->scsiio,
-						 /*sks_valid*/ 1,
-						 /*retry_count*/ 0xf051);
-			retval = CTL_RETVAL_COMPLETE;
+		if (cdb->how & SSS_START) {
+			if ((cdb->how & SSS_LOEJ) && be_lun->vn == NULL) {
+				retval = ctl_be_block_open(be_lun, &req);
+				cbe_lun->flags &= ~CTL_LUN_FLAG_EJECTED;
+				if (retval == 0) {
+					cbe_lun->flags &= ~CTL_LUN_FLAG_NO_MEDIA;
+					ctl_lun_has_media(cbe_lun);
+				} else {
+					cbe_lun->flags |= CTL_LUN_FLAG_NO_MEDIA;
+					ctl_lun_no_media(cbe_lun);
+				}
+			}
+			ctl_start_lun(cbe_lun);
 		} else {
-			ctl_set_success(&io->scsiio);
+			ctl_stop_lun(cbe_lun);
+			if (cdb->how & SSS_LOEJ) {
+				cbe_lun->flags |= CTL_LUN_FLAG_NO_MEDIA;
+				cbe_lun->flags |= CTL_LUN_FLAG_EJECTED;
+				ctl_lun_ejected(cbe_lun);
+				if (be_lun->vn != NULL)
+					ctl_be_block_close(be_lun);
+			}
 		}
+
+		ctl_set_success(&io->scsiio);
 		ctl_config_write_done(io);
 		break;
 	}
+	case PREVENT_ALLOW:
+		ctl_set_success(&io->scsiio);
+		ctl_config_write_done(io);
+		break;
 	default:
 		ctl_set_invalid_opcode(&io->scsiio);
 		ctl_config_write_done(io);
@@ -2892,22 +2852,16 @@ ctl_be_block_lun_info(void *be_lun, struct sbuf *sb)
 	int retval;
 
 	lun = (struct ctl_be_block_lun *)be_lun;
-	retval = 0;
 
 	retval = sbuf_printf(sb, "\t<num_threads>");
-
 	if (retval != 0)
 		goto bailout;
-
 	retval = sbuf_printf(sb, "%d", lun->num_threads);
-
 	if (retval != 0)
 		goto bailout;
-
 	retval = sbuf_printf(sb, "</num_threads>\n");
 
 bailout:
-
 	return (retval);
 }
 
diff --git a/sys/cam/ctl/ctl_backend_ramdisk.c b/sys/cam/ctl/ctl_backend_ramdisk.c
index 073b167650bb2..d1704462d47ee 100644
--- a/sys/cam/ctl/ctl_backend_ramdisk.c
+++ b/sys/cam/ctl/ctl_backend_ramdisk.c
@@ -1,6 +1,7 @@
 /*-
  * Copyright (c) 2003, 2008 Silicon Graphics International Corp.
  * Copyright (c) 2012 The FreeBSD Foundation
+ * Copyright (c) 2014-2015 Alexander Motin <mav@FreeBSD.org>
  * All rights reserved.
  *
  * Portions of this software were developed by Edward Tomasz Napierala
@@ -145,18 +146,13 @@ CTL_BACKEND_DECLARE(cbr, ctl_be_ramdisk_driver);
 int
 ctl_backend_ramdisk_init(void)
 {
-	struct ctl_be_ramdisk_softc *softc;
+	struct ctl_be_ramdisk_softc *softc = &rd_softc;
 #ifdef CTL_RAMDISK_PAGES
 	int i;
 #endif
 
-
-	softc = &rd_softc;
-
 	memset(softc, 0, sizeof(*softc));
-
 	mtx_init(&softc->lock, "ctlramdisk", NULL, MTX_DEF);
-
 	STAILQ_INIT(&softc->lun_list);
 	softc->rd_size = 1024 * 1024;
 #ifdef CTL_RAMDISK_PAGES
@@ -177,23 +173,14 @@ ctl_backend_ramdisk_init(void)
 void
 ctl_backend_ramdisk_shutdown(void)
 {
-	struct ctl_be_ramdisk_softc *softc;
+	struct ctl_be_ramdisk_softc *softc = &rd_softc;
 	struct ctl_be_ramdisk_lun *lun, *next_lun;
 #ifdef CTL_RAMDISK_PAGES
 	int i;
 #endif
 
-	softc = &rd_softc;
-
 	mtx_lock(&softc->lock);
-	for (lun = STAILQ_FIRST(&softc->lun_list); lun != NULL; lun = next_lun){
-		/*
-		 * Grab the next LUN.  The current LUN may get removed by
-		 * ctl_invalidate_lun(), which will call our LUN shutdown
-		 * routine, if there is no outstanding I/O for this LUN.
-		 */
-		next_lun = STAILQ_NEXT(lun, links);
-
+	STAILQ_FOREACH_SAFE(lun, &softc->lun_list, links, next_lun) {
 		/*
 		 * Drop our lock here.  Since ctl_invalidate_lun() can call
 		 * back into us, this could potentially lead to a recursive
@@ -235,11 +222,11 @@ ctl_backend_ramdisk_move_done(union ctl_io *io)
 		CTL_PRIV_BACKEND_LUN].ptr;
 	be_lun = (struct ctl_be_ramdisk_lun *)cbe_lun->be_lun;
 #ifdef CTL_TIME_IO
-	getbintime(&cur_bt);
+	getbinuptime(&cur_bt);
 	bintime_sub(&cur_bt, &io->io_hdr.dma_start_bt);
 	bintime_add(&io->io_hdr.dma_bt, &cur_bt);
-	io->io_hdr.num_dmas++;
 #endif
+	io->io_hdr.num_dmas++;
 	if (io->scsiio.kern_sg_entries > 0)
 		free(io->scsiio.kern_data_ptr, M_RAMDISK);
 	io->scsiio.kern_rel_offset += io->scsiio.kern_data_len;
@@ -321,7 +308,6 @@ ctl_backend_ramdisk_continue(union ctl_io *io)
 			sg_entries[i].len = MIN(PAGE_SIZE, len - len_filled);
 			len_filled += sg_entries[i].len;
 		}
-		io->io_hdr.flags |= CTL_FLAG_KDPTR_SGLIST;
 	} else {
 		sg_filled = 0;
 		len_filled = len;
@@ -340,7 +326,7 @@ ctl_backend_ramdisk_continue(union ctl_io *io)
 	io->io_hdr.flags |= CTL_FLAG_ALLOCATED;
 	io->io_hdr.ctl_private[CTL_PRIV_BACKEND].integer -= len_filled;
 #ifdef CTL_TIME_IO
-	getbintime(&io->io_hdr.dma_start_bt);
+	getbinuptime(&io->io_hdr.dma_start_bt);
 #endif
 	ctl_datamove(io);
 }
@@ -348,12 +334,10 @@ ctl_backend_ramdisk_continue(union ctl_io *io)
 static void
 ctl_backend_ramdisk_worker(void *context, int pending)
 {
-	struct ctl_be_ramdisk_softc *softc;
 	struct ctl_be_ramdisk_lun *be_lun;
 	union ctl_io *io;
 
 	be_lun = (struct ctl_be_ramdisk_lun *)context;
-	softc = be_lun->softc;
 
 	mtx_lock(&be_lun->queue_lock);
 	for (;;) {
@@ -361,11 +345,8 @@ ctl_backend_ramdisk_worker(void *context, int pending)
 		if (io != NULL) {
 			STAILQ_REMOVE(&be_lun->cont_queue, &io->io_hdr,
 				      ctl_io_hdr, links);
-
 			mtx_unlock(&be_lun->queue_lock);
-
 			ctl_backend_ramdisk_continue(io);
-
 			mtx_lock(&be_lun->queue_lock);
 			continue;
 		}
@@ -383,18 +364,14 @@ static int
 ctl_backend_ramdisk_ioctl(struct cdev *dev, u_long cmd, caddr_t addr,
 			  int flag, struct thread *td)
 {
-	struct ctl_be_ramdisk_softc *softc;
+	struct ctl_be_ramdisk_softc *softc = &rd_softc;
+	struct ctl_lun_req *lun_req;
 	int retval;
 
 	retval = 0;
-	softc = &rd_softc;
-
 	switch (cmd) {
-	case CTL_LUN_REQ: {
-		struct ctl_lun_req *lun_req;
-
+	case CTL_LUN_REQ:
 		lun_req = (struct ctl_lun_req *)addr;
-
 		switch (lun_req->reqtype) {
 		case CTL_LUNREQ_CREATE:
 			retval = ctl_backend_ramdisk_create(softc, lun_req);
@@ -413,7 +390,6 @@ ctl_backend_ramdisk_ioctl(struct cdev *dev, u_long cmd, caddr_t addr,
 			break;
 		}
 		break;
-	}
 	default:
 		retval = ENOTTY;
 		break;
@@ -430,20 +406,13 @@ ctl_backend_ramdisk_rm(struct ctl_be_ramdisk_softc *softc,
 	struct ctl_lun_rm_params *params;
 	int retval;
 
-
-	retval = 0;
 	params = &req->reqdata.rm;
-
-	be_lun = NULL;
-
 	mtx_lock(&softc->lock);
-
 	STAILQ_FOREACH(be_lun, &softc->lun_list, links) {
 		if (be_lun->cbe_lun.lun_id == params->lun_id)
 			break;
 	}
 	mtx_unlock(&softc->lock);
-
 	if (be_lun == NULL) {
 		snprintf(req->error_str, sizeof(req->error_str),
 			 "%s: LUN %u is not managed by the ramdisk backend",
@@ -452,7 +421,6 @@ ctl_backend_ramdisk_rm(struct ctl_be_ramdisk_softc *softc,
 	}
 
 	retval = ctl_disable_lun(&be_lun->cbe_lun);
-
 	if (retval != 0) {
 		snprintf(req->error_str, sizeof(req->error_str),
 			 "%s: error %d returned from ctl_disable_lun() for "
@@ -484,10 +452,9 @@ ctl_backend_ramdisk_rm(struct ctl_be_ramdisk_softc *softc,
 	}
 
 	mtx_lock(&softc->lock);
-
 	while ((be_lun->flags & CTL_BE_RAMDISK_LUN_UNCONFIGURED) == 0) {
 		retval = msleep(be_lun, &softc->lock, PCATCH, "ctlram", 0);
- 		if (retval == EINTR)   
+		if (retval == EINTR)
 			break;
 	}
 	be_lun->flags &= ~CTL_BE_RAMDISK_LUN_WAITING;
@@ -515,12 +482,10 @@ ctl_backend_ramdisk_rm(struct ctl_be_ramdisk_softc *softc,
 	}
 
 	req->status = CTL_LUN_OK;
-
 	return (retval);
 
 bailout_error:
 	req->status = CTL_LUN_ERROR;
-
 	return (0);
 }
 
@@ -559,9 +524,12 @@ ctl_backend_ramdisk_create(struct ctl_be_ramdisk_softc *softc,
 	} else if (control_softc->flags & CTL_FLAG_ACTIVE_SHELF)
 		cbe_lun->flags |= CTL_LUN_FLAG_PRIMARY;
 
-	if (cbe_lun->lun_type == T_DIRECT) {
+	if (cbe_lun->lun_type == T_DIRECT ||
+	    cbe_lun->lun_type == T_CDROM) {
 		if (params->blocksize_bytes != 0)
 			cbe_lun->blocksize = params->blocksize_bytes;
+		else if (cbe_lun->lun_type == T_CDROM)
+			cbe_lun->blocksize = 2048;
 		else
 			cbe_lun->blocksize = 512;
 		if (params->lun_size_bytes < cbe_lun->blocksize) {
@@ -585,7 +553,10 @@ ctl_backend_ramdisk_create(struct ctl_be_ramdisk_softc *softc,
 	if (value != NULL && strcmp(value, "on") == 0)
 		cbe_lun->flags |= CTL_LUN_FLAG_UNMAP;
 	value = ctl_get_opt(&cbe_lun->options, "readonly");
-	if (value != NULL && strcmp(value, "on") == 0)
+	if (value != NULL) {
+		if (strcmp(value, "on") == 0)
+			cbe_lun->flags |= CTL_LUN_FLAG_READONLY;
+	} else if (cbe_lun->lun_type != T_DIRECT)
 		cbe_lun->flags |= CTL_LUN_FLAG_READONLY;
 	cbe_lun->serseq = CTL_LUN_SERSEQ_OFF;
 	value = ctl_get_opt(&cbe_lun->options, "serseq");
@@ -657,7 +628,6 @@ ctl_backend_ramdisk_create(struct ctl_be_ramdisk_softc *softc,
 	mtx_lock(&softc->lock);
 	softc->num_luns++;
 	STAILQ_INSERT_TAIL(&softc->lun_list, be_lun, links);
-
 	mtx_unlock(&softc->lock);
 
 	retval = ctl_add_lun(&be_lun->cbe_lun);
@@ -704,7 +674,6 @@ ctl_backend_ramdisk_create(struct ctl_be_ramdisk_softc *softc,
 	mtx_unlock(&softc->lock);
 
 	req->status = CTL_LUN_OK;
-
 	return (retval);
 
 bailout_error:
@@ -717,7 +686,6 @@ bailout_error:
 		mtx_destroy(&be_lun->queue_lock);
 		free(be_lun, M_RAMDISK);
 	}
-
 	return (retval);
 }
 
@@ -740,7 +708,6 @@ ctl_backend_ramdisk_modify(struct ctl_be_ramdisk_softc *softc,
 			break;
 	}
 	mtx_unlock(&softc->lock);
-
 	if (be_lun == NULL) {
 		snprintf(req->error_str, sizeof(req->error_str),
 			 "%s: LUN %u is not managed by the ramdisk backend",
@@ -787,12 +754,10 @@ ctl_backend_ramdisk_modify(struct ctl_be_ramdisk_softc *softc,
 	params->lun_size_bytes = be_lun->size_bytes;
 
 	req->status = CTL_LUN_OK;
-
 	return (0);
 
 bailout_error:
 	req->status = CTL_LUN_ERROR;
-
 	return (0);
 }
 
@@ -808,9 +773,7 @@ ctl_backend_ramdisk_lun_shutdown(void *be_lun)
 	do_free = 0;
 
 	mtx_lock(&softc->lock);
-
 	lun->flags |= CTL_BE_RAMDISK_LUN_UNCONFIGURED;
-
 	if (lun->flags & CTL_BE_RAMDISK_LUN_WAITING) {
 		wakeup(lun);
 	} else {
@@ -819,7 +782,6 @@ ctl_backend_ramdisk_lun_shutdown(void *be_lun)
 		softc->num_luns--;
 		do_free = 1;
 	}
-
 	mtx_unlock(&softc->lock);
 
 	if (do_free != 0)
@@ -880,12 +842,12 @@ ctl_backend_ramdisk_lun_config_status(void *be_lun,
 static int
 ctl_backend_ramdisk_config_write(union ctl_io *io)
 {
-	struct ctl_be_ramdisk_softc *softc;
+	struct ctl_be_lun *cbe_lun;
 	int retval;
 
+	cbe_lun = (struct ctl_be_lun *)io->io_hdr.ctl_private[
+	    CTL_PRIV_BACKEND_LUN].ptr;
 	retval = 0;
-	softc = &rd_softc;
-
 	switch (io->scsiio.cdb[0]) {
 	case SYNCHRONIZE_CACHE:
 	case SYNCHRONIZE_CACHE_16:
@@ -909,42 +871,27 @@ ctl_backend_ramdisk_config_write(union ctl_io *io)
 		break;
 	case START_STOP_UNIT: {
 		struct scsi_start_stop_unit *cdb;
-		struct ctl_be_lun *cbe_lun;
-		struct ctl_be_ramdisk_lun *be_lun;
 
 		cdb = (struct scsi_start_stop_unit *)io->scsiio.cdb;
-
-		cbe_lun = (struct ctl_be_lun *)io->io_hdr.ctl_private[
-			CTL_PRIV_BACKEND_LUN].ptr;
-		be_lun = (struct ctl_be_ramdisk_lun *)cbe_lun->be_lun;
-
-		if (cdb->how & SSS_START)
-			retval = ctl_start_lun(cbe_lun);
-		else {
-			retval = ctl_stop_lun(cbe_lun);
-#ifdef NEEDTOPORT
-			if ((retval == 0)
-			 && (cdb->byte2 & SSS_ONOFFLINE))
-				retval = ctl_lun_offline(cbe_lun);
-#endif
+		if ((cdb->how & SSS_PC_MASK) != 0) {
+			ctl_set_success(&io->scsiio);
+			ctl_config_write_done(io);
+			break;
 		}
-
-		/*
-		 * In general, the above routines should not fail.  They
-		 * just set state for the LUN.  So we've got something
-		 * pretty wrong here if we can't start or stop the LUN.
-		 */
-		if (retval != 0) {
-			ctl_set_internal_failure(&io->scsiio,
-						 /*sks_valid*/ 1,
-						 /*retry_count*/ 0xf051);
-			retval = CTL_RETVAL_COMPLETE;
+		if (cdb->how & SSS_START) {
+			if (cdb->how & SSS_LOEJ)
+				ctl_lun_has_media(cbe_lun);
+			ctl_start_lun(cbe_lun);
 		} else {
-			ctl_set_success(&io->scsiio);
+			ctl_stop_lun(cbe_lun);
+			if (cdb->how & SSS_LOEJ)
+				ctl_lun_ejected(cbe_lun);
 		}
+		ctl_set_success(&io->scsiio);
 		ctl_config_write_done(io);
 		break;
 	}
+	case PREVENT_ALLOW:
 	case WRITE_SAME_10:
 	case WRITE_SAME_16:
 	case UNMAP:
diff --git a/sys/cam/ctl/ctl_cmd_table.c b/sys/cam/ctl/ctl_cmd_table.c
index 5b75468c1e40b..498f7c52d8446 100644
--- a/sys/cam/ctl/ctl_cmd_table.c
+++ b/sys/cam/ctl/ctl_cmd_table.c
@@ -1,5 +1,6 @@
 /*-
  * Copyright (c) 2003, 2004, 2005, 2009 Silicon Graphics International Corp.
+ * Copyright (c) 2014-2015 Alexander Motin <mav@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -67,8 +68,7 @@ const struct ctl_cmd_entry ctl_cmd_table_5e[32] =
 /* 00 READ KEYS */
 {ctl_persistent_reserve_in, CTL_SERIDX_RES, CTL_CMD_FLAG_ALLOW_ON_RESV |
 						CTL_CMD_FLAG_OK_ON_BOTH |
-						CTL_CMD_FLAG_OK_ON_STOPPED |
-						CTL_CMD_FLAG_OK_ON_INOPERABLE |
+						CTL_CMD_FLAG_OK_ON_NO_MEDIA |
 						CTL_CMD_FLAG_OK_ON_STANDBY |
 						CTL_FLAG_DATA_IN |
 						CTL_CMD_FLAG_ALLOW_ON_PR_RESV,
@@ -78,8 +78,7 @@ const struct ctl_cmd_entry ctl_cmd_table_5e[32] =
 /* 01 READ RESERVATION */
 {ctl_persistent_reserve_in, CTL_SERIDX_RES, CTL_CMD_FLAG_ALLOW_ON_RESV |
 						CTL_CMD_FLAG_OK_ON_BOTH |
-						CTL_CMD_FLAG_OK_ON_STOPPED |
-						CTL_CMD_FLAG_OK_ON_INOPERABLE |
+						CTL_CMD_FLAG_OK_ON_NO_MEDIA |
 						CTL_CMD_FLAG_OK_ON_STANDBY |
 						CTL_FLAG_DATA_IN |
 						CTL_CMD_FLAG_ALLOW_ON_PR_RESV,
@@ -89,8 +88,7 @@ const struct ctl_cmd_entry ctl_cmd_table_5e[32] =
 /* 02 REPORT CAPABILITIES */
 {ctl_persistent_reserve_in, CTL_SERIDX_INQ, CTL_CMD_FLAG_ALLOW_ON_RESV |
 					    CTL_CMD_FLAG_OK_ON_BOTH |
-					    CTL_CMD_FLAG_OK_ON_STOPPED |
-					    CTL_CMD_FLAG_OK_ON_INOPERABLE |
+					    CTL_CMD_FLAG_OK_ON_NO_MEDIA |
 					    CTL_CMD_FLAG_OK_ON_STANDBY |
 					    CTL_FLAG_DATA_IN |
 					    CTL_CMD_FLAG_ALLOW_ON_PR_RESV,
@@ -100,8 +98,7 @@ const struct ctl_cmd_entry ctl_cmd_table_5e[32] =
 /* 03 READ FULL STATUS */
 {ctl_persistent_reserve_in, CTL_SERIDX_INQ, CTL_CMD_FLAG_ALLOW_ON_RESV |
 					    CTL_CMD_FLAG_OK_ON_BOTH |
-					    CTL_CMD_FLAG_OK_ON_STOPPED |
-					    CTL_CMD_FLAG_OK_ON_INOPERABLE |
+					    CTL_CMD_FLAG_OK_ON_NO_MEDIA |
 					    CTL_CMD_FLAG_OK_ON_STANDBY |
 					    CTL_FLAG_DATA_IN |
 					    CTL_CMD_FLAG_ALLOW_ON_PR_RESV,
@@ -117,8 +114,7 @@ const struct ctl_cmd_entry ctl_cmd_table_5f[32] =
 /* 00 REGISTER */
 {ctl_persistent_reserve_out, CTL_SERIDX_RES, CTL_CMD_FLAG_ALLOW_ON_RESV |
 						CTL_CMD_FLAG_OK_ON_BOTH |
-						CTL_CMD_FLAG_OK_ON_STOPPED |
-						CTL_CMD_FLAG_OK_ON_INOPERABLE |
+						CTL_CMD_FLAG_OK_ON_NO_MEDIA |
 						CTL_CMD_FLAG_OK_ON_STANDBY |
 						CTL_FLAG_DATA_OUT |
 						CTL_CMD_FLAG_ALLOW_ON_PR_RESV,
@@ -128,8 +124,7 @@ const struct ctl_cmd_entry ctl_cmd_table_5f[32] =
 /* 01 RESERVE */
 {ctl_persistent_reserve_out, CTL_SERIDX_RES, CTL_CMD_FLAG_ALLOW_ON_RESV |
 						CTL_CMD_FLAG_OK_ON_BOTH |
-						CTL_CMD_FLAG_OK_ON_STOPPED |
-						CTL_CMD_FLAG_OK_ON_INOPERABLE |
+						CTL_CMD_FLAG_OK_ON_NO_MEDIA |
 						CTL_CMD_FLAG_OK_ON_STANDBY |
 						CTL_FLAG_DATA_OUT |
 						CTL_CMD_FLAG_ALLOW_ON_PR_RESV,
@@ -139,8 +134,7 @@ const struct ctl_cmd_entry ctl_cmd_table_5f[32] =
 /* 02 RELEASE */
 {ctl_persistent_reserve_out, CTL_SERIDX_RES, CTL_CMD_FLAG_ALLOW_ON_RESV |
 						CTL_CMD_FLAG_OK_ON_BOTH |
-						CTL_CMD_FLAG_OK_ON_STOPPED |
-						CTL_CMD_FLAG_OK_ON_INOPERABLE |
+						CTL_CMD_FLAG_OK_ON_NO_MEDIA |
 						CTL_CMD_FLAG_OK_ON_STANDBY |
 						CTL_FLAG_DATA_OUT |
 						CTL_CMD_FLAG_ALLOW_ON_PR_RESV,
@@ -150,8 +144,7 @@ const struct ctl_cmd_entry ctl_cmd_table_5f[32] =
 /* 03 CLEAR */
 {ctl_persistent_reserve_out, CTL_SERIDX_RES, CTL_CMD_FLAG_ALLOW_ON_RESV |
 						CTL_CMD_FLAG_OK_ON_BOTH |
-						CTL_CMD_FLAG_OK_ON_STOPPED |
-						CTL_CMD_FLAG_OK_ON_INOPERABLE |
+						CTL_CMD_FLAG_OK_ON_NO_MEDIA |
 						CTL_CMD_FLAG_OK_ON_STANDBY |
 						CTL_FLAG_DATA_OUT |
 						CTL_CMD_FLAG_ALLOW_ON_PR_RESV,
@@ -161,8 +154,7 @@ const struct ctl_cmd_entry ctl_cmd_table_5f[32] =
 /* 04 PREEMPT */
 {ctl_persistent_reserve_out, CTL_SERIDX_RES, CTL_CMD_FLAG_ALLOW_ON_RESV |
 						CTL_CMD_FLAG_OK_ON_BOTH |
-						CTL_CMD_FLAG_OK_ON_STOPPED |
-						CTL_CMD_FLAG_OK_ON_INOPERABLE |
+						CTL_CMD_FLAG_OK_ON_NO_MEDIA |
 						CTL_CMD_FLAG_OK_ON_STANDBY |
 						CTL_FLAG_DATA_OUT |
 						CTL_CMD_FLAG_ALLOW_ON_PR_RESV,
@@ -172,8 +164,7 @@ const struct ctl_cmd_entry ctl_cmd_table_5f[32] =
 /* 05 PREEMPT AND ABORT */
 {ctl_persistent_reserve_out, CTL_SERIDX_RES, CTL_CMD_FLAG_ALLOW_ON_RESV |
 						CTL_CMD_FLAG_OK_ON_BOTH |
-						CTL_CMD_FLAG_OK_ON_STOPPED |
-						CTL_CMD_FLAG_OK_ON_INOPERABLE |
+						CTL_CMD_FLAG_OK_ON_NO_MEDIA |
 						CTL_CMD_FLAG_OK_ON_STANDBY |
 						CTL_FLAG_DATA_OUT |
 						CTL_CMD_FLAG_ALLOW_ON_PR_RESV,
@@ -183,8 +174,7 @@ const struct ctl_cmd_entry ctl_cmd_table_5f[32] =
 /* 06 REGISTER AND IGNORE EXISTING KEY */
 {ctl_persistent_reserve_out, CTL_SERIDX_RES, CTL_CMD_FLAG_ALLOW_ON_RESV |
 						CTL_CMD_FLAG_OK_ON_BOTH |
-						CTL_CMD_FLAG_OK_ON_STOPPED |
-						CTL_CMD_FLAG_OK_ON_INOPERABLE |
+						CTL_CMD_FLAG_OK_ON_NO_MEDIA |
 						CTL_CMD_FLAG_OK_ON_STANDBY |
 						CTL_FLAG_DATA_OUT |
 						CTL_CMD_FLAG_ALLOW_ON_PR_RESV,
@@ -202,12 +192,14 @@ const struct ctl_cmd_entry ctl_cmd_table_83[32] =
 {
 /* 00 EXTENDED COPY (LID1) */
 {ctl_extended_copy_lid1, CTL_SERIDX_RD_CAP, CTL_CMD_FLAG_OK_ON_BOTH |
+					    CTL_CMD_FLAG_OK_ON_NO_MEDIA |
 					    CTL_FLAG_DATA_OUT,
  CTL_LUN_PAT_NONE,
  16, { 0x00, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff, 0xff, 0xff, 0, 0x07}},
 
 /* 01 EXTENDED COPY (LID4) */
 {ctl_extended_copy_lid4, CTL_SERIDX_RD_CAP, CTL_CMD_FLAG_OK_ON_BOTH |
+					    CTL_CMD_FLAG_OK_ON_NO_MEDIA |
 					    CTL_FLAG_DATA_OUT,
  CTL_LUN_PAT_NONE,
  16, { 0x01, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff, 0xff, 0xff, 0, 0x07}},
@@ -255,7 +247,7 @@ const struct ctl_cmd_entry ctl_cmd_table_83[32] =
 {NULL, CTL_SERIDX_INVLD, CTL_CMD_FLAG_NONE, CTL_LUN_PAT_NONE},
 
 /* 10 POPULATE TOKEN */
-{ctl_populate_token, CTL_SERIDX_RD_CAP, CTL_CMD_FLAG_OK_ON_SLUN |
+{ctl_populate_token, CTL_SERIDX_RD_CAP, CTL_CMD_FLAG_OK_ON_DIRECT |
 					CTL_FLAG_DATA_OUT |
 					CTL_CMD_FLAG_ALLOW_ON_PR_WRESV,
  CTL_LUN_PAT_NONE,
@@ -263,7 +255,7 @@ const struct ctl_cmd_entry ctl_cmd_table_83[32] =
        0xff, 0xff, 0xff, 0xff, 0, 0x07}},
 
 /* 11 WRITE USING TOKEN */
-{ctl_write_using_token, CTL_SERIDX_RD_CAP, CTL_CMD_FLAG_OK_ON_SLUN |
+{ctl_write_using_token, CTL_SERIDX_RD_CAP, CTL_CMD_FLAG_OK_ON_DIRECT |
 					CTL_FLAG_DATA_OUT,
  CTL_LUN_PAT_NONE,
  16, { 0x11, 0, 0, 0, 0, 0xff, 0xff, 0xff, 0xff,
@@ -301,6 +293,7 @@ const struct ctl_cmd_entry ctl_cmd_table_83[32] =
 
 /* 1C COPY OPERATION ABORT */
 {ctl_copy_operation_abort, CTL_SERIDX_RD_CAP, CTL_CMD_FLAG_OK_ON_BOTH |
+					      CTL_CMD_FLAG_OK_ON_NO_MEDIA |
 					      CTL_FLAG_DATA_NONE,
  CTL_LUN_PAT_NONE,
  16, { 0x1c, 0xff, 0xff, 0xff, 0xff, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x07}},
@@ -312,6 +305,7 @@ const struct ctl_cmd_entry ctl_cmd_table_84[32] =
 /* 00 RECEIVE COPY STATUS (LID1) */
 {ctl_receive_copy_status_lid1, CTL_SERIDX_RD_CAP,
  CTL_CMD_FLAG_OK_ON_BOTH |
+ CTL_CMD_FLAG_OK_ON_NO_MEDIA |
  CTL_FLAG_DATA_IN |
  CTL_CMD_FLAG_ALLOW_ON_PR_RESV,
  CTL_LUN_PAT_NONE,
@@ -326,8 +320,7 @@ const struct ctl_cmd_entry ctl_cmd_table_84[32] =
 /* 03 RECEIVE COPY OPERATING PARAMETERS */
 {ctl_receive_copy_operating_parameters, CTL_SERIDX_RD_CAP,
  CTL_CMD_FLAG_OK_ON_BOTH |
- CTL_CMD_FLAG_OK_ON_STOPPED |
- CTL_CMD_FLAG_OK_ON_INOPERABLE |
+ CTL_CMD_FLAG_OK_ON_NO_MEDIA |
  CTL_CMD_FLAG_OK_ON_STANDBY |
  CTL_FLAG_DATA_IN |
  CTL_CMD_FLAG_ALLOW_ON_PR_RESV,
@@ -337,6 +330,7 @@ const struct ctl_cmd_entry ctl_cmd_table_84[32] =
 /* 04 RECEIVE COPY FAILURE DETAILS (LID1) */
 {ctl_receive_copy_failure_details, CTL_SERIDX_RD_CAP,
  CTL_CMD_FLAG_OK_ON_BOTH |
+ CTL_CMD_FLAG_OK_ON_NO_MEDIA |
  CTL_FLAG_DATA_IN |
  CTL_CMD_FLAG_ALLOW_ON_PR_RESV,
  CTL_LUN_PAT_NONE,
@@ -345,6 +339,7 @@ const struct ctl_cmd_entry ctl_cmd_table_84[32] =
 /* 05 RECEIVE COPY STATUS (LID4) */
 {ctl_receive_copy_status_lid4, CTL_SERIDX_RD_CAP,
  CTL_CMD_FLAG_OK_ON_BOTH |
+ CTL_CMD_FLAG_OK_ON_NO_MEDIA |
  CTL_FLAG_DATA_IN |
  CTL_CMD_FLAG_ALLOW_ON_PR_RESV,
  CTL_LUN_PAT_NONE,
@@ -356,6 +351,7 @@ const struct ctl_cmd_entry ctl_cmd_table_84[32] =
 /* 07 RECEIVE ROD TOKEN INFORMATION */
 {ctl_receive_rod_token_information, CTL_SERIDX_RD_CAP,
  CTL_CMD_FLAG_OK_ON_BOTH |
+ CTL_CMD_FLAG_OK_ON_NO_MEDIA |
  CTL_FLAG_DATA_IN |
  CTL_CMD_FLAG_ALLOW_ON_PR_RESV,
  CTL_LUN_PAT_NONE,
@@ -364,6 +360,7 @@ const struct ctl_cmd_entry ctl_cmd_table_84[32] =
 /* 08 REPORT ALL ROD TOKENS */
 {ctl_report_all_rod_tokens, CTL_SERIDX_RD_CAP,
  CTL_CMD_FLAG_OK_ON_BOTH |
+ CTL_CMD_FLAG_OK_ON_NO_MEDIA |
  CTL_FLAG_DATA_IN |
  CTL_CMD_FLAG_ALLOW_ON_PR_RESV,
  CTL_LUN_PAT_NONE,
@@ -422,9 +419,7 @@ const struct ctl_cmd_entry ctl_cmd_table_9e[32] =
 {NULL, CTL_SERIDX_INVLD, CTL_CMD_FLAG_NONE, CTL_LUN_PAT_NONE},
 
 /* 10 READ CAPACITY(16) */
-{ctl_read_capacity_16, CTL_SERIDX_RD_CAP, CTL_CMD_FLAG_OK_ON_SLUN |
-					  CTL_CMD_FLAG_OK_ON_STOPPED |
-					  CTL_CMD_FLAG_OK_ON_INOPERABLE |
+{ctl_read_capacity_16, CTL_SERIDX_RD_CAP, CTL_CMD_FLAG_OK_ON_DIRECT |
 					  CTL_FLAG_DATA_IN |
 					  CTL_CMD_FLAG_ALLOW_ON_PR_RESV,
  CTL_LUN_PAT_READCAP,
@@ -434,7 +429,7 @@ const struct ctl_cmd_entry ctl_cmd_table_9e[32] =
 {NULL, CTL_SERIDX_INVLD, CTL_CMD_FLAG_NONE, CTL_LUN_PAT_NONE},
 
 /* 12 GET LBA STATUS */
-{ctl_get_lba_status, CTL_SERIDX_READ, CTL_CMD_FLAG_OK_ON_SLUN |
+{ctl_get_lba_status, CTL_SERIDX_READ, CTL_CMD_FLAG_OK_ON_DIRECT |
 				      CTL_FLAG_DATA_IN |
 				      CTL_CMD_FLAG_ALLOW_ON_PR_WRESV,
  CTL_LUN_PAT_READ | CTL_LUN_PAT_RANGE,
@@ -479,22 +474,20 @@ const struct ctl_cmd_entry ctl_cmd_table_a3[32] =
 
 /* 0A REPORT TARGET PORT GROUPS */
 {ctl_report_tagret_port_groups, CTL_SERIDX_INQ, CTL_CMD_FLAG_OK_ON_BOTH |
-						CTL_CMD_FLAG_OK_ON_STOPPED |
-						CTL_CMD_FLAG_OK_ON_INOPERABLE |
+						CTL_CMD_FLAG_OK_ON_NO_MEDIA |
 						CTL_CMD_FLAG_OK_ON_STANDBY |
 						CTL_CMD_FLAG_OK_ON_UNAVAIL |
 						CTL_FLAG_DATA_IN |
 						CTL_CMD_FLAG_ALLOW_ON_PR_RESV,
  CTL_LUN_PAT_NONE,
- 12, {0x0a, 0, 0, 0, 0, 0xff, 0xff, 0xff, 0xff, 0, 0x07}},
+ 12, {0xea, 0, 0, 0, 0, 0xff, 0xff, 0xff, 0xff, 0, 0x07}},
 
 /* 0B */
 {NULL, CTL_SERIDX_INVLD, CTL_CMD_FLAG_NONE, CTL_LUN_PAT_NONE},
 
 /* 0C REPORT SUPPORTED_OPCODES */
 {ctl_report_supported_opcodes, CTL_SERIDX_INQ, CTL_CMD_FLAG_OK_ON_BOTH |
-						CTL_CMD_FLAG_OK_ON_STOPPED |
-						CTL_CMD_FLAG_OK_ON_INOPERABLE |
+						CTL_CMD_FLAG_OK_ON_NO_MEDIA |
 						CTL_CMD_FLAG_OK_ON_STANDBY |
 						CTL_CMD_FLAG_OK_ON_UNAVAIL |
 						CTL_FLAG_DATA_IN |
@@ -504,8 +497,7 @@ const struct ctl_cmd_entry ctl_cmd_table_a3[32] =
 
 /* 0D REPORT SUPPORTED_TASK MANAGEMENT FUNCTIONS */
 {ctl_report_supported_tmf, CTL_SERIDX_INQ, CTL_CMD_FLAG_OK_ON_BOTH |
-					   CTL_CMD_FLAG_OK_ON_STOPPED |
-					   CTL_CMD_FLAG_OK_ON_INOPERABLE |
+					   CTL_CMD_FLAG_OK_ON_NO_MEDIA |
 					   CTL_CMD_FLAG_OK_ON_STANDBY |
 					   CTL_CMD_FLAG_OK_ON_UNAVAIL |
 					   CTL_FLAG_DATA_IN |
@@ -518,8 +510,7 @@ const struct ctl_cmd_entry ctl_cmd_table_a3[32] =
 
 /* 0F REPORT TIMESTAMP */
 {ctl_report_timestamp, CTL_SERIDX_INQ, CTL_CMD_FLAG_OK_ON_BOTH |
-					CTL_CMD_FLAG_OK_ON_STOPPED |
-					CTL_CMD_FLAG_OK_ON_INOPERABLE |
+					CTL_CMD_FLAG_OK_ON_NO_MEDIA |
 					CTL_CMD_FLAG_OK_ON_STANDBY |
 					CTL_CMD_FLAG_OK_ON_UNAVAIL |
 					CTL_FLAG_DATA_IN |
@@ -546,19 +537,19 @@ const struct ctl_cmd_entry ctl_cmd_table[256] =
 
 /* 03 REQUEST SENSE */
 {ctl_request_sense, CTL_SERIDX_RQ_SNS, CTL_FLAG_DATA_IN |
-				       CTL_CMD_FLAG_OK_ON_ALL_LUNS |
+				       CTL_CMD_FLAG_OK_ON_NO_LUN |
+				       CTL_CMD_FLAG_OK_ON_BOTH |
 				       CTL_CMD_FLAG_ALLOW_ON_RESV |
 				       CTL_CMD_FLAG_NO_SENSE |
-				       CTL_CMD_FLAG_OK_ON_STOPPED |
-				       CTL_CMD_FLAG_OK_ON_INOPERABLE |
+				       CTL_CMD_FLAG_OK_ON_NO_MEDIA |
 				       CTL_CMD_FLAG_OK_ON_STANDBY |
 				       CTL_CMD_FLAG_OK_ON_UNAVAIL |
-				       CTL_CMD_FLAG_ALLOW_ON_PR_RESV,
+				       CTL_CMD_FLAG_ALLOW_ON_PR_RESV |
+				       CTL_CMD_FLAG_RUN_HERE,
  CTL_LUN_PAT_NONE, 6, {0x01, 0, 0, 0xff, 0x07}},
 
 /* 04 FORMAT UNIT */
-{ctl_format, CTL_SERIDX_FORMAT, CTL_CMD_FLAG_OK_ON_SLUN |
-				CTL_CMD_FLAG_OK_ON_INOPERABLE |
+{ctl_format, CTL_SERIDX_FORMAT, CTL_CMD_FLAG_OK_ON_DIRECT |
 				CTL_FLAG_DATA_OUT,
  CTL_LUN_PAT_NONE, 6, {0xff, 0, 0, 0, 0x07}},
 
@@ -572,7 +563,7 @@ const struct ctl_cmd_entry ctl_cmd_table[256] =
 {NULL, CTL_SERIDX_INVLD, CTL_CMD_FLAG_NONE, CTL_LUN_PAT_NONE},
 
 /* 08 READ(6) */
-{ctl_read_write, CTL_SERIDX_READ, CTL_CMD_FLAG_OK_ON_SLUN |
+{ctl_read_write, CTL_SERIDX_READ, CTL_CMD_FLAG_OK_ON_DIRECT |
 				  CTL_FLAG_DATA_IN |
 				  CTL_CMD_FLAG_ALLOW_ON_PR_WRESV,
  CTL_LUN_PAT_READ | CTL_LUN_PAT_RANGE, 6, {0x1f, 0xff, 0xff, 0xff, 0x07}},
@@ -581,7 +572,7 @@ const struct ctl_cmd_entry ctl_cmd_table[256] =
 {NULL, CTL_SERIDX_INVLD, CTL_CMD_FLAG_NONE, CTL_LUN_PAT_NONE},
 
 /* 0A WRITE(6) */
-{ctl_read_write, CTL_SERIDX_WRITE, CTL_CMD_FLAG_OK_ON_SLUN |
+{ctl_read_write, CTL_SERIDX_WRITE, CTL_CMD_FLAG_OK_ON_DIRECT |
 				   CTL_FLAG_DATA_OUT,
  CTL_LUN_PAT_WRITE | CTL_LUN_PAT_RANGE, 6, {0x1f, 0xff, 0xff, 0xff, 0x07}},
 
@@ -607,11 +598,11 @@ const struct ctl_cmd_entry ctl_cmd_table[256] =
 {NULL, CTL_SERIDX_INVLD, CTL_CMD_FLAG_NONE, CTL_LUN_PAT_NONE},
 
 /* 12 INQUIRY */
-{ctl_inquiry, CTL_SERIDX_INQ, CTL_CMD_FLAG_OK_ON_ALL_LUNS |
+{ctl_inquiry, CTL_SERIDX_INQ, CTL_CMD_FLAG_OK_ON_NO_LUN |
+			      CTL_CMD_FLAG_OK_ON_BOTH |
 			      CTL_CMD_FLAG_ALLOW_ON_RESV |
 			      CTL_CMD_FLAG_NO_SENSE |
-			      CTL_CMD_FLAG_OK_ON_STOPPED |
-			      CTL_CMD_FLAG_OK_ON_INOPERABLE |
+			      CTL_CMD_FLAG_OK_ON_NO_MEDIA |
 			      CTL_CMD_FLAG_OK_ON_STANDBY |
 			      CTL_CMD_FLAG_OK_ON_UNAVAIL |
 			      CTL_FLAG_DATA_IN |
@@ -626,8 +617,7 @@ const struct ctl_cmd_entry ctl_cmd_table[256] =
 
 /* 15 MODE SELECT(6) */
 {ctl_mode_select, CTL_SERIDX_MD_SEL, CTL_CMD_FLAG_OK_ON_BOTH |
-				     CTL_CMD_FLAG_OK_ON_STOPPED |
-				     CTL_CMD_FLAG_OK_ON_INOPERABLE |
+				     CTL_CMD_FLAG_OK_ON_NO_MEDIA |
 				     CTL_CMD_FLAG_OK_ON_STANDBY |
 				     CTL_FLAG_DATA_OUT,
  CTL_LUN_PAT_NONE, 6, {0x11, 0, 0, 0xff, 0x07}},
@@ -635,8 +625,7 @@ const struct ctl_cmd_entry ctl_cmd_table[256] =
 /* 16 RESERVE(6) */
 {ctl_scsi_reserve, CTL_SERIDX_RES, CTL_CMD_FLAG_ALLOW_ON_RESV |
 				    CTL_CMD_FLAG_OK_ON_BOTH |
-				    CTL_CMD_FLAG_OK_ON_STOPPED |
-				    CTL_CMD_FLAG_OK_ON_INOPERABLE |
+				    CTL_CMD_FLAG_OK_ON_NO_MEDIA |
 				    CTL_CMD_FLAG_OK_ON_STANDBY |
 				    CTL_FLAG_DATA_OUT,
  CTL_LUN_PAT_NONE, 6, {0, 0, 0, 0, 0x07}},
@@ -644,8 +633,7 @@ const struct ctl_cmd_entry ctl_cmd_table[256] =
 /* 17 RELEASE(6) */
 {ctl_scsi_release, CTL_SERIDX_RES, CTL_CMD_FLAG_ALLOW_ON_RESV |
 				   CTL_CMD_FLAG_OK_ON_BOTH |
-				   CTL_CMD_FLAG_OK_ON_STOPPED |
-				   CTL_CMD_FLAG_OK_ON_INOPERABLE |
+				   CTL_CMD_FLAG_OK_ON_NO_MEDIA |
 				   CTL_CMD_FLAG_OK_ON_STANDBY |
 				   CTL_FLAG_DATA_NONE,
  CTL_LUN_PAT_NONE, 6, {0, 0, 0, 0, 0x07}},
@@ -658,20 +646,19 @@ const struct ctl_cmd_entry ctl_cmd_table[256] =
 
 /* 1A MODE SENSE(6) */
 {ctl_mode_sense, CTL_SERIDX_MD_SNS, CTL_CMD_FLAG_OK_ON_BOTH |
-				    CTL_CMD_FLAG_OK_ON_STOPPED |
-				    CTL_CMD_FLAG_OK_ON_INOPERABLE |
+				    CTL_CMD_FLAG_OK_ON_NO_MEDIA |
 				    CTL_CMD_FLAG_OK_ON_STANDBY |
 				    CTL_FLAG_DATA_IN |
 				    CTL_CMD_FLAG_ALLOW_ON_PR_WRESV,
  CTL_LUN_PAT_NONE, 6, {0x08, 0xff, 0xff, 0xff, 0x07}},
 
 /* 1B START STOP UNIT */
-{ctl_start_stop, CTL_SERIDX_START, CTL_CMD_FLAG_OK_ON_SLUN |
-				   CTL_CMD_FLAG_OK_ON_STOPPED |
-				   CTL_CMD_FLAG_OK_ON_INOPERABLE |
+{ctl_start_stop, CTL_SERIDX_START, CTL_CMD_FLAG_OK_ON_DIRECT |
+				   CTL_CMD_FLAG_OK_ON_CDROM |
+				   CTL_CMD_FLAG_OK_ON_NO_MEDIA |
 				   CTL_FLAG_DATA_NONE |
 				   CTL_CMD_FLAG_ALLOW_ON_PR_RESV,
- CTL_LUN_PAT_NONE, 6, {0x01, 0, 0, 0x03, 0x07}},
+ CTL_LUN_PAT_NONE, 6, {0x01, 0, 0x0f, 0xf7, 0x07}},
 
 /* 1C RECEIVE DIAGNOSTIC RESULTS */
 {NULL, CTL_SERIDX_INVLD, CTL_CMD_FLAG_NONE, CTL_LUN_PAT_NONE},
@@ -680,7 +667,11 @@ const struct ctl_cmd_entry ctl_cmd_table[256] =
 {NULL, CTL_SERIDX_INVLD, CTL_CMD_FLAG_NONE, CTL_LUN_PAT_NONE},
 
 /* 1E PREVENT ALLOW MEDIUM REMOVAL */
-{NULL, CTL_SERIDX_INVLD, CTL_CMD_FLAG_NONE, CTL_LUN_PAT_NONE},
+{ctl_prevent_allow, CTL_SERIDX_START, CTL_CMD_FLAG_OK_ON_DIRECT |
+				      CTL_CMD_FLAG_OK_ON_CDROM |
+				      CTL_CMD_FLAG_OK_ON_NO_MEDIA |
+				      CTL_FLAG_DATA_NONE,
+ CTL_LUN_PAT_NONE, 6, {0x01, 0, 0, 0x03, 0x07}},
 
 /* 1F */
 {NULL, CTL_SERIDX_INVLD, CTL_CMD_FLAG_NONE, CTL_LUN_PAT_NONE},
@@ -701,9 +692,8 @@ const struct ctl_cmd_entry ctl_cmd_table[256] =
 {NULL, CTL_SERIDX_INVLD, CTL_CMD_FLAG_NONE, CTL_LUN_PAT_NONE},
 
 /* 25 READ CAPACITY(10) */
-{ctl_read_capacity, CTL_SERIDX_RD_CAP, CTL_CMD_FLAG_OK_ON_SLUN|
-				       CTL_CMD_FLAG_OK_ON_STOPPED |
-				       CTL_CMD_FLAG_OK_ON_INOPERABLE |
+{ctl_read_capacity, CTL_SERIDX_RD_CAP, CTL_CMD_FLAG_OK_ON_DIRECT |
+				       CTL_CMD_FLAG_OK_ON_CDROM |
 				       CTL_FLAG_DATA_IN |
 				       CTL_CMD_FLAG_ALLOW_ON_PR_RESV,
  CTL_LUN_PAT_READCAP, 10, {0, 0, 0, 0, 0, 0, 0, 0, 0x07}},
@@ -715,7 +705,8 @@ const struct ctl_cmd_entry ctl_cmd_table[256] =
 {NULL, CTL_SERIDX_INVLD, CTL_CMD_FLAG_NONE, CTL_LUN_PAT_NONE},
 
 /* 28 READ(10) */
-{ctl_read_write, CTL_SERIDX_READ, CTL_CMD_FLAG_OK_ON_SLUN |
+{ctl_read_write, CTL_SERIDX_READ, CTL_CMD_FLAG_OK_ON_DIRECT |
+				  CTL_CMD_FLAG_OK_ON_CDROM |
 				  CTL_FLAG_DATA_IN |
 				  CTL_CMD_FLAG_ALLOW_ON_PR_WRESV,
  CTL_LUN_PAT_READ | CTL_LUN_PAT_RANGE,
@@ -725,7 +716,7 @@ const struct ctl_cmd_entry ctl_cmd_table[256] =
 {NULL, CTL_SERIDX_INVLD, CTL_CMD_FLAG_NONE, CTL_LUN_PAT_NONE},
 
 /* 2A WRITE(10) */
-{ctl_read_write, CTL_SERIDX_WRITE, CTL_CMD_FLAG_OK_ON_SLUN| CTL_FLAG_DATA_OUT,
+{ctl_read_write, CTL_SERIDX_WRITE, CTL_CMD_FLAG_OK_ON_DIRECT| CTL_FLAG_DATA_OUT,
  CTL_LUN_PAT_WRITE | CTL_LUN_PAT_RANGE,
  10, {0x1a, 0xff, 0xff, 0xff, 0xff, 0, 0xff, 0xff, 0x07}},
 
@@ -739,12 +730,12 @@ const struct ctl_cmd_entry ctl_cmd_table[256] =
 {NULL, CTL_SERIDX_INVLD, CTL_CMD_FLAG_NONE, CTL_LUN_PAT_NONE},
 
 /* 2E WRITE AND VERIFY(10) */
-{ctl_read_write, CTL_SERIDX_WRITE, CTL_CMD_FLAG_OK_ON_SLUN| CTL_FLAG_DATA_OUT,
+{ctl_read_write, CTL_SERIDX_WRITE, CTL_CMD_FLAG_OK_ON_DIRECT| CTL_FLAG_DATA_OUT,
  CTL_LUN_PAT_WRITE | CTL_LUN_PAT_RANGE,
  10, {0x12, 0xff, 0xff, 0xff, 0xff, 0, 0xff, 0xff, 0x07}},
 
 /* 2F VERIFY(10) */
-{ctl_verify, CTL_SERIDX_READ, CTL_CMD_FLAG_OK_ON_SLUN |
+{ctl_verify, CTL_SERIDX_READ, CTL_CMD_FLAG_OK_ON_DIRECT |
 			      CTL_FLAG_DATA_OUT |
 			      CTL_CMD_FLAG_ALLOW_ON_PR_WRESV,
  CTL_LUN_PAT_READ | CTL_LUN_PAT_RANGE,
@@ -766,16 +757,16 @@ const struct ctl_cmd_entry ctl_cmd_table[256] =
 {NULL, CTL_SERIDX_INVLD, CTL_CMD_FLAG_NONE, CTL_LUN_PAT_NONE},
 
 /* 35 SYNCHRONIZE CACHE(10) */
-{ctl_sync_cache, CTL_SERIDX_SYNC, CTL_CMD_FLAG_OK_ON_SLUN |
+{ctl_sync_cache, CTL_SERIDX_SYNC, CTL_CMD_FLAG_OK_ON_DIRECT |
 				  CTL_FLAG_DATA_NONE,
- CTL_LUN_PAT_NONE,
- 10, {0x02, 0xff, 0xff, 0xff, 0xff, 0, 0xff, 0xff, 0x07}},
+ CTL_LUN_PAT_WRITE,
+ 10, {0x06, 0xff, 0xff, 0xff, 0xff, 0, 0xff, 0xff, 0x07}},
 
 /* 36 LOCK UNLOCK CACHE(10) */
 {NULL, CTL_SERIDX_INVLD, CTL_CMD_FLAG_NONE, CTL_LUN_PAT_NONE},
 
 /* 37 READ DEFECT DATA(10) */
-{ctl_read_defect, CTL_SERIDX_MD_SNS, CTL_CMD_FLAG_OK_ON_SLUN |
+{ctl_read_defect, CTL_SERIDX_MD_SNS, CTL_CMD_FLAG_OK_ON_DIRECT |
 				     CTL_FLAG_DATA_IN |
 				     CTL_CMD_FLAG_ALLOW_ON_PR_WRESV,
  CTL_LUN_PAT_NONE,
@@ -792,8 +783,7 @@ const struct ctl_cmd_entry ctl_cmd_table[256] =
 
 /* 3B WRITE BUFFER */
 {ctl_write_buffer, CTL_SERIDX_MD_SEL, CTL_CMD_FLAG_OK_ON_BOTH |
-				      CTL_CMD_FLAG_OK_ON_STOPPED |
-				      CTL_CMD_FLAG_OK_ON_INOPERABLE |
+				      CTL_CMD_FLAG_OK_ON_NO_MEDIA |
 				      CTL_CMD_FLAG_OK_ON_STANDBY |
 				      CTL_FLAG_DATA_OUT,
  CTL_LUN_PAT_NONE,
@@ -801,8 +791,7 @@ const struct ctl_cmd_entry ctl_cmd_table[256] =
 
 /* 3C READ BUFFER */
 {ctl_read_buffer, CTL_SERIDX_MD_SNS, CTL_CMD_FLAG_OK_ON_BOTH |
-				     CTL_CMD_FLAG_OK_ON_STOPPED |
-				     CTL_CMD_FLAG_OK_ON_INOPERABLE |
+				     CTL_CMD_FLAG_OK_ON_NO_MEDIA |
 				     CTL_CMD_FLAG_OK_ON_STANDBY |
 				     CTL_FLAG_DATA_IN |
 				     CTL_CMD_FLAG_ALLOW_ON_PR_WRESV,
@@ -822,18 +811,22 @@ const struct ctl_cmd_entry ctl_cmd_table[256] =
 {NULL, CTL_SERIDX_INVLD, CTL_CMD_FLAG_NONE, CTL_LUN_PAT_NONE},
 
 /* 41 WRITE SAME(10) */
-{ctl_write_same, CTL_SERIDX_WRITE, CTL_CMD_FLAG_OK_ON_SLUN |
+{ctl_write_same, CTL_SERIDX_WRITE, CTL_CMD_FLAG_OK_ON_DIRECT |
 				   CTL_FLAG_DATA_OUT,
  CTL_LUN_PAT_WRITE | CTL_LUN_PAT_RANGE,
  10, {0x1a, 0xff, 0xff, 0xff, 0xff, 0, 0xff, 0xff, 0x07}},
 
 /* 42 READ SUB-CHANNEL / UNMAP */
-{ctl_unmap, CTL_SERIDX_UNMAP, CTL_CMD_FLAG_OK_ON_SLUN | CTL_FLAG_DATA_OUT,
+{ctl_unmap, CTL_SERIDX_UNMAP, CTL_CMD_FLAG_OK_ON_DIRECT | CTL_FLAG_DATA_OUT,
  CTL_LUN_PAT_WRITE,
  10, {1, 0, 0, 0, 0, 0, 0xff, 0xff, 0x07}},
 
 /* 43 READ TOC/PMA/ATIP */
-{NULL, CTL_SERIDX_INVLD, CTL_CMD_FLAG_NONE, CTL_LUN_PAT_NONE},
+{ctl_read_toc, CTL_SERIDX_RD_CAP, CTL_CMD_FLAG_OK_ON_CDROM |
+				  CTL_CMD_FLAG_ALLOW_ON_PR_WRESV |
+				  CTL_FLAG_DATA_IN,
+ CTL_LUN_PAT_NONE,
+ 10, {0x02, 0x01, 0, 0, 0, 0xff, 0xff, 0xff, 0x07}},
 
 /* 44 REPORT DENSITY SUPPORT */
 {NULL, CTL_SERIDX_INVLD, CTL_CMD_FLAG_NONE, CTL_LUN_PAT_NONE},
@@ -842,7 +835,12 @@ const struct ctl_cmd_entry ctl_cmd_table[256] =
 {NULL, CTL_SERIDX_INVLD, CTL_CMD_FLAG_NONE, CTL_LUN_PAT_NONE},
 
 /* 46 GET CONFIGURATION */
-{NULL, CTL_SERIDX_INVLD, CTL_CMD_FLAG_NONE, CTL_LUN_PAT_NONE},
+{ctl_get_config, CTL_SERIDX_INQ, CTL_CMD_FLAG_OK_ON_CDROM |
+				 CTL_CMD_FLAG_OK_ON_NO_MEDIA |
+				 CTL_CMD_FLAG_ALLOW_ON_PR_RESV |
+				 CTL_FLAG_DATA_IN,
+ CTL_LUN_PAT_NONE,
+ 10, {0x03, 0xff, 0xff, 0, 0, 0, 0xff, 0xff, 0x07}},
 
 /* 47 PLAY AUDIO MSF */
 {NULL, CTL_SERIDX_INVLD, CTL_CMD_FLAG_NONE, CTL_LUN_PAT_NONE},
@@ -854,7 +852,12 @@ const struct ctl_cmd_entry ctl_cmd_table[256] =
 {NULL, CTL_SERIDX_INVLD, CTL_CMD_FLAG_NONE, CTL_LUN_PAT_NONE},
 
 /* 4A GET EVENT STATUS NOTIFICATION */
-{NULL, CTL_SERIDX_INVLD, CTL_CMD_FLAG_NONE, CTL_LUN_PAT_NONE},
+{ctl_get_event_status, CTL_SERIDX_RD_CAP, CTL_CMD_FLAG_OK_ON_CDROM |
+					  CTL_CMD_FLAG_OK_ON_NO_MEDIA |
+					  CTL_CMD_FLAG_ALLOW_ON_PR_RESV |
+					  CTL_FLAG_DATA_IN,
+ CTL_LUN_PAT_NONE,
+ 10, {0x02, 0x01, 0, 0, 0, 0xff, 0xff, 0xff, 0x07}},
 
 /* 4B PAUSE/RESUME */
 {NULL, CTL_SERIDX_INVLD, CTL_CMD_FLAG_NONE, CTL_LUN_PAT_NONE},
@@ -863,7 +866,9 @@ const struct ctl_cmd_entry ctl_cmd_table[256] =
 {NULL, CTL_SERIDX_INVLD, CTL_CMD_FLAG_NONE, CTL_LUN_PAT_NONE},
 
 /* 4D LOG SENSE */
-{ctl_log_sense, CTL_SERIDX_LOG_SNS, CTL_CMD_FLAG_OK_ON_SLUN |
+{ctl_log_sense, CTL_SERIDX_LOG_SNS, CTL_CMD_FLAG_OK_ON_BOTH |
+				    CTL_CMD_FLAG_OK_ON_NO_MEDIA |
+				    CTL_CMD_FLAG_OK_ON_STANDBY |
 				    CTL_FLAG_DATA_IN |
 				    CTL_CMD_FLAG_ALLOW_ON_PR_RESV,
  CTL_LUN_PAT_NONE, 10, {0, 0xff, 0xff, 0, 0xff, 0xff, 0xff, 0xff, 0x07} },
@@ -891,8 +896,7 @@ const struct ctl_cmd_entry ctl_cmd_table[256] =
 
 /* 55 MODE SELECT(10) */
 {ctl_mode_select, CTL_SERIDX_MD_SEL, CTL_CMD_FLAG_OK_ON_BOTH |
-				     CTL_CMD_FLAG_OK_ON_STOPPED |
-				     CTL_CMD_FLAG_OK_ON_INOPERABLE |
+				     CTL_CMD_FLAG_OK_ON_NO_MEDIA |
 				     CTL_CMD_FLAG_OK_ON_STANDBY |
 				     CTL_FLAG_DATA_OUT,
  CTL_LUN_PAT_NONE, 10, {0x11, 0, 0, 0, 0, 0, 0xff, 0xff, 0x07} },
@@ -900,20 +904,18 @@ const struct ctl_cmd_entry ctl_cmd_table[256] =
 /* 56 RESERVE(10) */
 {ctl_scsi_reserve, CTL_SERIDX_RES, CTL_CMD_FLAG_ALLOW_ON_RESV |
 				    CTL_CMD_FLAG_OK_ON_BOTH |
-				    CTL_CMD_FLAG_OK_ON_STOPPED |
-				    CTL_CMD_FLAG_OK_ON_INOPERABLE |
+				    CTL_CMD_FLAG_OK_ON_NO_MEDIA |
 				    CTL_CMD_FLAG_OK_ON_STANDBY |
 				    CTL_FLAG_DATA_OUT,
- CTL_LUN_PAT_NONE, 10, {0x02, 0, 0xff, 0, 0, 0, 0xff, 0xff, 0x07} },
+ CTL_LUN_PAT_NONE, 10, {0, 0, 0, 0, 0, 0, 0, 0, 0x07} },
 
 /* 57 RELEASE(10) */
 {ctl_scsi_release, CTL_SERIDX_RES, CTL_CMD_FLAG_ALLOW_ON_RESV |
 				   CTL_CMD_FLAG_OK_ON_BOTH |
-				   CTL_CMD_FLAG_OK_ON_STOPPED |
-				   CTL_CMD_FLAG_OK_ON_INOPERABLE |
+				   CTL_CMD_FLAG_OK_ON_NO_MEDIA |
 				   CTL_CMD_FLAG_OK_ON_STANDBY |
 				   CTL_FLAG_DATA_OUT,
- CTL_LUN_PAT_NONE, 10, {0x02, 0, 0xff, 0, 0, 0, 0xff, 0xff, 0x07} },
+ CTL_LUN_PAT_NONE, 10, {0, 0, 0, 0, 0, 0, 0, 0, 0x07}},
 
 /* 58 REPAIR TRACK */
 {NULL, CTL_SERIDX_INVLD, CTL_CMD_FLAG_NONE, CTL_LUN_PAT_NONE},
@@ -923,8 +925,7 @@ const struct ctl_cmd_entry ctl_cmd_table[256] =
 
 /* 5A MODE SENSE(10) */
 {ctl_mode_sense, CTL_SERIDX_MD_SNS, CTL_CMD_FLAG_OK_ON_BOTH |
-				    CTL_CMD_FLAG_OK_ON_STOPPED |
-				    CTL_CMD_FLAG_OK_ON_INOPERABLE |
+				    CTL_CMD_FLAG_OK_ON_NO_MEDIA |
 				    CTL_CMD_FLAG_OK_ON_STANDBY |
 				    CTL_FLAG_DATA_IN |
 				    CTL_CMD_FLAG_ALLOW_ON_PR_WRESV,
@@ -1070,20 +1071,20 @@ const struct ctl_cmd_entry ctl_cmd_table[256] =
 {NULL, CTL_SERIDX_INVLD, CTL_CMD_FLAG_NONE, CTL_LUN_PAT_NONE},
 
 /* 88 READ(16) */
-{ctl_read_write, CTL_SERIDX_READ, CTL_CMD_FLAG_OK_ON_SLUN | CTL_FLAG_DATA_IN |
+{ctl_read_write, CTL_SERIDX_READ, CTL_CMD_FLAG_OK_ON_DIRECT | CTL_FLAG_DATA_IN |
                                   CTL_CMD_FLAG_ALLOW_ON_PR_WRESV,
  CTL_LUN_PAT_READ | CTL_LUN_PAT_RANGE,
  16, {0x1a, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
       0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0, 0x07}},
 
 /* 89 COMPARE AND WRITE */
-{ctl_cnw, CTL_SERIDX_WRITE, CTL_CMD_FLAG_OK_ON_SLUN| CTL_FLAG_DATA_OUT,
+{ctl_cnw, CTL_SERIDX_WRITE, CTL_CMD_FLAG_OK_ON_DIRECT| CTL_FLAG_DATA_OUT,
  CTL_LUN_PAT_WRITE | CTL_LUN_PAT_RANGE,
  16, {0x18, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
       0xff, 0xff, 0, 0, 0, 0xff, 0, 0x07}},
 
 /* 8A WRITE(16) */
-{ctl_read_write, CTL_SERIDX_WRITE, CTL_CMD_FLAG_OK_ON_SLUN| CTL_FLAG_DATA_OUT,
+{ctl_read_write, CTL_SERIDX_WRITE, CTL_CMD_FLAG_OK_ON_DIRECT| CTL_FLAG_DATA_OUT,
  CTL_LUN_PAT_WRITE | CTL_LUN_PAT_RANGE,
  16, {0x1a, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
       0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0, 0x07}},
@@ -1098,13 +1099,13 @@ const struct ctl_cmd_entry ctl_cmd_table[256] =
 {NULL, CTL_SERIDX_INVLD, CTL_CMD_FLAG_NONE, CTL_LUN_PAT_NONE},
 
 /* 8E WRITE AND VERIFY(16) */
-{ctl_read_write, CTL_SERIDX_WRITE, CTL_CMD_FLAG_OK_ON_SLUN| CTL_FLAG_DATA_OUT,
+{ctl_read_write, CTL_SERIDX_WRITE, CTL_CMD_FLAG_OK_ON_DIRECT| CTL_FLAG_DATA_OUT,
  CTL_LUN_PAT_WRITE | CTL_LUN_PAT_RANGE,
  16, {0x12, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
       0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0, 0x07}},
 
 /* 8F VERIFY(16) */
-{ctl_verify, CTL_SERIDX_READ, CTL_CMD_FLAG_OK_ON_SLUN |
+{ctl_verify, CTL_SERIDX_READ, CTL_CMD_FLAG_OK_ON_DIRECT |
 			      CTL_FLAG_DATA_OUT |
 			      CTL_CMD_FLAG_ALLOW_ON_PR_WRESV,
  CTL_LUN_PAT_READ | CTL_LUN_PAT_RANGE,
@@ -1115,17 +1116,17 @@ const struct ctl_cmd_entry ctl_cmd_table[256] =
 {NULL, CTL_SERIDX_INVLD, CTL_CMD_FLAG_NONE, CTL_LUN_PAT_NONE},
 
 /* 91 SYNCHRONIZE CACHE(16) */
-{ctl_sync_cache, CTL_SERIDX_SYNC, CTL_CMD_FLAG_OK_ON_SLUN |
+{ctl_sync_cache, CTL_SERIDX_SYNC, CTL_CMD_FLAG_OK_ON_DIRECT |
 				  CTL_FLAG_DATA_NONE,
- CTL_LUN_PAT_NONE,
- 16, {0x02, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ CTL_LUN_PAT_WRITE,
+ 16, {0x06, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
       0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0, 0x07}},
 
 /* 92 LOCK UNLOCK CACHE(16) */
 {NULL, CTL_SERIDX_INVLD, CTL_CMD_FLAG_NONE, CTL_LUN_PAT_NONE},
 
 /* 93 WRITE SAME(16) */
-{ctl_write_same, CTL_SERIDX_WRITE, CTL_CMD_FLAG_OK_ON_SLUN |
+{ctl_write_same, CTL_SERIDX_WRITE, CTL_CMD_FLAG_OK_ON_DIRECT |
 				   CTL_FLAG_DATA_OUT,
  CTL_LUN_PAT_WRITE | CTL_LUN_PAT_RANGE,
  16, {0x1b, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
@@ -1152,11 +1153,18 @@ const struct ctl_cmd_entry ctl_cmd_table[256] =
 /* 9A */
 {NULL, CTL_SERIDX_INVLD, CTL_CMD_FLAG_NONE, CTL_LUN_PAT_NONE},
 
-/* 9B */
-{NULL, CTL_SERIDX_INVLD, CTL_CMD_FLAG_NONE, CTL_LUN_PAT_NONE},
+/* 9B READ BUFFER(16) */
+{ctl_read_buffer, CTL_SERIDX_MD_SNS, CTL_CMD_FLAG_OK_ON_BOTH |
+				     CTL_CMD_FLAG_OK_ON_NO_MEDIA |
+				     CTL_CMD_FLAG_OK_ON_STANDBY |
+				     CTL_FLAG_DATA_IN |
+				     CTL_CMD_FLAG_ALLOW_ON_PR_WRESV,
+ CTL_LUN_PAT_NONE,
+ 10, {0x1f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+      0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0, 0x07}},
 
 /* 9C WRITE ATOMIC (16) */
-{ctl_read_write, CTL_SERIDX_WRITE, CTL_CMD_FLAG_OK_ON_SLUN| CTL_FLAG_DATA_OUT,
+{ctl_read_write, CTL_SERIDX_WRITE, CTL_CMD_FLAG_OK_ON_DIRECT| CTL_FLAG_DATA_OUT,
  CTL_LUN_PAT_WRITE | CTL_LUN_PAT_RANGE,
  16, {0x18, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
       0xff, 0xff, 0, 0, 0xff, 0xff, 0, 0x07}},
@@ -1172,15 +1180,16 @@ const struct ctl_cmd_entry ctl_cmd_table[256] =
 {NULL, CTL_SERIDX_INVLD, CTL_CMD_FLAG_NONE, CTL_LUN_PAT_NONE},
 
 /* A0 REPORT LUNS */
-{ctl_report_luns, CTL_SERIDX_INQ, CTL_CMD_FLAG_OK_ON_ALL_LUNS |
+{ctl_report_luns, CTL_SERIDX_INQ, CTL_FLAG_DATA_IN |
+				  CTL_CMD_FLAG_OK_ON_NO_LUN |
+				  CTL_CMD_FLAG_OK_ON_BOTH |
 				  CTL_CMD_FLAG_ALLOW_ON_RESV |
 				  CTL_CMD_FLAG_NO_SENSE |
-				  CTL_CMD_FLAG_OK_ON_STOPPED |
-				  CTL_CMD_FLAG_OK_ON_INOPERABLE |
+				  CTL_CMD_FLAG_OK_ON_NO_MEDIA |
 				  CTL_CMD_FLAG_OK_ON_STANDBY |
 				  CTL_CMD_FLAG_OK_ON_UNAVAIL |
-				  CTL_FLAG_DATA_IN |
-				  CTL_CMD_FLAG_ALLOW_ON_PR_RESV,
+				  CTL_CMD_FLAG_ALLOW_ON_PR_RESV |
+				  CTL_CMD_FLAG_RUN_HERE,
  CTL_LUN_PAT_NONE,
  12, {0, 0xff, 0, 0, 0, 0xff, 0xff, 0xff, 0xff, 0, 0x07}},
 
@@ -1207,8 +1216,10 @@ const struct ctl_cmd_entry ctl_cmd_table[256] =
 {NULL, CTL_SERIDX_INVLD, CTL_CMD_FLAG_NONE, CTL_LUN_PAT_NONE},
 
 /* A8 READ(12) */
-{ctl_read_write, CTL_SERIDX_READ, CTL_CMD_FLAG_OK_ON_SLUN | CTL_FLAG_DATA_IN |
-                                  CTL_CMD_FLAG_ALLOW_ON_PR_WRESV,
+{ctl_read_write, CTL_SERIDX_READ, CTL_CMD_FLAG_OK_ON_DIRECT |
+				  CTL_CMD_FLAG_OK_ON_CDROM |
+				  CTL_FLAG_DATA_IN |
+				  CTL_CMD_FLAG_ALLOW_ON_PR_WRESV,
  CTL_LUN_PAT_READ | CTL_LUN_PAT_RANGE,
  12, {0x1a, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0, 0x07}},
 
@@ -1216,7 +1227,7 @@ const struct ctl_cmd_entry ctl_cmd_table[256] =
 {NULL, CTL_SERIDX_INVLD, CTL_CMD_FLAG_NONE, CTL_LUN_PAT_NONE},
 
 /* AA WRITE(12) */
-{ctl_read_write, CTL_SERIDX_WRITE, CTL_CMD_FLAG_OK_ON_SLUN| CTL_FLAG_DATA_OUT,
+{ctl_read_write, CTL_SERIDX_WRITE, CTL_CMD_FLAG_OK_ON_DIRECT| CTL_FLAG_DATA_OUT,
  CTL_LUN_PAT_WRITE | CTL_LUN_PAT_RANGE,
  12, {0x1a, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0, 0x07}},
 
@@ -1230,12 +1241,12 @@ const struct ctl_cmd_entry ctl_cmd_table[256] =
 {NULL, CTL_SERIDX_INVLD, CTL_CMD_FLAG_NONE, CTL_LUN_PAT_NONE},
 
 /* AE WRITE AND VERIFY(12) */
-{ctl_read_write, CTL_SERIDX_WRITE, CTL_CMD_FLAG_OK_ON_SLUN| CTL_FLAG_DATA_OUT,
+{ctl_read_write, CTL_SERIDX_WRITE, CTL_CMD_FLAG_OK_ON_DIRECT| CTL_FLAG_DATA_OUT,
  CTL_LUN_PAT_WRITE | CTL_LUN_PAT_RANGE,
  12, {0x12, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0, 0x07}},
 
 /* AF VERIFY(12) */
-{ctl_verify, CTL_SERIDX_READ, CTL_CMD_FLAG_OK_ON_SLUN |
+{ctl_verify, CTL_SERIDX_READ, CTL_CMD_FLAG_OK_ON_DIRECT |
 			      CTL_FLAG_DATA_OUT |
 			      CTL_CMD_FLAG_ALLOW_ON_PR_WRESV,
  CTL_LUN_PAT_READ | CTL_LUN_PAT_RANGE,
@@ -1263,7 +1274,7 @@ const struct ctl_cmd_entry ctl_cmd_table[256] =
 {NULL, CTL_SERIDX_INVLD, CTL_CMD_FLAG_NONE, CTL_LUN_PAT_NONE},
 
 /* B7 READ DEFECT DATA(12) */
-{ctl_read_defect, CTL_SERIDX_MD_SNS, CTL_CMD_FLAG_OK_ON_SLUN |
+{ctl_read_defect, CTL_SERIDX_MD_SNS, CTL_CMD_FLAG_OK_ON_DIRECT |
 				     CTL_FLAG_DATA_IN |
 				     CTL_CMD_FLAG_ALLOW_ON_PR_WRESV,
  CTL_LUN_PAT_NONE,
@@ -1284,8 +1295,13 @@ const struct ctl_cmd_entry ctl_cmd_table[256] =
 /* BC SPARE IN */
 {NULL, CTL_SERIDX_INVLD, CTL_CMD_FLAG_NONE, CTL_LUN_PAT_NONE},
 
-/* BD SPARE OUT */
-{NULL, CTL_SERIDX_INVLD, CTL_CMD_FLAG_NONE, CTL_LUN_PAT_NONE},
+/* BD SPARE OUT / MECHANISM STATUS */
+{ctl_mechanism_status, CTL_SERIDX_RD_CAP, CTL_CMD_FLAG_OK_ON_CDROM |
+					  CTL_CMD_FLAG_OK_ON_NO_MEDIA |
+					  CTL_CMD_FLAG_ALLOW_ON_PR_RESV |
+					  CTL_FLAG_DATA_IN,
+ CTL_LUN_PAT_NONE,
+ 12, {0, 0, 0, 0, 0, 0, 0, 0xff, 0xff, 0, 0x07}},
 
 /* BE VOLUME SET IN */
 {NULL, CTL_SERIDX_INVLD, CTL_CMD_FLAG_NONE, CTL_LUN_PAT_NONE},
diff --git a/sys/cam/ctl/ctl_error.c b/sys/cam/ctl/ctl_error.c
index 8ef5176d455b9..61ec4ec3ea3d7 100644
--- a/sys/cam/ctl/ctl_error.c
+++ b/sys/cam/ctl/ctl_error.c
@@ -1,6 +1,7 @@
 /*-
  * Copyright (c) 2003-2009 Silicon Graphics International Corp.
  * Copyright (c) 2011 Spectra Logic Corporation
+ * Copyright (c) 2014-2015 Alexander Motin <mav@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -365,62 +366,35 @@ ctl_set_ua(struct ctl_scsiio *ctsio, int asc, int ascq)
 		      SSD_ELEM_NONE);
 }
 
-ctl_ua_type
-ctl_build_ua(struct ctl_lun *lun, uint32_t initidx,
-    struct scsi_sense_data *sense, scsi_sense_data_type sense_format)
+static void
+ctl_ua_to_acsq(struct ctl_lun *lun, ctl_ua_type ua_to_build, int *asc,
+    int *ascq, ctl_ua_type *ua_to_clear, uint8_t **info)
 {
-	ctl_ua_type *ua;
-	ctl_ua_type ua_to_build, ua_to_clear;
-	int asc, ascq;
-	uint32_t p, i;
-
-	mtx_assert(&lun->lun_lock, MA_OWNED);
-	p = initidx / CTL_MAX_INIT_PER_PORT;
-	if ((ua = lun->pending_ua[p]) == NULL) {
-		mtx_unlock(&lun->lun_lock);
-		ua = malloc(sizeof(ctl_ua_type) * CTL_MAX_INIT_PER_PORT,
-		    M_CTL, M_WAITOK);
-		mtx_lock(&lun->lun_lock);
-		if (lun->pending_ua[p] == NULL) {
-			lun->pending_ua[p] = ua;
-			for (i = 0; i < CTL_MAX_INIT_PER_PORT; i++)
-				ua[i] = CTL_UA_POWERON;
-		} else {
-			free(ua, M_CTL);
-			ua = lun->pending_ua[p];
-		}
-	}
-	i = initidx % CTL_MAX_INIT_PER_PORT;
-	if (ua[i] == CTL_UA_NONE)
-		return (CTL_UA_NONE);
-
-	ua_to_build = (1 << (ffs(ua[i]) - 1));
-	ua_to_clear = ua_to_build;
 
 	switch (ua_to_build) {
 	case CTL_UA_POWERON:
 		/* 29h/01h  POWER ON OCCURRED */
-		asc = 0x29;
-		ascq = 0x01;
-		ua_to_clear = ~0;
+		*asc = 0x29;
+		*ascq = 0x01;
+		*ua_to_clear = ~0;
 		break;
 	case CTL_UA_BUS_RESET:
 		/* 29h/02h  SCSI BUS RESET OCCURRED */
-		asc = 0x29;
-		ascq = 0x02;
-		ua_to_clear = ~0;
+		*asc = 0x29;
+		*ascq = 0x02;
+		*ua_to_clear = ~0;
 		break;
 	case CTL_UA_TARG_RESET:
 		/* 29h/03h  BUS DEVICE RESET FUNCTION OCCURRED*/
-		asc = 0x29;
-		ascq = 0x03;
-		ua_to_clear = ~0;
+		*asc = 0x29;
+		*ascq = 0x03;
+		*ua_to_clear = ~0;
 		break;
 	case CTL_UA_I_T_NEXUS_LOSS:
 		/* 29h/07h  I_T NEXUS LOSS OCCURRED */
-		asc = 0x29;
-		ascq = 0x07;
-		ua_to_clear = ~0;
+		*asc = 0x29;
+		*ascq = 0x07;
+		*ua_to_clear = ~0;
 		break;
 	case CTL_UA_LUN_RESET:
 		/* 29h/00h  POWER ON, RESET, OR BUS DEVICE RESET OCCURRED */
@@ -428,66 +402,143 @@ ctl_build_ua(struct ctl_lun *lun, uint32_t initidx,
 		 * Since we don't have a specific ASC/ASCQ pair for a LUN
 		 * reset, just return the generic reset code.
 		 */
-		asc = 0x29;
-		ascq = 0x00;
+		*asc = 0x29;
+		*ascq = 0x00;
 		break;
 	case CTL_UA_LUN_CHANGE:
 		/* 3Fh/0Eh  REPORTED LUNS DATA HAS CHANGED */
-		asc = 0x3F;
-		ascq = 0x0E;
+		*asc = 0x3F;
+		*ascq = 0x0E;
 		break;
 	case CTL_UA_MODE_CHANGE:
 		/* 2Ah/01h  MODE PARAMETERS CHANGED */
-		asc = 0x2A;
-		ascq = 0x01;
+		*asc = 0x2A;
+		*ascq = 0x01;
 		break;
 	case CTL_UA_LOG_CHANGE:
 		/* 2Ah/02h  LOG PARAMETERS CHANGED */
-		asc = 0x2A;
-		ascq = 0x02;
+		*asc = 0x2A;
+		*ascq = 0x02;
+		break;
+	case CTL_UA_INQ_CHANGE:
+		/* 3Fh/03h  INQUIRY DATA HAS CHANGED */
+		*asc = 0x3F;
+		*ascq = 0x03;
 		break;
 	case CTL_UA_RES_PREEMPT:
 		/* 2Ah/03h  RESERVATIONS PREEMPTED */
-		asc = 0x2A;
-		ascq = 0x03;
+		*asc = 0x2A;
+		*ascq = 0x03;
 		break;
 	case CTL_UA_RES_RELEASE:
 		/* 2Ah/04h  RESERVATIONS RELEASED */
-		asc = 0x2A;
-		ascq = 0x04;
+		*asc = 0x2A;
+		*ascq = 0x04;
 		break;
 	case CTL_UA_REG_PREEMPT:
 		/* 2Ah/05h  REGISTRATIONS PREEMPTED */
-		asc = 0x2A;
-		ascq = 0x05;
+		*asc = 0x2A;
+		*ascq = 0x05;
 		break;
 	case CTL_UA_ASYM_ACC_CHANGE:
-	        /* 2Ah/06n  ASYMMETRIC ACCESS STATE CHANGED */
-		asc = 0x2A;
-		ascq = 0x06;
+		/* 2Ah/06h  ASYMMETRIC ACCESS STATE CHANGED */
+		*asc = 0x2A;
+		*ascq = 0x06;
 		break;
-	case CTL_UA_CAPACITY_CHANGED:
-	        /* 2Ah/09n  CAPACITY DATA HAS CHANGED */
-		asc = 0x2A;
-		ascq = 0x09;
+	case CTL_UA_CAPACITY_CHANGE:
+		/* 2Ah/09h  CAPACITY DATA HAS CHANGED */
+		*asc = 0x2A;
+		*ascq = 0x09;
 		break;
 	case CTL_UA_THIN_PROV_THRES:
-		/* 38h/07n  THIN PROVISIONING SOFT THRESHOLD REACHED */
-		asc = 0x38;
-		ascq = 0x07;
+		/* 38h/07h  THIN PROVISIONING SOFT THRESHOLD REACHED */
+		*asc = 0x38;
+		*ascq = 0x07;
+		*info = lun->ua_tpt_info;
+		break;
+	case CTL_UA_MEDIUM_CHANGE:
+		/* 28h/00h  NOT READY TO READY CHANGE, MEDIUM MAY HAVE CHANGED */
+		*asc = 0x28;
+		*ascq = 0x00;
 		break;
 	default:
-		panic("ctl_build_ua: Unknown UA %x", ua_to_build);
+		panic("%s: Unknown UA %x", __func__, ua_to_build);
 	}
+}
 
-	ctl_set_sense_data(sense,
-			   /*lun*/ NULL,
-			   sense_format,
-			   /*current_error*/ 1,
-			   /*sense_key*/ SSD_KEY_UNIT_ATTENTION,
-			   asc,
-			   ascq,
-			   SSD_ELEM_NONE);
+ctl_ua_type
+ctl_build_qae(struct ctl_lun *lun, uint32_t initidx, uint8_t *resp)
+{
+	ctl_ua_type ua;
+	ctl_ua_type ua_to_build, ua_to_clear;
+	uint8_t *info;
+	int asc, ascq;
+	uint32_t p, i;
+
+	mtx_assert(&lun->lun_lock, MA_OWNED);
+	p = initidx / CTL_MAX_INIT_PER_PORT;
+	i = initidx % CTL_MAX_INIT_PER_PORT;
+	if (lun->pending_ua[p] == NULL)
+		ua = CTL_UA_POWERON;
+	else
+		ua = lun->pending_ua[p][i];
+	if (ua == CTL_UA_NONE)
+		return (CTL_UA_NONE);
+
+	ua_to_build = (1 << (ffs(ua) - 1));
+	ua_to_clear = ua_to_build;
+	info = NULL;
+	ctl_ua_to_acsq(lun, ua_to_build, &asc, &ascq, &ua_to_clear, &info);
+
+	resp[0] = SSD_KEY_UNIT_ATTENTION;
+	if (ua_to_build == ua)
+		resp[0] |= 0x10;
+	else
+		resp[0] |= 0x20;
+	resp[1] = asc;
+	resp[2] = ascq;
+	return (ua);
+}
+
+ctl_ua_type
+ctl_build_ua(struct ctl_lun *lun, uint32_t initidx,
+    struct scsi_sense_data *sense, scsi_sense_data_type sense_format)
+{
+	ctl_ua_type *ua;
+	ctl_ua_type ua_to_build, ua_to_clear;
+	uint8_t *info;
+	int asc, ascq;
+	uint32_t p, i;
+
+	mtx_assert(&lun->lun_lock, MA_OWNED);
+	p = initidx / CTL_MAX_INIT_PER_PORT;
+	if ((ua = lun->pending_ua[p]) == NULL) {
+		mtx_unlock(&lun->lun_lock);
+		ua = malloc(sizeof(ctl_ua_type) * CTL_MAX_INIT_PER_PORT,
+		    M_CTL, M_WAITOK);
+		mtx_lock(&lun->lun_lock);
+		if (lun->pending_ua[p] == NULL) {
+			lun->pending_ua[p] = ua;
+			for (i = 0; i < CTL_MAX_INIT_PER_PORT; i++)
+				ua[i] = CTL_UA_POWERON;
+		} else {
+			free(ua, M_CTL);
+			ua = lun->pending_ua[p];
+		}
+	}
+	i = initidx % CTL_MAX_INIT_PER_PORT;
+	if (ua[i] == CTL_UA_NONE)
+		return (CTL_UA_NONE);
+
+	ua_to_build = (1 << (ffs(ua[i]) - 1));
+	ua_to_clear = ua_to_build;
+	info = NULL;
+	ctl_ua_to_acsq(lun, ua_to_build, &asc, &ascq, &ua_to_clear, &info);
+
+	ctl_set_sense_data(sense, lun, sense_format, /*current_error*/ 1,
+	    /*sense_key*/ SSD_KEY_UNIT_ATTENTION, asc, ascq,
+	    ((info != NULL) ? SSD_ELEM_INFO : SSD_ELEM_SKIP), 8, info,
+	    SSD_ELEM_NONE);
 
 	/* We're reporting this UA, so clear it */
 	ua[i] &= ~ua_to_clear;
@@ -561,11 +612,8 @@ ctl_set_invalid_field(struct ctl_scsiio *ctsio, int sks_valid, int command,
 void
 ctl_set_invalid_opcode(struct ctl_scsiio *ctsio)
 {
-	struct scsi_sense_data *sense;
 	uint8_t sks[3];
 
-	sense = &ctsio->sense_data;
-
 	sks[0] = SSD_SCS_VALID | SSD_FIELDPTR_CMD;
 	scsi_ulto2b(0, &sks[1]);
 
@@ -642,9 +690,9 @@ ctl_set_internal_failure(struct ctl_scsiio *ctsio, int sks_valid,
 }
 
 void
-ctl_set_medium_error(struct ctl_scsiio *ctsio)
+ctl_set_medium_error(struct ctl_scsiio *ctsio, int read)
 {
-	if ((ctsio->io_hdr.flags & CTL_FLAG_DATA_MASK) == CTL_FLAG_DATA_IN) {
+	if (read) {
 		/* "Unrecovered read error" */
 		ctl_set_sense(ctsio,
 			      /*current_error*/ 1,
@@ -699,7 +747,7 @@ ctl_set_lun_stopped(struct ctl_scsiio *ctsio)
 }
 
 void
-ctl_set_lun_not_ready(struct ctl_scsiio *ctsio)
+ctl_set_lun_int_reqd(struct ctl_scsiio *ctsio)
 {
 	/* "Logical unit not ready, manual intervention required" */
 	ctl_set_sense(ctsio,
@@ -711,6 +759,30 @@ ctl_set_lun_not_ready(struct ctl_scsiio *ctsio)
 }
 
 void
+ctl_set_lun_ejected(struct ctl_scsiio *ctsio)
+{
+	/* "Medium not present - tray open" */
+	ctl_set_sense(ctsio,
+		      /*current_error*/ 1,
+		      /*sense_key*/ SSD_KEY_NOT_READY,
+		      /*asc*/ 0x3A,
+		      /*ascq*/ 0x02,
+		      SSD_ELEM_NONE);
+}
+
+void
+ctl_set_lun_no_media(struct ctl_scsiio *ctsio)
+{
+	/* "Medium not present - tray closed" */
+	ctl_set_sense(ctsio,
+		      /*current_error*/ 1,
+		      /*sense_key*/ SSD_KEY_NOT_READY,
+		      /*asc*/ 0x3A,
+		      /*ascq*/ 0x01,
+		      SSD_ELEM_NONE);
+}
+
+void
 ctl_set_illegal_pr_release(struct ctl_scsiio *ctsio)
 {
 	/* "Invalid release of persistent reservation" */
@@ -843,6 +915,18 @@ ctl_set_task_aborted(struct ctl_scsiio *ctsio)
 }
 
 void
+ctl_set_hw_write_protected(struct ctl_scsiio *ctsio)
+{
+	/* "Hardware write protected" */
+	ctl_set_sense(ctsio,
+		      /*current_error*/ 1,
+		      /*sense_key*/ SSD_KEY_DATA_PROTECT,
+		      /*asc*/ 0x27,
+		      /*ascq*/ 0x01,
+		      SSD_ELEM_NONE);
+}
+
+void
 ctl_set_space_alloc_fail(struct ctl_scsiio *ctsio)
 {
 	/* "Space allocation failed write protect" */
diff --git a/sys/cam/ctl/ctl_error.h b/sys/cam/ctl/ctl_error.h
index 4fa8060ea3a03..98706438e5674 100644
--- a/sys/cam/ctl/ctl_error.h
+++ b/sys/cam/ctl/ctl_error.h
@@ -1,5 +1,6 @@
 /*-
  * Copyright (c) 2003 Silicon Graphics International Corp.
+ * Copyright (c) 2014-2015 Alexander Motin <mav@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -57,6 +58,7 @@ void ctl_sense_to_desc(struct scsi_sense_data_fixed *sense_src,
 void ctl_sense_to_fixed(struct scsi_sense_data_desc *sense_src,
 			struct scsi_sense_data_fixed *sense_dest);
 void ctl_set_ua(struct ctl_scsiio *ctsio, int asc, int ascq);
+ctl_ua_type ctl_build_qae(struct ctl_lun *lun, uint32_t initidx, uint8_t *resp);
 ctl_ua_type ctl_build_ua(struct ctl_lun *lun, uint32_t initidx,
     struct scsi_sense_data *sense, scsi_sense_data_type sense_format);
 void ctl_set_overlapped_cmd(struct ctl_scsiio *ctsio);
@@ -72,11 +74,13 @@ void ctl_set_lun_standby(struct ctl_scsiio *ctsio);
 void ctl_set_lun_unavail(struct ctl_scsiio *ctsio);
 void ctl_set_internal_failure(struct ctl_scsiio *ctsio, int sks_valid,
 			      uint16_t retry_count);
-void ctl_set_medium_error(struct ctl_scsiio *ctsio);
+void ctl_set_medium_error(struct ctl_scsiio *ctsio, int read);
 void ctl_set_aborted(struct ctl_scsiio *ctsio);
 void ctl_set_lba_out_of_range(struct ctl_scsiio *ctsio);
 void ctl_set_lun_stopped(struct ctl_scsiio *ctsio);
-void ctl_set_lun_not_ready(struct ctl_scsiio *ctsio);
+void ctl_set_lun_int_reqd(struct ctl_scsiio *ctsio);
+void ctl_set_lun_ejected(struct ctl_scsiio *ctsio);
+void ctl_set_lun_no_media(struct ctl_scsiio *ctsio);
 void ctl_set_illegal_pr_release(struct ctl_scsiio *ctsio);
 void ctl_set_medium_format_corrupted(struct ctl_scsiio *ctsio);
 void ctl_set_medium_magazine_inaccessible(struct ctl_scsiio *ctsio);
@@ -85,6 +89,7 @@ void ctl_set_reservation_conflict(struct ctl_scsiio *ctsio);
 void ctl_set_queue_full(struct ctl_scsiio *ctsio);
 void ctl_set_busy(struct ctl_scsiio *ctsio);
 void ctl_set_task_aborted(struct ctl_scsiio *ctsio);
+void ctl_set_hw_write_protected(struct ctl_scsiio *ctsio);
 void ctl_set_space_alloc_fail(struct ctl_scsiio *ctsio);
 void ctl_set_success(struct ctl_scsiio *ctsio);
 
diff --git a/sys/cam/ctl/ctl_frontend.c b/sys/cam/ctl/ctl_frontend.c
index c9c75d4046611..87ed9d573856b 100644
--- a/sys/cam/ctl/ctl_frontend.c
+++ b/sys/cam/ctl/ctl_frontend.c
@@ -145,9 +145,8 @@ ctl_port_register(struct ctl_port *port)
 	int port_num;
 	int retval;
 
-	retval = 0;
-
 	KASSERT(softc != NULL, ("CTL is not initialized"));
+	port->ctl_softc = softc;
 
 	mtx_lock(&softc->ctl_lock);
 	if (port->targ_port >= 0)
@@ -218,7 +217,7 @@ error:
 int
 ctl_port_deregister(struct ctl_port *port)
 {
-	struct ctl_softc *softc = control_softc;
+	struct ctl_softc *softc = port->ctl_softc;
 	struct ctl_io_pool *pool;
 	int retval, i;
 
@@ -309,7 +308,7 @@ ctl_port_set_wwns(struct ctl_port *port, int wwnn_valid, uint64_t wwnn,
 void
 ctl_port_online(struct ctl_port *port)
 {
-	struct ctl_softc *softc = control_softc;
+	struct ctl_softc *softc = port->ctl_softc;
 	struct ctl_lun *lun;
 	uint32_t l;
 
@@ -328,15 +327,23 @@ ctl_port_online(struct ctl_port *port)
 	}
 	if (port->port_online != NULL)
 		port->port_online(port->onoff_arg);
-	/* XXX KDM need a lock here? */
+	mtx_lock(&softc->ctl_lock);
 	port->status |= CTL_PORT_STATUS_ONLINE;
+	STAILQ_FOREACH(lun, &softc->lun_list, links) {
+		if (ctl_lun_map_to_port(port, lun->lun) >= CTL_MAX_LUNS)
+			continue;
+		mtx_lock(&lun->lun_lock);
+		ctl_est_ua_all(lun, -1, CTL_UA_INQ_CHANGE);
+		mtx_unlock(&lun->lun_lock);
+	}
+	mtx_unlock(&softc->ctl_lock);
 	ctl_isc_announce_port(port);
 }
 
 void
 ctl_port_offline(struct ctl_port *port)
 {
-	struct ctl_softc *softc = control_softc;
+	struct ctl_softc *softc = port->ctl_softc;
 	struct ctl_lun *lun;
 	uint32_t l;
 
@@ -355,8 +362,16 @@ ctl_port_offline(struct ctl_port *port)
 				port->lun_disable(port->targ_lun_arg, lun->lun);
 		}
 	}
-	/* XXX KDM need a lock here? */
+	mtx_lock(&softc->ctl_lock);
 	port->status &= ~CTL_PORT_STATUS_ONLINE;
+	STAILQ_FOREACH(lun, &softc->lun_list, links) {
+		if (ctl_lun_map_to_port(port, lun->lun) >= CTL_MAX_LUNS)
+			continue;
+		mtx_lock(&lun->lun_lock);
+		ctl_est_ua_all(lun, -1, CTL_UA_INQ_CHANGE);
+		mtx_unlock(&lun->lun_lock);
+	}
+	mtx_unlock(&softc->ctl_lock);
 	ctl_isc_announce_port(port);
 }
 
diff --git a/sys/cam/ctl/ctl_frontend.h b/sys/cam/ctl/ctl_frontend.h
index a6ca9e16492f4..9a1a4ee03cbba 100644
--- a/sys/cam/ctl/ctl_frontend.h
+++ b/sys/cam/ctl/ctl_frontend.h
@@ -211,6 +211,7 @@ struct ctl_wwpn_iid {
  *			  shouldn't touch this field.
  */
 struct ctl_port {
+	struct ctl_softc *ctl_softc;
 	struct ctl_frontend *frontend;
 	ctl_port_type	port_type;		/* passed to CTL */
 	int		num_requested_ctl_io;	/* passed to CTL */
diff --git a/sys/cam/ctl/ctl_frontend_cam_sim.c b/sys/cam/ctl/ctl_frontend_cam_sim.c
index 01807de1a3553..428cf4460c83f 100644
--- a/sys/cam/ctl/ctl_frontend_cam_sim.c
+++ b/sys/cam/ctl/ctl_frontend_cam_sim.c
@@ -130,13 +130,9 @@ cfcs_init(void)
 	struct cfcs_softc *softc;
 	struct ccb_setasync csa;
 	struct ctl_port *port;
-#ifdef NEEDTOPORT
-	char wwnn[8];
-#endif
 	int retval;
 
 	softc = &cfcs_softc;
-	retval = 0;
 	bzero(softc, sizeof(*softc));
 	mtx_init(&softc->lock, "ctl2cam", NULL, MTX_DEF);
 	port = &softc->port;
@@ -168,15 +164,6 @@ cfcs_init(void)
 	}
 
 	/*
-	 * Get the WWNN out of the database, and create a WWPN as well.
-	 */
-#ifdef NEEDTOPORT
-	ddb_GetWWNN((char *)wwnn);
-	softc->wwnn = be64dec(wwnn);
-	softc->wwpn = softc->wwnn + (softc->port.targ_port & 0xff);
-#endif
-
-	/*
 	 * If the CTL frontend didn't tell us what our WWNN/WWPN is, go
 	 * ahead and set something random.
 	 */
@@ -435,6 +422,14 @@ cfcs_datamove(union ctl_io *io)
 
 	io->scsiio.ext_data_filled += len_copied;
 
+	if ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS) {
+		io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = NULL;
+		io->io_hdr.flags |= CTL_FLAG_STATUS_SENT;
+		ccb->ccb_h.status &= ~CAM_STATUS_MASK;
+		ccb->ccb_h.status |= CAM_REQ_CMP;
+		xpt_done(ccb);
+	}
+
 	io->scsiio.be_move_done(io);
 }
 
@@ -458,12 +453,13 @@ cfcs_done(union ctl_io *io)
 	/*
 	 * Translate CTL status to CAM status.
 	 */
+	ccb->ccb_h.status &= ~CAM_STATUS_MASK;
 	switch (io->io_hdr.status & CTL_STATUS_MASK) {
 	case CTL_SUCCESS:
-		ccb->ccb_h.status = CAM_REQ_CMP;
+		ccb->ccb_h.status |= CAM_REQ_CMP;
 		break;
 	case CTL_SCSI_ERROR:
-		ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR | CAM_AUTOSNS_VALID;
+		ccb->ccb_h.status |= CAM_SCSI_STATUS_ERROR | CAM_AUTOSNS_VALID;
 		ccb->csio.scsi_status = io->scsiio.scsi_status;
 		bcopy(&io->scsiio.sense_data, &ccb->csio.sense_data,
 		      min(io->scsiio.sense_len, ccb->csio.sense_len));
@@ -479,14 +475,18 @@ cfcs_done(union ctl_io *io)
 		}
 		break;
 	case CTL_CMD_ABORTED:
-		ccb->ccb_h.status = CAM_REQ_ABORTED;
+		ccb->ccb_h.status |= CAM_REQ_ABORTED;
 		break;
 	case CTL_ERROR:
 	default:
-		ccb->ccb_h.status = CAM_REQ_CMP_ERR;
+		ccb->ccb_h.status |= CAM_REQ_CMP_ERR;
 		break;
 	}
-
+	if ((ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP &&
+	    (ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
+		xpt_freeze_devq(ccb->ccb_h.path, 1);
+		ccb->ccb_h.status |= CAM_DEV_QFRZN;
+	}
 	xpt_done(ccb);
 	ctl_free_io(io);
 }
diff --git a/sys/cam/ctl/ctl_frontend_ioctl.c b/sys/cam/ctl/ctl_frontend_ioctl.c
index 67156bca57882..6ef2f666c1120 100644
--- a/sys/cam/ctl/ctl_frontend_ioctl.c
+++ b/sys/cam/ctl/ctl_frontend_ioctl.c
@@ -56,6 +56,18 @@ __FBSDID("$FreeBSD$");
 #include <cam/ctl/ctl_debug.h>
 #include <cam/ctl/ctl_error.h>
 
+typedef enum {
+	CTL_IOCTL_INPROG,
+	CTL_IOCTL_DATAMOVE,
+	CTL_IOCTL_DONE
+} ctl_fe_ioctl_state;
+
+struct ctl_fe_ioctl_params {
+	struct cv		sem;
+	struct mtx		ioctl_mtx;
+	ctl_fe_ioctl_state	state;
+};
+
 struct cfi_softc {
 	uint32_t		cur_tag_num;
 	struct ctl_port		port;
@@ -149,7 +161,7 @@ ctl_ioctl_do_datamove(struct ctl_scsiio *ctsio)
 	 * To simplify things here, if we have a single buffer, stick it in
 	 * a S/G entry and just make it a single entry S/G list.
 	 */
-	if (ctsio->io_hdr.flags & CTL_FLAG_EDPTR_SGLIST) {
+	if (ctsio->ext_sg_entries > 0) {
 		int len_seen;
 
 		ext_sglen = ctsio->ext_sg_entries * sizeof(*ext_sglist);
@@ -157,11 +169,8 @@ ctl_ioctl_do_datamove(struct ctl_scsiio *ctsio)
 		ext_sglist = (struct ctl_sg_entry *)malloc(ext_sglen, M_CTL,
 							   M_WAITOK);
 		ext_sglist_malloced = 1;
-		if (copyin(ctsio->ext_data_ptr, ext_sglist,
-				   ext_sglen) != 0) {
-			ctl_set_internal_failure(ctsio,
-						 /*sks_valid*/ 0,
-						 /*retry_count*/ 0);
+		if (copyin(ctsio->ext_data_ptr, ext_sglist, ext_sglen) != 0) {
+			ctsio->io_hdr.port_status = 31343;
 			goto bailout;
 		}
 		ext_sg_entries = ctsio->ext_sg_entries;
@@ -229,9 +238,7 @@ ctl_ioctl_do_datamove(struct ctl_scsiio *ctsio)
 			CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove: from %p "
 					 "to %p\n", kern_ptr, ext_ptr));
 			if (copyout(kern_ptr, ext_ptr, len_to_copy) != 0) {
-				ctl_set_internal_failure(ctsio,
-							 /*sks_valid*/ 0,
-							 /*retry_count*/ 0);
+				ctsio->io_hdr.port_status = 31344;
 				goto bailout;
 			}
 		} else {
@@ -240,9 +247,7 @@ ctl_ioctl_do_datamove(struct ctl_scsiio *ctsio)
 			CTL_DEBUG_PRINT(("ctl_ioctl_do_datamove: from %p "
 					 "to %p\n", ext_ptr, kern_ptr));
 			if (copyin(ext_ptr, kern_ptr, len_to_copy)!= 0){
-				ctl_set_internal_failure(ctsio,
-							 /*sks_valid*/ 0,
-							 /*retry_count*/0);
+				ctsio->io_hdr.port_status = 31345;
 				goto bailout;
 			}
 		}
@@ -314,10 +319,7 @@ cfi_submit_wait(union ctl_io *io)
 	ctl_fe_ioctl_state last_state;
 	int done, retval;
 
-	retval = 0;
-
 	bzero(&params, sizeof(params));
-
 	mtx_init(&params.ioctl_mtx, "ctliocmtx", NULL, MTX_DEF);
 	cv_init(&params.sem, "ctlioccv");
 	params.state = CTL_IOCTL_INPROG;
diff --git a/sys/cam/ctl/ctl_frontend_iscsi.c b/sys/cam/ctl/ctl_frontend_iscsi.c
index 6ea862f366380..30cca91efca94 100644
--- a/sys/cam/ctl/ctl_frontend_iscsi.c
+++ b/sys/cam/ctl/ctl_frontend_iscsi.c
@@ -174,7 +174,6 @@ static void	cfiscsi_target_release(struct cfiscsi_target *ct);
 static void	cfiscsi_session_delete(struct cfiscsi_session *cs);
 
 static struct cfiscsi_softc cfiscsi_softc;
-extern struct ctl_softc *control_softc;
 
 static struct ctl_frontend cfiscsi_frontend =
 {
@@ -462,6 +461,7 @@ cfiscsi_decode_lun(uint64_t encoded)
 			break;
 		}
 		result = (lun[1] << 16) + (lun[2] << 8) + lun[3];
+		break;
 	default:
 		CFISCSI_WARN("unsupported LUN format 0x%jx",
 		    (uintmax_t)encoded);
@@ -639,6 +639,12 @@ cfiscsi_pdu_handle_task_request(struct icl_pdu *request)
 #endif
 		io->taskio.task_action = CTL_TASK_ABORT_TASK_SET;
 		break;
+	case BHSTMR_FUNCTION_CLEAR_TASK_SET:
+#if 0
+		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_CLEAR_TASK_SET");
+#endif
+		io->taskio.task_action = CTL_TASK_CLEAR_TASK_SET;
+		break;
 	case BHSTMR_FUNCTION_LOGICAL_UNIT_RESET:
 #if 0
 		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_LOGICAL_UNIT_RESET");
@@ -651,6 +657,37 @@ cfiscsi_pdu_handle_task_request(struct icl_pdu *request)
 #endif
 		io->taskio.task_action = CTL_TASK_TARGET_RESET;
 		break;
+	case BHSTMR_FUNCTION_TARGET_COLD_RESET:
+#if 0
+		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_TARGET_COLD_RESET");
+#endif
+		io->taskio.task_action = CTL_TASK_TARGET_RESET;
+		break;
+	case BHSTMR_FUNCTION_QUERY_TASK:
+#if 0
+		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_QUERY_TASK");
+#endif
+		io->taskio.task_action = CTL_TASK_QUERY_TASK;
+		io->taskio.tag_num = bhstmr->bhstmr_referenced_task_tag;
+		break;
+	case BHSTMR_FUNCTION_QUERY_TASK_SET:
+#if 0
+		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_QUERY_TASK_SET");
+#endif
+		io->taskio.task_action = CTL_TASK_QUERY_TASK_SET;
+		break;
+	case BHSTMR_FUNCTION_I_T_NEXUS_RESET:
+#if 0
+		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_I_T_NEXUS_RESET");
+#endif
+		io->taskio.task_action = CTL_TASK_I_T_NEXUS_RESET;
+		break;
+	case BHSTMR_FUNCTION_QUERY_ASYNC_EVENT:
+#if 0
+		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_FUNCTION_QUERY_ASYNC_EVENT");
+#endif
+		io->taskio.task_action = CTL_TASK_QUERY_ASYNC_EVENT;
+		break;
 	default:
 		CFISCSI_SESSION_DEBUG(cs, "unsupported function 0x%x",
 		    bhstmr->bhstmr_function & ~0x80);
@@ -929,6 +966,7 @@ cfiscsi_pdu_handle_data_out(struct icl_pdu *request)
 		done = (io->scsiio.ext_data_filled != cdw->cdw_r2t_end ||
 		    io->scsiio.ext_data_filled == io->scsiio.kern_data_len);
 		cfiscsi_data_wait_free(cs, cdw);
+		io->io_hdr.flags &= ~CTL_FLAG_DMA_INPROG;
 		if (done)
 			io->scsiio.be_move_done(io);
 		else
@@ -1141,6 +1179,7 @@ cfiscsi_session_terminate_tasks(struct cfiscsi_session *cs)
 		 * assuming that the data transfer actually succeeded
 		 * and writing uninitialized data to disk.
 		 */
+		cdw->cdw_ctl_io->io_hdr.flags &= ~CTL_FLAG_DMA_INPROG;
 		cdw->cdw_ctl_io->scsiio.io_hdr.port_status = 42;
 		cdw->cdw_ctl_io->scsiio.be_move_done(cdw->cdw_ctl_io);
 		cfiscsi_data_wait_free(cs, cdw);
@@ -1339,10 +1378,8 @@ int
 cfiscsi_init(void)
 {
 	struct cfiscsi_softc *softc;
-	int retval;
 
 	softc = &cfiscsi_softc;
-	retval = 0;
 	bzero(softc, sizeof(*softc));
 	mtx_init(&softc->lock, "cfiscsi", NULL, MTX_DEF);
 
@@ -1572,6 +1609,16 @@ restart:
 			if (cs2 != cs && cs2->cs_tasks_aborted == false &&
 			    cs->cs_target == cs2->cs_target &&
 			    strcmp(cs->cs_initiator_id, cs2->cs_initiator_id) == 0) {
+				if (strcmp(cs->cs_initiator_addr,
+				    cs2->cs_initiator_addr) != 0) {
+					CFISCSI_SESSION_WARN(cs2,
+					    "session reinstatement from "
+					    "different address %s",
+					    cs->cs_initiator_addr);
+				} else {
+					CFISCSI_SESSION_DEBUG(cs2,
+					    "session reinstatement");
+				}
 				cfiscsi_session_terminate(cs2);
 				mtx_unlock(&softc->lock);
 				pause("cfiscsi_reinstate", 1);
@@ -2701,6 +2748,7 @@ cfiscsi_datamove_out(union ctl_io *io)
 		cfiscsi_session_terminate(cs);
 		return;
 	}
+	io->io_hdr.flags |= CTL_FLAG_DMA_INPROG;
 	bhsr2t = (struct iscsi_bhs_r2t *)response->ip_bhs;
 	bhsr2t->bhsr2t_opcode = ISCSI_BHS_OPCODE_R2T;
 	bhsr2t->bhsr2t_flags = 0x80;
@@ -2842,7 +2890,9 @@ cfiscsi_task_management_done(union ctl_io *io)
 	struct iscsi_bhs_task_management_request *bhstmr;
 	struct iscsi_bhs_task_management_response *bhstmr2;
 	struct cfiscsi_data_wait *cdw, *tmpcdw;
-	struct cfiscsi_session *cs;
+	struct cfiscsi_session *cs, *tcs;
+	struct cfiscsi_softc *softc;
+	int cold_reset = 0;
 
 	request = io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
 	cs = PDU_SESSION(request);
@@ -2875,34 +2925,55 @@ cfiscsi_task_management_done(union ctl_io *io)
 #endif
 			TAILQ_REMOVE(&cs->cs_waiting_for_data_out,
 			    cdw, cdw_next);
+			io->io_hdr.flags &= ~CTL_FLAG_DMA_INPROG;
+			cdw->cdw_ctl_io->scsiio.io_hdr.port_status = 43;
 			cdw->cdw_ctl_io->scsiio.be_move_done(cdw->cdw_ctl_io);
 			cfiscsi_data_wait_free(cs, cdw);
 		}
 		CFISCSI_SESSION_UNLOCK(cs);
 	}
+	if ((bhstmr->bhstmr_function & ~0x80) ==
+	    BHSTMR_FUNCTION_TARGET_COLD_RESET &&
+	    io->io_hdr.status == CTL_SUCCESS)
+		cold_reset = 1;
 
 	response = cfiscsi_pdu_new_response(request, M_WAITOK);
 	bhstmr2 = (struct iscsi_bhs_task_management_response *)
 	    response->ip_bhs;
 	bhstmr2->bhstmr_opcode = ISCSI_BHS_OPCODE_TASK_RESPONSE;
 	bhstmr2->bhstmr_flags = 0x80;
-	if (io->io_hdr.status == CTL_SUCCESS) {
+	switch (io->taskio.task_status) {
+	case CTL_TASK_FUNCTION_COMPLETE:
 		bhstmr2->bhstmr_response = BHSTMR_RESPONSE_FUNCTION_COMPLETE;
-	} else {
-		/*
-		 * XXX: How to figure out what exactly went wrong?  iSCSI spec
-		 * 	expects us to provide detailed error, e.g. "Task does
-		 * 	not exist" or "LUN does not exist".
-		 */
-		CFISCSI_SESSION_DEBUG(cs, "BHSTMR_RESPONSE_FUNCTION_NOT_SUPPORTED");
-		bhstmr2->bhstmr_response =
-		    BHSTMR_RESPONSE_FUNCTION_NOT_SUPPORTED;
+		break;
+	case CTL_TASK_FUNCTION_SUCCEEDED:
+		bhstmr2->bhstmr_response = BHSTMR_RESPONSE_FUNCTION_SUCCEEDED;
+		break;
+	case CTL_TASK_LUN_DOES_NOT_EXIST:
+		bhstmr2->bhstmr_response = BHSTMR_RESPONSE_LUN_DOES_NOT_EXIST;
+		break;
+	case CTL_TASK_FUNCTION_NOT_SUPPORTED:
+	default:
+		bhstmr2->bhstmr_response = BHSTMR_RESPONSE_FUNCTION_NOT_SUPPORTED;
+		break;
 	}
+	memcpy(bhstmr2->bhstmr_additional_reponse_information,
+	    io->taskio.task_resp, sizeof(io->taskio.task_resp));
 	bhstmr2->bhstmr_initiator_task_tag = bhstmr->bhstmr_initiator_task_tag;
 
 	ctl_free_io(io);
 	icl_pdu_free(request);
 	cfiscsi_pdu_queue(response);
+
+	if (cold_reset) {
+		softc = cs->cs_target->ct_softc;
+		mtx_lock(&softc->lock);
+		TAILQ_FOREACH(tcs, &softc->sessions, cs_next) {
+			if (tcs->cs_target == cs->cs_target)
+				cfiscsi_session_terminate(tcs);
+		}
+		mtx_unlock(&softc->lock);
+	}
 }
 
 static void
diff --git a/sys/cam/ctl/ctl_ha.c b/sys/cam/ctl/ctl_ha.c
index 8d6604fe2725a..5e31b41f948f8 100644
--- a/sys/cam/ctl/ctl_ha.c
+++ b/sys/cam/ctl/ctl_ha.c
@@ -155,11 +155,11 @@ struct ha_softc {
 	int		 ha_receiving;
 	int		 ha_wakeup;
 	int		 ha_disconnect;
+	int		 ha_shutdown;
+	eventhandler_tag ha_shutdown_eh;
 	TAILQ_HEAD(, ctl_ha_dt_req) ha_dts;
 } ha_softc;
 
-extern struct ctl_softc *control_softc;
-
 static void
 ctl_ha_conn_wake(struct ha_softc *softc)
 {
@@ -281,8 +281,9 @@ ctl_ha_rx_thread(void *arg)
 		else
 			next = sizeof(wire_hdr);
 		SOCKBUF_LOCK(&so->so_rcv);
-		while (sbavail(&so->so_rcv) < next) {
-			if (softc->ha_connected == 0 || so->so_error ||
+		while (sbavail(&so->so_rcv) < next || softc->ha_disconnect) {
+			if (softc->ha_connected == 0 || softc->ha_disconnect ||
+			    so->so_error ||
 			    (so->so_rcv.sb_state & SBS_CANTRCVMORE)) {
 				goto errout;
 			}
@@ -426,6 +427,7 @@ static int
 ctl_ha_connect(struct ha_softc *softc)
 {
 	struct thread *td = curthread;
+	struct sockaddr_in sa;
 	struct socket *so;
 	int error;
 
@@ -439,7 +441,8 @@ ctl_ha_connect(struct ha_softc *softc)
 	softc->ha_so = so;
 	ctl_ha_sock_setup(softc);
 
-	error = soconnect(so, (struct sockaddr *)&softc->ha_peer_in, td);
+	memcpy(&sa, &softc->ha_peer_in, sizeof(sa));
+	error = soconnect(so, (struct sockaddr *)&sa, td);
 	if (error != 0) {
 		printf("%s: soconnect() error %d\n", __func__, error);
 		goto out;
@@ -516,6 +519,7 @@ static int
 ctl_ha_listen(struct ha_softc *softc)
 {
 	struct thread *td = curthread;
+	struct sockaddr_in sa;
 	struct sockopt opt;
 	int error, val;
 
@@ -539,12 +543,25 @@ ctl_ha_listen(struct ha_softc *softc)
 			printf("%s: REUSEADDR setting failed %d\n",
 			    __func__, error);
 		}
+		bzero(&opt, sizeof(struct sockopt));
+		opt.sopt_dir = SOPT_SET;
+		opt.sopt_level = SOL_SOCKET;
+		opt.sopt_name = SO_REUSEPORT;
+		opt.sopt_val = &val;
+		opt.sopt_valsize = sizeof(val);
+		val = 1;
+		error = sosetopt(softc->ha_lso, &opt);
+		if (error) {
+			printf("%s: REUSEPORT setting failed %d\n",
+			    __func__, error);
+		}
 		SOCKBUF_LOCK(&softc->ha_lso->so_rcv);
 		soupcall_set(softc->ha_lso, SO_RCV, ctl_ha_lupcall, softc);
 		SOCKBUF_UNLOCK(&softc->ha_lso->so_rcv);
 	}
 
-	error = sobind(softc->ha_lso, (struct sockaddr *)&softc->ha_peer_in, td);
+	memcpy(&sa, &softc->ha_peer_in, sizeof(sa));
+	error = sobind(softc->ha_lso, (struct sockaddr *)&sa, td);
 	if (error != 0) {
 		printf("%s: sobind() error %d\n", __func__, error);
 		goto out;
@@ -568,10 +585,13 @@ ctl_ha_conn_thread(void *arg)
 	int error;
 
 	while (1) {
-		if (softc->ha_disconnect) {
+		if (softc->ha_disconnect || softc->ha_shutdown) {
 			ctl_ha_close(softc);
-			ctl_ha_lclose(softc);
+			if (softc->ha_disconnect == 2 || softc->ha_shutdown)
+				ctl_ha_lclose(softc);
 			softc->ha_disconnect = 0;
+			if (softc->ha_shutdown)
+				break;
 		} else if (softc->ha_so != NULL &&
 		    (softc->ha_so->so_error ||
 		     softc->ha_so->so_rcv.sb_state & SBS_CANTRCVMORE))
@@ -614,6 +634,11 @@ ctl_ha_conn_thread(void *arg)
 		softc->ha_wakeup = 0;
 		mtx_unlock(&softc->ha_lock);
 	}
+	mtx_lock(&softc->ha_lock);
+	softc->ha_shutdown = 2;
+	wakeup(&softc->ha_wakeup);
+	mtx_unlock(&softc->ha_lock);
+	kthread_exit();
 }
 
 static int
@@ -622,28 +647,33 @@ ctl_ha_peer_sysctl(SYSCTL_HANDLER_ARGS)
 	struct ha_softc *softc = (struct ha_softc *)arg1;
 	struct sockaddr_in *sa;
 	int error, b1, b2, b3, b4, p, num;
+	char buf[128];
 
-	error = sysctl_handle_string(oidp, softc->ha_peer,
-	    sizeof(softc->ha_peer), req);
-	if ((error != 0) || (req->newptr == NULL))
+	strlcpy(buf, softc->ha_peer, sizeof(buf));
+	error = sysctl_handle_string(oidp, buf, sizeof(buf), req);
+	if ((error != 0) || (req->newptr == NULL) ||
+	    strncmp(buf, softc->ha_peer, sizeof(buf)) == 0)
 		return (error);
 
 	sa = &softc->ha_peer_in;
 	mtx_lock(&softc->ha_lock);
-	if ((num = sscanf(softc->ha_peer, "connect %d.%d.%d.%d:%d",
+	if ((num = sscanf(buf, "connect %d.%d.%d.%d:%d",
 	    &b1, &b2, &b3, &b4, &p)) >= 4) {
 		softc->ha_connect = 1;
 		softc->ha_listen = 0;
-	} else if ((num = sscanf(softc->ha_peer, "listen %d.%d.%d.%d:%d",
+	} else if ((num = sscanf(buf, "listen %d.%d.%d.%d:%d",
 	    &b1, &b2, &b3, &b4, &p)) >= 4) {
 		softc->ha_connect = 0;
 		softc->ha_listen = 1;
 	} else {
 		softc->ha_connect = 0;
 		softc->ha_listen = 0;
-		if (softc->ha_peer[0] != 0)
+		if (buf[0] != 0) {
+			buf[0] = 0;
 			error = EINVAL;
+		}
 	}
+	strlcpy(softc->ha_peer, buf, sizeof(softc->ha_peer));
 	if (softc->ha_connect || softc->ha_listen) {
 		memset(sa, 0, sizeof(*sa));
 		sa->sin_len = sizeof(struct sockaddr_in);
@@ -652,7 +682,7 @@ ctl_ha_peer_sysctl(SYSCTL_HANDLER_ARGS)
 		sa->sin_addr.s_addr =
 		    htonl((b1 << 24) + (b2 << 16) + (b3 << 8) + b4);
 	}
-	softc->ha_disconnect = 1;
+	softc->ha_disconnect = 2;
 	softc->ha_wakeup = 1;
 	mtx_unlock(&softc->ha_lock);
 	wakeup(&softc->ha_wakeup);
@@ -797,6 +827,19 @@ ctl_ha_msg_send(ctl_ha_channel channel, const void *addr, size_t len,
 	return (ctl_ha_msg_send2(channel, addr, len, NULL, 0, wait));
 }
 
+ctl_ha_status
+ctl_ha_msg_abort(ctl_ha_channel channel)
+{
+	struct ha_softc *softc = &ha_softc;
+
+	mtx_lock(&softc->ha_lock);
+	softc->ha_disconnect = 1;
+	softc->ha_wakeup = 1;
+	mtx_unlock(&softc->ha_lock);
+	wakeup(&softc->ha_wakeup);
+	return (CTL_HA_STATUS_SUCCESS);
+}
+
 /*
  * Allocate a data transfer request structure.
  */
@@ -931,6 +974,8 @@ ctl_ha_msg_init(struct ctl_softc *ctl_softc)
 		mtx_destroy(&softc->ha_lock);
 		return (CTL_HA_STATUS_ERROR);
 	}
+	softc->ha_shutdown_eh = EVENTHANDLER_REGISTER(shutdown_pre_sync,
+	    ctl_ha_msg_shutdown, ctl_softc, SHUTDOWN_PRI_FIRST);
 	SYSCTL_ADD_PROC(&ctl_softc->sysctl_ctx,
 	    SYSCTL_CHILDREN(ctl_softc->sysctl_tree),
 	    OID_AUTO, "ha_peer", CTLTYPE_STRING | CTLFLAG_RWTUN,
@@ -944,15 +989,41 @@ ctl_ha_msg_init(struct ctl_softc *ctl_softc)
 	return (CTL_HA_STATUS_SUCCESS);
 };
 
-ctl_ha_status
+void
 ctl_ha_msg_shutdown(struct ctl_softc *ctl_softc)
 {
 	struct ha_softc *softc = &ha_softc;
 
-	if (ctl_ha_msg_deregister(CTL_HA_CHAN_DATA) != CTL_HA_STATUS_SUCCESS) {
-		printf("%s: ctl_ha_msg_deregister failed.\n", __func__);
+	/* Disconnect and shutdown threads. */
+	mtx_lock(&softc->ha_lock);
+	if (softc->ha_shutdown < 2) {
+		softc->ha_shutdown = 1;
+		softc->ha_wakeup = 1;
+		wakeup(&softc->ha_wakeup);
+		while (softc->ha_shutdown < 2) {
+			msleep(&softc->ha_wakeup, &softc->ha_lock, 0,
+			    "shutdown", hz);
+		}
+	}
+	mtx_unlock(&softc->ha_lock);
+};
+
+ctl_ha_status
+ctl_ha_msg_destroy(struct ctl_softc *ctl_softc)
+{
+	struct ha_softc *softc = &ha_softc;
+
+	if (softc->ha_shutdown_eh != NULL) {
+		EVENTHANDLER_DEREGISTER(shutdown_pre_sync,
+		    softc->ha_shutdown_eh);
+		softc->ha_shutdown_eh = NULL;
 	}
 
+	ctl_ha_msg_shutdown(ctl_softc);	/* Just in case. */
+
+	if (ctl_ha_msg_deregister(CTL_HA_CHAN_DATA) != CTL_HA_STATUS_SUCCESS)
+		printf("%s: ctl_ha_msg_deregister failed.\n", __func__);
+
 	mtx_destroy(&softc->ha_lock);
 	return (CTL_HA_STATUS_SUCCESS);
 };
diff --git a/sys/cam/ctl/ctl_ha.h b/sys/cam/ctl/ctl_ha.h
index 072492ae4771d..f38f6402267c9 100644
--- a/sys/cam/ctl/ctl_ha.h
+++ b/sys/cam/ctl/ctl_ha.h
@@ -109,7 +109,8 @@ struct ctl_ha_dt_req {
 
 struct ctl_softc;
 ctl_ha_status ctl_ha_msg_init(struct ctl_softc *softc);
-ctl_ha_status ctl_ha_msg_shutdown(struct ctl_softc *softc);
+void ctl_ha_msg_shutdown(struct ctl_softc *softc);
+ctl_ha_status ctl_ha_msg_destroy(struct ctl_softc *softc);
 
 typedef void (*ctl_evt_handler)(ctl_ha_channel channel, ctl_ha_event event,
 				int param);
@@ -124,6 +125,7 @@ ctl_ha_status ctl_ha_msg_send(ctl_ha_channel channel, const void *addr,
     size_t len, int wait);
 ctl_ha_status ctl_ha_msg_send2(ctl_ha_channel channel, const void *addr,
     size_t len, const void *addr2, size_t len2, int wait);
+ctl_ha_status ctl_ha_msg_abort(ctl_ha_channel channel);
 ctl_ha_status ctl_ha_msg_deregister(ctl_ha_channel channel);
 
 struct ctl_ha_dt_req * ctl_dt_req_alloc(void);
diff --git a/sys/cam/ctl/ctl_io.h b/sys/cam/ctl/ctl_io.h
index e85d0a50475ba..21db94a6f8322 100644
--- a/sys/cam/ctl/ctl_io.h
+++ b/sys/cam/ctl/ctl_io.h
@@ -1,5 +1,6 @@
 /*-
  * Copyright (c) 2003 Silicon Graphics International Corp.
+ * Copyright (c) 2014-2015 Alexander Motin <mav@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -88,8 +89,6 @@ typedef enum {
 	CTL_FLAG_DATA_OUT	= 0x00000002,	/* DATA OUT */
 	CTL_FLAG_DATA_NONE	= 0x00000003,	/* no data */
 	CTL_FLAG_DATA_MASK	= 0x00000003,
-	CTL_FLAG_KDPTR_SGLIST	= 0x00000008, 	/* kern_data_ptr is S/G list*/
-	CTL_FLAG_EDPTR_SGLIST	= 0x00000010,	/* ext_data_ptr is S/G list */
 	CTL_FLAG_DO_AUTOSENSE	= 0x00000020,	/* grab sense info */
 	CTL_FLAG_USER_REQ	= 0x00000040,	/* request came from userland */
 	CTL_FLAG_ALLOCATED	= 0x00000100,	/* data space allocated */
@@ -115,7 +114,8 @@ typedef enum {
 
 	CTL_FLAG_FAILOVER	= 0x04000000,	/* Killed by a failover */
 	CTL_FLAG_IO_ACTIVE	= 0x08000000,	/* I/O active on this SC */
-	CTL_FLAG_STATUS_SENT	= 0x10000000	/* Status sent by datamove */
+	CTL_FLAG_STATUS_SENT	= 0x10000000,	/* Status sent by datamove */
+	CTL_FLAG_SERSEQ_DONE	= 0x20000000	/* All storage I/O started */
 } ctl_io_flags;
 
 
@@ -197,6 +197,9 @@ typedef enum {
 	CTL_MSG_UA,			/* Set/clear UA on secondary. */
 	CTL_MSG_PORT_SYNC,		/* Information about port. */
 	CTL_MSG_LUN_SYNC,		/* Information about LUN. */
+	CTL_MSG_IID_SYNC,		/* Information about initiator. */
+	CTL_MSG_LOGIN,			/* Information about HA peer. */
+	CTL_MSG_MODE_SYNC,		/* Mode page current content. */
 	CTL_MSG_FAILOVER		/* Fake, never sent though the wire */
 } ctl_msg_type;
 
@@ -221,8 +224,8 @@ struct ctl_io_hdr {
 	struct bintime	  start_bt;	/* Timer start ticks */
 	struct bintime	  dma_start_bt;	/* DMA start ticks */
 	struct bintime	  dma_bt;	/* DMA total ticks */
-	uint32_t	  num_dmas;	/* Number of DMAs */
 #endif /* CTL_TIME_IO */
+	uint32_t	  num_dmas;	/* Number of DMAs */
 	union ctl_io	  *original_sc;
 	union ctl_io	  *serializing_sc;
 	void		  *pool;	/* I/O pool */
@@ -328,9 +331,20 @@ typedef enum {
 	CTL_TASK_TARGET_RESET,
 	CTL_TASK_BUS_RESET,
 	CTL_TASK_PORT_LOGIN,
-	CTL_TASK_PORT_LOGOUT
+	CTL_TASK_PORT_LOGOUT,
+	CTL_TASK_QUERY_TASK,
+	CTL_TASK_QUERY_TASK_SET,
+	CTL_TASK_QUERY_ASYNC_EVENT
 } ctl_task_type;
 
+typedef enum {
+	CTL_TASK_FUNCTION_COMPLETE,
+	CTL_TASK_FUNCTION_SUCCEEDED,
+	CTL_TASK_FUNCTION_REJECTED,
+	CTL_TASK_LUN_DOES_NOT_EXIST,
+	CTL_TASK_FUNCTION_NOT_SUPPORTED
+} ctl_task_status;
+
 /*
  * Task management I/O structure.  Aborts, bus resets, etc., are sent using
  * this structure.
@@ -343,6 +357,27 @@ struct ctl_taskio {
 	ctl_task_type		task_action; /* Target Reset, Abort, etc.  */
 	uint32_t		tag_num;     /* tag number */
 	ctl_tag_type		tag_type;    /* simple, ordered, etc. */
+	uint8_t			task_status; /* Complete, Succeeded, etc. */
+	uint8_t			task_resp[3];/* Response information */
+};
+
+
+/*
+ * HA link messages.
+ */
+#define	CTL_HA_VERSION		1
+
+/*
+ * Used for CTL_MSG_LOGIN.
+ */
+struct ctl_ha_msg_login {
+	ctl_msg_type		msg_type;
+	int			version;
+	int			ha_mode;
+	int			ha_id;
+	int			max_luns;
+	int			max_ports;
+	int			max_init_per_port;
 };
 
 typedef enum {
@@ -370,10 +405,10 @@ struct ctl_pr_info {
 
 struct ctl_ha_msg_hdr {
 	ctl_msg_type		msg_type;
+	uint32_t		status;	     /* transaction status */
 	union ctl_io		*original_sc;
 	union ctl_io		*serializing_sc;
 	struct ctl_nexus	nexus;	     /* Initiator, port, target, lun */
-	uint32_t		status;	     /* transaction status */
 };
 
 #define	CTL_HA_MAX_SG_ENTRIES	16
@@ -395,6 +430,7 @@ struct ctl_ha_msg_ua {
 	int			ua_all;
 	int			ua_set;
 	int			ua_type;
+	uint8_t			ua_info[8];
 };
 
 /*
@@ -439,7 +475,6 @@ struct ctl_ha_msg_scsi {
 	uint32_t		residual;    /* data residual length */
 	uint32_t		fetd_status; /* trans status, set by FETD,
 						0 = good*/
-	struct ctl_lba_len	lbalen;      /* used for stats */
 	struct scsi_sense_data	sense_data;  /* sense data */
 };
 
@@ -466,6 +501,7 @@ struct ctl_ha_msg_port {
 	int			lun_map_len;
 	int			port_devid_len;
 	int			target_devid_len;
+	int			init_devid_len;
 	uint8_t			data[];
 };
 
@@ -488,6 +524,28 @@ struct ctl_ha_msg_lun_pr_key {
 	uint64_t		pr_key;
 };
 
+/*
+ * Used for CTL_MSG_IID_SYNC.
+ */
+struct ctl_ha_msg_iid {
+	struct ctl_ha_msg_hdr	hdr;
+	int			in_use;
+	int			name_len;
+	uint64_t		wwpn;
+	uint8_t			data[];
+};
+
+/*
+ * Used for CTL_MSG_MODE_SYNC.
+ */
+struct ctl_ha_msg_mode {
+	struct ctl_ha_msg_hdr	hdr;
+	uint8_t			page_code;
+	uint8_t			subpage;
+	uint16_t		page_len;
+	uint8_t			data[];
+};
+
 union ctl_ha_msg {
 	struct ctl_ha_msg_hdr	hdr;
 	struct ctl_ha_msg_task	task;
@@ -497,16 +555,16 @@ union ctl_ha_msg {
 	struct ctl_ha_msg_ua	ua;
 	struct ctl_ha_msg_port	port;
 	struct ctl_ha_msg_lun	lun;
+	struct ctl_ha_msg_iid	iid;
+	struct ctl_ha_msg_login	login;
+	struct ctl_ha_msg_mode	mode;
 };
 
-
 struct ctl_prio {
 	struct ctl_io_hdr  io_hdr;
 	struct ctl_ha_msg_pr pr_msg;
 };
 
-
-
 union ctl_io {
 	struct ctl_io_hdr io_hdr;	/* common to all I/O types */
 	struct ctl_scsiio scsiio;	/* Normal SCSI commands */
@@ -520,7 +578,6 @@ union ctl_io *ctl_alloc_io(void *pool_ref);
 union ctl_io *ctl_alloc_io_nowait(void *pool_ref);
 void ctl_free_io(union ctl_io *io);
 void ctl_zero_io(union ctl_io *io);
-void ctl_copy_io(union ctl_io *src, union ctl_io *dest);
 
 #endif /* _KERNEL */
 
diff --git a/sys/cam/ctl/ctl_ioctl.h b/sys/cam/ctl/ctl_ioctl.h
index a691a1d00fb28..7a6fc785382e9 100644
--- a/sys/cam/ctl/ctl_ioctl.h
+++ b/sys/cam/ctl/ctl_ioctl.h
@@ -81,17 +81,6 @@
 #define	CTL_MINOR	225
 
 typedef enum {
-	CTL_OOA_INVALID_LUN,
-	CTL_OOA_SUCCESS
-} ctl_ooa_status;
-
-struct ctl_ooa_info {
-	uint32_t lun_id;	/* Passed in to CTL */
-	uint32_t num_entries;	/* Returned from CTL */
-	ctl_ooa_status status;	/* Returned from CTL */
-};
-
-typedef enum {
 	CTL_DELAY_TYPE_NONE,
 	CTL_DELAY_TYPE_CONT,
 	CTL_DELAY_TYPE_ONESHOT
@@ -121,22 +110,6 @@ struct ctl_io_delay_info {
 };
 
 typedef enum {
-	CTL_GS_SYNC_NONE,
-	CTL_GS_SYNC_OK,
-	CTL_GS_SYNC_NO_LUN
-} ctl_gs_sync_status;
-
-/*
- * The target and LUN id specify which device to modify.  The sync interval
- * means that we will let through every N SYNCHRONIZE CACHE commands.
- */
-struct ctl_sync_info {
-	uint32_t		lun_id;		/* passed to kernel */
-	int			sync_interval;	/* depends on whether get/set */
-	ctl_gs_sync_status	status;		/* passed from kernel */
-};
-
-typedef enum {
 	CTL_STATS_NO_IO,
 	CTL_STATS_READ,
 	CTL_STATS_WRITE
@@ -322,23 +295,6 @@ struct ctl_ooa {
 };
 
 typedef enum {
-	CTL_PORT_LIST_NONE,
-	CTL_PORT_LIST_OK,
-	CTL_PORT_LIST_NEED_MORE_SPACE,
-	CTL_PORT_LIST_ERROR
-} ctl_port_list_status;
-
-struct ctl_port_list {
-	uint32_t		alloc_len;	/* passed to kernel */
-	uint32_t		alloc_num;	/* passed to kernel */
-	struct ctl_port_entry   *entries;	/* filled in kernel */
-	uint32_t		fill_len;	/* passed to userland */
-	uint32_t		fill_num;	/* passed to userland */
-	uint32_t		dropped_num;	/* passed to userland */
-	ctl_port_list_status	status;		/* passed to userland */
-};
-
-typedef enum {
 	CTL_LUN_NOSTATUS,
 	CTL_LUN_OK,
 	CTL_LUN_ERROR,
@@ -804,18 +760,11 @@ struct ctl_lun_map {
 #define	CTL_IO			_IOWR(CTL_MINOR, 0x00, union ctl_io)
 #define	CTL_ENABLE_PORT		_IOW(CTL_MINOR, 0x04, struct ctl_port_entry)
 #define	CTL_DISABLE_PORT	_IOW(CTL_MINOR, 0x05, struct ctl_port_entry)
-#define	CTL_DUMP_OOA		_IO(CTL_MINOR, 0x06)
-#define	CTL_CHECK_OOA		_IOWR(CTL_MINOR, 0x07, struct ctl_ooa_info)
 #define	CTL_DELAY_IO		_IOWR(CTL_MINOR, 0x10, struct ctl_io_delay_info)
-#define	CTL_REALSYNC_GET	_IOR(CTL_MINOR, 0x11, int)
-#define	CTL_REALSYNC_SET	_IOW(CTL_MINOR, 0x12, int)
-#define	CTL_SETSYNC		_IOWR(CTL_MINOR, 0x13, struct ctl_sync_info)
-#define	CTL_GETSYNC		_IOWR(CTL_MINOR, 0x14, struct ctl_sync_info)
 #define	CTL_GETSTATS		_IOWR(CTL_MINOR, 0x15, struct ctl_stats)
 #define	CTL_ERROR_INJECT	_IOWR(CTL_MINOR, 0x16, struct ctl_error_desc)
 #define	CTL_GET_OOA		_IOWR(CTL_MINOR, 0x18, struct ctl_ooa)
 #define	CTL_DUMP_STRUCTS	_IO(CTL_MINOR, 0x19)
-#define	CTL_GET_PORT_LIST	_IOWR(CTL_MINOR, 0x20, struct ctl_port_list)
 #define	CTL_LUN_REQ		_IOWR(CTL_MINOR, 0x21, struct ctl_lun_req)
 #define	CTL_LUN_LIST		_IOWR(CTL_MINOR, 0x22, struct ctl_lun_list)
 #define	CTL_ERROR_INJECT_DELETE	_IOW(CTL_MINOR, 0x23, struct ctl_error_desc)
diff --git a/sys/cam/ctl/ctl_private.h b/sys/cam/ctl/ctl_private.h
index c42c2c9fec95c..826d00492e176 100644
--- a/sys/cam/ctl/ctl_private.h
+++ b/sys/cam/ctl/ctl_private.h
@@ -1,5 +1,6 @@
 /*-
  * Copyright (c) 2003, 2004, 2005, 2008 Silicon Graphics International Corp.
+ * Copyright (c) 2014-2015 Alexander Motin <mav@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -39,26 +40,19 @@
 #ifndef	_CTL_PRIVATE_H_
 #define	_CTL_PRIVATE_H_
 
+#include <cam/scsi/scsi_all.h>
+#include <cam/scsi/scsi_cd.h>
+#include <cam/scsi/scsi_da.h>
+
 /*
  * SCSI vendor and product names.
  */
 #define	CTL_VENDOR		"FREEBSD "
 #define	CTL_DIRECT_PRODUCT	"CTLDISK         "
 #define	CTL_PROCESSOR_PRODUCT	"CTLPROCESSOR    "
+#define	CTL_CDROM_PRODUCT	"CTLCDROM        "
 #define	CTL_UNKNOWN_PRODUCT	"CTLDEVICE       "
 
-typedef enum {
-	CTL_IOCTL_INPROG,
-	CTL_IOCTL_DATAMOVE,
-	CTL_IOCTL_DONE
-} ctl_fe_ioctl_state;
-
-struct ctl_fe_ioctl_params {
-	struct cv		sem;
-	struct mtx		ioctl_mtx;
-	ctl_fe_ioctl_state	state;
-};
-
 #define CTL_POOL_ENTRIES_OTHER_SC   200
 
 struct ctl_io_pool {
@@ -91,25 +85,26 @@ typedef enum {
  * WARNING:  Keep the bottom nibble here free, we OR in the data direction
  * flags for each command.
  *
- * Note:  "OK_ON_ALL_LUNS" == we don't have to have a lun configured
+ * Note:  "OK_ON_NO_LUN"   == we don't have to have a lun configured
  *        "OK_ON_BOTH"     == we have to have a lun configured
  *        "SA5"            == command has 5-bit service action at byte 1
  */
 typedef enum {
 	CTL_CMD_FLAG_NONE		= 0x0000,
 	CTL_CMD_FLAG_NO_SENSE		= 0x0010,
-	CTL_CMD_FLAG_OK_ON_ALL_LUNS	= 0x0020,
-	CTL_CMD_FLAG_ALLOW_ON_RESV	= 0x0040,
+	CTL_CMD_FLAG_ALLOW_ON_RESV	= 0x0020,
+	CTL_CMD_FLAG_ALLOW_ON_PR_RESV	= 0x0040,
 	CTL_CMD_FLAG_ALLOW_ON_PR_WRESV	= 0x0080,
 	CTL_CMD_FLAG_OK_ON_PROC		= 0x0100,
-	CTL_CMD_FLAG_OK_ON_SLUN		= 0x0200,
-	CTL_CMD_FLAG_OK_ON_BOTH		= 0x0300,
-	CTL_CMD_FLAG_OK_ON_STOPPED	= 0x0400,
-	CTL_CMD_FLAG_OK_ON_INOPERABLE	= 0x0800,
-	CTL_CMD_FLAG_OK_ON_STANDBY	= 0x1000,
-	CTL_CMD_FLAG_OK_ON_UNAVAIL	= 0x2000,
-	CTL_CMD_FLAG_ALLOW_ON_PR_RESV	= 0x4000,
-	CTL_CMD_FLAG_SA5		= 0x8000
+	CTL_CMD_FLAG_OK_ON_DIRECT	= 0x0200,
+	CTL_CMD_FLAG_OK_ON_CDROM	= 0x0400,
+	CTL_CMD_FLAG_OK_ON_BOTH		= 0x0700,
+	CTL_CMD_FLAG_OK_ON_NO_LUN	= 0x0800,
+	CTL_CMD_FLAG_OK_ON_NO_MEDIA	= 0x1000,
+	CTL_CMD_FLAG_OK_ON_STANDBY	= 0x2000,
+	CTL_CMD_FLAG_OK_ON_UNAVAIL	= 0x4000,
+	CTL_CMD_FLAG_SA5		= 0x8000,
+	CTL_CMD_FLAG_RUN_HERE		= 0x10000
 } ctl_cmd_flags;
 
 typedef enum {
@@ -152,13 +147,14 @@ typedef enum {
 	CTL_LUN_DISABLED	= 0x008,
 	CTL_LUN_MALLOCED	= 0x010,
 	CTL_LUN_STOPPED		= 0x020,
-	CTL_LUN_INOPERABLE	= 0x040,
-	CTL_LUN_OFFLINE		= 0x080,
+	CTL_LUN_NO_MEDIA	= 0x040,
+	CTL_LUN_EJECTED		= 0x080,
 	CTL_LUN_PR_RESERVED	= 0x100,
 	CTL_LUN_PRIMARY_SC	= 0x200,
 	CTL_LUN_SENSE_DESC	= 0x400,
 	CTL_LUN_READONLY	= 0x800,
-	CTL_LUN_PEER_SC_PRIMARY	= 0x1000
+	CTL_LUN_PEER_SC_PRIMARY	= 0x1000,
+	CTL_LUN_REMOVABLE	= 0x2000
 } ctl_lun_flags;
 
 typedef enum {
@@ -242,7 +238,10 @@ typedef int	ctl_modesel_handler(struct ctl_scsiio *ctsio,
 
 typedef enum {
 	CTL_PAGE_FLAG_NONE	 = 0x00,
-	CTL_PAGE_FLAG_DISK_ONLY	 = 0x01
+	CTL_PAGE_FLAG_DIRECT	 = 0x01,
+	CTL_PAGE_FLAG_PROC	 = 0x02,
+	CTL_PAGE_FLAG_CDROM	 = 0x04,
+	CTL_PAGE_FLAG_ALL	 = 0x07
 } ctl_page_flags;
 
 struct ctl_page_index {
@@ -273,22 +272,29 @@ struct ctl_logical_block_provisioning_page {
 
 static const struct ctl_page_index page_index_template[] = {
 	{SMS_RW_ERROR_RECOVERY_PAGE, 0, sizeof(struct scsi_da_rw_recovery_page), NULL,
-	 CTL_PAGE_FLAG_DISK_ONLY, NULL, NULL},
+	 CTL_PAGE_FLAG_DIRECT | CTL_PAGE_FLAG_CDROM, NULL, NULL},
 	{SMS_FORMAT_DEVICE_PAGE, 0, sizeof(struct scsi_format_page), NULL,
-	 CTL_PAGE_FLAG_DISK_ONLY, NULL, NULL},
+	 CTL_PAGE_FLAG_DIRECT, NULL, NULL},
 	{SMS_RIGID_DISK_PAGE, 0, sizeof(struct scsi_rigid_disk_page), NULL,
-	 CTL_PAGE_FLAG_DISK_ONLY, NULL, NULL},
+	 CTL_PAGE_FLAG_DIRECT, NULL, NULL},
 	{SMS_CACHING_PAGE, 0, sizeof(struct scsi_caching_page), NULL,
-	 CTL_PAGE_FLAG_DISK_ONLY, NULL, ctl_caching_sp_handler},
+	 CTL_PAGE_FLAG_DIRECT | CTL_PAGE_FLAG_CDROM,
+	 NULL, ctl_caching_sp_handler},
 	{SMS_CONTROL_MODE_PAGE, 0, sizeof(struct scsi_control_page), NULL,
-	 CTL_PAGE_FLAG_NONE, NULL, ctl_control_page_handler},
+	 CTL_PAGE_FLAG_ALL, NULL, ctl_control_page_handler},
+	{SMS_CONTROL_MODE_PAGE | SMPH_SPF, 0x01,
+	 sizeof(struct scsi_control_ext_page), NULL,
+	 CTL_PAGE_FLAG_ALL, NULL, NULL},
 	{SMS_INFO_EXCEPTIONS_PAGE, 0, sizeof(struct scsi_info_exceptions_page), NULL,
-	 CTL_PAGE_FLAG_NONE, NULL, NULL},
+	 CTL_PAGE_FLAG_ALL, NULL, NULL},
 	{SMS_INFO_EXCEPTIONS_PAGE | SMPH_SPF, 0x02,
 	 sizeof(struct ctl_logical_block_provisioning_page), NULL,
-	 CTL_PAGE_FLAG_DISK_ONLY, NULL, NULL},
+	 CTL_PAGE_FLAG_DIRECT, NULL, NULL},
+	{SMS_CDDVD_CAPS_PAGE, 0,
+	 sizeof(struct scsi_cddvd_capabilities_page), NULL,
+	 CTL_PAGE_FLAG_CDROM, NULL, NULL},
 	{SMS_VENDOR_SPECIFIC_PAGE | SMPH_SPF, DBGCNF_SUBPAGE_CODE,
-	 sizeof(struct copan_debugconf_subpage), NULL, CTL_PAGE_FLAG_NONE,
+	 sizeof(struct copan_debugconf_subpage), NULL, CTL_PAGE_FLAG_ALL,
 	 ctl_debugconf_sp_sense_handler, ctl_debugconf_sp_select_handler},
 };
 
@@ -301,21 +307,23 @@ struct ctl_mode_pages {
 	struct scsi_rigid_disk_page	rigid_disk_page[4];
 	struct scsi_caching_page	caching_page[4];
 	struct scsi_control_page	control_page[4];
+	struct scsi_control_ext_page	control_ext_page[4];
 	struct scsi_info_exceptions_page ie_page[4];
 	struct ctl_logical_block_provisioning_page lbp_page[4];
+	struct scsi_cddvd_capabilities_page cddvd_page[4];
 	struct copan_debugconf_subpage	debugconf_subpage[4];
 	struct ctl_page_index		index[CTL_NUM_MODE_PAGES];
 };
 
 static const struct ctl_page_index log_page_index_template[] = {
 	{SLS_SUPPORTED_PAGES_PAGE, 0, 0, NULL,
-	 CTL_PAGE_FLAG_NONE, NULL, NULL},
+	 CTL_PAGE_FLAG_ALL, NULL, NULL},
 	{SLS_SUPPORTED_PAGES_PAGE, SLS_SUPPORTED_SUBPAGES_SUBPAGE, 0, NULL,
-	 CTL_PAGE_FLAG_NONE, NULL, NULL},
+	 CTL_PAGE_FLAG_ALL, NULL, NULL},
 	{SLS_LOGICAL_BLOCK_PROVISIONING, 0, 0, NULL,
-	 CTL_PAGE_FLAG_NONE, ctl_lbp_log_sense_handler, NULL},
+	 CTL_PAGE_FLAG_DIRECT, ctl_lbp_log_sense_handler, NULL},
 	{SLS_STAT_AND_PERF, 0, 0, NULL,
-	 CTL_PAGE_FLAG_NONE, ctl_sap_log_sense_handler, NULL},
+	 CTL_PAGE_FLAG_ALL, ctl_sap_log_sense_handler, NULL},
 };
 
 #define	CTL_NUM_LOG_PAGES sizeof(log_page_index_template)/   \
@@ -340,17 +348,6 @@ struct ctl_lun_delay_info {
 	uint32_t		done_delay;
 };
 
-typedef enum {
-	CTL_ERR_INJ_NONE	= 0x00,
-	CTL_ERR_INJ_ABORTED	= 0x01
-} ctl_err_inject_flags;
-
-typedef enum {
-	CTL_PR_FLAG_NONE	= 0x00,
-	CTL_PR_FLAG_REGISTERED	= 0x01,
-	CTL_PR_FLAG_ACTIVE_RES	= 0x02
-} ctl_per_res_flags;
-
 #define CTL_PR_ALL_REGISTRANTS  0xFFFFFFFF
 #define CTL_PR_NO_RESERVATION   0xFFFFFFF0
 
@@ -376,10 +373,7 @@ struct ctl_lun {
 	struct ctl_softc		*ctl_softc;
 	struct ctl_be_lun		*be_lun;
 	struct ctl_backend_driver	*backend;
-	int				io_count;
 	struct ctl_lun_delay_info	delay_info;
-	int				sync_interval;
-	int				sync_count;
 #ifdef CTL_TIME_IO
 	sbintime_t			idle_time;
 	sbintime_t			last_busy;
@@ -387,12 +381,12 @@ struct ctl_lun {
 	TAILQ_HEAD(ctl_ooaq, ctl_io_hdr)  ooa_queue;
 	TAILQ_HEAD(ctl_blockq,ctl_io_hdr) blocked_queue;
 	STAILQ_ENTRY(ctl_lun)		links;
-	STAILQ_ENTRY(ctl_lun)		run_links;
 #ifdef CTL_WITH_CA
 	uint32_t			have_ca[CTL_MAX_INITIATORS >> 5];
 	struct scsi_sense_data		pending_sense[CTL_MAX_INITIATORS];
 #endif
 	ctl_ua_type			*pending_ua[CTL_MAX_PORTS];
+	uint8_t				ua_tpt_info[8];
 	time_t				lasttpt;
 	struct ctl_mode_pages		mode_pages;
 	struct ctl_log_pages		log_pages;
@@ -403,13 +397,14 @@ struct ctl_lun {
 	int				pr_key_count;
 	uint32_t			pr_res_idx;
 	uint8_t				res_type;
+	int				prevent_count;
+	uint32_t			prevent[(CTL_MAX_INITIATORS+31)/32];
 	uint8_t				*write_buffer;
 	struct ctl_devid		*lun_devid;
 	TAILQ_HEAD(tpc_lists, tpc_list) tpc_lists;
 };
 
 typedef enum {
-	CTL_FLAG_REAL_SYNC	= 0x02,
 	CTL_FLAG_ACTIVE_SHELF	= 0x04
 } ctl_gen_flags;
 
@@ -429,8 +424,6 @@ struct tpc_token;
 struct ctl_softc {
 	struct mtx ctl_lock;
 	struct cdev *dev;
-	int open_count;
-	int num_disks;
 	int num_luns;
 	ctl_gen_flags flags;
 	ctl_ha_mode ha_mode;
@@ -446,7 +439,6 @@ struct ctl_softc {
 	struct sysctl_oid *sysctl_tree;
 	void *othersc_pool;
 	struct proc *ctl_proc;
-	int targ_online;
 	uint32_t ctl_lun_mask[(CTL_MAX_LUNS + 31) / 32];
 	struct ctl_lun *ctl_luns[CTL_MAX_LUNS];
 	uint32_t ctl_port_mask[(CTL_MAX_PORTS + 31) / 32];
@@ -484,6 +476,7 @@ void ctl_pool_free(struct ctl_io_pool *pool);
 int ctl_scsi_release(struct ctl_scsiio *ctsio);
 int ctl_scsi_reserve(struct ctl_scsiio *ctsio);
 int ctl_start_stop(struct ctl_scsiio *ctsio);
+int ctl_prevent_allow(struct ctl_scsiio *ctsio);
 int ctl_sync_cache(struct ctl_scsiio *ctsio);
 int ctl_format(struct ctl_scsiio *ctsio);
 int ctl_read_buffer(struct ctl_scsiio *ctsio);
@@ -496,6 +489,7 @@ int ctl_log_sense(struct ctl_scsiio *ctsio);
 int ctl_read_capacity(struct ctl_scsiio *ctsio);
 int ctl_read_capacity_16(struct ctl_scsiio *ctsio);
 int ctl_read_defect(struct ctl_scsiio *ctsio);
+int ctl_read_toc(struct ctl_scsiio *ctsio);
 int ctl_read_write(struct ctl_scsiio *ctsio);
 int ctl_cnw(struct ctl_scsiio *ctsio);
 int ctl_report_luns(struct ctl_scsiio *ctsio);
@@ -503,6 +497,9 @@ int ctl_request_sense(struct ctl_scsiio *ctsio);
 int ctl_tur(struct ctl_scsiio *ctsio);
 int ctl_verify(struct ctl_scsiio *ctsio);
 int ctl_inquiry(struct ctl_scsiio *ctsio);
+int ctl_get_config(struct ctl_scsiio *ctsio);
+int ctl_get_event_status(struct ctl_scsiio *ctsio);
+int ctl_mechanism_status(struct ctl_scsiio *ctsio);
 int ctl_persistent_reserve_in(struct ctl_scsiio *ctsio);
 int ctl_persistent_reserve_out(struct ctl_scsiio *ctsio);
 int ctl_report_tagret_port_groups(struct ctl_scsiio *ctsio);
diff --git a/sys/cam/ctl/ctl_ser_table.c b/sys/cam/ctl/ctl_ser_table.c
index 8d5d6bce81bab..ef0ee2fbbddc2 100644
--- a/sys/cam/ctl/ctl_ser_table.c
+++ b/sys/cam/ctl/ctl_ser_table.c
@@ -61,7 +61,7 @@
 #define	xO	CTL_SER_EXTENTOPT	/* Optional extent check */
 #define	xS	CTL_SER_EXTENTSEQ	/* Sequential extent check */
 
-static ctl_serialize_action
+const static ctl_serialize_action
 ctl_serialize_table[CTL_SERIDX_COUNT][CTL_SERIDX_COUNT] = {
 /**>IDX_ :: 2nd:TUR RD  WRT UNM SYN MDSN MDSL RQSN INQ RDCP RES LSNS FMT STR*/
 /*TUR     */{   pS, pS, pS, pS, pS, bK,  bK,  bK,  pS, pS,  bK, pS,  bK, bK},
diff --git a/sys/cam/ctl/ctl_tpc.c b/sys/cam/ctl/ctl_tpc.c
index e4722fecb1c0b..33c7db8c03712 100644
--- a/sys/cam/ctl/ctl_tpc.c
+++ b/sys/cam/ctl/ctl_tpc.c
@@ -394,8 +394,7 @@ ctl_inquiry_evpd_tpc(struct ctl_scsiio *ctsio, int alloc_len)
 	scsi_ulto2b(0, rtfb_ptr->optimal_length_granularity);
 	scsi_u64to8b(0, rtfb_ptr->maximum_bytes);
 	scsi_u64to8b(0, rtfb_ptr->optimal_bytes);
-	scsi_u64to8b(TPC_MAX_IOCHUNK_SIZE,
-	    rtfb_ptr->optimal_bytes_to_token_per_segment);
+	scsi_u64to8b(UINT64_MAX, rtfb_ptr->optimal_bytes_to_token_per_segment);
 	scsi_u64to8b(TPC_MAX_IOCHUNK_SIZE,
 	    rtfb_ptr->optimal_bytes_from_token_per_segment);
 
@@ -821,7 +820,9 @@ tpc_process_b2b(struct tpc_list *list)
 	off_t srclba, dstlba, numbytes, donebytes, roundbytes;
 	int numlba;
 	uint32_t srcblock, dstblock, pb, pbo, adj;
+	uint8_t csi[4];
 
+	scsi_ulto4b(list->curseg, csi);
 	if (list->stage == 1) {
 		while ((tior = TAILQ_FIRST(&list->allio)) != NULL) {
 			TAILQ_REMOVE(&list->allio, tior, links);
@@ -835,7 +836,9 @@ tpc_process_b2b(struct tpc_list *list)
 		} else if (list->error) {
 			ctl_set_sense(list->ctsio, /*current_error*/ 1,
 			    /*sense_key*/ SSD_KEY_COPY_ABORTED,
-			    /*asc*/ 0x0d, /*ascq*/ 0x01, SSD_ELEM_NONE);
+			    /*asc*/ 0x0d, /*ascq*/ 0x01,
+			    SSD_ELEM_COMMAND, sizeof(csi), csi,
+			    SSD_ELEM_NONE);
 			return (CTL_RETVAL_ERROR);
 		}
 		list->cursectors += list->segsectors;
@@ -850,7 +853,9 @@ tpc_process_b2b(struct tpc_list *list)
 	if (sl >= CTL_MAX_LUNS || dl >= CTL_MAX_LUNS) {
 		ctl_set_sense(list->ctsio, /*current_error*/ 1,
 		    /*sense_key*/ SSD_KEY_COPY_ABORTED,
-		    /*asc*/ 0x08, /*ascq*/ 0x04, SSD_ELEM_NONE);
+		    /*asc*/ 0x08, /*ascq*/ 0x04,
+		    SSD_ELEM_COMMAND, sizeof(csi), csi,
+		    SSD_ELEM_NONE);
 		return (CTL_RETVAL_ERROR);
 	}
 	if (pbo > 0)
@@ -879,7 +884,9 @@ tpc_process_b2b(struct tpc_list *list)
 	if (numbytes % srcblock != 0 || numbytes % dstblock != 0) {
 		ctl_set_sense(list->ctsio, /*current_error*/ 1,
 		    /*sense_key*/ SSD_KEY_COPY_ABORTED,
-		    /*asc*/ 0x26, /*ascq*/ 0x0A, SSD_ELEM_NONE);
+		    /*asc*/ 0x26, /*ascq*/ 0x0A,
+		    SSD_ELEM_COMMAND, sizeof(csi), csi,
+		    SSD_ELEM_NONE);
 		return (CTL_RETVAL_ERROR);
 	}
 
@@ -963,7 +970,9 @@ tpc_process_verify(struct tpc_list *list)
 	struct scsi_ec_segment_verify *seg;
 	struct tpc_io *tio;
 	uint64_t sl;
+	uint8_t csi[4];
 
+	scsi_ulto4b(list->curseg, csi);
 	if (list->stage == 1) {
 		while ((tio = TAILQ_FIRST(&list->allio)) != NULL) {
 			TAILQ_REMOVE(&list->allio, tio, links);
@@ -976,7 +985,9 @@ tpc_process_verify(struct tpc_list *list)
 		} else if (list->error) {
 			ctl_set_sense(list->ctsio, /*current_error*/ 1,
 			    /*sense_key*/ SSD_KEY_COPY_ABORTED,
-			    /*asc*/ 0x0d, /*ascq*/ 0x01, SSD_ELEM_NONE);
+			    /*asc*/ 0x0d, /*ascq*/ 0x01,
+			    SSD_ELEM_COMMAND, sizeof(csi), csi,
+			    SSD_ELEM_NONE);
 			return (CTL_RETVAL_ERROR);
 		} else
 			return (CTL_RETVAL_COMPLETE);
@@ -988,7 +999,9 @@ tpc_process_verify(struct tpc_list *list)
 	if (sl >= CTL_MAX_LUNS) {
 		ctl_set_sense(list->ctsio, /*current_error*/ 1,
 		    /*sense_key*/ SSD_KEY_COPY_ABORTED,
-		    /*asc*/ 0x08, /*ascq*/ 0x04, SSD_ELEM_NONE);
+		    /*asc*/ 0x08, /*ascq*/ 0x04,
+		    SSD_ELEM_COMMAND, sizeof(csi), csi,
+		    SSD_ELEM_NONE);
 		return (CTL_RETVAL_ERROR);
 	}
 
@@ -1020,7 +1033,9 @@ tpc_process_register_key(struct tpc_list *list)
 	struct tpc_io *tio;
 	uint64_t dl;
 	int datalen;
+	uint8_t csi[4];
 
+	scsi_ulto4b(list->curseg, csi);
 	if (list->stage == 1) {
 		while ((tio = TAILQ_FIRST(&list->allio)) != NULL) {
 			TAILQ_REMOVE(&list->allio, tio, links);
@@ -1034,7 +1049,9 @@ tpc_process_register_key(struct tpc_list *list)
 		} else if (list->error) {
 			ctl_set_sense(list->ctsio, /*current_error*/ 1,
 			    /*sense_key*/ SSD_KEY_COPY_ABORTED,
-			    /*asc*/ 0x0d, /*ascq*/ 0x01, SSD_ELEM_NONE);
+			    /*asc*/ 0x0d, /*ascq*/ 0x01,
+			    SSD_ELEM_COMMAND, sizeof(csi), csi,
+			    SSD_ELEM_NONE);
 			return (CTL_RETVAL_ERROR);
 		} else
 			return (CTL_RETVAL_COMPLETE);
@@ -1046,7 +1063,9 @@ tpc_process_register_key(struct tpc_list *list)
 	if (dl >= CTL_MAX_LUNS) {
 		ctl_set_sense(list->ctsio, /*current_error*/ 1,
 		    /*sense_key*/ SSD_KEY_COPY_ABORTED,
-		    /*asc*/ 0x08, /*ascq*/ 0x04, SSD_ELEM_NONE);
+		    /*asc*/ 0x08, /*ascq*/ 0x04,
+		    SSD_ELEM_COMMAND, sizeof(csi), csi,
+		    SSD_ELEM_NONE);
 		return (CTL_RETVAL_ERROR);
 	}
 
@@ -1347,6 +1366,7 @@ tpc_process(struct tpc_list *list)
 	struct scsi_ec_segment *seg;
 	struct ctl_scsiio *ctsio = list->ctsio;
 	int retval = CTL_RETVAL_COMPLETE;
+	uint8_t csi[4];
 
 	if (list->service_action == EC_WUT) {
 		if (list->token != NULL)
@@ -1374,9 +1394,12 @@ tpc_process(struct tpc_list *list)
 				retval = tpc_process_register_key(list);
 				break;
 			default:
+				scsi_ulto4b(list->curseg, csi);
 				ctl_set_sense(ctsio, /*current_error*/ 1,
 				    /*sense_key*/ SSD_KEY_COPY_ABORTED,
-				    /*asc*/ 0x26, /*ascq*/ 0x09, SSD_ELEM_NONE);
+				    /*asc*/ 0x26, /*ascq*/ 0x09,
+				    SSD_ELEM_COMMAND, sizeof(csi), csi,
+				    SSD_ELEM_NONE);
 				goto done;
 			}
 			if (retval == CTL_RETVAL_QUEUED)
@@ -1590,6 +1613,10 @@ ctl_extended_copy_lid1(struct ctl_scsiio *ctsio)
 	cdb = (struct scsi_extended_copy *)ctsio->cdb;
 	len = scsi_4btoul(cdb->length);
 
+	if (len == 0) {
+		ctl_set_success(ctsio);
+		goto done;
+	}
 	if (len < sizeof(struct scsi_extended_copy_lid1_data) ||
 	    len > sizeof(struct scsi_extended_copy_lid1_data) +
 	    TPC_MAX_LIST + TPC_MAX_INLINE) {
@@ -1620,20 +1647,22 @@ ctl_extended_copy_lid1(struct ctl_scsiio *ctsio)
 	lencscd = scsi_2btoul(data->cscd_list_length);
 	lenseg = scsi_4btoul(data->segment_list_length);
 	leninl = scsi_4btoul(data->inline_data_length);
-	if (len < sizeof(struct scsi_extended_copy_lid1_data) +
-	    lencscd + lenseg + leninl ||
-	    leninl > TPC_MAX_INLINE) {
-		ctl_set_invalid_field(ctsio, /*sks_valid*/ 1, /*command*/ 0,
-		    /*field*/ 2, /*bit_valid*/ 0, /*bit*/ 0);
-		goto done;
-	}
 	if (lencscd > TPC_MAX_CSCDS * sizeof(struct scsi_ec_cscd)) {
 		ctl_set_sense(ctsio, /*current_error*/ 1,
 		    /*sense_key*/ SSD_KEY_ILLEGAL_REQUEST,
 		    /*asc*/ 0x26, /*ascq*/ 0x06, SSD_ELEM_NONE);
 		goto done;
 	}
-	if (lencscd + lenseg > TPC_MAX_LIST) {
+	if (lenseg > TPC_MAX_SEGS * sizeof(struct scsi_ec_segment)) {
+		ctl_set_sense(ctsio, /*current_error*/ 1,
+		    /*sense_key*/ SSD_KEY_ILLEGAL_REQUEST,
+		    /*asc*/ 0x26, /*ascq*/ 0x08, SSD_ELEM_NONE);
+		goto done;
+	}
+	if (lencscd + lenseg > TPC_MAX_LIST ||
+	    leninl > TPC_MAX_INLINE ||
+	    len < sizeof(struct scsi_extended_copy_lid1_data) +
+	     lencscd + lenseg + leninl) {
 		ctl_set_param_len_error(ctsio);
 		goto done;
 	}
@@ -1717,6 +1746,10 @@ ctl_extended_copy_lid4(struct ctl_scsiio *ctsio)
 	cdb = (struct scsi_extended_copy *)ctsio->cdb;
 	len = scsi_4btoul(cdb->length);
 
+	if (len == 0) {
+		ctl_set_success(ctsio);
+		goto done;
+	}
 	if (len < sizeof(struct scsi_extended_copy_lid4_data) ||
 	    len > sizeof(struct scsi_extended_copy_lid4_data) +
 	    TPC_MAX_LIST + TPC_MAX_INLINE) {
@@ -1747,20 +1780,22 @@ ctl_extended_copy_lid4(struct ctl_scsiio *ctsio)
 	lencscd = scsi_2btoul(data->cscd_list_length);
 	lenseg = scsi_2btoul(data->segment_list_length);
 	leninl = scsi_2btoul(data->inline_data_length);
-	if (len < sizeof(struct scsi_extended_copy_lid4_data) +
-	    lencscd + lenseg + leninl ||
-	    leninl > TPC_MAX_INLINE) {
-		ctl_set_invalid_field(ctsio, /*sks_valid*/ 1, /*command*/ 0,
-		    /*field*/ 2, /*bit_valid*/ 0, /*bit*/ 0);
-		goto done;
-	}
 	if (lencscd > TPC_MAX_CSCDS * sizeof(struct scsi_ec_cscd)) {
 		ctl_set_sense(ctsio, /*current_error*/ 1,
 		    /*sense_key*/ SSD_KEY_ILLEGAL_REQUEST,
 		    /*asc*/ 0x26, /*ascq*/ 0x06, SSD_ELEM_NONE);
 		goto done;
 	}
-	if (lencscd + lenseg > TPC_MAX_LIST) {
+	if (lenseg > TPC_MAX_SEGS * sizeof(struct scsi_ec_segment)) {
+		ctl_set_sense(ctsio, /*current_error*/ 1,
+		    /*sense_key*/ SSD_KEY_ILLEGAL_REQUEST,
+		    /*asc*/ 0x26, /*ascq*/ 0x08, SSD_ELEM_NONE);
+		goto done;
+	}
+	if (lencscd + lenseg > TPC_MAX_LIST ||
+	    leninl > TPC_MAX_INLINE ||
+	    len < sizeof(struct scsi_extended_copy_lid1_data) +
+	     lencscd + lenseg + leninl) {
 		ctl_set_param_len_error(ctsio);
 		goto done;
 	}
diff --git a/sys/cam/ctl/ctl_tpc_local.c b/sys/cam/ctl/ctl_tpc_local.c
index 7664cb94b5bf9..4f368f95ff02b 100644
--- a/sys/cam/ctl/ctl_tpc_local.c
+++ b/sys/cam/ctl/ctl_tpc_local.c
@@ -142,10 +142,6 @@ tpcl_datamove(union ctl_io *io)
 	struct ctl_scsiio *ctsio;
 	int i, j;
 
-	ext_sg_start = 0;
-	ext_offset = 0;
-	ext_sglist = NULL;
-
 	CTL_DEBUG_PRINT(("%s\n", __func__));
 
 	ctsio = &io->scsiio;
@@ -162,7 +158,7 @@ tpcl_datamove(union ctl_io *io)
 	 * To simplify things here, if we have a single buffer, stick it in
 	 * a S/G entry and just make it a single entry S/G list.
 	 */
-	if (ctsio->io_hdr.flags & CTL_FLAG_EDPTR_SGLIST) {
+	if (ctsio->ext_sg_entries > 0) {
 		int len_seen;
 
 		ext_sglist = (struct ctl_sg_entry *)ctsio->ext_data_ptr;
@@ -281,7 +277,9 @@ tpcl_resolve(struct ctl_softc *softc, int init_port,
 	struct ctl_lun *lun;
 	uint64_t lunid = UINT64_MAX;
 
-	if (cscd->type_code != EC_CSCD_ID)
+	if (cscd->type_code != EC_CSCD_ID ||
+	    (cscd->luidt_pdt & EC_LUIDT_MASK) != EC_LUIDT_LUN ||
+	    (cscd->luidt_pdt & EC_NUL) != 0)
 		return (lunid);
 
 	cscdid = (struct scsi_ec_cscd_id *)cscd;
diff --git a/sys/cam/ctl/ctl_util.c b/sys/cam/ctl/ctl_util.c
index 8c57a223855a7..6fcec03f78d1c 100644
--- a/sys/cam/ctl/ctl_util.c
+++ b/sys/cam/ctl/ctl_util.c
@@ -89,7 +89,10 @@ static struct ctl_task_desc ctl_task_table[] = {
 	{CTL_TASK_TARGET_RESET, "Target Reset"},
 	{CTL_TASK_BUS_RESET, "Bus Reset"},
 	{CTL_TASK_PORT_LOGIN, "Port Login"},
-	{CTL_TASK_PORT_LOGOUT, "Port Logout"}
+	{CTL_TASK_PORT_LOGOUT, "Port Logout"},
+	{CTL_TASK_QUERY_TASK, "Query Task"},
+	{CTL_TASK_QUERY_TASK_SET, "Query Task Set"},
+	{CTL_TASK_QUERY_ASYNC_EVENT, "Query Async Event"}
 };
 
 void
@@ -486,8 +489,7 @@ ctl_scsi_mode_sense(union ctl_io *io, uint8_t *data_ptr, uint32_t data_len,
 
 void
 ctl_scsi_start_stop(union ctl_io *io, int start, int load_eject, int immediate,
-		    int power_conditions, int onoffline __unused,
-		    ctl_tag_type tag_type, uint8_t control)
+    int power_conditions, ctl_tag_type tag_type, uint8_t control)
 {
 	struct scsi_start_stop_unit *cdb;
 
@@ -498,10 +500,6 @@ ctl_scsi_start_stop(union ctl_io *io, int start, int load_eject, int immediate,
 	cdb->opcode = START_STOP_UNIT;
 	if (immediate)
 		cdb->byte2 |= SSS_IMMED;
-#ifdef NEEDTOPORT
-	if (onoffline)
-		cdb->byte2 |= SSS_ONOFFLINE;
-#endif
 	cdb->how = power_conditions;
 	if (load_eject)
 		cdb->how |= SSS_LOEJ;
@@ -846,24 +844,8 @@ void
 ctl_io_error_print(union ctl_io *io, struct scsi_inquiry_data *inq_data)
 {
 	char str[512];
-#ifdef NEEDTOPORT
-	char *message;
-	char *line;
-
-	message = io_error_string(io, inq_data, str, sizeof(str));
 
-	for (line = strsep(&message, "\n"); line != NULL;
-	     line = strsep(&message, "\n")) {
-		csevent_log(CSC_CTL | CSC_SHELF_SW | CTL_ERROR_REPORT,
-                            csevent_LogType_Trace,
-                            csevent_Severity_Information,
-                            csevent_AlertLevel_Green,
-                            csevent_FRU_Firmware,
-                            csevent_FRU_Unknown, "%s", line);
-	}
-#else
 	printf("%s", ctl_io_error_string(io, inq_data, str, sizeof(str)));
-#endif
 
 }
 
@@ -879,7 +861,7 @@ ctl_data_print(union ctl_io *io)
 		return;
 	if (io->io_hdr.flags & CTL_FLAG_BUS_ADDR)
 		return;
-	if (io->io_hdr.flags & CTL_FLAG_EDPTR_SGLIST)	/* XXX: Implement */
+	if (io->scsiio.ext_sg_entries > 0)	/* XXX: Implement */
 		return;
 	ctl_scsi_path_string(io, path_str, sizeof(path_str));
 	len = min(io->scsiio.kern_data_len, 4096);
diff --git a/sys/cam/ctl/ctl_util.h b/sys/cam/ctl/ctl_util.h
index b1c0d846f2bc8..2966b49164706 100644
--- a/sys/cam/ctl/ctl_util.h
+++ b/sys/cam/ctl/ctl_util.h
@@ -77,7 +77,7 @@ void ctl_scsi_mode_sense(union ctl_io *io, uint8_t *data_ptr,
 			 int minimum_cdb_size, ctl_tag_type tag_type,
 			 uint8_t control);
 void ctl_scsi_start_stop(union ctl_io *io, int start, int load_eject,
-			 int immediate, int power_conditions, int onoffline,
+			 int immediate, int power_conditions,
 			 ctl_tag_type tag_type, uint8_t control);
 void ctl_scsi_sync_cache(union ctl_io *io, int immed, int reladr,
 			 int minimum_cdb_size, uint64_t starting_lba,
diff --git a/sys/cam/ctl/scsi_ctl.c b/sys/cam/ctl/scsi_ctl.c
index 655afd8399811..25c745b5f85f8 100644
--- a/sys/cam/ctl/scsi_ctl.c
+++ b/sys/cam/ctl/scsi_ctl.c
@@ -1068,7 +1068,6 @@ ctlfe_adjust_cdb(struct ccb_accept_tio *atio, uint32_t offset)
 	}
 	case READ_16:
 	case WRITE_16:
-	case WRITE_ATOMIC_16:
 	{
 		struct scsi_rw_16 *cdb = (struct scsi_rw_16 *)cmdbyt;
 		lba = scsi_8btou64(cdb->addr);
diff --git a/sys/cam/scsi/scsi_all.c b/sys/cam/scsi/scsi_all.c
index a54c03b2b0906..023103bb9e1a4 100644
--- a/sys/cam/scsi/scsi_all.c
+++ b/sys/cam/scsi/scsi_all.c
@@ -175,7 +175,7 @@ static struct op_table_entry scsi_op_codes[] = {
 	 *
 	 * SCSI Operation Codes
 	 * Numeric Sorted Listing
-	 * as of  3/11/08
+	 * as of  5/26/15
 	 *
 	 *     D - DIRECT ACCESS DEVICE (SBC-2)                device column key
 	 *     .T - SEQUENTIAL ACCESS DEVICE (SSC-2)           -----------------
@@ -501,16 +501,22 @@ static struct op_table_entry scsi_op_codes[] = {
 	{ 0x93,	D, "WRITE SAME(16)" },
 	/* 93   M              ERASE(16) */
 	{ 0x93,	T, "ERASE(16)" },
-	/* 94 [usage proposed by SCSI Socket Services project] */
-	/* 95 [usage proposed by SCSI Socket Services project] */
-	/* 96 [usage proposed by SCSI Socket Services project] */
-	/* 97 [usage proposed by SCSI Socket Services project] */
+	/* 94  O               ZBC OUT */
+	{ 0x94,	D, "ZBC OUT" },
+	/* 95  O               ZBC OUT */
+	{ 0x95,	D, "ZBC OUT" },
+	/* 96 */
+	/* 97 */
 	/* 98 */
 	/* 99 */
-	/* 9A */
-	/* 9B */
-	/* 9C */
-	/* 9D */
+	/* 9A  O               WRITE STREAM(16) */
+	{ 0x9A,	D, "WRITE STREAM(16)" },
+	/* 9B  OOOOOOOOOO OOO  READ BUFFER(16) */
+	{ 0x9B,	ALL & ~(B) , "READ BUFFER(16)" },
+	/* 9C  O              WRITE ATOMIC(16) */
+	{ 0x9C, D, "WRITE ATOMIC(16)" },
+	/* 9D                  SERVICE ACTION BIDIRECTIONAL */
+	{ 0x9D, ALL, "SERVICE ACTION BIDIRECTIONAL" },
 	/* XXX KDM ALL for this?  op-num.txt defines it for none.. */
 	/* 9E                  SERVICE ACTION IN(16) */
 	{ 0x9E, ALL, "SERVICE ACTION IN(16)" },
@@ -968,7 +974,7 @@ static struct asc_table_entry asc_table[] = {
 	 *
 	 * SCSI ASC/ASCQ Assignments
 	 * Numeric Sorted Listing
-	 * as of  5/20/12
+	 * as of  8/12/15
 	 *
 	 * D - DIRECT ACCESS DEVICE (SBC-2)                   device column key
 	 * .T - SEQUENTIAL ACCESS DEVICE (SSC)               -------------------
@@ -1060,6 +1066,9 @@ static struct asc_table_entry asc_table[] = {
 	/* DT P      B    */
 	{ SST(0x00, 0x20, SS_RDEF,	/* XXX TBD */
 	    "Extended copy information available") },
+	/* D              */
+	{ SST(0x00, 0x21, SS_RDEF,	/* XXX TBD */
+	    "Atomic command aborted due to ACA") },
 	/* D   W O   BK   */
 	{ SST(0x01, 0x00, SS_RDEF,
 	    "No index/sector signal") },
@@ -1079,7 +1088,7 @@ static struct asc_table_entry asc_table[] = {
 	{ SST(0x04, 0x00, SS_RDEF,
 	    "Logical unit not ready, cause not reportable") },
 	/* DTLPWROMAEBKVF */
-	{ SST(0x04, 0x01, SS_TUR | SSQ_MANY | SSQ_DECREMENT_COUNT | EBUSY,
+	{ SST(0x04, 0x01, SS_WAIT | EBUSY,
 	    "Logical unit is in process of becoming ready") },
 	/* DTLPWROMAEBKVF */
 	{ SST(0x04, 0x02, SS_START | SSQ_DECREMENT_COUNT | ENXIO,
@@ -1106,7 +1115,7 @@ static struct asc_table_entry asc_table[] = {
 	{ SST(0x04, 0x09, SS_RDEF,	/* XXX TBD */
 	    "Logical unit not ready, self-test in progress") },
 	/* DTLPWROMAEBKVF */
-	{ SST(0x04, 0x0A, SS_TUR | SSQ_MANY | SSQ_DECREMENT_COUNT | ENXIO,
+	{ SST(0x04, 0x0A, SS_WAIT | ENXIO,
 	    "Logical unit not accessible, asymmetric access state transition")},
 	/* DTLPWROMAEBKVF */
 	{ SST(0x04, 0x0B, SS_FATAL | ENXIO,
@@ -1117,11 +1126,14 @@ static struct asc_table_entry asc_table[] = {
 	/*              F */
 	{ SST(0x04, 0x0D, SS_RDEF,	/* XXX TBD */
 	    "Logical unit not ready, structure check required") },
+	/* DTL WR MAEBKVF */
+	{ SST(0x04, 0x0E, SS_RDEF,	/* XXX TBD */
+	    "Logical unit not ready, security session in progress") },
 	/* DT  WROM  B    */
 	{ SST(0x04, 0x10, SS_RDEF,	/* XXX TBD */
 	    "Logical unit not ready, auxiliary memory not accessible") },
 	/* DT  WRO AEB VF */
-	{ SST(0x04, 0x11, SS_TUR | SSQ_MANY | SSQ_DECREMENT_COUNT | EBUSY,
+	{ SST(0x04, 0x11, SS_WAIT | EBUSY,
 	    "Logical unit not ready, notify (enable spinup) required") },
 	/*        M    V  */
 	{ SST(0x04, 0x12, SS_RDEF,	/* XXX TBD */
@@ -1156,6 +1168,24 @@ static struct asc_table_entry asc_table[] = {
 	/* DT     MAEB    */
 	{ SST(0x04, 0x1C, SS_RDEF,	/* XXX TBD */
 	    "Logical unit not ready, additional power use not yet granted") },
+	/* D              */
+	{ SST(0x04, 0x1D, SS_RDEF,	/* XXX TBD */
+	    "Logical unit not ready, configuration in progress") },
+	/* D              */
+	{ SST(0x04, 0x1E, SS_FATAL | ENXIO,
+	    "Logical unit not ready, microcode activation required") },
+	/* DTLPWROMAEBKVF */
+	{ SST(0x04, 0x1F, SS_FATAL | ENXIO,
+	    "Logical unit not ready, microcode download required") },
+	/* DTLPWROMAEBKVF */
+	{ SST(0x04, 0x20, SS_RDEF,	/* XXX TBD */
+	    "Logical unit not ready, logical unit reset required") },
+	/* DTLPWROMAEBKVF */
+	{ SST(0x04, 0x21, SS_RDEF,	/* XXX TBD */
+	    "Logical unit not ready, hard reset required") },
+	/* DTLPWROMAEBKVF */
+	{ SST(0x04, 0x22, SS_RDEF,	/* XXX TBD */
+	    "Logical unit not ready, power cycle required") },
 	/* DTL WROMAEBKVF */
 	{ SST(0x05, 0x00, SS_RDEF,
 	    "Logical unit does not respond to selection") },
@@ -1195,6 +1225,9 @@ static struct asc_table_entry asc_table[] = {
 	/* DT  WRO   B    */
 	{ SST(0x09, 0x04, SS_RDEF,
 	    "Head select fault") },
+	/* DT   RO   B    */
+	{ SST(0x09, 0x05, SS_RDEF,
+	    "Vibration induced tracking error") },
 	/* DTLPWROMAEBKVF */
 	{ SST(0x0A, 0x00, SS_FATAL | ENOSPC,
 	    "Error log overflow") },
@@ -1228,6 +1261,30 @@ static struct asc_table_entry asc_table[] = {
 	/* D              */
 	{ SST(0x0B, 0x09, SS_RDEF,	/* XXX TBD */
 	    "Warning - device statistics notification available") },
+	/* DTLPWROMAEBKVF */
+	{ SST(0x0B, 0x0A, SS_RDEF,	/* XXX TBD */
+	    "Warning - High critical temperature limit exceeded") },
+	/* DTLPWROMAEBKVF */
+	{ SST(0x0B, 0x0B, SS_RDEF,	/* XXX TBD */
+	    "Warning - Low critical temperature limit exceeded") },
+	/* DTLPWROMAEBKVF */
+	{ SST(0x0B, 0x0C, SS_RDEF,	/* XXX TBD */
+	    "Warning - High operating temperature limit exceeded") },
+	/* DTLPWROMAEBKVF */
+	{ SST(0x0B, 0x0D, SS_RDEF,	/* XXX TBD */
+	    "Warning - Low operating temperature limit exceeded") },
+	/* DTLPWROMAEBKVF */
+	{ SST(0x0B, 0x0E, SS_RDEF,	/* XXX TBD */
+	    "Warning - High citical humidity limit exceeded") },
+	/* DTLPWROMAEBKVF */
+	{ SST(0x0B, 0x0F, SS_RDEF,	/* XXX TBD */
+	    "Warning - Low citical humidity limit exceeded") },
+	/* DTLPWROMAEBKVF */
+	{ SST(0x0B, 0x10, SS_RDEF,	/* XXX TBD */
+	    "Warning - High operating humidity limit exceeded") },
+	/* DTLPWROMAEBKVF */
+	{ SST(0x0B, 0x11, SS_RDEF,	/* XXX TBD */
+	    "Warning - Low operating humidity limit exceeded") },
 	/*  T   R         */
 	{ SST(0x0C, 0x00, SS_RDEF,
 	    "Write error") },
@@ -1276,6 +1333,15 @@ static struct asc_table_entry asc_table[] = {
 	/*      R         */
 	{ SST(0x0C, 0x0F, SS_RDEF,	/* XXX TBD */
 	    "Defects in error window") },
+	/* D              */
+	{ SST(0x0C, 0x10, SS_RDEF,	/* XXX TBD */
+	    "Incomplete multiple atomic write operations") },
+	/* D              */
+	{ SST(0x0C, 0x11, SS_RDEF,	/* XXX TBD */
+	    "Write error - recovery scan needed") },
+	/* D              */
+	{ SST(0x0C, 0x12, SS_RDEF,	/* XXX TBD */
+	    "Write error - insufficient zone resources") },
 	/* DTLPWRO A  K   */
 	{ SST(0x0D, 0x00, SS_RDEF,	/* XXX TBD */
 	    "Error detected by third party temporary initiator") },
@@ -1387,6 +1453,9 @@ static struct asc_table_entry asc_table[] = {
 	/* D              */
 	{ SST(0x11, 0x14, SS_RDEF,	/* XXX TBD */
 	    "Read error - LBA marked bad by application client") },
+	/* D              */
+	{ SST(0x11, 0x15, SS_RDEF,	/* XXX TBD */
+	    "Write after sanitize required") },
 	/* D   W O   BK   */
 	{ SST(0x12, 0x00, SS_RDEF,
 	    "Address mark not found for ID field") },
@@ -1589,6 +1658,18 @@ static struct asc_table_entry asc_table[] = {
 	{ SST(0x21, 0x03, SS_RDEF,	/* XXX TBD */
 	    "Invalid write crossing layer jump") },
 	/* D              */
+	{ SST(0x21, 0x04, SS_RDEF,	/* XXX TBD */
+	    "Unaligned write command") },
+	/* D              */
+	{ SST(0x21, 0x05, SS_RDEF,	/* XXX TBD */
+	    "Write boundary violation") },
+	/* D              */
+	{ SST(0x21, 0x06, SS_RDEF,	/* XXX TBD */
+	    "Attempt to read invalid data") },
+	/* D              */
+	{ SST(0x21, 0x07, SS_RDEF,	/* XXX TBD */
+	    "Read boundary violation") },
+	/* D              */
 	{ SST(0x22, 0x00, SS_FATAL | EINVAL,
 	    "Illegal function (use 20 00, 24 00, or 26 00)") },
 	/* DT P      B    */
@@ -1711,6 +1792,9 @@ static struct asc_table_entry asc_table[] = {
 	/*  T             */
 	{ SST(0x26, 0x12, SS_RDEF,	/* XXX TBD */
 	    "Vendor specific key reference not found") },
+	/* D              */
+	{ SST(0x26, 0x13, SS_RDEF,	/* XXX TBD */
+	    "Application tag mode page is invalid") },
 	/* DT  WRO   BK   */
 	{ SST(0x27, 0x00, SS_FATAL | EACCES,
 	    "Write protected") },
@@ -1735,6 +1819,9 @@ static struct asc_table_entry asc_table[] = {
 	/* D         B    */
 	{ SST(0x27, 0x07, SS_FATAL | ENOSPC,
 	    "Space allocation failed write protect") },
+	/* D              */
+	{ SST(0x27, 0x08, SS_FATAL | EACCES,
+	    "Zone is read only") },
 	/* DTLPWROMAEBKVF */
 	{ SST(0x28, 0x00, SS_FATAL | ENXIO,
 	    "Not ready to ready change, medium may have changed") },
@@ -1878,12 +1965,33 @@ static struct asc_table_entry asc_table[] = {
 	/* D              */
 	{ SST(0x2C, 0x0C, SS_RDEF,	/* XXX TBD */
 	    "ORWRITE generation does not match") },
+	/* D              */
+	{ SST(0x2C, 0x0D, SS_RDEF,	/* XXX TBD */
+	    "Reset write pointer not allowed") },
+	/* D              */
+	{ SST(0x2C, 0x0E, SS_RDEF,	/* XXX TBD */
+	    "Zone is offline") },
+	/* D              */
+	{ SST(0x2C, 0x0F, SS_RDEF,	/* XXX TBD */
+	    "Stream not open") },
+	/* D              */
+	{ SST(0x2C, 0x10, SS_RDEF,	/* XXX TBD */
+	    "Unwritten data in zone") },
 	/*  T             */
 	{ SST(0x2D, 0x00, SS_RDEF,
 	    "Overwrite error on update in place") },
 	/*      R         */
 	{ SST(0x2E, 0x00, SS_RDEF,	/* XXX TBD */
 	    "Insufficient time for operation") },
+	/* D              */
+	{ SST(0x2E, 0x01, SS_RDEF,	/* XXX TBD */
+	    "Command timeout before processing") },
+	/* D              */
+	{ SST(0x2E, 0x02, SS_RDEF,	/* XXX TBD */
+	    "Command timeout during processing") },
+	/* D              */
+	{ SST(0x2E, 0x03, SS_RDEF,	/* XXX TBD */
+	    "Command timeout during processing due to error recovery") },
 	/* DTLPWROMAEBKVF */
 	{ SST(0x2F, 0x00, SS_RDEF,
 	    "Commands cleared by another initiator") },
@@ -1893,6 +2001,9 @@ static struct asc_table_entry asc_table[] = {
 	/* DTLPWROMAEBKVF */
 	{ SST(0x2F, 0x02, SS_RDEF,	/* XXX TBD */
 	    "Commands cleared by device server") },
+	/* DTLPWROMAEBKVF */
+	{ SST(0x2F, 0x03, SS_RDEF,	/* XXX TBD */
+	    "Some commands cleared by queuing layer event") },
 	/* DT  WROM  BK   */
 	{ SST(0x30, 0x00, SS_RDEF,
 	    "Incompatible medium installed") },
@@ -2190,6 +2301,15 @@ static struct asc_table_entry asc_table[] = {
 	/* DTLPWR MAEBK F */
 	{ SST(0x3F, 0x14, SS_RDEF,	/* XXX TBD */
 	    "iSCSI IP address changed") },
+	/* DTLPWR MAEBK   */
+	{ SST(0x3F, 0x15, SS_RDEF,	/* XXX TBD */
+	    "Inspect referrals sense descriptors") },
+	/* DTLPWROMAEBKVF */
+	{ SST(0x3F, 0x16, SS_RDEF,	/* XXX TBD */
+	    "Microcode has been changed without reset") },
+	/* D              */
+	{ SST(0x3F, 0x17, SS_RDEF,	/* XXX TBD */
+	    "Zone transition to full") },
 	/* D              */
 	{ SST(0x40, 0x00, SS_RDEF,
 	    "RAM failure") },		/* deprecated - use 40 NN instead */
@@ -2299,6 +2419,30 @@ static struct asc_table_entry asc_table[] = {
 	/* DT PWROMAEBK F */
 	{ SST(0x4B, 0x0D, SS_RDEF,	/* XXX TBD */
 	    "Data-out buffer error") },
+	/* DT PWROMAEBK F */
+	{ SST(0x4B, 0x0E, SS_RDEF,	/* XXX TBD */
+	    "PCIe fabric error") },
+	/* DT PWROMAEBK F */
+	{ SST(0x4B, 0x0F, SS_RDEF,	/* XXX TBD */
+	    "PCIe completion timeout") },
+	/* DT PWROMAEBK F */
+	{ SST(0x4B, 0x10, SS_RDEF,	/* XXX TBD */
+	    "PCIe completer abort") },
+	/* DT PWROMAEBK F */
+	{ SST(0x4B, 0x11, SS_RDEF,	/* XXX TBD */
+	    "PCIe poisoned TLP received") },
+	/* DT PWROMAEBK F */
+	{ SST(0x4B, 0x12, SS_RDEF,	/* XXX TBD */
+	    "PCIe ECRC check failed") },
+	/* DT PWROMAEBK F */
+	{ SST(0x4B, 0x13, SS_RDEF,	/* XXX TBD */
+	    "PCIe unsupported request") },
+	/* DT PWROMAEBK F */
+	{ SST(0x4B, 0x14, SS_RDEF,	/* XXX TBD */
+	    "PCIe ACS violation") },
+	/* DT PWROMAEBK F */
+	{ SST(0x4B, 0x15, SS_RDEF,	/* XXX TBD */
+	    "PCIe TLP prefix blocket") },
 	/* DTLPWROMAEBKVF */
 	{ SST(0x4C, 0x00, SS_RDEF,
 	    "Logical unit failed self-configuration") },
@@ -2356,6 +2500,21 @@ static struct asc_table_entry asc_table[] = {
 	/*        M       */
 	{ SST(0x53, 0x08, SS_RDEF,	/* XXX TBD */
 	    "Element status unknown") },
+	/*        M       */
+	{ SST(0x53, 0x09, SS_RDEF,	/* XXX TBD */
+	    "Data transfer device error - load failed") },
+	/*        M       */
+	{ SST(0x53, 0x0A, SS_RDEF,	/* XXX TBD */
+	    "Data transfer device error - unload failed") },
+	/*        M       */
+	{ SST(0x53, 0x0B, SS_RDEF,	/* XXX TBD */
+	    "Data transfer device error - unload missing") },
+	/*        M       */
+	{ SST(0x53, 0x0C, SS_RDEF,	/* XXX TBD */
+	    "Data transfer device error - eject failed") },
+	/*        M       */
+	{ SST(0x53, 0x0D, SS_RDEF,	/* XXX TBD */
+	    "Data transfer device error - library communication failed") },
 	/*    P           */
 	{ SST(0x54, 0x00, SS_RDEF,
 	    "SCSI to host system interface failure") },
@@ -2401,6 +2560,15 @@ static struct asc_table_entry asc_table[] = {
 	/* DT P      B    */
 	{ SST(0x55, 0x0D, SS_RDEF,	/* XXX TBD */
 	    "Insufficient resources to create ROD token") },
+	/* D              */
+	{ SST(0x55, 0x0E, SS_RDEF,	/* XXX TBD */
+	    "Insufficient zone resources") },
+	/* D              */
+	{ SST(0x55, 0x0F, SS_RDEF,	/* XXX TBD */
+	    "Insufficient zone resources to complete write") },
+	/* D              */
+	{ SST(0x55, 0x10, SS_RDEF,	/* XXX TBD */
+	    "Maximum number of streams open") },
 	/*      R         */
 	{ SST(0x57, 0x00, SS_RDEF,
 	    "Unable to recover table-of-contents") },
@@ -2821,6 +2989,9 @@ static struct asc_table_entry asc_table[] = {
 	/*         A      */
 	{ SST(0x68, 0x00, SS_RDEF,
 	    "Logical unit not configured") },
+	/* D              */
+	{ SST(0x68, 0x01, SS_RDEF,
+	    "Subsidiary logical unit not configured") },
 	/*         A      */
 	{ SST(0x69, 0x00, SS_RDEF,
 	    "Data loss on logical unit") },
@@ -3803,8 +3974,6 @@ scsi_set_sense_data_va(struct scsi_sense_data *sense_data,
 			 */
 			sense->extra_len = 10;
 			sense_len = (int)va_arg(ap, int);
-			len_to_copy = MIN(sense_len, SSD_EXTRA_MAX -
-					  sense->extra_len);
 			data = (uint8_t *)va_arg(ap, uint8_t *);
 
 			switch (elem_type) {
@@ -3822,10 +3991,14 @@ scsi_set_sense_data_va(struct scsi_sense_data *sense_data,
 				uint8_t *data_dest;
 				int i;
 
-				if (elem_type == SSD_ELEM_COMMAND)
+				if (elem_type == SSD_ELEM_COMMAND) {
 					data_dest = &sense->cmd_spec_info[0];
-				else {
+					len_to_copy = MIN(sense_len,
+					    sizeof(sense->cmd_spec_info));
+				} else {
 					data_dest = &sense->info[0];
+					len_to_copy = MIN(sense_len,
+					    sizeof(sense->info));
 					/*
 					 * We're setting the info field, so
 					 * set the valid bit.
diff --git a/sys/cam/scsi/scsi_all.h b/sys/cam/scsi/scsi_all.h
index 253e28e085107..f2b4b21d4fcb2 100644
--- a/sys/cam/scsi/scsi_all.h
+++ b/sys/cam/scsi/scsi_all.h
@@ -103,6 +103,9 @@ typedef enum {
 /* The retyable, error action, with table specified error code */
 #define	SS_RET		SS_RETRY|SSQ_DECREMENT_COUNT|SSQ_PRINT_SENSE
 
+/* Wait for transient error status to change */
+#define	SS_WAIT		SS_TUR|SSQ_MANY|SSQ_DECREMENT_COUNT|SSQ_PRINT_SENSE
+
 /* Fatal error action, with table specified error code */
 #define	SS_FATAL	SS_FAIL|SSQ_PRINT_SENSE
 
@@ -697,6 +700,19 @@ struct scsi_control_page {
 	u_int8_t extended_selftest_completion_time[2];
 };
 
+struct scsi_control_ext_page {
+	uint8_t page_code;
+	uint8_t subpage_code;
+	uint8_t page_length[2];
+	uint8_t flags;
+#define	SCEP_TCMOS			0x04	/* Timestamp Changeable by */
+#define	SCEP_SCSIP			0x02	/* SCSI Precedence (clock) */
+#define	SCEP_IALUAE			0x01	/* Implicit ALUA Enabled */
+	uint8_t prio;
+	uint8_t max_sense;
+	uint8_t reserve[25];
+};
+
 struct scsi_cache_page {
 	u_int8_t page_code;
 #define	SCHP_PAGE_SAVABLE		0x80	/* Page is savable */
@@ -986,6 +1002,16 @@ struct scsi_read_buffer
         u_int8_t control;
 };
 
+struct scsi_read_buffer_16
+{
+	uint8_t opcode;
+	uint8_t byte2;
+	uint8_t offset[8];
+	uint8_t length[4];
+	uint8_t buffer_id;
+	uint8_t control;
+};
+
 struct scsi_write_buffer
 {
 	u_int8_t opcode;
@@ -1257,6 +1283,17 @@ struct scsi_rw_16
 	u_int8_t control;
 };
 
+struct scsi_write_atomic_16
+{
+	uint8_t	opcode;
+	uint8_t	byte2;
+	uint8_t	addr[8];
+	uint8_t	boundary[2];
+	uint8_t	length[2];
+	uint8_t	group;
+	uint8_t	control;
+};
+
 struct scsi_write_same_10
 {
 	uint8_t	opcode;
@@ -1666,6 +1703,7 @@ struct scsi_ec_cscd
 	uint8_t  type_code;
 #define EC_CSCD_EXT		0xff
 	uint8_t  luidt_pdt;
+#define EC_NUL			0x20
 #define EC_LUIDT_MASK		0xc0
 #define EC_LUIDT_LUN		0x00
 #define EC_LUIDT_PROXY_TOKEN	0x40
@@ -1971,6 +2009,7 @@ struct ata_pass_16 {
 #define	VERIFY_16		0x8F
 #define	SYNCHRONIZE_CACHE_16	0x91
 #define	WRITE_SAME_16		0x93
+#define	READ_BUFFER_16		0x9B
 #define	WRITE_ATOMIC_16		0x9C
 #define	SERVICE_ACTION_IN	0x9E
 #define	REPORT_LUNS		0xA0
@@ -2729,7 +2768,8 @@ struct scsi_vpd_block_limits
 	u_int8_t max_atomic_transfer_length[4];
 	u_int8_t atomic_alignment[4];
 	u_int8_t atomic_transfer_length_granularity[4];
-	u_int8_t reserved2[8];
+	u_int8_t max_atomic_transfer_length_with_atomic_boundary[4];
+	u_int8_t max_atomic_boundary_size[4];
 };
 
 struct scsi_read_capacity
@@ -2824,6 +2864,9 @@ struct scsi_report_luns
 #define	RPL_REPORT_DEFAULT	0x00
 #define	RPL_REPORT_WELLKNOWN	0x01
 #define	RPL_REPORT_ALL		0x02
+#define	RPL_REPORT_ADMIN	0x10
+#define	RPL_REPORT_NONSUBSID	0x11
+#define	RPL_REPORT_CONGLOM	0x12
 	uint8_t select_report;
 	uint8_t reserved2[3];
 	uint8_t length[4];
diff --git a/sys/cam/scsi/scsi_cd.h b/sys/cam/scsi/scsi_cd.h
index f502d664391a3..cf8baaf7ed787 100644
--- a/sys/cam/scsi/scsi_cd.h
+++ b/sys/cam/scsi/scsi_cd.h
@@ -56,6 +56,83 @@
  * SCSI command format
  */
 
+struct scsi_get_config
+{
+	uint8_t opcode;
+	uint8_t rt;
+#define	SGC_RT_ALL		0x00
+#define	SGC_RT_CURRENT		0x01
+#define	SGC_RT_SPECIFIC		0x02
+#define	SGC_RT_MASK		0x03
+	uint8_t starting_feature[2];
+	uint8_t reserved[3];
+	uint8_t length[2];
+	uint8_t control;
+};
+
+struct scsi_get_config_header
+{
+	uint8_t data_length[4];
+	uint8_t reserved[2];
+	uint8_t current_profile[2];
+};
+
+struct scsi_get_config_feature
+{
+	uint8_t feature_code[2];
+	uint8_t flags;
+#define	SGC_F_CURRENT		0x01
+#define	SGC_F_PERSISTENT	0x02
+#define	SGC_F_VERSION_MASK	0x2C
+#define	SGC_F_VERSION_SHIFT	2
+	uint8_t add_length;
+	uint8_t feature_data[];
+};
+
+struct scsi_get_event_status
+{
+	uint8_t opcode;
+	uint8_t byte2;
+#define	SGESN_POLLED		1
+	uint8_t reserved[2];
+	uint8_t notif_class;
+	uint8_t reserved2[2];
+	uint8_t length[2];
+	uint8_t control;
+};
+
+struct scsi_get_event_status_header
+{
+	uint8_t descr_length[4];
+	uint8_t nea_class;
+#define	SGESN_NEA		0x80
+	uint8_t supported_class;
+};
+
+struct scsi_get_event_status_descr
+{
+	uint8_t event_code;
+	uint8_t event_info[];
+};
+
+struct scsi_mechanism_status
+{
+	uint8_t opcode;
+	uint8_t reserved[7];
+	uint8_t length[2];
+	uint8_t reserved2;
+	uint8_t control;
+};
+
+struct scsi_mechanism_status_header
+{
+	uint8_t state1;
+	uint8_t state2;
+	uint8_t lba[3];
+	uint8_t slots_num;
+	uint8_t slots_length[2];
+};
+
 struct scsi_pause
 {
 	u_int8_t op_code;
@@ -151,12 +228,29 @@ struct scsi_read_toc
 {
 	u_int8_t op_code;
 	u_int8_t byte2;
-	u_int8_t unused[4];
+	u_int8_t format;
+	u_int8_t unused[3];
 	u_int8_t from_track;
 	u_int8_t data_len[2];
 	u_int8_t control;
 };
 
+struct scsi_read_toc_hdr
+{
+	uint8_t data_length[2];
+	uint8_t first;
+	uint8_t last;
+};
+
+struct scsi_read_toc_type01_descr
+{
+	uint8_t reserved;
+	uint8_t addr_ctl;
+	uint8_t track_number;
+	uint8_t reserved2;
+	uint8_t track_start[4];
+};
+
 struct scsi_read_cd_capacity
 {
 	u_int8_t op_code;
@@ -252,9 +346,11 @@ struct scsi_read_dvd_structure
 #define READ_TOC		0x43	/* cdrom read TOC */
 #define READ_HEADER		0x44	/* cdrom read header */
 #define PLAY_10			0x45	/* cdrom play  'play audio' mode */
+#define GET_CONFIGURATION	0x46	/* Get device configuration */
 #define PLAY_MSF		0x47	/* cdrom play Min,Sec,Frames mode */
 #define PLAY_TRACK		0x48	/* cdrom play track/index mode */
 #define PLAY_TRACK_REL		0x49	/* cdrom play track/index mode */
+#define GET_EVENT_STATUS	0x4a	/* Get event status notification */
 #define PAUSE			0x4b	/* cdrom pause in 'play audio' mode */
 #define SEND_KEY		0xa3	/* dvd send key command */
 #define REPORT_KEY		0xa4	/* dvd report key command */
@@ -262,6 +358,7 @@ struct scsi_read_dvd_structure
 #define PLAY_TRACK_REL_BIG	0xa9	/* cdrom play track/index mode */
 #define READ_DVD_STRUCTURE	0xad	/* read dvd structure */
 #define SET_CD_SPEED		0xbb	/* set c/dvd speed */
+#define MECHANISM_STATUS	0xbd	/* get status of c/dvd mechanics */
 
 struct scsi_report_key_data_header
 {
@@ -686,6 +783,37 @@ struct cd_audio_page
 #define	RIGHT_PORT		1
 };
 
+struct scsi_cddvd_capabilities_page_sd {
+	uint8_t reserved;
+	uint8_t rotation_control;
+	uint8_t write_speed_supported[2];
+};
+
+struct scsi_cddvd_capabilities_page {
+	uint8_t page_code;
+#define	SMS_CDDVD_CAPS_PAGE		0x2a
+	uint8_t page_length;
+	uint8_t caps1;
+	uint8_t caps2;
+	uint8_t caps3;
+	uint8_t caps4;
+	uint8_t caps5;
+	uint8_t caps6;
+	uint8_t obsolete[2];
+	uint8_t nvol_levels[2];
+	uint8_t buffer_size[2];
+	uint8_t obsolete2[2];
+	uint8_t reserved;
+	uint8_t digital;
+	uint8_t obsolete3;
+	uint8_t copy_management;
+	uint8_t reserved2;
+	uint8_t rotation_control;
+	uint8_t cur_write_speed;
+	uint8_t num_speed_descr;
+	struct scsi_cddvd_capabilities_page_sd speed_descr[];
+};
+
 union cd_pages
 {
 	struct cd_audio_page audio;
diff --git a/sys/cam/scsi/scsi_ch.c b/sys/cam/scsi/scsi_ch.c
index 38acb0197213a..bc398e7f924f4 100644
--- a/sys/cam/scsi/scsi_ch.c
+++ b/sys/cam/scsi/scsi_ch.c
@@ -655,11 +655,13 @@ chdone(struct cam_periph *periph, union ccb *done_ccb)
 				 */
 				return;
 			} else if (error != 0) {
-				int retry_scheduled;
 				struct scsi_mode_sense_6 *sms;
+				int frozen, retry_scheduled;
 
 				sms = (struct scsi_mode_sense_6 *)
 					done_ccb->csio.cdb_io.cdb_bytes;
+				frozen = (done_ccb->ccb_h.status &
+				    CAM_DEV_QFRZN) != 0;
 
 				/*
 				 * Check to see if block descriptors were
@@ -670,7 +672,8 @@ chdone(struct cam_periph *periph, union ccb *done_ccb)
 				 * block descriptors were disabled, enable
 				 * them and re-send the command.
 				 */
-				if (sms->byte2 & SMS_DBD) {
+				if ((sms->byte2 & SMS_DBD) != 0 &&
+				    (periph->flags & CAM_PERIPH_INVALID) == 0) {
 					sms->byte2 &= ~SMS_DBD;
 					xpt_action(done_ccb);
 					softc->quirks |= CH_Q_NO_DBD;
@@ -679,7 +682,7 @@ chdone(struct cam_periph *periph, union ccb *done_ccb)
 					retry_scheduled = 0;
 
 				/* Don't wedge this device's queue */
-				if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0)
+				if (frozen)
 					cam_release_devq(done_ccb->ccb_h.path,
 						 /*relsim_flags*/0,
 						 /*reduction*/0,
diff --git a/sys/cam/scsi/scsi_pass.c b/sys/cam/scsi/scsi_pass.c
index 584b6b35e7833..055291b1bd88c 100644
--- a/sys/cam/scsi/scsi_pass.c
+++ b/sys/cam/scsi/scsi_pass.c
@@ -77,6 +77,7 @@ struct pass_softc {
 	u_int8_t	 pd_type;
 	union ccb	 saved_ccb;
 	int		 open_count;
+	u_int		 maxio;
 	struct devstat	*device_stats;
 	struct cdev	*dev;
 	struct cdev	*alias_dev;
@@ -366,6 +367,13 @@ passregister(struct cam_periph *periph, void *arg)
 	cpi.ccb_h.func_code = XPT_PATH_INQ;
 	xpt_action((union ccb *)&cpi);
 
+	if (cpi.maxio == 0)
+		softc->maxio = DFLTPHYS;	/* traditional default */
+	else if (cpi.maxio > MAXPHYS)
+		softc->maxio = MAXPHYS;		/* for safety */
+	else
+		softc->maxio = cpi.maxio;	/* real value */
+
 	/*
 	 * We pass in 0 for a blocksize, since we don't 
 	 * know what the blocksize of this device is, if 
@@ -657,7 +665,7 @@ passsendccb(struct cam_periph *periph, union ccb *ccb, union ccb *inccb)
 		 * Dropping it here is reasonably safe.
 		 */
 		cam_periph_unlock(periph);
-		error = cam_periph_mapmem(ccb, &mapinfo); 
+		error = cam_periph_mapmem(ccb, &mapinfo, softc->maxio);
 		cam_periph_lock(periph);
 
 		/*
diff --git a/sys/cam/scsi/scsi_sg.c b/sys/cam/scsi/scsi_sg.c
index 3e80ac3e763f6..3e130036ec02f 100644
--- a/sys/cam/scsi/scsi_sg.c
+++ b/sys/cam/scsi/scsi_sg.c
@@ -99,6 +99,7 @@ struct sg_softc {
 	sg_state		state;
 	sg_flags		flags;
 	int			open_count;
+	u_int			maxio;
 	struct devstat		*device_stats;
 	TAILQ_HEAD(, sg_rdwr)	rdwr_done;
 	struct cdev		*dev;
@@ -325,6 +326,13 @@ sgregister(struct cam_periph *periph, void *arg)
 	cpi.ccb_h.func_code = XPT_PATH_INQ;
 	xpt_action((union ccb *)&cpi);
 
+	if (cpi.maxio == 0)
+		softc->maxio = DFLTPHYS;	/* traditional default */
+	else if (cpi.maxio > MAXPHYS)
+		softc->maxio = MAXPHYS;		/* for safety */
+	else
+		softc->maxio = cpi.maxio;	/* real value */
+
 	/*
 	 * We pass in 0 for all blocksize, since we don't know what the
 	 * blocksize of the device is, if it even has a blocksize.
@@ -894,7 +902,7 @@ sgsendccb(struct cam_periph *periph, union ccb *ccb)
 	 * need for additional checks.
 	 */
 	cam_periph_unlock(periph);
-	error = cam_periph_mapmem(ccb, &mapinfo);
+	error = cam_periph_mapmem(ccb, &mapinfo, softc->maxio);
 	cam_periph_lock(periph);
 	if (error)
 		return (error);
diff --git a/sys/cam/scsi/scsi_target.c b/sys/cam/scsi/scsi_target.c
index 42dc15275e832..f2504abb8c5f6 100644
--- a/sys/cam/scsi/scsi_target.c
+++ b/sys/cam/scsi/scsi_target.c
@@ -94,6 +94,7 @@ struct targ_softc {
 	struct cam_periph	*periph;
 	struct cam_path		*path;
 	targ_state		 state;
+	u_int			 maxio;
 	struct selinfo		 read_select;
 	struct devstat		 device_stats;
 };
@@ -403,6 +404,12 @@ targenable(struct targ_softc *softc, struct cam_path *path, int grp6_len,
 		status = CAM_FUNC_NOTAVAIL;
 		goto enable_fail;
 	}
+	if (cpi.maxio == 0)
+		softc->maxio = DFLTPHYS;	/* traditional default */
+	else if (cpi.maxio > MAXPHYS)
+		softc->maxio = MAXPHYS;		/* for safety */
+	else
+		softc->maxio = cpi.maxio;	/* real value */
 
 	/* Destroy any periph on our path if it is disabled */
 	periph = cam_periph_find(path, "targ");
@@ -725,7 +732,7 @@ targsendccb(struct targ_softc *softc, union ccb *ccb,
 	if ((ccb_h->func_code == XPT_CONT_TARGET_IO) ||
 	    (ccb_h->func_code == XPT_DEV_MATCH)) {
 
-		error = cam_periph_mapmem(ccb, mapinfo);
+		error = cam_periph_mapmem(ccb, mapinfo, softc->maxio);
 
 		/*
 		 * cam_periph_mapmem returned an error, we can't continue.
diff --git a/sys/cddl/contrib/opensolaris/common/nvpair/opensolaris_nvpair.c b/sys/cddl/contrib/opensolaris/common/nvpair/opensolaris_nvpair.c
index cd6af4e0de8f2..7038f7f9f1568 100644
--- a/sys/cddl/contrib/opensolaris/common/nvpair/opensolaris_nvpair.c
+++ b/sys/cddl/contrib/opensolaris/common/nvpair/opensolaris_nvpair.c
@@ -44,6 +44,14 @@
 #endif
 #define	skip_whitespace(p)	while ((*(p) == ' ') || (*(p) == '\t')) p++
 
+#if defined(__FreeBSD__) && !defined(_KERNEL)
+/*
+ * libnvpair is the lowest commen denominator for ZFS related libraries,
+ * defining aok here makes it usable by all ZFS related libraries
+ */
+int aok;
+#endif
+
 /*
  * nvpair.c - Provides kernel & userland interfaces for manipulating
  *	name-value pairs.
diff --git a/sys/cddl/contrib/opensolaris/uts/common/dtrace/dtrace.c b/sys/cddl/contrib/opensolaris/uts/common/dtrace/dtrace.c
index cf2f7bbd144c5..4b356d7b1a45f 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/dtrace/dtrace.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/dtrace/dtrace.c
@@ -155,7 +155,7 @@ int		dtrace_destructive_disallow = 0;
 dtrace_optval_t	dtrace_nonroot_maxsize = (16 * 1024 * 1024);
 size_t		dtrace_difo_maxsize = (256 * 1024);
 dtrace_optval_t	dtrace_dof_maxsize = (8 * 1024 * 1024);
-size_t		dtrace_global_maxsize = (16 * 1024);
+size_t		dtrace_statvar_maxsize = (16 * 1024);
 size_t		dtrace_actions_max = (16 * 1024);
 size_t		dtrace_retain_max = 1024;
 dtrace_optval_t	dtrace_helper_actions_max = 128;
@@ -699,13 +699,33 @@ dtrace_canstore_statvar(uint64_t addr, size_t sz,
     dtrace_statvar_t **svars, int nsvars)
 {
 	int i;
+	size_t maxglobalsize, maxlocalsize;
+
+	if (nsvars == 0)
+		return (0);
+
+	maxglobalsize = dtrace_statvar_maxsize;
+	maxlocalsize = (maxglobalsize + sizeof (uint64_t)) * NCPU;
 
 	for (i = 0; i < nsvars; i++) {
 		dtrace_statvar_t *svar = svars[i];
+		uint8_t scope;
+		size_t size;
 
-		if (svar == NULL || svar->dtsv_size == 0)
+		if (svar == NULL || (size = svar->dtsv_size) == 0)
 			continue;
 
+		scope = svar->dtsv_var.dtdv_scope;
+
+		/*
+		 * We verify that our size is valid in the spirit of providing
+		 * defense in depth:  we want to prevent attackers from using
+		 * DTrace to escalate an orthogonal kernel heap corruption bug
+		 * into the ability to store to arbitrary locations in memory.
+		 */
+		VERIFY((scope == DIFV_SCOPE_GLOBAL && size < maxglobalsize) ||
+		    (scope == DIFV_SCOPE_LOCAL && size < maxlocalsize));
+
 		if (DTRACE_INRANGE(addr, sz, svar->dtsv_data, svar->dtsv_size))
 			return (1);
 	}
@@ -4455,7 +4475,8 @@ dtrace_dif_subr(uint_t subr, uint_t rd, uint64_t *regs,
 
 		if (!dtrace_destructive_disallow &&
 		    dtrace_priv_proc_control(state) &&
-		    !dtrace_istoxic(kaddr, size)) {
+		    !dtrace_istoxic(kaddr, size) &&
+		    dtrace_canload(kaddr, size, mstate, vstate)) {
 			DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
 			dtrace_copyout(kaddr, uaddr, size, flags);
 			DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
@@ -4470,7 +4491,8 @@ dtrace_dif_subr(uint_t subr, uint_t rd, uint64_t *regs,
 
 		if (!dtrace_destructive_disallow &&
 		    dtrace_priv_proc_control(state) &&
-		    !dtrace_istoxic(kaddr, size)) {
+		    !dtrace_istoxic(kaddr, size) &&
+		    dtrace_strcanload(kaddr, size, mstate, vstate)) {
 			DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
 			dtrace_copyoutstr(kaddr, uaddr, size, flags);
 			DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
@@ -6458,6 +6480,11 @@ dtrace_dif_emulate(dtrace_difo_t *difo, dtrace_mstate_t *mstate,
 				    regs[r2] ? regs[r2] :
 				    dtrace_strsize_default) + 1;
 			} else {
+				if (regs[r2] > LONG_MAX) {
+					*flags |= CPU_DTRACE_ILLOP;
+					break;
+				}
+
 				tupregs[ttop].dttk_size = regs[r2];
 			}
 
@@ -9915,9 +9942,10 @@ dtrace_difo_validate(dtrace_difo_t *dp, dtrace_vstate_t *vstate, uint_t nregs,
 				break;
 			}
 
-			if (v->dtdv_scope == DIFV_SCOPE_GLOBAL &&
-			    vt->dtdt_size > dtrace_global_maxsize) {
-				err += efunc(i, "oversized by-ref global\n");
+			if ((v->dtdv_scope == DIFV_SCOPE_GLOBAL ||
+			    v->dtdv_scope == DIFV_SCOPE_LOCAL) &&
+			    vt->dtdt_size > dtrace_statvar_maxsize) {
+				err += efunc(i, "oversized by-ref static\n");
 				break;
 			}
 		}
@@ -10261,6 +10289,9 @@ dtrace_difo_chunksize(dtrace_difo_t *dp, dtrace_vstate_t *vstate)
 				if (srd == 0)
 					return;
 
+				if (sval > LONG_MAX)
+					return;
+
 				tupregs[ttop++].dttk_size = sval;
 			}
 
@@ -10322,6 +10353,19 @@ dtrace_difo_chunksize(dtrace_difo_t *dp, dtrace_vstate_t *vstate)
 		 */
 		size = P2ROUNDUP(size, sizeof (uint64_t));
 
+		/*
+		 * Before setting the chunk size, check that we're not going
+		 * to set it to a negative value...
+		 */
+		if (size > LONG_MAX)
+			return;
+
+		/*
+		 * ...and make certain that we didn't badly overflow.
+		 */
+		if (size < ksize || size < sizeof (dtrace_dynvar_t))
+			return;
+
 		if (size > vstate->dtvs_dynvars.dtds_chunksize)
 			vstate->dtvs_dynvars.dtds_chunksize = size;
 	}
@@ -13942,6 +13986,8 @@ dtrace_dstate_init(dtrace_dstate_t *dstate, size_t size)
 	if ((dstate->dtds_chunksize = chunksize) == 0)
 		dstate->dtds_chunksize = DTRACE_DYNVAR_CHUNKSIZE;
 
+	VERIFY(dstate->dtds_chunksize < LONG_MAX);
+
 	if (size < (min = dstate->dtds_chunksize + sizeof (dtrace_dynhash_t)))
 		size = min;
 
@@ -13982,6 +14028,9 @@ dtrace_dstate_init(dtrace_dstate_t *dstate, size_t size)
 	    ((uintptr_t)base + hashsize * sizeof (dtrace_dynhash_t));
 	limit = (uintptr_t)base + size;
 
+	VERIFY((uintptr_t)start < limit);
+	VERIFY((uintptr_t)start >= (uintptr_t)base);
+
 	maxper = (limit - (uintptr_t)start) / NCPU;
 	maxper = (maxper / dstate->dtds_chunksize) * dstate->dtds_chunksize;
 
@@ -14007,7 +14056,7 @@ dtrace_dstate_init(dtrace_dstate_t *dstate, size_t size)
 			start = (dtrace_dynvar_t *)limit;
 		}
 
-		ASSERT(limit <= (uintptr_t)base + size);
+		VERIFY(limit <= (uintptr_t)base + size);
 
 		for (;;) {
 			next = (dtrace_dynvar_t *)((uintptr_t)dvar +
@@ -14016,6 +14065,8 @@ dtrace_dstate_init(dtrace_dstate_t *dstate, size_t size)
 			if ((uintptr_t)next + dstate->dtds_chunksize >= limit)
 				break;
 
+			VERIFY((uintptr_t)dvar >= (uintptr_t)base &&
+			    (uintptr_t)dvar <= (uintptr_t)base + size);
 			dvar->dtdv_next = next;
 			dvar = next;
 		}
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c
index 6d1792cce5669..42c001a7cda37 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c
@@ -1485,6 +1485,13 @@ arc_hdr_realloc(arc_buf_hdr_t *hdr, kmem_cache_t *old, kmem_cache_t *new)
 		VERIFY(!HDR_L2_WRITING(hdr));
 		VERIFY3P(hdr->b_l1hdr.b_tmp_cdata, ==, NULL);
 
+#ifdef ZFS_DEBUG
+		if (hdr->b_l1hdr.b_thawed != NULL) {
+			kmem_free(hdr->b_l1hdr.b_thawed, 1);
+			hdr->b_l1hdr.b_thawed = NULL;
+		}
+#endif
+
 		nhdr->b_flags &= ~ARC_FLAG_HAS_L1HDR;
 	}
 	/*
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_objset.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_objset.c
index 9ad18f0d11fba..f84ff378c947b 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_objset.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_objset.c
@@ -680,40 +680,8 @@ dmu_objset_evict(objset_t *os)
 	for (int t = 0; t < TXG_SIZE; t++)
 		ASSERT(!dmu_objset_is_dirty(os, t));
 
-	if (ds) {
-		if (!ds->ds_is_snapshot) {
-			VERIFY0(dsl_prop_unregister(ds,
-			    zfs_prop_to_name(ZFS_PROP_CHECKSUM),
-			    checksum_changed_cb, os));
-			VERIFY0(dsl_prop_unregister(ds,
-			    zfs_prop_to_name(ZFS_PROP_COMPRESSION),
-			    compression_changed_cb, os));
-			VERIFY0(dsl_prop_unregister(ds,
-			    zfs_prop_to_name(ZFS_PROP_COPIES),
-			    copies_changed_cb, os));
-			VERIFY0(dsl_prop_unregister(ds,
-			    zfs_prop_to_name(ZFS_PROP_DEDUP),
-			    dedup_changed_cb, os));
-			VERIFY0(dsl_prop_unregister(ds,
-			    zfs_prop_to_name(ZFS_PROP_LOGBIAS),
-			    logbias_changed_cb, os));
-			VERIFY0(dsl_prop_unregister(ds,
-			    zfs_prop_to_name(ZFS_PROP_SYNC),
-			    sync_changed_cb, os));
-			VERIFY0(dsl_prop_unregister(ds,
-			    zfs_prop_to_name(ZFS_PROP_REDUNDANT_METADATA),
-			    redundant_metadata_changed_cb, os));
-			VERIFY0(dsl_prop_unregister(ds,
-			    zfs_prop_to_name(ZFS_PROP_RECORDSIZE),
-			    recordsize_changed_cb, os));
-		}
-		VERIFY0(dsl_prop_unregister(ds,
-		    zfs_prop_to_name(ZFS_PROP_PRIMARYCACHE),
-		    primary_cache_changed_cb, os));
-		VERIFY0(dsl_prop_unregister(ds,
-		    zfs_prop_to_name(ZFS_PROP_SECONDARYCACHE),
-		    secondary_cache_changed_cb, os));
-	}
+	if (ds)
+		dsl_prop_unregister_all(ds, os);
 
 	if (os->os_sa)
 		sa_tear_down(os);
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_dataset.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_dataset.c
index f4fdaf9d5eed6..4fbbe7c4e8056 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_dataset.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_dataset.c
@@ -288,6 +288,7 @@ dsl_dataset_evict(void *dbu)
 
 	ASSERT(!list_link_active(&ds->ds_synced_link));
 
+	list_destroy(&ds->ds_prop_cbs);
 	if (mutex_owned(&ds->ds_lock))
 		mutex_exit(&ds->ds_lock);
 	mutex_destroy(&ds->ds_lock);
@@ -434,6 +435,9 @@ dsl_dataset_hold_obj(dsl_pool_t *dp, uint64_t dsobj, void *tag,
 		list_create(&ds->ds_sendstreams, sizeof (dmu_sendarg_t),
 		    offsetof(dmu_sendarg_t, dsa_link));
 
+		list_create(&ds->ds_prop_cbs, sizeof (dsl_prop_cb_record_t),
+		    offsetof(dsl_prop_cb_record_t, cbr_ds_node));
+
 		if (doi.doi_type == DMU_OTN_ZAP_METADATA) {
 			for (spa_feature_t f = 0; f < SPA_FEATURES; f++) {
 				if (!(spa_feature_table[f].fi_flags &
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_dir.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_dir.c
index 4b268da88d092..00341ad927645 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_dir.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_dir.c
@@ -152,11 +152,7 @@ dsl_dir_evict(void *dbu)
 
 	spa_async_close(dd->dd_pool->dp_spa, dd);
 
-	/*
-	 * The props callback list should have been cleaned up by
-	 * objset_evict().
-	 */
-	list_destroy(&dd->dd_prop_cbs);
+	dsl_prop_fini(dd);
 	mutex_destroy(&dd->dd_lock);
 	kmem_free(dd, sizeof (dsl_dir_t));
 }
@@ -191,9 +187,7 @@ dsl_dir_hold_obj(dsl_pool_t *dp, uint64_t ddobj,
 		dd->dd_dbuf = dbuf;
 		dd->dd_pool = dp;
 		mutex_init(&dd->dd_lock, NULL, MUTEX_DEFAULT, NULL);
-
-		list_create(&dd->dd_prop_cbs, sizeof (dsl_prop_cb_record_t),
-		    offsetof(dsl_prop_cb_record_t, cbr_node));
+		dsl_prop_init(dd);
 
 		dsl_dir_snap_cmtime_update(dd);
 
@@ -251,6 +245,7 @@ dsl_dir_hold_obj(dsl_pool_t *dp, uint64_t ddobj,
 		if (winner != NULL) {
 			if (dd->dd_parent)
 				dsl_dir_rele(dd->dd_parent, dd);
+			dsl_prop_fini(dd);
 			mutex_destroy(&dd->dd_lock);
 			kmem_free(dd, sizeof (dsl_dir_t));
 			dd = winner;
@@ -278,6 +273,7 @@ dsl_dir_hold_obj(dsl_pool_t *dp, uint64_t ddobj,
 errout:
 	if (dd->dd_parent)
 		dsl_dir_rele(dd->dd_parent, dd);
+	dsl_prop_fini(dd);
 	mutex_destroy(&dd->dd_lock);
 	kmem_free(dd, sizeof (dsl_dir_t));
 	dmu_buf_rele(dbuf, tag);
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_prop.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_prop.c
index 094b07d21fe33..12e27711b6c41 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_prop.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_prop.c
@@ -215,6 +215,58 @@ dsl_prop_get_ds(dsl_dataset_t *ds, const char *propname,
 	    intsz, numints, buf, setpoint, ds->ds_is_snapshot));
 }
 
+static dsl_prop_record_t *
+dsl_prop_record_find(dsl_dir_t *dd, const char *propname)
+{
+	dsl_prop_record_t *pr = NULL;
+
+	ASSERT(MUTEX_HELD(&dd->dd_lock));
+
+	for (pr = list_head(&dd->dd_props);
+	    pr != NULL; pr = list_next(&dd->dd_props, pr)) {
+		if (strcmp(pr->pr_propname, propname) == 0)
+			break;
+	}
+
+	return (pr);
+}
+
+static dsl_prop_record_t *
+dsl_prop_record_create(dsl_dir_t *dd, const char *propname)
+{
+	dsl_prop_record_t *pr;
+
+	ASSERT(MUTEX_HELD(&dd->dd_lock));
+
+	pr = kmem_alloc(sizeof (dsl_prop_record_t), KM_SLEEP);
+	pr->pr_propname = spa_strdup(propname);
+	list_create(&pr->pr_cbs, sizeof (dsl_prop_cb_record_t),
+	    offsetof(dsl_prop_cb_record_t, cbr_pr_node));
+	list_insert_head(&dd->dd_props, pr);
+
+	return (pr);
+}
+
+void
+dsl_prop_init(dsl_dir_t *dd)
+{
+	list_create(&dd->dd_props, sizeof (dsl_prop_record_t),
+	    offsetof(dsl_prop_record_t, pr_node));
+}
+
+void
+dsl_prop_fini(dsl_dir_t *dd)
+{
+	dsl_prop_record_t *pr;
+
+	while ((pr = list_remove_head(&dd->dd_props)) != NULL) {
+		list_destroy(&pr->pr_cbs);
+		strfree((char *)pr->pr_propname);
+		kmem_free(pr, sizeof (dsl_prop_record_t));
+	}
+	list_destroy(&dd->dd_props);
+}
+
 /*
  * Register interest in the named property.  We'll call the callback
  * once to notify it of the current property value, and again each time
@@ -229,6 +281,7 @@ dsl_prop_register(dsl_dataset_t *ds, const char *propname,
 	dsl_dir_t *dd = ds->ds_dir;
 	dsl_pool_t *dp = dd->dd_pool;
 	uint64_t value;
+	dsl_prop_record_t *pr;
 	dsl_prop_cb_record_t *cbr;
 	int err;
 
@@ -240,12 +293,16 @@ dsl_prop_register(dsl_dataset_t *ds, const char *propname,
 
 	cbr = kmem_alloc(sizeof (dsl_prop_cb_record_t), KM_SLEEP);
 	cbr->cbr_ds = ds;
-	cbr->cbr_propname = kmem_alloc(strlen(propname)+1, KM_SLEEP);
-	(void) strcpy((char *)cbr->cbr_propname, propname);
 	cbr->cbr_func = callback;
 	cbr->cbr_arg = cbarg;
+
 	mutex_enter(&dd->dd_lock);
-	list_insert_head(&dd->dd_prop_cbs, cbr);
+	pr = dsl_prop_record_find(dd, propname);
+	if (pr == NULL)
+		pr = dsl_prop_record_create(dd, propname);
+	cbr->cbr_pr = pr;
+	list_insert_head(&pr->pr_cbs, cbr);
+	list_insert_head(&ds->ds_prop_cbs, cbr);
 	mutex_exit(&dd->dd_lock);
 
 	cbr->cbr_func(cbr->cbr_arg, value);
@@ -376,56 +433,34 @@ dsl_prop_predict(dsl_dir_t *dd, const char *propname,
 }
 
 /*
- * Unregister this callback.  Return 0 on success, ENOENT if ddname is
- * invalid, or ENOMSG if no matching callback registered.
+ * Unregister all callbacks that are registered with the
+ * given callback argument.
  */
-int
-dsl_prop_unregister(dsl_dataset_t *ds, const char *propname,
-    dsl_prop_changed_cb_t *callback, void *cbarg)
+void
+dsl_prop_unregister_all(dsl_dataset_t *ds, void *cbarg)
 {
+	dsl_prop_cb_record_t *cbr, *next_cbr;
+
 	dsl_dir_t *dd = ds->ds_dir;
-	dsl_prop_cb_record_t *cbr;
 
 	mutex_enter(&dd->dd_lock);
-	for (cbr = list_head(&dd->dd_prop_cbs);
-	    cbr; cbr = list_next(&dd->dd_prop_cbs, cbr)) {
-		if (cbr->cbr_ds == ds &&
-		    cbr->cbr_func == callback &&
-		    cbr->cbr_arg == cbarg &&
-		    strcmp(cbr->cbr_propname, propname) == 0)
-			break;
-	}
-
-	if (cbr == NULL) {
-		mutex_exit(&dd->dd_lock);
-		return (SET_ERROR(ENOMSG));
+	next_cbr = list_head(&ds->ds_prop_cbs);
+	while (next_cbr != NULL) {
+		cbr = next_cbr;
+		next_cbr = list_next(&ds->ds_prop_cbs, cbr);
+		if (cbr->cbr_arg == cbarg) {
+			list_remove(&ds->ds_prop_cbs, cbr);
+			list_remove(&cbr->cbr_pr->pr_cbs, cbr);
+			kmem_free(cbr, sizeof (dsl_prop_cb_record_t));
+		}
 	}
-
-	list_remove(&dd->dd_prop_cbs, cbr);
 	mutex_exit(&dd->dd_lock);
-	kmem_free((void*)cbr->cbr_propname, strlen(cbr->cbr_propname)+1);
-	kmem_free(cbr, sizeof (dsl_prop_cb_record_t));
-
-	return (0);
 }
 
 boolean_t
 dsl_prop_hascb(dsl_dataset_t *ds)
 {
-	dsl_dir_t *dd = ds->ds_dir;
-	boolean_t rv = B_FALSE;
-	dsl_prop_cb_record_t *cbr;
-
-	mutex_enter(&dd->dd_lock);
-	for (cbr = list_head(&dd->dd_prop_cbs); cbr;
-	    cbr = list_next(&dd->dd_prop_cbs, cbr)) {
-		if (cbr->cbr_ds == ds) {
-			rv = B_TRUE;
-			break;
-		}
-	}
-	mutex_exit(&dd->dd_lock);
-	return (rv);
+	return (!list_is_empty(&ds->ds_prop_cbs));
 }
 
 /* ARGSUSED */
@@ -433,38 +468,50 @@ static int
 dsl_prop_notify_all_cb(dsl_pool_t *dp, dsl_dataset_t *ds, void *arg)
 {
 	dsl_dir_t *dd = ds->ds_dir;
+	dsl_prop_record_t *pr;
 	dsl_prop_cb_record_t *cbr;
 
 	mutex_enter(&dd->dd_lock);
-	for (cbr = list_head(&dd->dd_prop_cbs); cbr;
-	    cbr = list_next(&dd->dd_prop_cbs, cbr)) {
-		uint64_t value;
+	for (pr = list_head(&dd->dd_props);
+	    pr; pr = list_next(&dd->dd_props, pr)) {
+		for (cbr = list_head(&pr->pr_cbs); cbr;
+		    cbr = list_next(&pr->pr_cbs, cbr)) {
+			uint64_t value;
 
-		/*
-		 * Callback entries do not have holds on their datasets
-		 * so that datasets with registered callbacks are still
-		 * eligible for eviction.  Unlike operations on callbacks
-		 * for a single dataset, we are performing a recursive
-		 * descent of related datasets and the calling context
-		 * for this iteration only has a dataset hold on the root.
-		 * Without a hold, the callback's pointer to the dataset
-		 * could be invalidated by eviction at any time.
-		 *
-		 * Use dsl_dataset_try_add_ref() to verify that the
-		 * dataset has not begun eviction processing and to
-		 * prevent eviction from occurring for the duration
-		 * of the callback.  If the hold attempt fails, this
-		 * object is already being evicted and the callback can
-		 * be safely ignored.
-		 */
-		if (!dsl_dataset_try_add_ref(dp, cbr->cbr_ds, FTAG))
-			continue;
+			/*
+			 * Callback entries do not have holds on their
+			 * datasets so that datasets with registered
+			 * callbacks are still eligible for eviction.
+			 * Unlike operations to update properties on a
+			 * single dataset, we are performing a recursive
+			 * descent of related head datasets.  The caller
+			 * of this function only has a dataset hold on
+			 * the passed in head dataset, not the snapshots
+			 * associated with this dataset.  Without a hold,
+			 * the dataset pointer within callback records
+			 * for snapshots can be invalidated by eviction
+			 * at any time.
+			 *
+			 * Use dsl_dataset_try_add_ref() to verify
+			 * that the dataset for a snapshot has not
+			 * begun eviction processing and to prevent
+			 * eviction from occurring for the duration of
+			 * the callback.  If the hold attempt fails,
+			 * this object is already being evicted and the
+			 * callback can be safely ignored.
+			 */
+			if (ds != cbr->cbr_ds &&
+			    !dsl_dataset_try_add_ref(dp, cbr->cbr_ds, FTAG))
+				continue;
 
-		if (dsl_prop_get_ds(cbr->cbr_ds, cbr->cbr_propname,
-		    sizeof (value), 1, &value, NULL) == 0)
-			cbr->cbr_func(cbr->cbr_arg, value);
+			if (dsl_prop_get_ds(cbr->cbr_ds,
+			    cbr->cbr_pr->pr_propname, sizeof (value), 1,
+			    &value, NULL) == 0)
+				cbr->cbr_func(cbr->cbr_arg, value);
 
-		dsl_dataset_rele(cbr->cbr_ds, FTAG);
+			if (ds != cbr->cbr_ds)
+				dsl_dataset_rele(cbr->cbr_ds, FTAG);
+		}
 	}
 	mutex_exit(&dd->dd_lock);
 
@@ -489,6 +536,7 @@ dsl_prop_changed_notify(dsl_pool_t *dp, uint64_t ddobj,
     const char *propname, uint64_t value, int first)
 {
 	dsl_dir_t *dd;
+	dsl_prop_record_t *pr;
 	dsl_prop_cb_record_t *cbr;
 	objset_t *mos = dp->dp_meta_objset;
 	zap_cursor_t zc;
@@ -515,30 +563,33 @@ dsl_prop_changed_notify(dsl_pool_t *dp, uint64_t ddobj,
 	}
 
 	mutex_enter(&dd->dd_lock);
-	for (cbr = list_head(&dd->dd_prop_cbs); cbr;
-	    cbr = list_next(&dd->dd_prop_cbs, cbr)) {
-		uint64_t propobj;
+	pr = dsl_prop_record_find(dd, propname);
+	if (pr != NULL) {
+		for (cbr = list_head(&pr->pr_cbs); cbr;
+		    cbr = list_next(&pr->pr_cbs, cbr)) {
+			uint64_t propobj;
 
-		/*
-		 * cbr->cbf_ds may be invalidated due to eviction,
-		 * requiring the use of dsl_dataset_try_add_ref().
-		 * See comment block in dsl_prop_notify_all_cb()
-		 * for details.
-		 */
-		if (strcmp(cbr->cbr_propname, propname) != 0 ||
-		    !dsl_dataset_try_add_ref(dp, cbr->cbr_ds, FTAG))
-			continue;
+			/*
+			 * cbr->cbr_ds may be invalidated due to eviction,
+			 * requiring the use of dsl_dataset_try_add_ref().
+			 * See comment block in dsl_prop_notify_all_cb()
+			 * for details.
+			 */
+			if (!dsl_dataset_try_add_ref(dp, cbr->cbr_ds, FTAG))
+				continue;
 
-		propobj = dsl_dataset_phys(cbr->cbr_ds)->ds_props_obj;
+			propobj = dsl_dataset_phys(cbr->cbr_ds)->ds_props_obj;
 
-		/*
-		 * If the property is not set on this ds, then it is
-		 * inherited here; call the callback.
-		 */
-		if (propobj == 0 || zap_contains(mos, propobj, propname) != 0)
-			cbr->cbr_func(cbr->cbr_arg, value);
+			/*
+			 * If the property is not set on this ds, then it is
+			 * inherited here; call the callback.
+			 */
+			if (propobj == 0 ||
+			    zap_contains(mos, propobj, propname) != 0)
+				cbr->cbr_func(cbr->cbr_arg, value);
 
-		dsl_dataset_rele(cbr->cbr_ds, FTAG);
+			dsl_dataset_rele(cbr->cbr_ds, FTAG);
+		}
 	}
 	mutex_exit(&dd->dd_lock);
 
@@ -678,10 +729,10 @@ dsl_prop_set_sync_impl(dsl_dataset_t *ds, const char *propname,
 			 * ds here.
 			 */
 			mutex_enter(&ds->ds_dir->dd_lock);
-			for (cbr = list_head(&ds->ds_dir->dd_prop_cbs); cbr;
-			    cbr = list_next(&ds->ds_dir->dd_prop_cbs, cbr)) {
-				if (cbr->cbr_ds == ds &&
-				    strcmp(cbr->cbr_propname, propname) == 0)
+			for (cbr = list_head(&ds->ds_prop_cbs); cbr;
+			    cbr = list_next(&ds->ds_prop_cbs, cbr)) {
+				if (strcmp(cbr->cbr_pr->pr_propname,
+				    propname) == 0)
 					cbr->cbr_func(cbr->cbr_arg, intval);
 			}
 			mutex_exit(&ds->ds_dir->dd_lock);
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c
index d2085bb2f2dfd..61c83c1c927b2 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c
@@ -22,7 +22,7 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
- * Copyright (c) 2013, 2014, Nexenta Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015, Nexenta Systems, Inc.  All rights reserved.
  * Copyright (c) 2013 Martin Matuska <mm@FreeBSD.org>. All rights reserved.
  * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
@@ -1633,6 +1633,7 @@ load_nvlist(spa_t *spa, uint64_t obj, nvlist_t **value)
 	error = dmu_bonus_hold(spa->spa_meta_objset, obj, FTAG, &db);
 	if (error != 0)
 		return (error);
+
 	nvsize = *(uint64_t *)db->db_data;
 	dmu_buf_rele(db, FTAG);
 
@@ -3773,6 +3774,7 @@ spa_create(const char *pool, nvlist_t *nvroot, nvlist_t *props,
 	txg_wait_synced(spa->spa_dsl_pool, txg);
 
 	spa_config_sync(spa, B_FALSE, B_TRUE);
+	spa_event_notify(spa, NULL, ESC_ZFS_POOL_CREATE);
 
 	spa_history_log_version(spa, "create");
 
@@ -4233,6 +4235,7 @@ spa_import(const char *pool, nvlist_t *config, nvlist_t *props, uint64_t flags)
 			spa_configfile_set(spa, props, B_FALSE);
 
 		spa_config_sync(spa, B_FALSE, B_TRUE);
+		spa_event_notify(spa, NULL, ESC_ZFS_POOL_IMPORT);
 
 		mutex_exit(&spa_namespace_lock);
 		return (0);
@@ -4363,9 +4366,12 @@ spa_import(const char *pool, nvlist_t *config, nvlist_t *props, uint64_t flags)
 	 */
 	spa_async_request(spa, SPA_ASYNC_AUTOEXPAND);
 
-	mutex_exit(&spa_namespace_lock);
 	spa_history_log_version(spa, "import");
 
+	spa_event_notify(spa, NULL, ESC_ZFS_POOL_IMPORT);
+
+	mutex_exit(&spa_namespace_lock);
+
 #ifdef __FreeBSD__
 #ifdef _KERNEL
 	zvol_create_minors(pool);
@@ -4711,6 +4717,7 @@ spa_vdev_add(spa_t *spa, nvlist_t *nvroot)
 
 	mutex_enter(&spa_namespace_lock);
 	spa_config_update(spa, SPA_CONFIG_UPDATE_POOL);
+	spa_event_notify(spa, NULL, ESC_ZFS_VDEV_ADD);
 	mutex_exit(&spa_namespace_lock);
 
 	return (0);
@@ -4905,6 +4912,11 @@ spa_vdev_attach(spa_t *spa, uint64_t guid, nvlist_t *nvroot, int replacing)
 	 */
 	dsl_resilver_restart(spa->spa_dsl_pool, dtl_max_txg);
 
+	if (spa->spa_bootfs)
+		spa_event_notify(spa, newvd, ESC_ZFS_BOOTFS_VDEV_ATTACH);
+
+	spa_event_notify(spa, newvd, ESC_ZFS_VDEV_ATTACH);
+
 	/*
 	 * Commit the config
 	 */
@@ -4919,9 +4931,6 @@ spa_vdev_attach(spa_t *spa, uint64_t guid, nvlist_t *nvroot, int replacing)
 	spa_strfree(oldvdpath);
 	spa_strfree(newvdpath);
 
-	if (spa->spa_bootfs)
-		spa_event_notify(spa, newvd, ESC_ZFS_BOOTFS_VDEV_ATTACH);
-
 	return (0);
 }
 
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dsl_dataset.h b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dsl_dataset.h
index 001bff566a888..c9cd5890aa743 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dsl_dataset.h
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dsl_dataset.h
@@ -184,6 +184,9 @@ typedef struct dsl_dataset {
 	kmutex_t ds_sendstream_lock;
 	list_t ds_sendstreams;
 
+	/* Protected by our dsl_dir's dd_lock */
+	list_t ds_prop_cbs;
+
 	/*
 	 * For ZFEATURE_FLAG_PER_DATASET features, set if this dataset
 	 * uses this feature.
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dsl_dir.h b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dsl_dir.h
index f50014d95a810..93fb73f37e6b2 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dsl_dir.h
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dsl_dir.h
@@ -102,7 +102,7 @@ struct dsl_dir {
 
 	/* Protected by dd_lock */
 	kmutex_t dd_lock;
-	list_t dd_prop_cbs; /* list of dsl_prop_cb_record_t's */
+	list_t dd_props; /* list of dsl_prop_record_t's */
 	timestruc_t dd_snap_cmtime; /* last time snapshot namespace changed */
 	uint64_t dd_origin_txg;
 
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dsl_prop.h b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dsl_prop.h
index 5fe18d6a7c550..21e6f4674be9e 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dsl_prop.h
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/dsl_prop.h
@@ -41,10 +41,17 @@ struct dsl_dir;
 /* The callback func may not call into the DMU or DSL! */
 typedef void (dsl_prop_changed_cb_t)(void *arg, uint64_t newval);
 
+typedef struct dsl_prop_record {
+	list_node_t pr_node; /* link on dd_props */
+	const char *pr_propname;
+	list_t pr_cbs;
+} dsl_prop_record_t;
+
 typedef struct dsl_prop_cb_record {
-	list_node_t cbr_node; /* link on dd_prop_cbs */
+	list_node_t cbr_pr_node; /* link on pr_cbs */
+	list_node_t cbr_ds_node; /* link on ds_prop_cbs */
+	dsl_prop_record_t *cbr_pr;
 	struct dsl_dataset *cbr_ds;
-	const char *cbr_propname;
 	dsl_prop_changed_cb_t *cbr_func;
 	void *cbr_arg;
 } dsl_prop_cb_record_t;
@@ -54,10 +61,11 @@ typedef struct dsl_props_arg {
 	zprop_source_t pa_source;
 } dsl_props_arg_t;
 
+void dsl_prop_init(dsl_dir_t *dd);
+void dsl_prop_fini(dsl_dir_t *dd);
 int dsl_prop_register(struct dsl_dataset *ds, const char *propname,
     dsl_prop_changed_cb_t *callback, void *cbarg);
-int dsl_prop_unregister(struct dsl_dataset *ds, const char *propname,
-    dsl_prop_changed_cb_t *callback, void *cbarg);
+void dsl_prop_unregister_all(struct dsl_dataset *ds, void *cbarg);
 void dsl_prop_notify_all(struct dsl_dir *dd);
 boolean_t dsl_prop_hascb(struct dsl_dataset *ds);
 
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev.c
index 663b395d709fb..a8f1c5f5ca7fc 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev.c
@@ -2503,6 +2503,7 @@ int
 vdev_online(spa_t *spa, uint64_t guid, uint64_t flags, vdev_state_t *newstate)
 {
 	vdev_t *vd, *tvd, *pvd, *rvd = spa->spa_root_vdev;
+	boolean_t postevent = B_FALSE;
 
 	spa_vdev_state_enter(spa, SCL_NONE);
 
@@ -2512,6 +2513,10 @@ vdev_online(spa_t *spa, uint64_t guid, uint64_t flags, vdev_state_t *newstate)
 	if (!vd->vdev_ops->vdev_op_leaf)
 		return (spa_vdev_state_exit(spa, NULL, ENOTSUP));
 
+	postevent =
+	    (vd->vdev_offline == B_TRUE || vd->vdev_tmpoffline == B_TRUE) ?
+	    B_TRUE : B_FALSE;
+
 	tvd = vd->vdev_top;
 	vd->vdev_offline = B_FALSE;
 	vd->vdev_tmpoffline = B_FALSE;
@@ -2547,6 +2552,10 @@ vdev_online(spa_t *spa, uint64_t guid, uint64_t flags, vdev_state_t *newstate)
 			return (spa_vdev_state_exit(spa, vd, ENOTSUP));
 		spa_async_request(spa, SPA_ASYNC_CONFIG_UPDATE);
 	}
+
+	if (postevent)
+		spa_event_notify(spa, vd, ESC_ZFS_VDEV_ONLINE);
+
 	return (spa_vdev_state_exit(spa, vd, 0));
 }
 
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vfsops.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vfsops.c
index b45d958a030d3..e8bcaf53d3b09 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vfsops.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vfsops.c
@@ -555,35 +555,7 @@ zfs_register_callbacks(vfs_t *vfsp)
 	return (0);
 
 unregister:
-	/*
-	 * We may attempt to unregister some callbacks that are not
-	 * registered, but this is OK; it will simply return ENOMSG,
-	 * which we will ignore.
-	 */
-	(void) dsl_prop_unregister(ds, zfs_prop_to_name(ZFS_PROP_ATIME),
-	    atime_changed_cb, zfsvfs);
-	(void) dsl_prop_unregister(ds, zfs_prop_to_name(ZFS_PROP_XATTR),
-	    xattr_changed_cb, zfsvfs);
-	(void) dsl_prop_unregister(ds, zfs_prop_to_name(ZFS_PROP_RECORDSIZE),
-	    blksz_changed_cb, zfsvfs);
-	(void) dsl_prop_unregister(ds, zfs_prop_to_name(ZFS_PROP_READONLY),
-	    readonly_changed_cb, zfsvfs);
-#ifdef illumos
-	(void) dsl_prop_unregister(ds, zfs_prop_to_name(ZFS_PROP_DEVICES),
-	    devices_changed_cb, zfsvfs);
-#endif
-	(void) dsl_prop_unregister(ds, zfs_prop_to_name(ZFS_PROP_SETUID),
-	    setuid_changed_cb, zfsvfs);
-	(void) dsl_prop_unregister(ds, zfs_prop_to_name(ZFS_PROP_EXEC),
-	    exec_changed_cb, zfsvfs);
-	(void) dsl_prop_unregister(ds, zfs_prop_to_name(ZFS_PROP_SNAPDIR),
-	    snapdir_changed_cb, zfsvfs);
-	(void) dsl_prop_unregister(ds, zfs_prop_to_name(ZFS_PROP_ACLMODE),
-	    acl_mode_changed_cb, zfsvfs);
-	(void) dsl_prop_unregister(ds, zfs_prop_to_name(ZFS_PROP_ACLINHERIT),
-	    acl_inherit_changed_cb, zfsvfs);
-	(void) dsl_prop_unregister(ds, zfs_prop_to_name(ZFS_PROP_VSCAN),
-	    vscan_changed_cb, zfsvfs);
+	dsl_prop_unregister_all(ds, zfsvfs);
 	return (error);
 }
 
@@ -1263,43 +1235,9 @@ void
 zfs_unregister_callbacks(zfsvfs_t *zfsvfs)
 {
 	objset_t *os = zfsvfs->z_os;
-	struct dsl_dataset *ds;
-
-	/*
-	 * Unregister properties.
-	 */
-	if (!dmu_objset_is_snapshot(os)) {
-		ds = dmu_objset_ds(os);
-		VERIFY(dsl_prop_unregister(ds, "atime", atime_changed_cb,
-		    zfsvfs) == 0);
-
-		VERIFY(dsl_prop_unregister(ds, "xattr", xattr_changed_cb,
-		    zfsvfs) == 0);
 
-		VERIFY(dsl_prop_unregister(ds, "recordsize", blksz_changed_cb,
-		    zfsvfs) == 0);
-
-		VERIFY(dsl_prop_unregister(ds, "readonly", readonly_changed_cb,
-		    zfsvfs) == 0);
-
-		VERIFY(dsl_prop_unregister(ds, "setuid", setuid_changed_cb,
-		    zfsvfs) == 0);
-
-		VERIFY(dsl_prop_unregister(ds, "exec", exec_changed_cb,
-		    zfsvfs) == 0);
-
-		VERIFY(dsl_prop_unregister(ds, "snapdir", snapdir_changed_cb,
-		    zfsvfs) == 0);
-
-		VERIFY(dsl_prop_unregister(ds, "aclmode", acl_mode_changed_cb,
-		    zfsvfs) == 0);
-
-		VERIFY(dsl_prop_unregister(ds, "aclinherit",
-		    acl_inherit_changed_cb, zfsvfs) == 0);
-
-		VERIFY(dsl_prop_unregister(ds, "vscan",
-		    vscan_changed_cb, zfsvfs) == 0);
-	}
+	if (!dmu_objset_is_snapshot(os))
+		dsl_prop_unregister_all(dmu_objset_ds(os), zfsvfs);
 }
 
 #ifdef SECLABEL
diff --git a/sys/cddl/contrib/opensolaris/uts/common/sys/dtrace_impl.h b/sys/cddl/contrib/opensolaris/uts/common/sys/dtrace_impl.h
index a65d1ae16c182..1ec90914d5be0 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/sys/dtrace_impl.h
+++ b/sys/cddl/contrib/opensolaris/uts/common/sys/dtrace_impl.h
@@ -1317,16 +1317,19 @@ extern void dtrace_copystr(uintptr_t, uintptr_t, size_t, volatile uint16_t *);
 /*
  * DTrace Assertions
  *
- * DTrace calls ASSERT from probe context.  To assure that a failed ASSERT
- * does not induce a markedly more catastrophic failure (e.g., one from which
- * a dump cannot be gleaned), DTrace must define its own ASSERT to be one that
- * may safely be called from probe context.  This header file must thus be
- * included by any DTrace component that calls ASSERT from probe context, and
- * _only_ by those components.  (The only exception to this is kernel
- * debugging infrastructure at user-level that doesn't depend on calling
- * ASSERT.)
+ * DTrace calls ASSERT and VERIFY from probe context.  To assure that a failed
+ * ASSERT or VERIFY does not induce a markedly more catastrophic failure (e.g.,
+ * one from which a dump cannot be gleaned), DTrace must define its own ASSERT
+ * and VERIFY macros to be ones that may safely be called from probe context.
+ * This header file must thus be included by any DTrace component that calls
+ * ASSERT and/or VERIFY from probe context, and _only_ by those components.
+ * (The only exception to this is kernel debugging infrastructure at user-level
+ * that doesn't depend on calling ASSERT.)
  */
 #undef ASSERT
+#undef VERIFY
+#define	VERIFY(EX)	((void)((EX) || \
+			dtrace_assfail(#EX, __FILE__, __LINE__)))
 #ifdef DEBUG
 #define	ASSERT(EX)	((void)((EX) || \
 			dtrace_assfail(#EX, __FILE__, __LINE__)))
diff --git a/sys/cddl/contrib/opensolaris/uts/common/sys/sysevent/eventdefs.h b/sys/cddl/contrib/opensolaris/uts/common/sys/sysevent/eventdefs.h
index 89bb06e50a08f..5760a8564ab94 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/sys/sysevent/eventdefs.h
+++ b/sys/cddl/contrib/opensolaris/uts/common/sys/sysevent/eventdefs.h
@@ -20,7 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
+ * Copyright 2015 Nexenta Systems, Inc.  All rights reserved.
  */
 
 #ifndef	_SYS_SYSEVENT_EVENTDEFS_H
@@ -249,9 +249,14 @@ extern "C" {
 #define	ESC_ZFS_RESILVER_START		"ESC_ZFS_resilver_start"
 #define	ESC_ZFS_RESILVER_FINISH		"ESC_ZFS_resilver_finish"
 #define	ESC_ZFS_VDEV_REMOVE		"ESC_ZFS_vdev_remove"
+#define	ESC_ZFS_POOL_CREATE		"ESC_ZFS_pool_create"
 #define	ESC_ZFS_POOL_DESTROY		"ESC_ZFS_pool_destroy"
+#define	ESC_ZFS_POOL_IMPORT		"ESC_ZFS_pool_import"
+#define	ESC_ZFS_VDEV_ADD		"ESC_ZFS_vdev_add"
+#define	ESC_ZFS_VDEV_ATTACH		"ESC_ZFS_vdev_attach"
 #define	ESC_ZFS_VDEV_CLEAR		"ESC_ZFS_vdev_clear"
 #define	ESC_ZFS_VDEV_CHECK		"ESC_ZFS_vdev_check"
+#define	ESC_ZFS_VDEV_ONLINE		"ESC_ZFS_vdev_online"
 #define	ESC_ZFS_CONFIG_SYNC		"ESC_ZFS_config_sync"
 #define	ESC_ZFS_SCRUB_START		"ESC_ZFS_scrub_start"
 #define	ESC_ZFS_SCRUB_FINISH		"ESC_ZFS_scrub_finish"
diff --git a/sys/cddl/dev/dtrace/amd64/dtrace_isa.c b/sys/cddl/dev/dtrace/amd64/dtrace_isa.c
index e05932d77e3ca..d233be4fd2c18 100644
--- a/sys/cddl/dev/dtrace/amd64/dtrace_isa.c
+++ b/sys/cddl/dev/dtrace/amd64/dtrace_isa.c
@@ -448,7 +448,6 @@ load:
 	DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
 
 	return (val);
-	return (0);
 }
 
 int
diff --git a/sys/cddl/dev/dtrace/powerpc/dtrace_isa.c b/sys/cddl/dev/dtrace/powerpc/dtrace_isa.c
index d35bf773eb231..d80e950048617 100644
--- a/sys/cddl/dev/dtrace/powerpc/dtrace_isa.c
+++ b/sys/cddl/dev/dtrace/powerpc/dtrace_isa.c
@@ -520,7 +520,6 @@ load:
 	DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
 
 	return (val);
-	return (0);
 }
 
 int
diff --git a/sys/cddl/dev/sdt/sdt.c b/sys/cddl/dev/sdt/sdt.c
index a321317408900..4e5dd71f9b947 100644
--- a/sys/cddl/dev/sdt/sdt.c
+++ b/sys/cddl/dev/sdt/sdt.c
@@ -99,8 +99,8 @@ static dtrace_pops_t sdt_pops = {
 
 static TAILQ_HEAD(, sdt_provider) sdt_prov_list;
 
-eventhandler_tag	sdt_kld_load_tag;
-eventhandler_tag	sdt_kld_unload_try_tag;
+static eventhandler_tag	sdt_kld_load_tag;
+static eventhandler_tag	sdt_kld_unload_try_tag;
 
 static void
 sdt_create_provider(struct sdt_provider *prov)
@@ -141,6 +141,12 @@ sdt_create_probe(struct sdt_probe *probe)
 	char *to;
 	size_t len;
 
+	if (probe->version != (int)sizeof(*probe)) {
+		printf("ignoring probe %p, version %u expected %u\n",
+		    probe, probe->version, (int)sizeof(*probe));
+		return;
+	}
+
 	TAILQ_FOREACH(prov, &sdt_prov_list, prov_entry)
 		if (strcmp(prov->name, probe->prov->name) == 0)
 			break;
@@ -162,6 +168,8 @@ sdt_create_probe(struct sdt_probe *probe)
 	 * in the C compiler, so we have to respect const vs non-const.
 	 */
 	strlcpy(func, probe->func, sizeof(func));
+	if (func[0] == '\0')
+		strcpy(func, "none");
 
 	from = probe->name;
 	to = name;
@@ -401,4 +409,3 @@ sdt_modevent(module_t mod __unused, int type, void *data __unused)
 DEV_MODULE(sdt, sdt_modevent, NULL);
 MODULE_VERSION(sdt, 1);
 MODULE_DEPEND(sdt, dtrace, 1, 1, 1);
-MODULE_DEPEND(sdt, opensolaris, 1, 1, 1);
diff --git a/sys/compat/linux/linux_dtrace.h b/sys/compat/linux/linux_dtrace.h
index b713f16c82a6e..c446b3e067fb0 100644
--- a/sys/compat/linux/linux_dtrace.h
+++ b/sys/compat/linux/linux_dtrace.h
@@ -82,7 +82,7 @@
     c, d, e, f)
 #define	LIN_SDT_PROBE4(a, b, c, d, e, f, g)	SDT_PROBE4(LINUX_DTRACE, a, b, \
     c, d, e, f, g)
-#define	_LIN_SDT_PROBE5(a, b, c, d, e, f, g, h, i)	SDT_PROBE(a, b, c, d, \
+#define	_LIN_SDT_PROBE5(a, b, c, d, e, f, g, h, i)	SDT_PROBE5(a, b, c, d, \
     e, f, g, h, i)
 #define	LIN_SDT_PROBE5(a, b, c, d, e, f, g, h)	_LIN_SDT_PROBE5(LINUX_DTRACE, \
     a, b, c, d, e, f, g, h)
diff --git a/sys/conf/files b/sys/conf/files
index df8233f472edc..4f13269f3ebdf 100644
--- a/sys/conf/files
+++ b/sys/conf/files
@@ -1881,6 +1881,7 @@ dev/ixgbe/ixgbe_dcb_82599.c	optional ix inet | ixv inet \
 dev/jme/if_jme.c		optional jme pci
 dev/joy/joy.c			optional joy
 dev/joy/joy_isa.c		optional joy isa
+dev/kbd/kbd.c			optional atkbd | pckbd | sc | ukbd | vt
 dev/kbdmux/kbdmux.c		optional kbdmux
 dev/ksyms/ksyms.c		optional ksyms
 dev/le/am7990.c			optional le
diff --git a/sys/conf/files.amd64 b/sys/conf/files.amd64
index 3def78fc3c214..0788b4669efaa 100644
--- a/sys/conf/files.amd64
+++ b/sys/conf/files.amd64
@@ -275,7 +275,6 @@ dev/hyperv/vmbus/hv_connection.c			optional	hyperv
 dev/hyperv/vmbus/hv_hv.c				optional	hyperv
 dev/hyperv/vmbus/hv_ring_buffer.c			optional	hyperv
 dev/hyperv/vmbus/hv_vmbus_drv_freebsd.c			optional	hyperv
-dev/kbd/kbd.c			optional	atkbd | sc | ukbd | vt
 dev/nfe/if_nfe.c		optional	nfe pci
 dev/ntb/if_ntb/if_ntb.c		optional	if_ntb
 dev/ntb/ntb_hw/ntb_hw.c		optional	if_ntb | ntb_hw
diff --git a/sys/conf/files.arm b/sys/conf/files.arm
index 743fd867cfe74..84267a0c8d14d 100644
--- a/sys/conf/files.arm
+++ b/sys/conf/files.arm
@@ -84,11 +84,12 @@ cddl/dev/dtrace/arm/dtrace_subr.c			optional dtrace compile-with "${DTRACE_C}"
 cddl/dev/fbt/arm/fbt_isa.c				optional dtrace_fbt | dtraceall compile-with "${FBT_C}"
 crypto/blowfish/bf_enc.c	optional	crypto | ipsec 
 crypto/des/des_enc.c		optional	crypto | ipsec | netsmb
+dev/dwc/if_dwc.c		optional	dwc
+dev/dwc/if_dwc_if.m		optional	dwc
 dev/fb/fb.c			optional	sc
 dev/fdt/fdt_arm_platform.c	optional	platform fdt
 dev/hwpmc/hwpmc_arm.c		optional	hwpmc
 dev/hwpmc/hwpmc_armv7.c		optional	hwpmc armv6
-dev/kbd/kbd.c			optional	sc | vt
 dev/psci/psci.c			optional	psci
 dev/psci/psci_arm.S		optional	psci
 dev/syscons/scgfbrndr.c		optional	sc
diff --git a/sys/conf/files.i386 b/sys/conf/files.i386
index 1fae3a51d0871..da2097b0cfe2d 100644
--- a/sys/conf/files.i386
+++ b/sys/conf/files.i386
@@ -265,7 +265,6 @@ dev/ipmi/ipmi_smbios.c		optional ipmi
 dev/ipmi/ipmi_ssif.c		optional ipmi smbus
 dev/ipmi/ipmi_pci.c		optional ipmi pci
 dev/ipmi/ipmi_linux.c		optional ipmi compat_linux
-dev/kbd/kbd.c			optional atkbd | sc | ukbd | vt
 dev/le/if_le_isa.c		optional le isa
 dev/mse/mse.c			optional mse
 dev/mse/mse_isa.c		optional mse isa
diff --git a/sys/conf/files.mips b/sys/conf/files.mips
index f47630710d2c4..76ad371d932d9 100644
--- a/sys/conf/files.mips
+++ b/sys/conf/files.mips
@@ -71,7 +71,6 @@ dev/cfe/cfe_env.c			optional	cfe_env
 
 # syscons support
 dev/fb/fb.c				optional	sc
-dev/kbd/kbd.c				optional	sc | vt
 dev/syscons/scgfbrndr.c			optional	sc
 dev/syscons/scterm-teken.c		optional	sc
 dev/syscons/scvtb.c			optional	sc
diff --git a/sys/conf/files.pc98 b/sys/conf/files.pc98
index ee6cf714ad5bb..78f85baeb349c 100644
--- a/sys/conf/files.pc98
+++ b/sys/conf/files.pc98
@@ -127,7 +127,6 @@ dev/hwpmc/hwpmc_ppro.c		optional hwpmc
 dev/hwpmc/hwpmc_tsc.c		optional hwpmc
 dev/hwpmc/hwpmc_x86.c		optional hwpmc
 dev/io/iodev.c			optional io
-dev/kbd/kbd.c			optional pckbd | sc | ukbd
 dev/le/if_le_cbus.c		optional le isa
 dev/mse/mse.c			optional mse
 dev/mse/mse_cbus.c		optional mse isa
diff --git a/sys/conf/files.powerpc b/sys/conf/files.powerpc
index d647accf80b9c..be17b6ee5bdb1 100644
--- a/sys/conf/files.powerpc
+++ b/sys/conf/files.powerpc
@@ -44,7 +44,6 @@ dev/iicbus/adt746x.c		optional        adt746x powermac
 dev/iicbus/ds1631.c		optional	ds1631 powermac
 dev/iicbus/ds1775.c		optional	ds1775 powermac
 dev/iicbus/max6690.c		optional	max6690 powermac
-dev/kbd/kbd.c			optional	sc | vt
 dev/nand/nfc_fsl.c		optional	nand mpc85xx
 dev/nand/nfc_rb.c		optional	nand mpc85xx
 # ofw can be either aim or fdt: fdt case handled in files. aim only powerpc specific.
diff --git a/sys/conf/files.sparc64 b/sys/conf/files.sparc64
index 0c160d4a13c9c..0a0b721c91f89 100644
--- a/sys/conf/files.sparc64
+++ b/sys/conf/files.sparc64
@@ -40,7 +40,6 @@ dev/fb/fb.c			optional	sc
 dev/fb/gallant12x22.c		optional	sc
 dev/fb/machfb.c			optional	machfb sc
 dev/hwpmc/hwpmc_sparc64.c	optional	hwpmc
-dev/kbd/kbd.c			optional	atkbd | sc | ukbd | vt
 dev/le/if_le_lebuffer.c		optional	le sbus
 dev/le/if_le_ledma.c		optional	le sbus
 dev/le/lebuffer_sbus.c		optional	le sbus
@@ -82,6 +81,7 @@ sparc64/isa/isa_dma.c		optional	isa
 sparc64/isa/ofw_isa.c		optional	ebus | isa
 sparc64/pci/apb.c		optional	pci
 sparc64/pci/fire.c		optional	pci
+sparc64/pci/ofw_pci.c		optional	pci
 sparc64/pci/ofw_pcib.c		optional	pci
 sparc64/pci/ofw_pcib_subr.c	optional	pci
 sparc64/pci/ofw_pcibus.c	optional	pci
diff --git a/sys/conf/kern.post.mk b/sys/conf/kern.post.mk
index d87aa5c21e61d..0703cc82347b6 100644
--- a/sys/conf/kern.post.mk
+++ b/sys/conf/kern.post.mk
@@ -23,6 +23,11 @@ MKMODULESENV+=	CONF_CFLAGS="${CONF_CFLAGS}"
 MKMODULESENV+=	WITH_CTF="${WITH_CTF}"
 .endif
 
+# Allow overriding the kernel debug directory, so kernel and user debug may be
+# installed in different directories. Setting it to "" restores the historical
+# behavior of installing debug files in the kernel directory.
+KERN_DEBUGDIR?=	${DEBUGDIR}
+
 .MAIN: all
 
 .for target in all clean cleandepend cleandir clobber depend install \
@@ -101,11 +106,11 @@ modules-all modules-depend: modules-obj
 .if !defined(DEBUG)
 FULLKERNEL=	${KERNEL_KO}
 .else
-FULLKERNEL=	${KERNEL_KO}.debug
-${KERNEL_KO}: ${FULLKERNEL} ${KERNEL_KO}.symbols
-	${OBJCOPY} --strip-debug --add-gnu-debuglink=${KERNEL_KO}.symbols\
+FULLKERNEL=	${KERNEL_KO}.full
+${KERNEL_KO}: ${FULLKERNEL} ${KERNEL_KO}.debug
+	${OBJCOPY} --strip-debug --add-gnu-debuglink=${KERNEL_KO}.debug \
 	    ${FULLKERNEL} ${.TARGET}
-${KERNEL_KO}.symbols: ${FULLKERNEL}
+${KERNEL_KO}.debug: ${FULLKERNEL}
 	${OBJCOPY} --only-keep-debug ${FULLKERNEL} ${.TARGET}
 install.debug reinstall.debug: gdbinit
 	cd ${.CURDIR}; ${MAKE} ${.TARGET:R}
@@ -151,7 +156,7 @@ ${mfile:T:S/.m$/.h/}: ${mfile}
 
 kernel-clean:
 	rm -f *.o *.so *.So *.ko *.s eddep errs \
-	    ${FULLKERNEL} ${KERNEL_KO} ${KERNEL_KO}.symbols \
+	    ${FULLKERNEL} ${KERNEL_KO} ${KERNEL_KO}.debug \
 	    linterrs tags vers.c \
 	    vnode_if.c vnode_if.h vnode_if_newproto.h vnode_if_typedef.h \
 	    ${MFILES:T:S/.m$/.c/} ${MFILES:T:S/.m$/.h/} \
@@ -249,19 +254,26 @@ kernel-install:
 	if [ ! "`dirname "$$thiskernel"`" -ef ${DESTDIR}${KODIR} ] ; then \
 		chflags -R noschg ${DESTDIR}${KODIR} ; \
 		rm -rf ${DESTDIR}${KODIR} ; \
+		rm -rf ${DESTDIR}${KERN_DEBUGDIR}${KODIR} ; \
 	else \
 		if [ -d ${DESTDIR}${KODIR}.old ] ; then \
 			chflags -R noschg ${DESTDIR}${KODIR}.old ; \
 			rm -rf ${DESTDIR}${KODIR}.old ; \
 		fi ; \
 		mv ${DESTDIR}${KODIR} ${DESTDIR}${KODIR}.old ; \
+		if [ -n "${KERN_DEBUGDIR}" -a \
+		     -d ${DESTDIR}${KERN_DEBUGDIR}${KODIR} ]; then \
+			rm -rf ${DESTDIR}${KERN_DEBUGDIR}${KODIR}.old ; \
+			mv ${DESTDIR}${KERN_DEBUGDIR}${KODIR} ${DESTDIR}${KERN_DEBUGDIR}${KODIR}.old ; \
+		fi ; \
 		sysctl kern.bootfile=${DESTDIR}${KODIR}.old/"`basename "$$thiskernel"`" ; \
 	fi
 .endif
 	mkdir -p ${DESTDIR}${KODIR}
-	${INSTALL} -T release -p -m 555 -o ${KMODOWN} -g ${KMODGRP} ${KERNEL_KO} ${DESTDIR}${KODIR}
+	${INSTALL} -p -m 555 -o ${KMODOWN} -g ${KMODGRP} ${KERNEL_KO} ${DESTDIR}${KODIR}
 .if defined(DEBUG) && !defined(INSTALL_NODEBUG) && ${MK_KERNEL_SYMBOLS} != "no"
-	${INSTALL} -T debug -p -m 555 -o ${KMODOWN} -g ${KMODGRP} ${KERNEL_KO}.symbols ${DESTDIR}${KODIR}
+	mkdir -p ${DESTDIR}${KERN_DEBUGDIR}${KODIR}
+	${INSTALL} -p -m 555 -o ${KMODOWN} -g ${KMODGRP} ${KERNEL_KO}.debug ${DESTDIR}${KERN_DEBUGDIR}${KODIR}
 .endif
 .if defined(KERNEL_EXTRA_INSTALL)
 	${INSTALL} -p -m 555 -o ${KMODOWN} -g ${KMODGRP} ${KERNEL_EXTRA_INSTALL} ${DESTDIR}${KODIR}
@@ -271,9 +283,9 @@ kernel-install:
 
 kernel-reinstall:
 	@-chflags -R noschg ${DESTDIR}${KODIR}
-	${INSTALL} -T release -p -m 555 -o ${KMODOWN} -g ${KMODGRP} ${KERNEL_KO} ${DESTDIR}${KODIR}
+	${INSTALL} -p -m 555 -o ${KMODOWN} -g ${KMODGRP} ${KERNEL_KO} ${DESTDIR}${KODIR}
 .if defined(DEBUG) && !defined(INSTALL_NODEBUG) && ${MK_KERNEL_SYMBOLS} != "no"
-	${INSTALL} -T debug -p -m 555 -o ${KMODOWN} -g ${KMODGRP} ${KERNEL_KO}.symbols ${DESTDIR}${KODIR}
+	${INSTALL} -p -m 555 -o ${KMODOWN} -g ${KMODGRP} ${KERNEL_KO}.debug ${DESTDIR}${KERN_DEBUGDIR}${KODIR}
 .endif
 
 config.o env.o hints.o vers.o vnode_if.o:
diff --git a/sys/conf/kmod.mk b/sys/conf/kmod.mk
index 4b632a7639bab..c06ae51317e0b 100644
--- a/sys/conf/kmod.mk
+++ b/sys/conf/kmod.mk
@@ -172,11 +172,11 @@ PROG=	${KMOD}.ko
 .if !defined(DEBUG_FLAGS)
 FULLPROG=	${PROG}
 .else
-FULLPROG=	${PROG}.debug
-${PROG}: ${FULLPROG} ${PROG}.symbols
-	${OBJCOPY} --strip-debug --add-gnu-debuglink=${PROG}.symbols\
+FULLPROG=	${PROG}.full
+${PROG}: ${FULLPROG} ${PROG}.debug
+	${OBJCOPY} --strip-debug --add-gnu-debuglink=${PROG}.debug \
 	    ${FULLPROG} ${.TARGET}
-${PROG}.symbols: ${FULLPROG}
+${PROG}.debug: ${FULLPROG}
 	${OBJCOPY} --only-keep-debug ${FULLPROG} ${.TARGET}
 .endif
 
@@ -266,7 +266,7 @@ ${_ILINKS}:
 CLEANFILES+= ${PROG} ${KMOD}.kld ${OBJS}
 
 .if defined(DEBUG_FLAGS)
-CLEANFILES+= ${FULLPROG} ${PROG}.symbols
+CLEANFILES+= ${FULLPROG} ${PROG}.debug
 .endif
 
 .if !target(install)
@@ -277,6 +277,7 @@ _INSTALLFLAGS:=	${_INSTALLFLAGS${ie}}
 .endfor
 
 .if !target(realinstall)
+KERN_DEBUGDIR?=	${DEBUGDIR}
 realinstall: _kmodinstall
 .ORDER: beforeinstall _kmodinstall
 _kmodinstall:
@@ -284,7 +285,7 @@ _kmodinstall:
 	    ${_INSTALLFLAGS} ${PROG} ${DESTDIR}${KMODDIR}
 .if defined(DEBUG_FLAGS) && !defined(INSTALL_NODEBUG) && ${MK_KERNEL_SYMBOLS} != "no"
 	${INSTALL} -T debug -o ${KMODOWN} -g ${KMODGRP} -m ${KMODMODE} \
-	    ${_INSTALLFLAGS} ${PROG}.symbols ${DESTDIR}${KMODDIR}
+	    ${_INSTALLFLAGS} ${PROG}.debug ${DESTDIR}${KERN_DEBUGDIR}${KMODDIR}
 .endif
 
 .include <bsd.links.mk>
diff --git a/sys/contrib/dev/otus/otus-init b/sys/contrib/dev/otus/otus-init
new file mode 100644
index 0000000000000..d422bdbdf2284
--- /dev/null
+++ b/sys/contrib/dev/otus/otus-init
diff --git a/sys/contrib/dev/otus/otus-license b/sys/contrib/dev/otus/otus-license
new file mode 100644
index 0000000000000..0fc7576c40617
--- /dev/null
+++ b/sys/contrib/dev/otus/otus-license
@@ -0,0 +1,47 @@
+Copyright (c) 2008, Atheros Communications, Inc.
+All rights reserved.
+
+Redistribution.  Redistribution and use in binary form, without
+modification, are permitted provided that the following conditions are
+met:
+
+* Redistributions must reproduce the above copyright notice and the
+  following disclaimer in the documentation and/or other materials
+  provided with the distribution.
+
+* Neither the name of Atheros Communications, Inc. nor the names of
+  its suppliers may be used to endorse or promote products derived
+  from this software without specific prior written permission.
+
+* No reverse engineering, decompilation, or disassembly of this
+  software is permitted.
+
+Limited patent license.  Atheros Communications, Inc. grants a
+world-wide, royalty-free, non-exclusive license under patents it
+now or hereafter owns or controls to make, have made, use, import,
+offer to sell and sell ("Utilize") this software, but solely to
+the extent that any such patent is necessary to Utilize the software
+alone, or in combination with an operating system licensed under an
+approved Open Source license as listed by the Open Source Initiative
+at http://opensource.org/licenses.  The patent license shall not
+apply to any other combinations which include this software. No
+hardware per se is licensed hereunder.
+
+DISCLAIMER.  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,
+BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
+TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+The following files are under this license:
+
+otus-init otus-main
+
+These files are needed by otus(4) devices.
diff --git a/sys/contrib/dev/otus/otus-main b/sys/contrib/dev/otus/otus-main
new file mode 100644
index 0000000000000..aa0e80717b1b4
--- /dev/null
+++ b/sys/contrib/dev/otus/otus-main
diff --git a/sys/contrib/ipfilter/netinet/ip_nat.c b/sys/contrib/ipfilter/netinet/ip_nat.c
index 36b4fe3bd2d70..9ec450b5bd27a 100644
--- a/sys/contrib/ipfilter/netinet/ip_nat.c
+++ b/sys/contrib/ipfilter/netinet/ip_nat.c
@@ -8075,13 +8075,13 @@ ipf_nat_rehash(softc, t, p)
 	 * the outbound lookup table and the hash chain length for each.
 	 */
 	KMALLOCS(newtab[0], nat_t **, newsize * sizeof(nat_t *));
-	if (newtab == NULL) {
+	if (newtab[0] == NULL) {
 		error = 60063;
 		goto badrehash;
 	}
 
 	KMALLOCS(newtab[1], nat_t **, newsize * sizeof(nat_t *));
-	if (newtab == NULL) {
+	if (newtab[1] == NULL) {
 		error = 60064;
 		goto badrehash;
 	}
diff --git a/sys/dev/acpica/acpi_pcib.c b/sys/dev/acpica/acpi_pcib.c
index 93dada5703ef7..c2c24992f89a5 100644
--- a/sys/dev/acpica/acpi_pcib.c
+++ b/sys/dev/acpica/acpi_pcib.c
@@ -150,7 +150,7 @@ acpi_pcib_attach(device_t dev, ACPI_BUFFER *prt, int busno)
     /*
      * Attach the PCI bus proper.
      */
-    if (device_add_child(dev, "pci", busno) == NULL) {
+    if (device_add_child(dev, "pci", -1) == NULL) {
 	device_printf(device_get_parent(dev), "couldn't attach pci bus\n");
 	return_VALUE(ENXIO);
     }
diff --git a/sys/dev/ahci/ahci_pci.c b/sys/dev/ahci/ahci_pci.c
index a8f49a1cc3659..5c236cea692c2 100644
--- a/sys/dev/ahci/ahci_pci.c
+++ b/sys/dev/ahci/ahci_pci.c
@@ -327,6 +327,9 @@ ahci_probe(device_t dev)
 	    pci_get_subclass(dev) == PCIS_STORAGE_SATA &&
 	    pci_get_progif(dev) == PCIP_STORAGE_SATA_AHCI_1_0)
 		valid = 1;
+	else if (pci_get_class(dev) == PCIC_STORAGE &&
+	    pci_get_subclass(dev) == PCIS_STORAGE_RAID)
+		valid = 2;
 	/* Is this a known AHCI chip? */
 	for (i = 0; ahci_ids[i].id != 0; i++) {
 		if (ahci_ids[i].id == devid &&
@@ -343,7 +346,7 @@ ahci_probe(device_t dev)
 			return (BUS_PROBE_DEFAULT);
 		}
 	}
-	if (!valid)
+	if (valid != 1)
 		return (ENXIO);
 	device_set_desc_copy(dev, "AHCI SATA controller");
 	return (BUS_PROBE_DEFAULT);
diff --git a/sys/dev/ath/if_ath.c b/sys/dev/ath/if_ath.c
index 374e1d03422f1..770badd9175e2 100644
--- a/sys/dev/ath/if_ath.c
+++ b/sys/dev/ath/if_ath.c
@@ -4127,7 +4127,6 @@ ath_txq_init(struct ath_softc *sc, struct ath_txq *txq, int qnum)
 static struct ath_txq *
 ath_txq_setup(struct ath_softc *sc, int qtype, int subtype)
 {
-#define	N(a)	(sizeof(a)/sizeof(a[0]))
 	struct ath_hal *ah = sc->sc_ah;
 	HAL_TXQ_INFO qi;
 	int qnum;
@@ -4164,10 +4163,10 @@ ath_txq_setup(struct ath_softc *sc, int qtype, int subtype)
 		 */
 		return NULL;
 	}
-	if (qnum >= N(sc->sc_txq)) {
+	if (qnum >= nitems(sc->sc_txq)) {
 		device_printf(sc->sc_dev,
 			"hal qnum %u out of range, max %zu!\n",
-			qnum, N(sc->sc_txq));
+			qnum, nitems(sc->sc_txq));
 		ath_hal_releasetxqueue(ah, qnum);
 		return NULL;
 	}
@@ -4176,7 +4175,6 @@ ath_txq_setup(struct ath_softc *sc, int qtype, int subtype)
 		sc->sc_txqsetup |= 1<<qnum;
 	}
 	return &sc->sc_txq[qnum];
-#undef N
 }
 
 /*
@@ -4191,12 +4189,11 @@ ath_txq_setup(struct ath_softc *sc, int qtype, int subtype)
 static int
 ath_tx_setup(struct ath_softc *sc, int ac, int haltype)
 {
-#define	N(a)	(sizeof(a)/sizeof(a[0]))
 	struct ath_txq *txq;
 
-	if (ac >= N(sc->sc_ac2q)) {
+	if (ac >= nitems(sc->sc_ac2q)) {
 		device_printf(sc->sc_dev, "AC %u out of range, max %zu!\n",
-			ac, N(sc->sc_ac2q));
+			ac, nitems(sc->sc_ac2q));
 		return 0;
 	}
 	txq = ath_txq_setup(sc, HAL_TX_QUEUE_DATA, haltype);
@@ -4206,7 +4203,6 @@ ath_tx_setup(struct ath_softc *sc, int ac, int haltype)
 		return 1;
 	} else
 		return 0;
-#undef N
 }
 
 /*
@@ -4216,7 +4212,6 @@ static int
 ath_txq_update(struct ath_softc *sc, int ac)
 {
 #define	ATH_EXPONENT_TO_VALUE(v)	((1<<v)-1)
-#define	ATH_TXOP_TO_US(v)		(v<<5)
 	struct ieee80211com *ic = &sc->sc_ic;
 	struct ath_txq *txq = sc->sc_ac2q[ac];
 	struct wmeParams *wmep = &ic->ic_wme.wme_chanParams.cap_wmeParams[ac];
@@ -4262,7 +4257,7 @@ ath_txq_update(struct ath_softc *sc, int ac)
 		qi.tqi_cwmin = ATH_EXPONENT_TO_VALUE(wmep->wmep_logcwmin);
 		qi.tqi_cwmax = ATH_EXPONENT_TO_VALUE(wmep->wmep_logcwmax);
 		qi.tqi_readyTime = 0;
-		qi.tqi_burstTime = ATH_TXOP_TO_US(wmep->wmep_txopLimit);
+		qi.tqi_burstTime = IEEE80211_TXOP_TO_US(wmep->wmep_txopLimit);
 #ifdef IEEE80211_SUPPORT_TDMA
 	}
 #endif
@@ -4280,7 +4275,6 @@ ath_txq_update(struct ath_softc *sc, int ac)
 		ath_hal_resettxqueue(ah, txq->axq_qnum); /* push to h/w */
 		return 1;
 	}
-#undef ATH_TXOP_TO_US
 #undef ATH_EXPONENT_TO_VALUE
 }
 
@@ -6302,7 +6296,6 @@ ath_rate_setup(struct ath_softc *sc, u_int mode)
 static void
 ath_setcurmode(struct ath_softc *sc, enum ieee80211_phymode mode)
 {
-#define	N(a)	(sizeof(a)/sizeof(a[0]))
 	/* NB: on/off times from the Atheros NDIS driver, w/ permission */
 	static const struct {
 		u_int		rate;		/* tx/rx 802.11 rate */
@@ -6339,7 +6332,7 @@ ath_setcurmode(struct ath_softc *sc, enum ieee80211_phymode mode)
 			sc->sc_rixmap[ieeerate | IEEE80211_RATE_MCS] = i;
 	}
 	memset(sc->sc_hwmap, 0, sizeof(sc->sc_hwmap));
-	for (i = 0; i < N(sc->sc_hwmap); i++) {
+	for (i = 0; i < nitems(sc->sc_hwmap); i++) {
 		if (i >= rt->rateCount) {
 			sc->sc_hwmap[i].ledon = (500 * hz) / 1000;
 			sc->sc_hwmap[i].ledoff = (130 * hz) / 1000;
@@ -6354,7 +6347,7 @@ ath_setcurmode(struct ath_softc *sc, enum ieee80211_phymode mode)
 		    rt->info[i].phy == IEEE80211_T_OFDM)
 			sc->sc_hwmap[i].txflags |= IEEE80211_RADIOTAP_F_SHORTPRE;
 		sc->sc_hwmap[i].rxflags = sc->sc_hwmap[i].txflags;
-		for (j = 0; j < N(blinkrates)-1; j++)
+		for (j = 0; j < nitems(blinkrates)-1; j++)
 			if (blinkrates[j].rate == sc->sc_hwmap[i].ieeerate)
 				break;
 		/* NB: this uses the last entry if the rate isn't found */
@@ -6373,7 +6366,6 @@ ath_setcurmode(struct ath_softc *sc, enum ieee80211_phymode mode)
 	else
 		sc->sc_protrix = ath_tx_findrix(sc, 2*1);
 	/* NB: caller is responsible for resetting rate control state */
-#undef N
 }
 
 static void
diff --git a/sys/dev/ath/if_ath_ahb.c b/sys/dev/ath/if_ath_ahb.c
index fe6420c09c014..0b546a069b8b6 100644
--- a/sys/dev/ath/if_ath_ahb.c
+++ b/sys/dev/ath/if_ath_ahb.c
@@ -261,7 +261,6 @@ ath_ahb_attach(device_t dev)
 	ATH_PCU_LOCK_INIT(sc);
 	ATH_RX_LOCK_INIT(sc);
 	ATH_TX_LOCK_INIT(sc);
-	ATH_TX_IC_LOCK_INIT(sc);
 	ATH_TXSTATUS_LOCK_INIT(sc);
 
 	error = ath_attach(device_id, sc);
@@ -271,7 +270,6 @@ ath_ahb_attach(device_t dev)
 	ATH_TXSTATUS_LOCK_DESTROY(sc);
 	ATH_RX_LOCK_DESTROY(sc);
 	ATH_TX_LOCK_DESTROY(sc);
-	ATH_TX_IC_LOCK_DESTROY(sc);
 	ATH_PCU_LOCK_DESTROY(sc);
 	ATH_LOCK_DESTROY(sc);
 	bus_dma_tag_destroy(sc->sc_dmat);
@@ -315,7 +313,6 @@ ath_ahb_detach(device_t dev)
 	ATH_TXSTATUS_LOCK_DESTROY(sc);
 	ATH_RX_LOCK_DESTROY(sc);
 	ATH_TX_LOCK_DESTROY(sc);
-	ATH_TX_IC_LOCK_DESTROY(sc);
 	ATH_PCU_LOCK_DESTROY(sc);
 	ATH_LOCK_DESTROY(sc);
 
diff --git a/sys/dev/ath/if_ath_beacon.c b/sys/dev/ath/if_ath_beacon.c
index 41267b31f885b..1ef3d58d6068a 100644
--- a/sys/dev/ath/if_ath_beacon.c
+++ b/sys/dev/ath/if_ath_beacon.c
@@ -199,7 +199,7 @@ ath_beacon_alloc(struct ath_softc *sc, struct ieee80211_node *ni)
 	 * we assume the mbuf routines will return us something
 	 * with this alignment (perhaps should assert).
 	 */
-	m = ieee80211_beacon_alloc(ni, &avp->av_boff);
+	m = ieee80211_beacon_alloc(ni, &vap->iv_bcn_off);
 	if (m == NULL) {
 		device_printf(sc->sc_dev, "%s: cannot get mbuf\n", __func__);
 		sc->sc_stats.ast_be_nombuf++;
@@ -374,7 +374,7 @@ ath_beacon_setup(struct ath_softc *sc, struct ath_buf *bf)
 void
 ath_beacon_update(struct ieee80211vap *vap, int item)
 {
-	struct ieee80211_beacon_offsets *bo = &ATH_VAP(vap)->av_boff;
+	struct ieee80211_beacon_offsets *bo = &vap->iv_bcn_off;
 
 	setbit(bo->bo_flags, item);
 }
@@ -713,7 +713,7 @@ ath_beacon_generate(struct ath_softc *sc, struct ieee80211vap *vap)
 	/* XXX lock mcastq? */
 	nmcastq = avp->av_mcastq.axq_depth;
 
-	if (ieee80211_beacon_update(bf->bf_node, &avp->av_boff, m, nmcastq)) {
+	if (ieee80211_beacon_update(bf->bf_node, &vap->iv_bcn_off, m, nmcastq)) {
 		/* XXX too conservative? */
 		bus_dmamap_unload(sc->sc_dmat, bf->bf_dmamap);
 		error = bus_dmamap_load_mbuf_sg(sc->sc_dmat, bf->bf_dmamap, m,
@@ -726,7 +726,7 @@ ath_beacon_generate(struct ath_softc *sc, struct ieee80211vap *vap)
 			return NULL;
 		}
 	}
-	if ((avp->av_boff.bo_tim[4] & 1) && cabq->axq_depth) {
+	if ((vap->iv_bcn_off.bo_tim[4] & 1) && cabq->axq_depth) {
 		DPRINTF(sc, ATH_DEBUG_BEACON,
 		    "%s: cabq did not drain, mcastq %u cabq %u\n",
 		    __func__, nmcastq, cabq->axq_depth);
@@ -764,7 +764,7 @@ ath_beacon_generate(struct ath_softc *sc, struct ieee80211vap *vap)
 	 * Enable the CAB queue before the beacon queue to
 	 * insure cab frames are triggered by this beacon.
 	 */
-	if (avp->av_boff.bo_tim[4] & 1) {
+	if (vap->iv_bcn_off.bo_tim[4] & 1) {
 
 		/* NB: only at DTIM */
 		ATH_TXQ_LOCK(&avp->av_mcastq);
@@ -829,7 +829,7 @@ ath_beacon_start_adhoc(struct ath_softc *sc, struct ieee80211vap *vap)
 	 */
 	bf = avp->av_bcbuf;
 	m = bf->bf_m;
-	if (ieee80211_beacon_update(bf->bf_node, &avp->av_boff, m, 0)) {
+	if (ieee80211_beacon_update(bf->bf_node, &vap->iv_bcn_off, m, 0)) {
 		/* XXX too conservative? */
 		bus_dmamap_unload(sc->sc_dmat, bf->bf_dmamap);
 		error = bus_dmamap_load_mbuf_sg(sc->sc_dmat, bf->bf_dmamap, m,
diff --git a/sys/dev/ath/if_ath_keycache.c b/sys/dev/ath/if_ath_keycache.c
index b8a77e89343a1..f81785fa4ce16 100644
--- a/sys/dev/ath/if_ath_keycache.c
+++ b/sys/dev/ath/if_ath_keycache.c
@@ -184,7 +184,6 @@ ath_keyset(struct ath_softc *sc, struct ieee80211vap *vap,
 	const struct ieee80211_key *k,
 	struct ieee80211_node *bss)
 {
-#define	N(a)	(sizeof(a)/sizeof(a[0]))
 	static const u_int8_t ciphermap[] = {
 		HAL_CIPHER_WEP,		/* IEEE80211_CIPHER_WEP */
 		HAL_CIPHER_TKIP,	/* IEEE80211_CIPHER_TKIP */
@@ -208,7 +207,7 @@ ath_keyset(struct ath_softc *sc, struct ieee80211vap *vap,
 	 * so that rx frames have an entry to match.
 	 */
 	if ((k->wk_flags & IEEE80211_KEY_SWCRYPT) == 0) {
-		KASSERT(cip->ic_cipher < N(ciphermap),
+		KASSERT(cip->ic_cipher < nitems(ciphermap),
 			("invalid cipher type %u", cip->ic_cipher));
 		hk.kv_type = ciphermap[cip->ic_cipher];
 		hk.kv_len = k->wk_keylen;
@@ -266,7 +265,6 @@ ath_keyset(struct ath_softc *sc, struct ieee80211vap *vap,
 	ATH_UNLOCK(sc);
 
 	return (ret);
-#undef N
 }
 
 /*
@@ -277,12 +275,11 @@ static u_int16_t
 key_alloc_2pair(struct ath_softc *sc,
 	ieee80211_keyix *txkeyix, ieee80211_keyix *rxkeyix)
 {
-#define	N(a)	(sizeof(a)/sizeof(a[0]))
 	u_int i, keyix;
 
 	KASSERT(sc->sc_splitmic, ("key cache !split"));
 	/* XXX could optimize */
-	for (i = 0; i < N(sc->sc_keymap)/4; i++) {
+	for (i = 0; i < nitems(sc->sc_keymap)/4; i++) {
 		u_int8_t b = sc->sc_keymap[i];
 		if (b != 0xff) {
 			/*
@@ -321,7 +318,6 @@ key_alloc_2pair(struct ath_softc *sc,
 	}
 	DPRINTF(sc, ATH_DEBUG_KEYCACHE, "%s: out of pair space\n", __func__);
 	return 0;
-#undef N
 }
 
 /*
@@ -332,12 +328,11 @@ static u_int16_t
 key_alloc_pair(struct ath_softc *sc,
 	ieee80211_keyix *txkeyix, ieee80211_keyix *rxkeyix)
 {
-#define	N(a)	(sizeof(a)/sizeof(a[0]))
 	u_int i, keyix;
 
 	KASSERT(!sc->sc_splitmic, ("key cache split"));
 	/* XXX could optimize */
-	for (i = 0; i < N(sc->sc_keymap)/4; i++) {
+	for (i = 0; i < nitems(sc->sc_keymap)/4; i++) {
 		u_int8_t b = sc->sc_keymap[i];
 		if (b != 0xff) {
 			/*
@@ -369,7 +364,6 @@ key_alloc_pair(struct ath_softc *sc,
 	}
 	DPRINTF(sc, ATH_DEBUG_KEYCACHE, "%s: out of pair space\n", __func__);
 	return 0;
-#undef N
 }
 
 /*
@@ -379,7 +373,6 @@ static int
 key_alloc_single(struct ath_softc *sc,
 	ieee80211_keyix *txkeyix, ieee80211_keyix *rxkeyix)
 {
-#define	N(a)	(sizeof(a)/sizeof(a[0]))
 	u_int i, keyix;
 
 	if (sc->sc_hasclrkey == 0) {
@@ -391,7 +384,7 @@ key_alloc_single(struct ath_softc *sc,
 	}
 
 	/* XXX try i,i+32,i+64,i+32+64 to minimize key pair conflicts */
-	for (i = 0; i < N(sc->sc_keymap); i++) {
+	for (i = 0; i < nitems(sc->sc_keymap); i++) {
 		u_int8_t b = sc->sc_keymap[i];
 		if (b != 0xff) {
 			/*
@@ -409,7 +402,6 @@ key_alloc_single(struct ath_softc *sc,
 	}
 	DPRINTF(sc, ATH_DEBUG_KEYCACHE, "%s: out of space\n", __func__);
 	return 0;
-#undef N
 }
 
 /*
diff --git a/sys/dev/ath/if_ath_pci.c b/sys/dev/ath/if_ath_pci.c
index 5610882f315db..bc4f7f0229b7e 100644
--- a/sys/dev/ath/if_ath_pci.c
+++ b/sys/dev/ath/if_ath_pci.c
@@ -283,7 +283,6 @@ ath_pci_attach(device_t dev)
 	ATH_PCU_LOCK_INIT(sc);
 	ATH_RX_LOCK_INIT(sc);
 	ATH_TX_LOCK_INIT(sc);
-	ATH_TX_IC_LOCK_INIT(sc);
 	ATH_TXSTATUS_LOCK_INIT(sc);
 
 	/*
@@ -371,7 +370,6 @@ bad1:
 	ATH_TXSTATUS_LOCK_DESTROY(sc);
 	ATH_PCU_LOCK_DESTROY(sc);
 	ATH_RX_LOCK_DESTROY(sc);
-	ATH_TX_IC_LOCK_DESTROY(sc);
 	ATH_TX_LOCK_DESTROY(sc);
 	ATH_LOCK_DESTROY(sc);
 
@@ -408,7 +406,6 @@ ath_pci_detach(device_t dev)
 	ATH_TXSTATUS_LOCK_DESTROY(sc);
 	ATH_PCU_LOCK_DESTROY(sc);
 	ATH_RX_LOCK_DESTROY(sc);
-	ATH_TX_IC_LOCK_DESTROY(sc);
 	ATH_TX_LOCK_DESTROY(sc);
 	ATH_LOCK_DESTROY(sc);
 
diff --git a/sys/dev/ath/if_ath_tdma.c b/sys/dev/ath/if_ath_tdma.c
index 99cd61eb51a6f..e3460c6374c3a 100644
--- a/sys/dev/ath/if_ath_tdma.c
+++ b/sys/dev/ath/if_ath_tdma.c
@@ -548,7 +548,7 @@ ath_tdma_update(struct ieee80211_node *ni,
 	 *     slot position changes) because ieee80211_add_tdma
 	 *     skips over the data.
 	 */
-	memcpy(ATH_VAP(vap)->av_boff.bo_tdma +
+	memcpy(vap->iv_bcn_off.bo_tdma +
 		__offsetof(struct ieee80211_tdma_param, tdma_tstamp),
 		&ni->ni_tstamp.data, 8);
 #if 0
diff --git a/sys/dev/ath/if_athvar.h b/sys/dev/ath/if_athvar.h
index 2e71ff6374d4b..1af99aa9c334e 100644
--- a/sys/dev/ath/if_athvar.h
+++ b/sys/dev/ath/if_athvar.h
@@ -477,7 +477,6 @@ struct ath_vap {
 	struct ieee80211vap av_vap;	/* base class */
 	int		av_bslot;	/* beacon slot index */
 	struct ath_buf	*av_bcbuf;	/* beacon buffer */
-	struct ieee80211_beacon_offsets av_boff;/* dynamic update state */
 	struct ath_txq	av_mcastq;	/* buffered mcast s/w queue */
 
 	void		(*av_recv_mgmt)(struct ieee80211_node *,
@@ -942,26 +941,6 @@ struct ath_softc {
 					mtx_trylock(&(_sc)->sc_tx_mtx))
 
 /*
- * The IC TX lock is non-reentrant and serialises packet queuing from
- * the upper layers.
- */
-#define	ATH_TX_IC_LOCK_INIT(_sc) do {\
-	snprintf((_sc)->sc_tx_ic_mtx_name,				\
-	    sizeof((_sc)->sc_tx_ic_mtx_name),				\
-	    "%s IC TX lock",						\
-	    device_get_nameunit((_sc)->sc_dev));			\
-	mtx_init(&(_sc)->sc_tx_ic_mtx, (_sc)->sc_tx_ic_mtx_name,	\
-		 NULL, MTX_DEF);					\
-	} while (0)
-#define	ATH_TX_IC_LOCK_DESTROY(_sc)	mtx_destroy(&(_sc)->sc_tx_ic_mtx)
-#define	ATH_TX_IC_LOCK(_sc)		mtx_lock(&(_sc)->sc_tx_ic_mtx)
-#define	ATH_TX_IC_UNLOCK(_sc)		mtx_unlock(&(_sc)->sc_tx_ic_mtx)
-#define	ATH_TX_IC_LOCK_ASSERT(_sc)	mtx_assert(&(_sc)->sc_tx_ic_mtx,	\
-		MA_OWNED)
-#define	ATH_TX_IC_UNLOCK_ASSERT(_sc)	mtx_assert(&(_sc)->sc_tx_ic_mtx,	\
-		MA_NOTOWNED)
-
-/*
  * The PCU lock is non-recursive and should be treated as a spinlock.
  * Although currently the interrupt code is run in netisr context and
  * doesn't require this, this may change in the future.
diff --git a/sys/dev/bwi/bwimac.c b/sys/dev/bwi/bwimac.c
index baad7fef550fb..54d2fde57cc11 100644
--- a/sys/dev/bwi/bwimac.c
+++ b/sys/dev/bwi/bwimac.c
@@ -1657,17 +1657,15 @@ bwi_mac_attach(struct bwi_softc *sc, int id, uint8_t rev)
 	/*
 	 * Test whether the revision of this MAC is supported
 	 */
-#define N(arr)	(int)(sizeof(arr) / sizeof(arr[0]))
-	for (i = 0; i < N(bwi_sup_macrev); ++i) {
+	for (i = 0; i < nitems(bwi_sup_macrev); ++i) {
 		if (bwi_sup_macrev[i] == rev)
 			break;
 	}
-	if (i == N(bwi_sup_macrev)) {
+	if (i == nitems(bwi_sup_macrev)) {
 		device_printf(sc->sc_dev, "MAC rev %u is "
 			      "not supported\n", rev);
 		return ENXIO;
 	}
-#undef N
 
 	BWI_CREATE_MAC(mac, sc, id, rev);
 	sc->sc_nmac++;
diff --git a/sys/dev/bwi/bwiphy.c b/sys/dev/bwi/bwiphy.c
index 60d6bf569bafc..60afb191d240e 100644
--- a/sys/dev/bwi/bwiphy.c
+++ b/sys/dev/bwi/bwiphy.c
@@ -185,19 +185,17 @@ bwi_phy_attach(struct bwi_mac *mac)
 		phy->phy_tbl_data_hi = BWI_PHYR_TBL_DATA_HI_11A;
 		break;
 	case BWI_PHYINFO_TYPE_11B:
-#define N(arr)	(int)(sizeof(arr) / sizeof(arr[0]))
-		for (i = 0; i < N(bwi_sup_bphy); ++i) {
+		for (i = 0; i < nitems(bwi_sup_bphy); ++i) {
 			if (phyrev == bwi_sup_bphy[i].rev) {
 				phy->phy_init = bwi_sup_bphy[i].init;
 				break;
 			}
 		}
-		if (i == N(bwi_sup_bphy)) {
+		if (i == nitems(bwi_sup_bphy)) {
 			device_printf(sc->sc_dev, "unsupported 11B PHY, "
 				      "rev %u\n", phyrev);
 			return ENXIO;
 		}
-#undef N
 		phy->phy_mode = IEEE80211_MODE_11B;
 		break;
 	case BWI_PHYINFO_TYPE_11G:
@@ -745,8 +743,6 @@ bwi_phy_init_11b_rev6(struct bwi_mac *mac)
 	}
 }
 
-#define N(arr)	(int)(sizeof(arr) / sizeof(arr[0]))
-
 static void
 bwi_phy_config_11g(struct bwi_mac *mac)
 {
@@ -763,19 +759,19 @@ bwi_phy_config_11g(struct bwi_mac *mac)
 		PHY_WRITE(mac, 0x427, 0x1a);
 
 		/* Fill frequency table */
-		for (i = 0; i < N(bwi_phy_freq_11g_rev1); ++i) {
+		for (i = 0; i < nitems(bwi_phy_freq_11g_rev1); ++i) {
 			bwi_tbl_write_2(mac, BWI_PHYTBL_FREQ + i,
 					bwi_phy_freq_11g_rev1[i]);
 		}
 
 		/* Fill noise table */
-		for (i = 0; i < N(bwi_phy_noise_11g_rev1); ++i) {
+		for (i = 0; i < nitems(bwi_phy_noise_11g_rev1); ++i) {
 			bwi_tbl_write_2(mac, BWI_PHYTBL_NOISE + i,
 					bwi_phy_noise_11g_rev1[i]);
 		}
 
 		/* Fill rotor table */
-		for (i = 0; i < N(bwi_phy_rotor_11g_rev1); ++i) {
+		for (i = 0; i < nitems(bwi_phy_rotor_11g_rev1); ++i) {
 			/* NB: data length is 4 bytes */
 			bwi_tbl_write_4(mac, BWI_PHYTBL_ROTOR + i,
 					bwi_phy_rotor_11g_rev1[i]);
@@ -798,7 +794,7 @@ bwi_phy_config_11g(struct bwi_mac *mac)
 			bwi_tbl_write_2(mac, BWI_PHYTBL_RSSI + i, i);
 
 		/* Fill noise table */
-		for (i = 0; i < N(bwi_phy_noise_11g); ++i) {
+		for (i = 0; i < nitems(bwi_phy_noise_11g); ++i) {
 			bwi_tbl_write_2(mac, BWI_PHYTBL_NOISE + i,
 					bwi_phy_noise_11g[i]);
 		}
@@ -809,13 +805,13 @@ bwi_phy_config_11g(struct bwi_mac *mac)
 	 */
 	if (phy->phy_rev <= 2) {
 		tbl = bwi_phy_noise_scale_11g_rev2;
-		n = N(bwi_phy_noise_scale_11g_rev2);
+		n = nitems(bwi_phy_noise_scale_11g_rev2);
 	} else if (phy->phy_rev >= 7 && (PHY_READ(mac, 0x449) & 0x200)) {
 		tbl = bwi_phy_noise_scale_11g_rev7;
-		n = N(bwi_phy_noise_scale_11g_rev7);
+		n = nitems(bwi_phy_noise_scale_11g_rev7);
 	} else {
 		tbl = bwi_phy_noise_scale_11g;
-		n = N(bwi_phy_noise_scale_11g);
+		n = nitems(bwi_phy_noise_scale_11g);
 	}
 	for (i = 0; i < n; ++i)
 		bwi_tbl_write_2(mac, BWI_PHYTBL_NOISE_SCALE + i, tbl[i]);
@@ -825,10 +821,10 @@ bwi_phy_config_11g(struct bwi_mac *mac)
 	 */
 	if (phy->phy_rev == 2) {
 		tbl = bwi_phy_sigma_sq_11g_rev2;
-		n = N(bwi_phy_sigma_sq_11g_rev2);
+		n = nitems(bwi_phy_sigma_sq_11g_rev2);
 	} else if (phy->phy_rev > 2 && phy->phy_rev <= 8) {
 		tbl = bwi_phy_sigma_sq_11g_rev7;
-		n = N(bwi_phy_sigma_sq_11g_rev7);
+		n = nitems(bwi_phy_sigma_sq_11g_rev7);
 	} else {
 		tbl = NULL;
 		n = 0;
@@ -838,7 +834,7 @@ bwi_phy_config_11g(struct bwi_mac *mac)
 
 	if (phy->phy_rev == 1) {
 		/* Fill delay table */
-		for (i = 0; i < N(bwi_phy_delay_11g_rev1); ++i) {
+		for (i = 0; i < nitems(bwi_phy_delay_11g_rev1); ++i) {
 			bwi_tbl_write_4(mac, BWI_PHYTBL_DELAY + i,
 					bwi_phy_delay_11g_rev1[i]);
 		}
@@ -877,8 +873,6 @@ bwi_phy_config_11g(struct bwi_mac *mac)
 		PHY_WRITE(mac, 0x46e, 0x3cf);
 }
 
-#undef N
-
 /*
  * Configure Automatic Gain Controller
  */
diff --git a/sys/dev/bwi/bwirf.c b/sys/dev/bwi/bwirf.c
index 615e6dff69023..341e929f6a5fb 100644
--- a/sys/dev/bwi/bwirf.c
+++ b/sys/dev/bwi/bwirf.c
@@ -1155,7 +1155,6 @@ bwi_rf_map_txpower(struct bwi_mac *mac)
 	}
 
 #define IS_VALID_PA_PARAM(p)	((p) != 0 && (p) != -1)
-#define N(arr)	(int)(sizeof(arr) / sizeof(arr[0]))
 
 	/*
 	 * Extract PA parameters
@@ -1164,10 +1163,10 @@ bwi_rf_map_txpower(struct bwi_mac *mac)
 		sprom_ofs = BWI_SPROM_PA_PARAM_11A;
 	else
 		sprom_ofs = BWI_SPROM_PA_PARAM_11BG;
-	for (i = 0; i < N(pa_params); ++i)
+	for (i = 0; i < nitems(pa_params); ++i)
 		pa_params[i] = (int16_t)bwi_read_sprom(sc, sprom_ofs + (i * 2));
 
-	for (i = 0; i < N(pa_params); ++i) {
+	for (i = 0; i < nitems(pa_params); ++i) {
 		/*
 		 * If one of the PA parameters from SPROM is not valid,
 		 * fall back to the default values, if there are any.
@@ -1200,8 +1199,6 @@ bwi_rf_map_txpower(struct bwi_mac *mac)
 		}
 	}
 
-#undef N
-
 	/*
 	 * All of the PA parameters from SPROM are valid.
 	 */
diff --git a/sys/dev/bwi/if_bwi.c b/sys/dev/bwi/if_bwi.c
index 741f5f1d922bc..0a220a1de5764 100644
--- a/sys/dev/bwi/if_bwi.c
+++ b/sys/dev/bwi/if_bwi.c
@@ -762,7 +762,6 @@ bwi_regwin_info(struct bwi_softc *sc, uint16_t *type, uint8_t *rev)
 static int
 bwi_bbp_attach(struct bwi_softc *sc)
 {
-#define N(arr)	(int)(sizeof(arr) / sizeof(arr[0]))
 	uint16_t bbp_id, rw_type;
 	uint8_t rw_rev;
 	uint32_t info;
@@ -792,7 +791,7 @@ bwi_bbp_attach(struct bwi_softc *sc)
 
 		sc->sc_cap = CSR_READ_4(sc, BWI_CAPABILITY);
 	} else {
-		for (i = 0; i < N(bwi_bbpid_map); ++i) {
+		for (i = 0; i < nitems(bwi_bbpid_map); ++i) {
 			if (sc->sc_pci_did >= bwi_bbpid_map[i].did_min &&
 			    sc->sc_pci_did <= bwi_bbpid_map[i].did_max) {
 				bbp_id = bwi_bbpid_map[i].bbp_id;
@@ -816,7 +815,7 @@ bwi_bbp_attach(struct bwi_softc *sc)
 	if (rw_type == BWI_REGWIN_T_COM && rw_rev >= 4) {
 		nregwin = __SHIFTOUT(info, BWI_INFO_NREGWIN_MASK);
 	} else {
-		for (i = 0; i < N(bwi_regwin_count); ++i) {
+		for (i = 0; i < nitems(bwi_regwin_count); ++i) {
 			if (bwi_regwin_count[i].bbp_id == bbp_id) {
 				nregwin = bwi_regwin_count[i].nregwin;
 				break;
@@ -898,7 +897,6 @@ bwi_bbp_attach(struct bwi_softc *sc)
 		return error;
 
 	return 0;
-#undef N
 }
 
 int
@@ -3765,9 +3763,7 @@ bwi_led_attach(struct bwi_softc *sc)
 	uint16_t gpio, val[BWI_LED_MAX];
 	int i;
 
-#define N(arr)	(int)(sizeof(arr) / sizeof(arr[0]))
-
-	for (i = 0; i < N(bwi_vendor_led_act); ++i) {
+	for (i = 0; i < nitems(bwi_vendor_led_act); ++i) {
 		if (sc->sc_pci_subvid == bwi_vendor_led_act[i].vid) {
 			led_act = bwi_vendor_led_act[i].led_act;
 			break;
@@ -3776,8 +3772,6 @@ bwi_led_attach(struct bwi_softc *sc)
 	if (led_act == NULL)
 		led_act = bwi_default_led_act;
 
-#undef N
-
 	gpio = bwi_read_sprom(sc, BWI_SPROM_GPIO01);
 	val[0] = __SHIFTOUT(gpio, BWI_SPROM_GPIO_0);
 	val[1] = __SHIFTOUT(gpio, BWI_SPROM_GPIO_1);
diff --git a/sys/dev/drm2/drm_lock.c b/sys/dev/drm2/drm_lock.c
index 1bc681ca44d9f..8b593c41035b0 100644
--- a/sys/dev/drm2/drm_lock.c
+++ b/sys/dev/drm2/drm_lock.c
@@ -38,7 +38,9 @@ __FBSDID("$FreeBSD$");
 
 #include <dev/drm2/drmP.h>
 
+#if defined(__linux__)
 static int drm_notifier(void *priv);
+#endif
 
 static int drm_lock_take(struct drm_lock_data *lock_data, unsigned int context);
 
@@ -284,6 +286,7 @@ int drm_lock_free(struct drm_lock_data *lock_data, unsigned int context)
 	return 0;
 }
 
+#if defined(__linux__)
 /**
  * If we get here, it means that the process has called DRM_IOCTL_LOCK
  * without calling DRM_IOCTL_UNLOCK.
@@ -314,6 +317,7 @@ static int drm_notifier(void *priv)
 	} while (prev != old);
 	return 0;
 }
+#endif
 
 /**
  * This function returns immediately and takes the hw lock
diff --git a/sys/dev/dwc/if_dwc.c b/sys/dev/dwc/if_dwc.c
index 939ce541a5b50..e25d7f5a3d20f 100644
--- a/sys/dev/dwc/if_dwc.c
+++ b/sys/dev/dwc/if_dwc.c
@@ -43,21 +43,14 @@ __FBSDID("$FreeBSD$");
 #include <sys/systm.h>
 #include <sys/bus.h>
 #include <sys/kernel.h>
-#include <sys/module.h>
-#include <sys/malloc.h>
-#include <sys/rman.h>
-#include <sys/endian.h>
 #include <sys/lock.h>
+#include <sys/malloc.h>
 #include <sys/mbuf.h>
+#include <sys/module.h>
 #include <sys/mutex.h>
+#include <sys/rman.h>
 #include <sys/socket.h>
 #include <sys/sockio.h>
-#include <sys/sysctl.h>
-
-#include <dev/fdt/fdt_common.h>
-#include <dev/ofw/openfirm.h>
-#include <dev/ofw/ofw_bus.h>
-#include <dev/ofw/ofw_bus_subr.h>
 
 #include <net/bpf.h>
 #include <net/if.h>
@@ -66,13 +59,17 @@ __FBSDID("$FreeBSD$");
 #include <net/if_media.h>
 #include <net/if_types.h>
 #include <net/if_var.h>
-#include <net/if_vlan_var.h>
 
 #include <machine/bus.h>
-#include <machine/fdt.h>
 
+#include <dev/dwc/if_dwc.h>
+#include <dev/dwc/if_dwcvar.h>
 #include <dev/mii/mii.h>
 #include <dev/mii/miivar.h>
+#include <dev/ofw/ofw_bus.h>
+#include <dev/ofw/ofw_bus_subr.h>
+
+#include "if_dwc_if.h"
 #include "miibus_if.h"
 
 #define	READ4(_sc, _reg) \
@@ -83,9 +80,6 @@ __FBSDID("$FreeBSD$");
 #define	MAC_RESET_TIMEOUT	100
 #define	WATCHDOG_TIMEOUT_SECS	5
 #define	STATS_HARVEST_INTERVAL	2
-#define	MII_CLK_VAL		2
-
-#include <dev/dwc/if_dwc.h>
 
 #define	DWC_LOCK(sc)			mtx_lock(&(sc)->mtx)
 #define	DWC_UNLOCK(sc)			mtx_unlock(&(sc)->mtx)
@@ -105,77 +99,34 @@ __FBSDID("$FreeBSD$");
 #define	DDESC_RDES0_FL_SHIFT		16	/* Frame Length */
 #define	DDESC_RDES1_CHAINED		(1U << 14)
 
-struct dwc_bufmap {
-	bus_dmamap_t	map;
-	struct mbuf	*mbuf;
-};
+/* Alt descriptor bits. */
+#define	DDESC_CNTL_TXINT		(1U << 31)
+#define	DDESC_CNTL_TXLAST		(1U << 30)
+#define	DDESC_CNTL_TXFIRST		(1U << 29)
+#define	DDESC_CNTL_TXCRCDIS		(1U << 26)
+#define	DDESC_CNTL_TXRINGEND		(1U << 25)
+#define	DDESC_CNTL_TXCHAIN		(1U << 24)
+
+#define	DDESC_CNTL_CHAINED		(1U << 24)
 
 /*
  * A hardware buffer descriptor.  Rx and Tx buffers have the same descriptor
- * layout, but the bits in the flags field have different meanings.
+ * layout, but the bits in the fields have different meanings.
  */
 struct dwc_hwdesc
 {
-	uint32_t tdes0;
-	uint32_t tdes1;
+	uint32_t tdes0;		/* status for alt layout */
+	uint32_t tdes1;		/* cntl for alt layout */
 	uint32_t addr;		/* pointer to buffer data */
 	uint32_t addr_next;	/* link to next descriptor */
 };
 
 /*
- * Driver data and defines.
- */
-#define	RX_DESC_COUNT	1024
-#define	RX_DESC_SIZE	(sizeof(struct dwc_hwdesc) * RX_DESC_COUNT)
-#define	TX_DESC_COUNT	1024
-#define	TX_DESC_SIZE	(sizeof(struct dwc_hwdesc) * TX_DESC_COUNT)
-
-/*
  * The hardware imposes alignment restrictions on various objects involved in
  * DMA transfers.  These values are expressed in bytes (not bits).
  */
 #define	DWC_DESC_RING_ALIGN		2048
 
-struct dwc_softc {
-	struct resource		*res[2];
-	bus_space_tag_t		bst;
-	bus_space_handle_t	bsh;
-	device_t		dev;
-	int			mii_clk;
-	device_t		miibus;
-	struct mii_data *	mii_softc;
-	struct ifnet		*ifp;
-	int			if_flags;
-	struct mtx		mtx;
-	void *			intr_cookie;
-	struct callout		dwc_callout;
-	boolean_t		link_is_up;
-	boolean_t		is_attached;
-	boolean_t		is_detaching;
-	int			tx_watchdog_count;
-	int			stats_harvest_count;
-
-	/* RX */
-	bus_dma_tag_t		rxdesc_tag;
-	bus_dmamap_t		rxdesc_map;
-	struct dwc_hwdesc	*rxdesc_ring;
-	bus_addr_t		rxdesc_ring_paddr;
-	bus_dma_tag_t		rxbuf_tag;
-	struct dwc_bufmap	rxbuf_map[RX_DESC_COUNT];
-	uint32_t		rx_idx;
-
-	/* TX */
-	bus_dma_tag_t		txdesc_tag;
-	bus_dmamap_t		txdesc_map;
-	struct dwc_hwdesc	*txdesc_ring;
-	bus_addr_t		txdesc_ring_paddr;
-	bus_dma_tag_t		txbuf_tag;
-	struct dwc_bufmap	txbuf_map[RX_DESC_COUNT];
-	uint32_t		tx_idx_head;
-	uint32_t		tx_idx_tail;
-	int			txcount;
-};
-
 static struct resource_spec dwc_spec[] = {
 	{ SYS_RES_MEMORY,	0,	RF_ACTIVE },
 	{ SYS_RES_IRQ,		0,	RF_ACTIVE },
@@ -224,14 +175,23 @@ dwc_setup_txdesc(struct dwc_softc *sc, int idx, bus_addr_t paddr,
 		flags = 0;
 		--sc->txcount;
 	} else {
-		flags = DDESC_TDES0_TXCHAIN | DDESC_TDES0_TXFIRST
-		    | DDESC_TDES0_TXLAST | DDESC_TDES0_TXINT;
+		if (sc->mactype == DWC_GMAC_ALT_DESC)
+			flags = DDESC_CNTL_TXCHAIN | DDESC_CNTL_TXFIRST
+			    | DDESC_CNTL_TXLAST | DDESC_CNTL_TXINT;
+		else
+			flags = DDESC_TDES0_TXCHAIN | DDESC_TDES0_TXFIRST
+			    | DDESC_TDES0_TXLAST | DDESC_TDES0_TXINT;
 		++sc->txcount;
 	}
 
 	sc->txdesc_ring[idx].addr = (uint32_t)(paddr);
-	sc->txdesc_ring[idx].tdes0 = flags;
-	sc->txdesc_ring[idx].tdes1 = len;
+	if (sc->mactype == DWC_GMAC_ALT_DESC) {
+		sc->txdesc_ring[idx].tdes0 = 0;
+		sc->txdesc_ring[idx].tdes1 = flags | len;
+	} else {
+		sc->txdesc_ring[idx].tdes0 = flags;
+		sc->txdesc_ring[idx].tdes1 = len;
+	}
 
 	if (paddr && len) {
 		wmb();
@@ -504,7 +464,10 @@ dwc_setup_rxdesc(struct dwc_softc *sc, int idx, bus_addr_t paddr)
 	nidx = next_rxidx(sc, idx);
 	sc->rxdesc_ring[idx].addr_next = sc->rxdesc_ring_paddr +	\
 	    (nidx * sizeof(struct dwc_hwdesc));
-	sc->rxdesc_ring[idx].tdes1 = DDESC_RDES1_CHAINED | MCLBYTES;
+	if (sc->mactype == DWC_GMAC_ALT_DESC)
+		sc->rxdesc_ring[idx].tdes1 = DDESC_CNTL_CHAINED | RX_MAX_PACKET;
+	else
+		sc->rxdesc_ring[idx].tdes1 = DDESC_RDES1_CHAINED | MCLBYTES;
 
 	wmb();
 	sc->rxdesc_ring[idx].tdes0 = DDESC_RDES0_OWN;
@@ -821,10 +784,8 @@ dwc_intr(void *arg)
 	DWC_LOCK(sc);
 
 	reg = READ4(sc, INTERRUPT_STATUS);
-	if (reg) {
-		mii_mediachg(sc->mii_softc);
+	if (reg)
 		READ4(sc, SGMII_RGMII_SMII_CTRL_STATUS);
-	}
 
 	reg = READ4(sc, DMA_STATUS);
 	if (reg & DMA_STATUS_NIS) {
@@ -1074,10 +1035,13 @@ dwc_attach(device_t dev)
 
 	sc = device_get_softc(dev);
 	sc->dev = dev;
-	sc->mii_clk = MII_CLK_VAL;
 	sc->rx_idx = 0;
-
 	sc->txcount = TX_DESC_COUNT;
+	sc->mii_clk = IF_DWC_MII_CLK(dev);
+	sc->mactype = IF_DWC_MAC_TYPE(dev);
+
+	if (IF_DWC_INIT(dev) != 0)
+		return (ENXIO);
 
 	if (bus_alloc_resources(dev, dwc_spec, sc->res)) {
 		device_printf(dev, "could not allocate resources\n");
@@ -1109,8 +1073,11 @@ dwc_attach(device_t dev)
 		return (ENXIO);
 	}
 
-	reg = READ4(sc, BUS_MODE);
-	reg |= (BUS_MODE_EIGHTXPBL);
+	if (sc->mactype == DWC_GMAC_ALT_DESC) {
+		reg = BUS_MODE_FIXEDBURST;
+		reg |= (BUS_MODE_PRIORXTX_41 << BUS_MODE_PRIORXTX_SHIFT);
+	} else
+		reg = (BUS_MODE_EIGHTXPBL);
 	reg |= (BUS_MODE_PBL_BEATS_8 << BUS_MODE_PBL_SHIFT);
 	WRITE4(sc, BUS_MODE, reg);
 
@@ -1155,7 +1122,6 @@ dwc_attach(device_t dev)
 	IFQ_SET_MAXLEN(&ifp->if_snd, TX_DESC_COUNT - 1);
 	ifp->if_snd.ifq_drv_maxlen = TX_DESC_COUNT - 1;
 	IFQ_SET_READY(&ifp->if_snd);
-	ifp->if_hdrlen = sizeof(struct ether_vlan_header);
 
 	/* Attach the mii driver. */
 	error = mii_attach(dev, &sc->miibus, ifp, dwc_media_change,
@@ -1294,7 +1260,7 @@ static device_method_t dwc_methods[] = {
 	{ 0, 0 }
 };
 
-static driver_t dwc_driver = {
+driver_t dwc_driver = {
 	"dwc",
 	dwc_methods,
 	sizeof(struct dwc_softc),
diff --git a/sys/dev/dwc/if_dwc.h b/sys/dev/dwc/if_dwc.h
index f6179bfa68496..c9403da71000e 100644
--- a/sys/dev/dwc/if_dwc.h
+++ b/sys/dev/dwc/if_dwc.h
@@ -34,6 +34,9 @@
  * Register names were taken almost as is from the documentation.
  */
 
+#ifndef __IF_DWC_H__
+#define __IF_DWC_H__
+
 #define	MAC_CONFIGURATION	0x0
 #define	 CONF_JD		(1 << 22)	/* jabber timer disable */
 #define	 CONF_BE		(1 << 21)	/* Frame Burst Enable */
@@ -207,6 +210,12 @@
 /* DMA */
 #define	BUS_MODE		0x1000
 #define	 BUS_MODE_EIGHTXPBL	(1 << 24) /* Multiplies PBL by 8 */
+#define	 BUS_MODE_FIXEDBURST	(1 << 16)
+#define	 BUS_MODE_PRIORXTX_SHIFT	14
+#define	 BUS_MODE_PRIORXTX_41	3
+#define	 BUS_MODE_PRIORXTX_31	2
+#define	 BUS_MODE_PRIORXTX_21	1
+#define	 BUS_MODE_PRIORXTX_11	0
 #define	 BUS_MODE_PBL_SHIFT	8 /* Single block transfer size */
 #define	 BUS_MODE_PBL_BEATS_8	8
 #define	 BUS_MODE_SWR		(1 << 0) /* Reset */
@@ -260,3 +269,22 @@
 #define	CURRENT_HOST_TRANSMIT_BUF_ADDR	0x1050
 #define	CURRENT_HOST_RECEIVE_BUF_ADDR	0x1054
 #define	HW_FEATURE			0x1058
+
+#define	DWC_GMAC			0x1
+#define	DWC_GMAC_ALT_DESC		0x2
+#define	GMAC_MII_CLK_60_100M_DIV42	0x0
+#define	GMAC_MII_CLK_100_150M_DIV62	0x1
+#define	GMAC_MII_CLK_25_35M_DIV16	0x2
+#define	GMAC_MII_CLK_35_60M_DIV26	0x3
+#define	GMAC_MII_CLK_150_250M_DIV102	0x4
+#define	GMAC_MII_CLK_250_300M_DIV124	0x5
+#define	GMAC_MII_CLK_DIV4		0x8
+#define	GMAC_MII_CLK_DIV6		0x9
+#define	GMAC_MII_CLK_DIV8		0xa
+#define	GMAC_MII_CLK_DIV10		0xb
+#define	GMAC_MII_CLK_DIV12		0xc
+#define	GMAC_MII_CLK_DIV14		0xd
+#define	GMAC_MII_CLK_DIV16		0xe
+#define	GMAC_MII_CLK_DIV18		0xf
+
+#endif	/* __IF_DWC_H__ */
diff --git a/sys/dev/dwc/if_dwc_if.m b/sys/dev/dwc/if_dwc_if.m
new file mode 100644
index 0000000000000..feecb4e1f599f
--- /dev/null
+++ b/sys/dev/dwc/if_dwc_if.m
@@ -0,0 +1,76 @@
+#-
+# Copyright (c) 2015 Luiz Otavio O Souza <loos@FreeBSD.org>
+# Copyright (c) 2014 Ruslan Bukin <br@bsdpad.com>
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+# $FreeBSD$
+#
+
+INTERFACE if_dwc;
+
+#include <dev/dwc/if_dwc.h>
+
+CODE {
+	static int
+	if_dwc_default_init(device_t dev)
+	{
+		return (0);
+	}
+
+	static int
+	if_dwc_default_mac_type(device_t dev)
+	{
+		return (DWC_GMAC);
+	}
+
+	static int
+	if_dwc_default_mii_clk(device_t dev)
+	{
+		return (GMAC_MII_CLK_25_35M_DIV16);
+	}
+};
+
+HEADER {
+};
+
+#
+# Initialize the SoC specific registers.
+#
+METHOD int init {
+	device_t dev;
+} DEFAULT if_dwc_default_init;
+
+#
+# Return the DWC MAC type (descriptor type).
+#
+METHOD int mac_type {
+	device_t dev;
+} DEFAULT if_dwc_default_mac_type;
+
+#
+# Return the DWC MII clock for a specific hardware.
+#
+METHOD int mii_clk {
+	device_t dev;
+} DEFAULT if_dwc_default_mii_clk;
diff --git a/sys/dev/dwc/if_dwcvar.h b/sys/dev/dwc/if_dwcvar.h
new file mode 100644
index 0000000000000..3dd6a8dda2b53
--- /dev/null
+++ b/sys/dev/dwc/if_dwcvar.h
@@ -0,0 +1,99 @@
+/*-
+ * Copyright (c) 2014 Ruslan Bukin <br@bsdpad.com>
+ * All rights reserved.
+ *
+ * This software was developed by SRI International and the University of
+ * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237)
+ * ("CTSRD"), as part of the DARPA CRASH research programme.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+/*
+ * Ethernet media access controller (EMAC)
+ * Chapter 17, Altera Cyclone V Device Handbook (CV-5V2 2014.07.22)
+ *
+ * EMAC is an instance of the Synopsys DesignWare 3504-0
+ * Universal 10/100/1000 Ethernet MAC (DWC_gmac).
+ */
+
+#ifndef	__IF_DWCVAR_H__
+#define	__IF_DWCVAR_H__
+
+/*
+ * Driver data and defines.
+ */
+#define	RX_MAX_PACKET	0x7ff
+#define	RX_DESC_COUNT	1024
+#define	RX_DESC_SIZE	(sizeof(struct dwc_hwdesc) * RX_DESC_COUNT)
+#define	TX_DESC_COUNT	1024
+#define	TX_DESC_SIZE	(sizeof(struct dwc_hwdesc) * TX_DESC_COUNT)
+
+struct dwc_bufmap {
+	bus_dmamap_t		map;
+	struct mbuf		*mbuf;
+};
+
+struct dwc_softc {
+	struct resource		*res[2];
+	bus_space_tag_t		bst;
+	bus_space_handle_t	bsh;
+	device_t		dev;
+	int			mactype;
+	int			mii_clk;
+	device_t		miibus;
+	struct mii_data *	mii_softc;
+	struct ifnet		*ifp;
+	int			if_flags;
+	struct mtx		mtx;
+	void *			intr_cookie;
+	struct callout		dwc_callout;
+	boolean_t		link_is_up;
+	boolean_t		is_attached;
+	boolean_t		is_detaching;
+	int			tx_watchdog_count;
+	int			stats_harvest_count;
+
+	/* RX */
+	bus_dma_tag_t		rxdesc_tag;
+	bus_dmamap_t		rxdesc_map;
+	struct dwc_hwdesc	*rxdesc_ring;
+	bus_addr_t		rxdesc_ring_paddr;
+	bus_dma_tag_t		rxbuf_tag;
+	struct dwc_bufmap	rxbuf_map[RX_DESC_COUNT];
+	uint32_t		rx_idx;
+
+	/* TX */
+	bus_dma_tag_t		txdesc_tag;
+	bus_dmamap_t		txdesc_map;
+	struct dwc_hwdesc	*txdesc_ring;
+	bus_addr_t		txdesc_ring_paddr;
+	bus_dma_tag_t		txbuf_tag;
+	struct dwc_bufmap	txbuf_map[RX_DESC_COUNT];
+	uint32_t		tx_idx_head;
+	uint32_t		tx_idx_tail;
+	int			txcount;
+};
+
+#endif	/* __IF_DWCVAR_H__ */
diff --git a/sys/dev/ipw/if_ipw.c b/sys/dev/ipw/if_ipw.c
index 1ffd24c5243d6..c4f47e4deed44 100644
--- a/sys/dev/ipw/if_ipw.c
+++ b/sys/dev/ipw/if_ipw.c
@@ -1417,7 +1417,6 @@ ipw_dma_map_addr(void *arg, bus_dma_segment_t *segs, int nseg, int error)
 static const char *
 ipw_cmdname(int cmd)
 {
-#define	N(a)	(sizeof(a) / sizeof(a[0]))
 	static const struct {
 		int	cmd;
 		const char *name;
@@ -1454,12 +1453,11 @@ ipw_cmdname(int cmd)
 	static char buf[12];
 	int i;
 
-	for (i = 0; i < N(cmds); i++)
+	for (i = 0; i < nitems(cmds); i++)
 		if (cmds[i].cmd == cmd)
 			return cmds[i].name;
 	snprintf(buf, sizeof(buf), "%u", cmd);
 	return buf;
-#undef N
 }
 
 /*
diff --git a/sys/dev/iwi/if_iwi.c b/sys/dev/iwi/if_iwi.c
index 73631eaa1e5f4..c45a37ffe55e5 100644
--- a/sys/dev/iwi/if_iwi.c
+++ b/sys/dev/iwi/if_iwi.c
@@ -3415,7 +3415,6 @@ iwi_led_blink(struct iwi_softc *sc, int on, int off)
 static void
 iwi_led_event(struct iwi_softc *sc, int event)
 {
-#define	N(a)	(sizeof(a)/sizeof(a[0]))
 	/* NB: on/off times from the Atheros NDIS driver, w/ permission */
 	static const struct {
 		u_int		rate;		/* tx/rx iwi rate */
@@ -3445,13 +3444,13 @@ iwi_led_event(struct iwi_softc *sc, int event)
 		return;
 	switch (event) {
 	case IWI_LED_POLL:
-		j = N(blinkrates)-1;
+		j = nitems(blinkrates)-1;
 		break;
 	case IWI_LED_TX:
 		/* read current transmission rate from adapter */
 		txrate = CSR_READ_4(sc, IWI_CSR_CURRENT_TX_RATE);
 		if (blinkrates[sc->sc_txrix].rate != txrate) {
-			for (j = 0; j < N(blinkrates)-1; j++)
+			for (j = 0; j < nitems(blinkrates)-1; j++)
 				if (blinkrates[j].rate == txrate)
 					break;
 			sc->sc_txrix = j;
@@ -3460,7 +3459,7 @@ iwi_led_event(struct iwi_softc *sc, int event)
 		break;
 	case IWI_LED_RX:
 		if (blinkrates[sc->sc_rxrix].rate != sc->sc_rxrate) {
-			for (j = 0; j < N(blinkrates)-1; j++)
+			for (j = 0; j < nitems(blinkrates)-1; j++)
 				if (blinkrates[j].rate == sc->sc_rxrate)
 					break;
 			sc->sc_rxrix = j;
@@ -3471,7 +3470,6 @@ iwi_led_event(struct iwi_softc *sc, int event)
 	/* XXX beware of overflow */
 	iwi_led_blink(sc, (blinkrates[j].timeOn * hz) / 1000,
 		(blinkrates[j].timeOff * hz) / 1000);
-#undef N
 }
 
 static int
diff --git a/sys/dev/iwm/if_iwm.c b/sys/dev/iwm/if_iwm.c
index 46abd16ed6132..cd2ccd2814f3a 100644
--- a/sys/dev/iwm/if_iwm.c
+++ b/sys/dev/iwm/if_iwm.c
@@ -884,7 +884,9 @@ iwm_reset_rx_ring(struct iwm_softc *sc, struct iwm_rx_ring *ring)
 		(void) iwm_pcie_rx_stop(sc);
 		iwm_nic_unlock(sc);
 	}
+	/* Reset the ring state */
 	ring->cur = 0;
+	memset(sc->rxq.stat, 0, sizeof(*sc->rxq.stat));
 }
 
 static void
diff --git a/sys/dev/iwn/if_iwn.c b/sys/dev/iwn/if_iwn.c
index ecaab2dea0fa3..c1ed6ab004811 100644
--- a/sys/dev/iwn/if_iwn.c
+++ b/sys/dev/iwn/if_iwn.c
@@ -2716,7 +2716,6 @@ static uint32_t
 iwn_rate_to_plcp(struct iwn_softc *sc, struct ieee80211_node *ni,
     uint8_t rate)
 {
-#define	RV(v)	((v) & IEEE80211_RATE_VAL)
 	struct ieee80211com *ic = ni->ni_ic;
 	uint32_t plcp = 0;
 	int ridx;
@@ -2731,7 +2730,7 @@ iwn_rate_to_plcp(struct iwn_softc *sc, struct ieee80211_node *ni,
 		 * MCS 0 -> MCS 31, then set the "I'm an MCS rate!"
 		 * flag.
 		 */
-		plcp = RV(rate) | IWN_RFLAG_MCS;
+		plcp = IEEE80211_RV(rate) | IWN_RFLAG_MCS;
 
 		/*
 		 * XXX the following should only occur if both
@@ -2792,7 +2791,6 @@ iwn_rate_to_plcp(struct iwn_softc *sc, struct ieee80211_node *ni,
 	    plcp);
 
 	return (htole32(plcp));
-#undef	RV
 }
 
 static void
@@ -4370,7 +4368,6 @@ iwn_tx_data(struct iwn_softc *sc, struct mbuf *m, struct ieee80211_node *ni)
 		struct ieee80211_tx_ampdu *tap = &ni->ni_tx_ampdu[ac];
 
 		if (!IEEE80211_AMPDU_RUNNING(tap)) {
-			m_freem(m);
 			return EINVAL;
 		}
 
@@ -4422,7 +4419,6 @@ iwn_tx_data(struct iwn_softc *sc, struct mbuf *m, struct ieee80211_node *ni)
 		/* Retrieve key for TX. */
 		k = ieee80211_crypto_encap(ni, m);
 		if (k == NULL) {
-			m_freem(m);
 			return ENOBUFS;
 		}
 		/* 802.11 header may have moved. */
@@ -4553,7 +4549,6 @@ iwn_tx_data(struct iwn_softc *sc, struct mbuf *m, struct ieee80211_node *ni)
 		if (error != EFBIG) {
 			device_printf(sc->sc_dev,
 			    "%s: can't map mbuf (error %d)\n", __func__, error);
-			m_freem(m);
 			return error;
 		}
 		/* Too many DMA segments, linearize mbuf. */
@@ -4561,7 +4556,6 @@ iwn_tx_data(struct iwn_softc *sc, struct mbuf *m, struct ieee80211_node *ni)
 		if (m1 == NULL) {
 			device_printf(sc->sc_dev,
 			    "%s: could not defrag mbuf\n", __func__);
-			m_freem(m);
 			return ENOBUFS;
 		}
 		m = m1;
@@ -4571,7 +4565,6 @@ iwn_tx_data(struct iwn_softc *sc, struct mbuf *m, struct ieee80211_node *ni)
 		if (error != 0) {
 			device_printf(sc->sc_dev,
 			    "%s: can't map mbuf (error %d)\n", __func__, error);
-			m_freem(m);
 			return error;
 		}
 	}
@@ -4757,7 +4750,6 @@ iwn_tx_data_raw(struct iwn_softc *sc, struct mbuf *m,
 		if (error != EFBIG) {
 			device_printf(sc->sc_dev,
 			    "%s: can't map mbuf (error %d)\n", __func__, error);
-			m_freem(m);
 			return error;
 		}
 		/* Too many DMA segments, linearize mbuf. */
@@ -4765,7 +4757,6 @@ iwn_tx_data_raw(struct iwn_softc *sc, struct mbuf *m,
 		if (m1 == NULL) {
 			device_printf(sc->sc_dev,
 			    "%s: could not defrag mbuf\n", __func__);
-			m_freem(m);
 			return ENOBUFS;
 		}
 		m = m1;
@@ -4775,7 +4766,6 @@ iwn_tx_data_raw(struct iwn_softc *sc, struct mbuf *m,
 		if (error != 0) {
 			device_printf(sc->sc_dev,
 			    "%s: can't map mbuf (error %d)\n", __func__, error);
-			m_freem(m);
 			return error;
 		}
 	}
@@ -4871,6 +4861,9 @@ iwn_xmit_task(void *arg0, int pending)
 	IWN_UNLOCK(sc);
 }
 
+/*
+ * raw frame xmit - free node/reference if failed.
+ */
 static int
 iwn_raw_xmit(struct ieee80211_node *ni, struct mbuf *m,
     const struct ieee80211_bpf_params *params)
@@ -4933,6 +4926,9 @@ iwn_raw_xmit(struct ieee80211_node *ni, struct mbuf *m,
 	return error;
 }
 
+/*
+ * transmit - don't free mbuf if failed; don't free node ref if failed.
+ */
 static int
 iwn_transmit(struct ieee80211com *ic, struct mbuf *m)
 {
@@ -4940,6 +4936,8 @@ iwn_transmit(struct ieee80211com *ic, struct mbuf *m)
 	struct ieee80211_node *ni;
 	int error;
 
+	ni = (struct ieee80211_node *)m->m_pkthdr.rcvif;
+
 	IWN_LOCK(sc);
 	if ((sc->sc_flags & IWN_FLAG_RUNNING) == 0 || sc->sc_beacon_wait) {
 		IWN_UNLOCK(sc);
@@ -4951,11 +4949,9 @@ iwn_transmit(struct ieee80211com *ic, struct mbuf *m)
 		return (ENOBUFS);
 	}
 
-	ni = (struct ieee80211_node *)m->m_pkthdr.rcvif;
 	error = iwn_tx_data(sc, m, ni);
 	if (error) {
 		if_inc_counter(ni->ni_vap->iv_ifp, IFCOUNTER_OERRORS, 1);
-		ieee80211_free_node(ni);
 	} else
 		sc->sc_tx_timer = 5;
 	IWN_UNLOCK(sc);
@@ -5179,7 +5175,6 @@ iwn5000_add_node(struct iwn_softc *sc, struct iwn_node_info *node, int async)
 static int
 iwn_set_link_quality(struct iwn_softc *sc, struct ieee80211_node *ni)
 {
-#define	RV(v)	((v) & IEEE80211_RATE_VAL)
 	struct iwn_node *wn = (void *)ni;
 	struct ieee80211_rateset *rs;
 	struct iwn_cmd_link_quality linkq;
@@ -5238,7 +5233,7 @@ iwn_set_link_quality(struct iwn_softc *sc, struct ieee80211_node *ni)
 		if (is_11n)
 			rate = IEEE80211_RATE_MCS | rs->rs_rates[txrate];
 		else
-			rate = RV(rs->rs_rates[txrate]);
+			rate = IEEE80211_RV(rs->rs_rates[txrate]);
 
 		/* Do rate -> PLCP config mapping */
 		plcp = iwn_rate_to_plcp(sc, ni, rate);
@@ -5263,7 +5258,7 @@ iwn_set_link_quality(struct iwn_softc *sc, struct ieee80211_node *ni)
 		 * entry, we're already pointing at it.
 		 */
 		if ((le32toh(plcp) & IWN_RFLAG_MCS) &&
-		    RV(le32toh(plcp)) > 7)
+		    IEEE80211_RV(le32toh(plcp)) > 7)
 			linkq.mimo = i + 1;
 
 		/* Next retry at immediate lower bit-rate. */
@@ -5283,7 +5278,6 @@ iwn_set_link_quality(struct iwn_softc *sc, struct ieee80211_node *ni)
 	DPRINTF(sc, IWN_DEBUG_TRACE, "->%s: end\n",__func__);
 
 	return iwn_cmd(sc, IWN_CMD_LINK_QUALITY, &linkq, sizeof linkq, 1);
-#undef	RV
 }
 
 /*
@@ -5350,6 +5344,8 @@ iwn_updateedca(struct ieee80211com *ic)
 
 	memset(&cmd, 0, sizeof cmd);
 	cmd.flags = htole32(IWN_EDCA_UPDATE);
+
+	IEEE80211_LOCK(ic);
 	for (aci = 0; aci < WME_NUM_AC; aci++) {
 		const struct wmeParams *ac =
 		    &ic->ic_wme.wme_chanParams.cap_wmeParams[aci];
@@ -5360,10 +5356,10 @@ iwn_updateedca(struct ieee80211com *ic)
 		    htole16(IEEE80211_TXOP_TO_US(ac->wmep_txopLimit));
 	}
 	IEEE80211_UNLOCK(ic);
+
 	IWN_LOCK(sc);
 	(void)iwn_cmd(sc, IWN_CMD_EDCA_PARAMS, &cmd, sizeof cmd, 1);
 	IWN_UNLOCK(sc);
-	IEEE80211_LOCK(ic);
 
 	DPRINTF(sc, IWN_DEBUG_TRACE, "->%s: end\n",__func__);
 
diff --git a/sys/dev/malo/if_malo.c b/sys/dev/malo/if_malo.c
index 2790fdc8400c9..251ddc60d3273 100644
--- a/sys/dev/malo/if_malo.c
+++ b/sys/dev/malo/if_malo.c
@@ -901,22 +901,18 @@ malo_printtxbuf(const struct malo_txbuf *bf, u_int qnum, u_int ix)
 static __inline void
 malo_updatetxrate(struct ieee80211_node *ni, int rix)
 {
-#define	N(x)	(sizeof(x)/sizeof(x[0]))
 	static const int ieeerates[] =
 	    { 2, 4, 11, 22, 44, 12, 18, 24, 36, 48, 96, 108 };
-	if (rix < N(ieeerates))
+	if (rix < nitems(ieeerates))
 		ni->ni_txrate = ieeerates[rix];
-#undef N
 }
 
 static int
 malo_fix2rate(int fix_rate)
 {
-#define	N(x)	(sizeof(x)/sizeof(x[0]))
 	static const int rates[] =
 	    { 2, 4, 11, 22, 12, 18, 24, 36, 48, 96, 108 };
-	return (fix_rate < N(rates) ? rates[fix_rate] : 0);
-#undef N
+	return (fix_rate < nitems(rates) ? rates[fix_rate] : 0);
 }
 
 /* idiomatic shorthands: MS = mask+shift, SM = shift+mask */
@@ -1034,8 +1030,6 @@ static int
 malo_tx_start(struct malo_softc *sc, struct ieee80211_node *ni,
     struct malo_txbuf *bf, struct mbuf *m0)
 {
-#define	IEEE80211_DIR_DSTODS(wh) \
-	((wh->i_fc[1] & IEEE80211_FC1_DIR_MASK) == IEEE80211_FC1_DIR_DSTODS)
 #define	IS_DATA_FRAME(wh)						\
 	((wh->i_fc[0] & (IEEE80211_FC0_TYPE_MASK)) == IEEE80211_FC0_TYPE_DATA)
 	int error, ismcast, iswep;
@@ -1054,7 +1048,7 @@ malo_tx_start(struct malo_softc *sc, struct ieee80211_node *ni,
 	copyhdrlen = hdrlen = ieee80211_anyhdrsize(wh);
 	pktlen = m0->m_pkthdr.len;
 	if (IEEE80211_QOS_HAS_SEQ(wh)) {
-		if (IEEE80211_DIR_DSTODS(wh)) {
+		if (IEEE80211_IS_DSTODS(wh)) {
 			qos = *(uint16_t *)
 			    (((struct ieee80211_qosframe_addr4 *) wh)->i_qos);
 			copyhdrlen -= sizeof(qos);
@@ -1210,7 +1204,6 @@ malo_tx_start(struct malo_softc *sc, struct ieee80211_node *ni,
 	sc->malo_timer = 5;
 	MALO_TXQ_UNLOCK(txq);
 	return 0;
-#undef IEEE80211_DIR_DSTODS
 }
 
 static int
@@ -1960,9 +1953,6 @@ malo_set_channel(struct ieee80211com *ic)
 static void
 malo_rx_proc(void *arg, int npending)
 {
-#define	IEEE80211_DIR_DSTODS(wh)					\
-	((((const struct ieee80211_frame *)wh)->i_fc[1] &		\
-	    IEEE80211_FC1_DIR_MASK) == IEEE80211_FC1_DIR_DSTODS)
 	struct malo_softc *sc = arg;
 	struct ieee80211com *ic = &sc->malo_ic;
 	struct malo_rxbuf *bf;
@@ -2080,7 +2070,7 @@ malo_rx_proc(void *arg, int npending)
 		/* XXX special case so we can memcpy after m_devget? */
 		ovbcopy(data + sizeof(uint16_t), wh, hdrlen);
 		if (IEEE80211_QOS_HAS_SEQ(wh)) {
-			if (IEEE80211_DIR_DSTODS(wh)) {
+			if (IEEE80211_IS_DSTODS(wh)) {
 				wh4 = mtod(m,
 				    struct ieee80211_qosframe_addr4*);
 				*(uint16_t *)wh4->i_qos = ds->qosctrl;
@@ -2119,7 +2109,6 @@ rx_next:
 
 	if (mbufq_first(&sc->malo_snd) != NULL)
 		malo_start(sc);
-#undef IEEE80211_DIR_DSTODS
 }
 
 /*
diff --git a/sys/dev/malo/if_malo_pci.c b/sys/dev/malo/if_malo_pci.c
index 26324c09f0916..01b519e849884 100644
--- a/sys/dev/malo/if_malo_pci.c
+++ b/sys/dev/malo/if_malo_pci.c
@@ -131,7 +131,6 @@ static int	malo_pci_detach(device_t);
 static int
 malo_pci_probe(device_t dev)
 {
-#define N(a)	(sizeof(a) / sizeof((a)[0]))
 	struct malo_product *mp;
 	uint16_t vendor, devid;
 	int i;
@@ -140,7 +139,7 @@ malo_pci_probe(device_t dev)
 	devid = pci_get_device(dev);
 	mp = malo_products;
 
-	for (i = 0; i < N(malo_products); i++, mp++) {
+	for (i = 0; i < nitems(malo_products); i++, mp++) {
 		if (vendor == mp->mp_vendorid && devid == mp->mp_deviceid) {
 			device_set_desc(dev, mp->mp_name);
 			return (BUS_PROBE_DEFAULT);
@@ -148,7 +147,6 @@ malo_pci_probe(device_t dev)
 	}
 
 	return (ENXIO);
-#undef N
 }
 
 static int
diff --git a/sys/dev/mwl/if_mwl.c b/sys/dev/mwl/if_mwl.c
index be80e465cd39c..463b5c9c15867 100644
--- a/sys/dev/mwl/if_mwl.c
+++ b/sys/dev/mwl/if_mwl.c
@@ -70,6 +70,7 @@ __FBSDID("$FreeBSD$");
 #include <net/bpf.h>
 
 #include <net80211/ieee80211_var.h>
+#include <net80211/ieee80211_input.h>
 #include <net80211/ieee80211_regdomain.h>
 
 #ifdef INET
@@ -1707,18 +1708,6 @@ mwl_key_set(struct ieee80211vap *vap, const struct ieee80211_key *k,
 #undef GRPXMIT
 }
 
-/* unaligned little endian access */
-#define LE_READ_2(p)				\
-	((uint16_t)				\
-	 ((((const uint8_t *)(p))[0]      ) |	\
-	  (((const uint8_t *)(p))[1] <<  8)))
-#define LE_READ_4(p)				\
-	((uint32_t)				\
-	 ((((const uint8_t *)(p))[0]      ) |	\
-	  (((const uint8_t *)(p))[1] <<  8) |	\
-	  (((const uint8_t *)(p))[2] << 16) |	\
-	  (((const uint8_t *)(p))[3] << 24)))
-
 /*
  * Set the multicast filter contents into the hardware.
  * XXX f/w has no support; just defer to the os.
@@ -1845,10 +1834,10 @@ mwl_beacon_setup(struct ieee80211vap *vap)
 {
 	struct mwl_hal_vap *hvap = MWL_VAP(vap)->mv_hvap;
 	struct ieee80211_node *ni = vap->iv_bss;
-	struct ieee80211_beacon_offsets bo;
+	struct ieee80211_beacon_offsets *bo = &vap->iv_bcn_off;
 	struct mbuf *m;
 
-	m = ieee80211_beacon_alloc(ni, &bo);
+	m = ieee80211_beacon_alloc(ni, bo);
 	if (m == NULL)
 		return ENOBUFS;
 	mwl_hal_setbeacon(hvap, mtod(m, const void *), m->m_len);
@@ -2619,8 +2608,6 @@ cvtrssi(uint8_t ssi)
 static void
 mwl_rx_proc(void *arg, int npending)
 {
-#define	IEEE80211_DIR_DSTODS(wh) \
-	((((const struct ieee80211_frame *)wh)->i_fc[1] & IEEE80211_FC1_DIR_MASK) == IEEE80211_FC1_DIR_DSTODS)
 	struct mwl_softc *sc = arg;
 	struct ieee80211com *ic = &sc->sc_ic;
 	struct mwl_rxbuf *bf;
@@ -2775,7 +2762,7 @@ mwl_rx_proc(void *arg, int npending)
 		/* XXX special case so we can memcpy after m_devget? */
 		ovbcopy(data + sizeof(uint16_t), wh, hdrlen);
 		if (IEEE80211_QOS_HAS_SEQ(wh)) {
-			if (IEEE80211_DIR_DSTODS(wh)) {
+			if (IEEE80211_IS_DSTODS(wh)) {
 				wh4 = mtod(m,
 				    struct ieee80211_qosframe_addr4*);
 				*(uint16_t *)wh4->i_qos = ds->QosCtrl;
@@ -2845,7 +2832,6 @@ rx_stop:
 		mwl_hal_txstart(sc->sc_mh, 0);
 		mwl_start(sc);
 	}
-#undef IEEE80211_DIR_DSTODS
 }
 
 static void
@@ -2881,12 +2867,11 @@ mwl_txq_init(struct mwl_softc *sc, struct mwl_txq *txq, int qnum)
 static int
 mwl_tx_setup(struct mwl_softc *sc, int ac, int mvtype)
 {
-#define	N(a)	(sizeof(a)/sizeof(a[0]))
 	struct mwl_txq *txq;
 
-	if (ac >= N(sc->sc_ac2q)) {
+	if (ac >= nitems(sc->sc_ac2q)) {
 		device_printf(sc->sc_dev, "AC %u out of range, max %zu!\n",
-			ac, N(sc->sc_ac2q));
+			ac, nitems(sc->sc_ac2q));
 		return 0;
 	}
 	if (mvtype >= MWL_NUM_TX_QUEUES) {
@@ -2898,7 +2883,6 @@ mwl_tx_setup(struct mwl_softc *sc, int ac, int mvtype)
 	mwl_txq_init(sc, txq, mvtype);
 	sc->sc_ac2q[ac] = txq;
 	return 1;
-#undef N
 }
 
 /*
@@ -3091,8 +3075,6 @@ static int
 mwl_tx_start(struct mwl_softc *sc, struct ieee80211_node *ni, struct mwl_txbuf *bf,
     struct mbuf *m0)
 {
-#define	IEEE80211_DIR_DSTODS(wh) \
-	((wh->i_fc[1] & IEEE80211_FC1_DIR_MASK) == IEEE80211_FC1_DIR_DSTODS)
 	struct ieee80211com *ic = &sc->sc_ic;
 	struct ieee80211vap *vap = ni->ni_vap;
 	int error, iswep, ismcast;
@@ -3114,7 +3096,7 @@ mwl_tx_start(struct mwl_softc *sc, struct ieee80211_node *ni, struct mwl_txbuf *
 	copyhdrlen = hdrlen;
 	pktlen = m0->m_pkthdr.len;
 	if (IEEE80211_QOS_HAS_SEQ(wh)) {
-		if (IEEE80211_DIR_DSTODS(wh)) {
+		if (IEEE80211_IS_DSTODS(wh)) {
 			qos = *(uint16_t *)
 			    (((struct ieee80211_qosframe_addr4 *) wh)->i_qos);
 			copyhdrlen -= sizeof(qos);
@@ -3331,17 +3313,14 @@ mwl_tx_start(struct mwl_softc *sc, struct ieee80211_node *ni, struct mwl_txbuf *
 	MWL_TXQ_UNLOCK(txq);
 
 	return 0;
-#undef	IEEE80211_DIR_DSTODS
 }
 
 static __inline int
 mwl_cvtlegacyrix(int rix)
 {
-#define	N(x)	(sizeof(x)/sizeof(x[0]))
 	static const int ieeerates[] =
 	    { 2, 4, 11, 22, 44, 12, 18, 24, 36, 48, 72, 96, 108 };
-	return (rix < N(ieeerates) ? ieeerates[rix] : 0);
-#undef N
+	return (rix < nitems(ieeerates) ? ieeerates[rix] : 0);
 }
 
 /*
diff --git a/sys/dev/nxge/if_nxge.c b/sys/dev/nxge/if_nxge.c
index 90871aec9e88e..af95e8bc2f87d 100644
--- a/sys/dev/nxge/if_nxge.c
+++ b/sys/dev/nxge/if_nxge.c
@@ -1457,7 +1457,7 @@ xge_ioctl_stats(xge_lldev_t *lldev, struct ifreq *ifreqp)
 
 	    case XGE_READ_VERSION:
 	        info = xge_os_malloc(NULL, XGE_BUFFER_SIZE);
-	        if(version != NULL) {
+	        if(info != NULL) {
 	            strcpy(info, XGE_DRIVER_VERSION);
 	            if(copyout(info, ifreqp->ifr_data, XGE_BUFFER_SIZE) == 0)
 	                retValue = 0;
diff --git a/sys/dev/nxge/xgehal/xgehal-fifo.c b/sys/dev/nxge/xgehal/xgehal-fifo.c
index 303afafbf7ce6..78c0923dee0f4 100644
--- a/sys/dev/nxge/xgehal/xgehal-fifo.c
+++ b/sys/dev/nxge/xgehal/xgehal-fifo.c
@@ -464,7 +464,7 @@ __hal_fifo_hw_initialize(xge_hal_device_h devh)
 
 	    if (!hldev->config.fifo.queue[i].configured ||
 	        !hldev->config.fifo.queue[i].intr_vector ||
-	        !hldev->config.intr_mode != XGE_HAL_INTR_MODE_MSIX)
+	        hldev->config.intr_mode != XGE_HAL_INTR_MODE_MSIX)
 	        continue;
 
 	    /* find channel */
diff --git a/sys/dev/nxge/xgehal/xgehal-ring.c b/sys/dev/nxge/xgehal/xgehal-ring.c
index 444e10c4f06d3..061b773842138 100644
--- a/sys/dev/nxge/xgehal/xgehal-ring.c
+++ b/sys/dev/nxge/xgehal/xgehal-ring.c
@@ -609,7 +609,7 @@ __hal_ring_hw_initialize(xge_hal_device_h devh)
 
 	    if (!hldev->config.ring.queue[i].configured ||
 	        !hldev->config.ring.queue[i].intr_vector ||
-	        !hldev->config.intr_mode != XGE_HAL_INTR_MODE_MSIX)
+	        hldev->config.intr_mode != XGE_HAL_INTR_MODE_MSIX)
 	        continue;
 
 	    /* find channel */
diff --git a/sys/dev/otus/if_otus.c b/sys/dev/otus/if_otus.c
new file mode 100644
index 0000000000000..e60157adda9d7
--- /dev/null
+++ b/sys/dev/otus/if_otus.c
@@ -0,0 +1,3156 @@
+/*	$OpenBSD: if_otus.c,v 1.46 2015/03/14 03:38:49 jsg Exp $	*/
+
+/*-
+ * Copyright (c) 2009 Damien Bergamini <damien.bergamini@free.fr>
+ * Copyright (c) 2015 Adrian Chadd <adrian@FreeBSD.org>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+/*
+ * Driver for Atheros AR9001U chipset.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include "opt_wlan.h"
+
+#include <sys/param.h>
+#include <sys/endian.h>
+#include <sys/sockio.h>
+#include <sys/mbuf.h>
+#include <sys/kernel.h>
+#include <sys/socket.h>
+#include <sys/systm.h>
+#include <sys/conf.h>
+#include <sys/bus.h>
+#include <sys/rman.h>
+#include <sys/firmware.h>
+#include <sys/module.h>
+#include <sys/taskqueue.h>
+
+#include <machine/bus.h>
+#include <machine/resource.h>
+
+#include <net/bpf.h>
+#include <net/if.h>
+#include <net/if_var.h>
+#include <net/if_arp.h>
+#include <net/if_dl.h>
+#include <net/if_media.h>
+#include <net/if_types.h>
+
+#include <netinet/in.h>
+#include <netinet/in_systm.h>
+#include <netinet/in_var.h>
+#include <netinet/if_ether.h>
+#include <netinet/ip.h>
+
+#include <net80211/ieee80211_var.h>
+#include <net80211/ieee80211_regdomain.h>
+#include <net80211/ieee80211_radiotap.h>
+#include <net80211/ieee80211_ratectl.h>
+#include <net80211/ieee80211_input.h>
+#ifdef	IEEE80211_SUPPORT_SUPERG
+#include <net80211/ieee80211_superg.h>
+#endif
+
+#include <dev/usb/usb.h>
+#include <dev/usb/usbdi.h>
+#include "usbdevs.h"
+
+#define USB_DEBUG_VAR otus_debug
+#include <dev/usb/usb_debug.h>
+
+#include "if_otusreg.h"
+
+static int otus_debug = 0;
+static SYSCTL_NODE(_hw_usb, OID_AUTO, otus, CTLFLAG_RW, 0, "USB otus");
+SYSCTL_INT(_hw_usb_otus, OID_AUTO, debug, CTLFLAG_RWTUN, &otus_debug, 0,
+    "Debug level");
+#define	OTUS_DEBUG_XMIT		0x00000001
+#define	OTUS_DEBUG_RECV		0x00000002
+#define	OTUS_DEBUG_TXDONE	0x00000004
+#define	OTUS_DEBUG_RXDONE	0x00000008
+#define	OTUS_DEBUG_CMD		0x00000010
+#define	OTUS_DEBUG_CMDDONE	0x00000020
+#define	OTUS_DEBUG_RESET	0x00000040
+#define	OTUS_DEBUG_STATE	0x00000080
+#define	OTUS_DEBUG_CMDNOTIFY	0x00000100
+#define	OTUS_DEBUG_REGIO	0x00000200
+#define	OTUS_DEBUG_IRQ		0x00000400
+#define	OTUS_DEBUG_TXCOMP	0x00000800
+#define	OTUS_DEBUG_ANY		0xffffffff
+
+#define	OTUS_DPRINTF(sc, dm, ...) \
+	do { \
+		if ((dm == OTUS_DEBUG_ANY) || (dm & otus_debug)) \
+			device_printf(sc->sc_dev, __VA_ARGS__); \
+	} while (0)
+
+#define	OTUS_DEV(v, p) { USB_VPI(v, p, 0) }
+static const STRUCT_USB_HOST_ID otus_devs[] = {
+	OTUS_DEV(USB_VENDOR_ACCTON,		USB_PRODUCT_ACCTON_WN7512),
+	OTUS_DEV(USB_VENDOR_ATHEROS2,		USB_PRODUCT_ATHEROS2_3CRUSBN275),
+	OTUS_DEV(USB_VENDOR_ATHEROS2,		USB_PRODUCT_ATHEROS2_TG121N),
+	OTUS_DEV(USB_VENDOR_ATHEROS2,		USB_PRODUCT_ATHEROS2_AR9170),
+	OTUS_DEV(USB_VENDOR_ATHEROS2,		USB_PRODUCT_ATHEROS2_WN612),
+	OTUS_DEV(USB_VENDOR_ATHEROS2,		USB_PRODUCT_ATHEROS2_WN821NV2),
+	OTUS_DEV(USB_VENDOR_AVM,		USB_PRODUCT_AVM_FRITZWLAN),
+	OTUS_DEV(USB_VENDOR_CACE,		USB_PRODUCT_CACE_AIRPCAPNX),
+	OTUS_DEV(USB_VENDOR_DLINK2,		USB_PRODUCT_DLINK2_DWA130D1),
+	OTUS_DEV(USB_VENDOR_DLINK2,		USB_PRODUCT_DLINK2_DWA160A1),
+	OTUS_DEV(USB_VENDOR_DLINK2,		USB_PRODUCT_DLINK2_DWA160A2),
+	OTUS_DEV(USB_VENDOR_IODATA,		USB_PRODUCT_IODATA_WNGDNUS2),
+	OTUS_DEV(USB_VENDOR_NEC,		USB_PRODUCT_NEC_WL300NUG),
+	OTUS_DEV(USB_VENDOR_NETGEAR,		USB_PRODUCT_NETGEAR_WN111V2),
+	OTUS_DEV(USB_VENDOR_NETGEAR,		USB_PRODUCT_NETGEAR_WNA1000),
+	OTUS_DEV(USB_VENDOR_NETGEAR,		USB_PRODUCT_NETGEAR_WNDA3100),
+	OTUS_DEV(USB_VENDOR_PLANEX2,		USB_PRODUCT_PLANEX2_GW_US300),
+	OTUS_DEV(USB_VENDOR_WISTRONNEWEB,	USB_PRODUCT_WISTRONNEWEB_O8494),
+	OTUS_DEV(USB_VENDOR_WISTRONNEWEB,	USB_PRODUCT_WISTRONNEWEB_WNC0600),
+	OTUS_DEV(USB_VENDOR_ZCOM,		USB_PRODUCT_ZCOM_UB81),
+	OTUS_DEV(USB_VENDOR_ZCOM,		USB_PRODUCT_ZCOM_UB82),
+	OTUS_DEV(USB_VENDOR_ZYDAS,		USB_PRODUCT_ZYDAS_ZD1221),
+	OTUS_DEV(USB_VENDOR_ZYXEL,		USB_PRODUCT_ZYXEL_NWD271N),
+};
+
+static device_probe_t otus_match;
+static device_attach_t otus_attach;
+static device_detach_t otus_detach;
+
+static int	otus_attachhook(struct otus_softc *);
+void		otus_get_chanlist(struct otus_softc *);
+int		otus_load_firmware(struct otus_softc *, const char *,
+		    uint32_t);
+int		otus_open_pipes(struct otus_softc *);
+void		otus_close_pipes(struct otus_softc *);
+
+static int	otus_alloc_tx_cmd_list(struct otus_softc *);
+static void	otus_free_tx_cmd_list(struct otus_softc *);
+
+static int	otus_alloc_rx_list(struct otus_softc *);
+static void	otus_free_rx_list(struct otus_softc *);
+static int	otus_alloc_tx_list(struct otus_softc *);
+static void	otus_free_tx_list(struct otus_softc *);
+static void	otus_free_list(struct otus_softc *, struct otus_data [], int);
+static struct otus_data *_otus_getbuf(struct otus_softc *);
+static struct otus_data *otus_getbuf(struct otus_softc *);
+static void	otus_freebuf(struct otus_softc *, struct otus_data *);
+
+static struct otus_tx_cmd *_otus_get_txcmd(struct otus_softc *);
+static struct otus_tx_cmd *otus_get_txcmd(struct otus_softc *);
+static void	otus_free_txcmd(struct otus_softc *, struct otus_tx_cmd *);
+
+void		otus_next_scan(void *, int);
+static void	otus_tx_task(void *, int pending);
+static void	otus_wme_update_task(void *, int pending);
+void		otus_do_async(struct otus_softc *,
+		    void (*)(struct otus_softc *, void *), void *, int);
+int		otus_newstate(struct ieee80211vap *, enum ieee80211_state,
+		    int);
+int		otus_cmd(struct otus_softc *, uint8_t, const void *, int,
+		    void *, int);
+void		otus_write(struct otus_softc *, uint32_t, uint32_t);
+int		otus_write_barrier(struct otus_softc *);
+struct		ieee80211_node *otus_node_alloc(struct ieee80211com *);
+int		otus_media_change(struct ifnet *);
+int		otus_read_eeprom(struct otus_softc *);
+void		otus_newassoc(struct ieee80211_node *, int);
+void		otus_cmd_rxeof(struct otus_softc *, uint8_t *, int);
+void		otus_sub_rxeof(struct otus_softc *, uint8_t *, int,
+		    struct mbufq *);
+static int	otus_tx(struct otus_softc *, struct ieee80211_node *,
+		    struct mbuf *, struct otus_data *);
+int		otus_ioctl(struct ifnet *, u_long, caddr_t);
+int		otus_set_multi(struct otus_softc *);
+static void	otus_updateedca(struct otus_softc *sc);
+static void	otus_updateslot(struct otus_softc *sc);
+int		otus_init_mac(struct otus_softc *);
+uint32_t	otus_phy_get_def(struct otus_softc *, uint32_t);
+int		otus_set_board_values(struct otus_softc *,
+		    struct ieee80211_channel *);
+int		otus_program_phy(struct otus_softc *,
+		    struct ieee80211_channel *);
+int		otus_set_rf_bank4(struct otus_softc *,
+		    struct ieee80211_channel *);
+void		otus_get_delta_slope(uint32_t, uint32_t *, uint32_t *);
+static int	otus_set_chan(struct otus_softc *, struct ieee80211_channel *,
+		    int);
+int		otus_set_key(struct ieee80211com *, struct ieee80211_node *,
+		    struct ieee80211_key *);
+void		otus_set_key_cb(struct otus_softc *, void *);
+void		otus_delete_key(struct ieee80211com *, struct ieee80211_node *,
+		    struct ieee80211_key *);
+void		otus_delete_key_cb(struct otus_softc *, void *);
+void		otus_calibrate_to(void *, int);
+int		otus_set_bssid(struct otus_softc *, const uint8_t *);
+int		otus_set_macaddr(struct otus_softc *, const uint8_t *);
+void		otus_led_newstate_type1(struct otus_softc *);
+void		otus_led_newstate_type2(struct otus_softc *);
+void		otus_led_newstate_type3(struct otus_softc *);
+int		otus_init(struct otus_softc *sc);
+void		otus_stop(struct otus_softc *sc);
+
+static device_method_t otus_methods[] = {
+	DEVMETHOD(device_probe,		otus_match),
+	DEVMETHOD(device_attach,	otus_attach),
+	DEVMETHOD(device_detach,	otus_detach),
+
+	DEVMETHOD_END
+};
+
+static driver_t otus_driver = {
+	.name = "otus",
+	.methods = otus_methods,
+	.size = sizeof(struct otus_softc)
+};
+
+static devclass_t otus_devclass;
+
+DRIVER_MODULE(otus, uhub, otus_driver, otus_devclass, NULL, 0);
+MODULE_DEPEND(otus, wlan, 1, 1, 1);
+MODULE_DEPEND(otus, usb, 1, 1, 1);
+MODULE_DEPEND(otus, firmware, 1, 1, 1);
+MODULE_VERSION(otus, 1);
+
+static usb_callback_t	otus_bulk_tx_callback;
+static usb_callback_t	otus_bulk_rx_callback;
+static usb_callback_t	otus_bulk_irq_callback;
+static usb_callback_t	otus_bulk_cmd_callback;
+
+static const struct usb_config otus_config[OTUS_N_XFER] = {
+	[OTUS_BULK_TX] = {
+	.type = UE_BULK,
+	.endpoint = UE_ADDR_ANY,
+	.direction = UE_DIR_OUT,
+	.bufsize = 0x200,
+	.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
+	.callback = otus_bulk_tx_callback,
+	.timeout = 5000,	/* ms */
+	},
+	[OTUS_BULK_RX] = {
+	.type = UE_BULK,
+	.endpoint = UE_ADDR_ANY,
+	.direction = UE_DIR_IN,
+	.bufsize = OTUS_RXBUFSZ,
+	.flags = { .ext_buffer = 1, .pipe_bof = 1,.short_xfer_ok = 1,},
+	.callback = otus_bulk_rx_callback,
+	},
+	[OTUS_BULK_IRQ] = {
+	.type = UE_INTERRUPT,
+	.endpoint = UE_ADDR_ANY,
+	.direction = UE_DIR_IN,
+	.bufsize = OTUS_MAX_CTRLSZ,
+	.flags = {.pipe_bof = 1,.short_xfer_ok = 1,},
+	.callback = otus_bulk_irq_callback,
+	},
+	[OTUS_BULK_CMD] = {
+	.type = UE_INTERRUPT,
+	.endpoint = UE_ADDR_ANY,
+	.direction = UE_DIR_OUT,
+	.bufsize = OTUS_MAX_CTRLSZ,
+	.flags = {.pipe_bof = 1,.force_short_xfer = 1,},
+	.callback = otus_bulk_cmd_callback,
+	.timeout = 5000,	/* ms */
+	},
+};
+
+static int
+otus_match(device_t self)
+{
+	struct usb_attach_arg *uaa = device_get_ivars(self);
+
+	if (uaa->usb_mode != USB_MODE_HOST ||
+	    uaa->info.bIfaceIndex != 0 ||
+	    uaa->info.bConfigIndex != 0)
+	return (ENXIO);
+
+	return (usbd_lookup_id_by_uaa(otus_devs, sizeof(otus_devs), uaa));
+}
+
+static int
+otus_attach(device_t self)
+{
+	struct usb_attach_arg *uaa = device_get_ivars(self);
+	struct otus_softc *sc = device_get_softc(self);
+	int error;
+	uint8_t iface_index;
+
+	device_set_usb_desc(self);
+	sc->sc_udev = uaa->device;
+	sc->sc_dev = self;
+
+	mtx_init(&sc->sc_mtx, device_get_nameunit(self), MTX_NETWORK_LOCK,
+	    MTX_DEF);
+
+	TIMEOUT_TASK_INIT(taskqueue_thread, &sc->scan_to, 0, otus_next_scan, sc);
+	TIMEOUT_TASK_INIT(taskqueue_thread, &sc->calib_to, 0, otus_calibrate_to, sc);
+	TASK_INIT(&sc->tx_task, 0, otus_tx_task, sc);
+	TASK_INIT(&sc->wme_update_task, 0, otus_wme_update_task, sc);
+	mbufq_init(&sc->sc_snd, ifqmaxlen);
+
+	iface_index = 0;
+	error = usbd_transfer_setup(uaa->device, &iface_index, sc->sc_xfer,
+	    otus_config, OTUS_N_XFER, sc, &sc->sc_mtx);
+	if (error) {
+		device_printf(sc->sc_dev,
+		    "could not allocate USB transfers, err=%s\n",
+		    usbd_errstr(error));
+		goto fail_usb;
+	}
+
+	if ((error = otus_open_pipes(sc)) != 0) {
+		device_printf(sc->sc_dev, "%s: could not open pipes\n",
+		    __func__);
+		goto fail;
+	}
+
+	/* XXX check return status; fail out if appropriate */
+	if (otus_attachhook(sc) != 0)
+		goto fail;
+
+	return (0);
+
+fail:
+	otus_close_pipes(sc);
+fail_usb:
+	mtx_destroy(&sc->sc_mtx);
+	return (ENXIO);
+}
+
+static int
+otus_detach(device_t self)
+{
+	struct otus_softc *sc = device_get_softc(self);
+	struct ieee80211com *ic = &sc->sc_ic;
+
+	otus_stop(sc);
+
+	usbd_transfer_unsetup(sc->sc_xfer, OTUS_N_XFER);
+
+	taskqueue_drain_timeout(taskqueue_thread, &sc->scan_to);
+	taskqueue_drain_timeout(taskqueue_thread, &sc->calib_to);
+	taskqueue_drain(taskqueue_thread, &sc->tx_task);
+	taskqueue_drain(taskqueue_thread, &sc->wme_update_task);
+
+	otus_close_pipes(sc);
+#if 0
+	/* Wait for all queued asynchronous commands to complete. */
+	usb_rem_wait_task(sc->sc_udev, &sc->sc_task);
+
+	usbd_ref_wait(sc->sc_udev);
+#endif
+
+	ieee80211_ifdetach(ic);
+	mtx_destroy(&sc->sc_mtx);
+	return 0;
+}
+
+static void
+otus_delay_ms(struct otus_softc *sc, int ms)
+{
+
+	DELAY(1000 * ms);
+}
+
+static struct ieee80211vap *
+otus_vap_create(struct ieee80211com *ic, const char name[IFNAMSIZ], int unit,
+    enum ieee80211_opmode opmode, int flags,
+    const uint8_t bssid[IEEE80211_ADDR_LEN],
+    const uint8_t mac[IEEE80211_ADDR_LEN])
+{
+	struct otus_vap *uvp;
+	struct ieee80211vap *vap;
+
+	if (!TAILQ_EMPTY(&ic->ic_vaps))	 /* only one at a time */
+		return (NULL);
+
+	uvp =  malloc(sizeof(struct otus_vap), M_80211_VAP, M_WAITOK | M_ZERO);
+	vap = &uvp->vap;
+
+	if (ieee80211_vap_setup(ic, vap, name, unit, opmode,
+	    flags, bssid) != 0) {
+		/* out of memory */
+		free(uvp, M_80211_VAP);
+		return (NULL);
+	}
+
+	/* override state transition machine */
+	uvp->newstate = vap->iv_newstate;
+	vap->iv_newstate = otus_newstate;
+
+	/* XXX TODO: double-check */
+	vap->iv_ampdu_density = IEEE80211_HTCAP_MPDUDENSITY_16;
+	vap->iv_ampdu_rxmax = IEEE80211_HTCAP_MAXRXAMPDU_32K;
+
+	ieee80211_ratectl_init(vap);
+
+	/* complete setup */
+	ieee80211_vap_attach(vap, ieee80211_media_change,
+	    ieee80211_media_status, mac);
+	ic->ic_opmode = opmode;
+
+	return (vap);
+}
+
+static void
+otus_vap_delete(struct ieee80211vap *vap)
+{
+	struct otus_vap *uvp = OTUS_VAP(vap);
+
+	ieee80211_ratectl_deinit(vap);
+	ieee80211_vap_detach(vap);
+	free(uvp, M_80211_VAP);
+}
+
+static void
+otus_parent(struct ieee80211com *ic)
+{
+	struct otus_softc *sc = ic->ic_softc;
+	int startall = 0;
+
+	if (ic->ic_nrunning > 0) {
+		if (!sc->sc_running) {
+			otus_init(sc);
+			startall = 1;
+		} else {
+			(void) otus_set_multi(sc);
+		}
+	} else if (sc->sc_running)
+		otus_stop(sc);
+
+	if (startall)
+		ieee80211_start_all(ic);
+}
+
+static void
+otus_drain_mbufq(struct otus_softc *sc)
+{
+	struct mbuf *m;
+	struct ieee80211_node *ni;
+
+	OTUS_LOCK_ASSERT(sc);
+	while ((m = mbufq_dequeue(&sc->sc_snd)) != NULL) {
+		ni = (struct ieee80211_node *) m->m_pkthdr.rcvif;
+		m->m_pkthdr.rcvif = NULL;
+		ieee80211_free_node(ni);
+		m_freem(m);
+	}
+}
+
+static void
+otus_tx_start(struct otus_softc *sc)
+{
+
+	taskqueue_enqueue(taskqueue_thread, &sc->tx_task);
+}
+
+static int
+otus_transmit(struct ieee80211com *ic, struct mbuf *m)
+{
+	struct otus_softc *sc = ic->ic_softc;
+	int error;
+
+	OTUS_LOCK(sc);
+	if (! sc->sc_running) {
+		OTUS_UNLOCK(sc);
+		return (ENXIO);
+	}
+
+	/* XXX TODO: handle fragments */
+	error = mbufq_enqueue(&sc->sc_snd, m);
+	if (error) {
+		OTUS_DPRINTF(sc, OTUS_DEBUG_XMIT,
+		    "%s: mbufq_enqueue failed: %d\n",
+		    __func__,
+		    error);
+		OTUS_UNLOCK(sc);
+		return (error);
+	}
+	OTUS_UNLOCK(sc);
+
+	/* Kick TX */
+	otus_tx_start(sc);
+
+	return (0);
+}
+
+static void
+_otus_start(struct otus_softc *sc)
+{
+	struct ieee80211_node *ni;
+	struct otus_data *bf;
+	struct mbuf *m;
+
+	OTUS_LOCK_ASSERT(sc);
+
+	while ((m = mbufq_dequeue(&sc->sc_snd)) != NULL) {
+		bf = otus_getbuf(sc);
+		if (bf == NULL) {
+			OTUS_DPRINTF(sc, OTUS_DEBUG_XMIT,
+			    "%s: failed to get buffer\n", __func__);
+			mbufq_prepend(&sc->sc_snd, m);
+			break;
+		}
+
+		ni = (struct ieee80211_node *)m->m_pkthdr.rcvif;
+		m->m_pkthdr.rcvif = NULL;
+
+		if (otus_tx(sc, ni, m, bf) != 0) {
+			OTUS_DPRINTF(sc, OTUS_DEBUG_XMIT,
+			    "%s: failed to transmit\n", __func__);
+			if_inc_counter(ni->ni_vap->iv_ifp,
+			    IFCOUNTER_OERRORS, 1);
+			otus_freebuf(sc, bf);
+			ieee80211_free_node(ni);
+			m_freem(m);
+			break;
+		}
+	}
+}
+
+static void
+otus_tx_task(void *arg, int pending)
+{
+	struct otus_softc *sc = arg;
+
+	OTUS_LOCK(sc);
+	_otus_start(sc);
+	OTUS_UNLOCK(sc);
+}
+
+static int
+otus_raw_xmit(struct ieee80211_node *ni, struct mbuf *m,
+    const struct ieee80211_bpf_params *params)
+{
+	struct ieee80211com *ic= ni->ni_ic;
+	struct otus_softc *sc = ic->ic_softc;
+	struct otus_data *bf = NULL;
+	int error = 0;
+
+	/* Don't transmit if we're not running */
+	OTUS_LOCK(sc);
+	if (! sc->sc_running) {
+		error = ENETDOWN;
+		goto error;
+	}
+
+	bf = otus_getbuf(sc);
+	if (bf == NULL) {
+		error = ENOBUFS;
+		goto error;
+	}
+
+	/*
+	 * XXX TODO: support TX bpf params
+	 */
+	if (otus_tx(sc, ni, m, bf) != 0) {
+		error = EIO;
+		goto error;
+	}
+
+	OTUS_UNLOCK(sc);
+	return (0);
+error:
+	if (bf)
+		otus_freebuf(sc, bf);
+	OTUS_UNLOCK(sc);
+	ieee80211_free_node(ni);
+	m_freem(m);
+	return (ENXIO);
+}
+
+static void
+otus_update_chw(struct ieee80211com *ic)
+{
+
+	printf("%s: TODO\n", __func__);
+}
+
+static void
+otus_set_channel(struct ieee80211com *ic)
+{
+	struct otus_softc *sc = ic->ic_softc;
+	OTUS_DPRINTF(sc, OTUS_DEBUG_RESET, "%s: set channel: %d\n",
+	    __func__,
+	    ic->ic_curchan->ic_freq);
+
+	OTUS_LOCK(sc);
+	(void) otus_set_chan(sc, ic->ic_curchan, 0);
+	OTUS_UNLOCK(sc);
+}
+
+static void
+otus_wme_update_task(void *arg, int pending)
+{
+	struct otus_softc *sc = arg;
+
+	OTUS_LOCK(sc);
+	/*
+	 * XXX TODO: take temporary copy of EDCA information
+	 * when scheduling this so we have a more time-correct view
+	 * of things.
+	 */
+	otus_updateedca(sc);
+	OTUS_UNLOCK(sc);
+}
+
+static void
+otus_wme_schedule_update(struct otus_softc *sc)
+{
+
+	taskqueue_enqueue(taskqueue_thread, &sc->wme_update_task);
+}
+
+/*
+ * This is called by net80211 in RX packet context, so we
+ * can't sleep here.
+ *
+ * TODO: have net80211 schedule an update itself for its
+ * own internal taskqueue.
+ */
+static int
+otus_wme_update(struct ieee80211com *ic)
+{
+	struct otus_softc *sc = ic->ic_softc;
+
+	otus_wme_schedule_update(sc);
+	return (0);
+}
+
+static int
+otus_ampdu_enable(struct ieee80211_node *ni, struct ieee80211_tx_ampdu *tap)
+{
+
+	/* For now, no A-MPDU TX support in the driver */
+	return (0);
+}
+
+static void
+otus_scan_start(struct ieee80211com *ic)
+{
+
+//	printf("%s: TODO\n", __func__);
+}
+
+static void
+otus_scan_end(struct ieee80211com *ic)
+{
+
+//	printf("%s: TODO\n", __func__);
+}
+
+static void
+otus_update_mcast(struct ieee80211com *ic)
+{
+	struct otus_softc *sc = ic->ic_softc;
+
+	(void) otus_set_multi(sc);
+}
+
+static int
+otus_attachhook(struct otus_softc *sc)
+{
+	struct ieee80211com *ic = &sc->sc_ic;
+	usb_device_request_t req;
+	uint32_t in, out;
+	int error;
+	uint8_t bands;
+
+	/* Not locked */
+	error = otus_load_firmware(sc, "otusfw_init", AR_FW_INIT_ADDR);
+	if (error != 0) {
+		device_printf(sc->sc_dev, "%s: could not load %s firmware\n",
+		    __func__, "init");
+		return (ENXIO);
+	}
+
+	/* XXX not locked? */
+	otus_delay_ms(sc, 1000);
+
+	/* Not locked */
+	error = otus_load_firmware(sc, "otusfw_main", AR_FW_MAIN_ADDR);
+	if (error != 0) {
+		device_printf(sc->sc_dev, "%s: could not load %s firmware\n",
+		    __func__, "main");
+		return (ENXIO);
+	}
+
+	OTUS_LOCK(sc);
+
+	/* Tell device that firmware transfer is complete. */
+	req.bmRequestType = UT_WRITE_VENDOR_DEVICE;
+	req.bRequest = AR_FW_DOWNLOAD_COMPLETE;
+	USETW(req.wValue, 0);
+	USETW(req.wIndex, 0);
+	USETW(req.wLength, 0);
+	if (usbd_do_request_flags(sc->sc_udev, &sc->sc_mtx, &req, NULL,
+	    0, NULL, 250) != 0) {
+		OTUS_UNLOCK(sc);
+		device_printf(sc->sc_dev,
+		    "%s: firmware initialization failed\n",
+		    __func__);
+		return (ENXIO);
+	}
+
+	/* Send an ECHO command to check that everything is settled. */
+	in = 0xbadc0ffe;
+	if (otus_cmd(sc, AR_CMD_ECHO, &in, sizeof in, &out, sizeof(out)) != 0) {
+		OTUS_UNLOCK(sc);
+		device_printf(sc->sc_dev,
+		    "%s: echo command failed\n", __func__);
+		return (ENXIO);
+	}
+	if (in != out) {
+		OTUS_UNLOCK(sc);
+		device_printf(sc->sc_dev,
+		    "%s: echo reply mismatch: 0x%08x!=0x%08x\n",
+		    __func__, in, out);
+		return (ENXIO);
+	}
+
+	/* Read entire EEPROM. */
+	if (otus_read_eeprom(sc) != 0) {
+		OTUS_UNLOCK(sc);
+		device_printf(sc->sc_dev,
+		    "%s: could not read EEPROM\n",
+		    __func__);
+		return (ENXIO);
+	}
+
+	OTUS_UNLOCK(sc);
+
+	sc->txmask = sc->eeprom.baseEepHeader.txMask;
+	sc->rxmask = sc->eeprom.baseEepHeader.rxMask;
+	sc->capflags = sc->eeprom.baseEepHeader.opCapFlags;
+	IEEE80211_ADDR_COPY(ic->ic_macaddr, sc->eeprom.baseEepHeader.macAddr);
+	sc->sc_led_newstate = otus_led_newstate_type3;	/* XXX */
+
+	device_printf(sc->sc_dev,
+	    "MAC/BBP AR9170, RF AR%X, MIMO %dT%dR, address %s\n",
+	    (sc->capflags & AR5416_OPFLAGS_11A) ?
+		0x9104 : ((sc->txmask == 0x5) ? 0x9102 : 0x9101),
+	    (sc->txmask == 0x5) ? 2 : 1, (sc->rxmask == 0x5) ? 2 : 1,
+	    ether_sprintf(ic->ic_macaddr));
+
+	ic->ic_softc = sc;
+	ic->ic_name = device_get_nameunit(sc->sc_dev);
+	ic->ic_phytype = IEEE80211_T_OFDM;	/* not only, but not used */
+	ic->ic_opmode = IEEE80211_M_STA;	/* default to BSS mode */
+
+	/* Set device capabilities. */
+	ic->ic_caps =
+	    IEEE80211_C_STA |		/* station mode */
+#if 0
+	    IEEE80211_C_BGSCAN |	/* Background scan. */
+#endif
+	    IEEE80211_C_SHPREAMBLE |	/* Short preamble supported. */
+	    IEEE80211_C_WME |		/* WME/QoS */
+	    IEEE80211_C_SHSLOT |	/* Short slot time supported. */
+	    IEEE80211_C_FF |		/* Atheros fast-frames supported. */
+	    IEEE80211_C_WPA;		/* WPA/RSN. */
+
+	/* XXX TODO: 11n */
+
+#if 0
+	if (sc->eeprom.baseEepHeader.opCapFlags & AR5416_OPFLAGS_11G) {
+		/* Set supported .11b and .11g rates. */
+		ic->ic_sup_rates[IEEE80211_MODE_11B] =
+		    ieee80211_std_rateset_11b;
+		ic->ic_sup_rates[IEEE80211_MODE_11G] =
+		    ieee80211_std_rateset_11g;
+	}
+	if (sc->eeprom.baseEepHeader.opCapFlags & AR5416_OPFLAGS_11A) {
+		/* Set supported .11a rates. */
+		ic->ic_sup_rates[IEEE80211_MODE_11A] =
+		    ieee80211_std_rateset_11a;
+	}
+#endif
+
+#if 0
+	/* Build the list of supported channels. */
+	otus_get_chanlist(sc);
+#else
+	/* Set supported .11b and .11g rates. */
+	bands = 0;
+	if (sc->eeprom.baseEepHeader.opCapFlags & AR5416_OPFLAGS_11G) {
+		setbit(&bands, IEEE80211_MODE_11B);
+		setbit(&bands, IEEE80211_MODE_11G);
+	}
+	if (sc->eeprom.baseEepHeader.opCapFlags & AR5416_OPFLAGS_11A) {
+		setbit(&bands, IEEE80211_MODE_11A);
+	}
+#if 0
+	if (sc->sc_ht)
+		setbit(&bands, IEEE80211_MODE_11NG);
+#endif
+	ieee80211_init_channels(ic, NULL, &bands);
+#endif
+
+	ieee80211_ifattach(ic);
+	ic->ic_raw_xmit = otus_raw_xmit;
+	ic->ic_scan_start = otus_scan_start;
+	ic->ic_scan_end = otus_scan_end;
+	ic->ic_set_channel = otus_set_channel;
+	ic->ic_vap_create = otus_vap_create;
+	ic->ic_vap_delete = otus_vap_delete;
+	ic->ic_update_mcast = otus_update_mcast;
+	ic->ic_update_promisc = otus_update_mcast;
+	ic->ic_parent = otus_parent;
+	ic->ic_transmit = otus_transmit;
+	ic->ic_update_chw = otus_update_chw;
+	ic->ic_ampdu_enable = otus_ampdu_enable;
+	ic->ic_wme.wme_update = otus_wme_update;
+	ic->ic_newassoc = otus_newassoc;
+
+#ifdef notyet
+	ic->ic_set_key = otus_set_key;
+	ic->ic_delete_key = otus_delete_key;
+#endif
+
+	ieee80211_radiotap_attach(ic, &sc->sc_txtap.wt_ihdr,
+	    sizeof(sc->sc_txtap), OTUS_TX_RADIOTAP_PRESENT,
+	    &sc->sc_rxtap.wr_ihdr, sizeof(sc->sc_rxtap),
+	    OTUS_RX_RADIOTAP_PRESENT);
+
+	return (0);
+}
+
+void
+otus_get_chanlist(struct otus_softc *sc)
+{
+	struct ieee80211com *ic = &sc->sc_ic;
+	uint16_t domain;
+	uint8_t chan;
+	int i;
+
+	/* XXX regulatory domain. */
+	domain = le16toh(sc->eeprom.baseEepHeader.regDmn[0]);
+	OTUS_DPRINTF(sc, OTUS_DEBUG_RESET, "regdomain=0x%04x\n", domain);
+
+	if (sc->eeprom.baseEepHeader.opCapFlags & AR5416_OPFLAGS_11G) {
+		for (i = 0; i < 14; i++) {
+			chan = ar_chans[i];
+			ic->ic_channels[chan].ic_freq =
+			    ieee80211_ieee2mhz(chan, IEEE80211_CHAN_2GHZ);
+			ic->ic_channels[chan].ic_flags =
+			    IEEE80211_CHAN_CCK | IEEE80211_CHAN_OFDM |
+			    IEEE80211_CHAN_DYN | IEEE80211_CHAN_2GHZ;
+		}
+	}
+	if (sc->eeprom.baseEepHeader.opCapFlags & AR5416_OPFLAGS_11A) {
+		for (i = 14; i < nitems(ar_chans); i++) {
+			chan = ar_chans[i];
+			ic->ic_channels[chan].ic_freq =
+			    ieee80211_ieee2mhz(chan, IEEE80211_CHAN_5GHZ);
+			ic->ic_channels[chan].ic_flags = IEEE80211_CHAN_A;
+		}
+	}
+}
+
+int
+otus_load_firmware(struct otus_softc *sc, const char *name, uint32_t addr)
+{
+	usb_device_request_t req;
+	char *ptr;
+	const struct firmware *fw;
+	int mlen, error, size;
+
+	error = 0;
+
+	/* Read firmware image from the filesystem. */
+	if ((fw = firmware_get(name)) == NULL) {
+		device_printf(sc->sc_dev,
+		    "%s: failed loadfirmware of file %s\n", __func__, name);
+		return (ENXIO);
+	}
+	req.bmRequestType = UT_WRITE_VENDOR_DEVICE;
+	req.bRequest = AR_FW_DOWNLOAD;
+	USETW(req.wIndex, 0);
+
+	OTUS_LOCK(sc);
+
+	/* XXX const */
+	ptr = __DECONST(char *, fw->data);
+	size = fw->datasize;
+	addr >>= 8;
+	while (size > 0) {
+		mlen = MIN(size, 4096);
+
+		USETW(req.wValue, addr);
+		USETW(req.wLength, mlen);
+		if (usbd_do_request_flags(sc->sc_udev, &sc->sc_mtx,
+		    &req, ptr, 0, NULL, 250) != 0) {
+			error = EIO;
+			break;
+		}
+		addr += mlen >> 8;
+		ptr  += mlen;
+		size -= mlen;
+	}
+
+	OTUS_UNLOCK(sc);
+
+	firmware_put(fw, FIRMWARE_UNLOAD);
+	if (error != 0)
+		device_printf(sc->sc_dev,
+		    "%s: %s: error=%d\n", __func__, name, error);
+	return error;
+}
+
+int
+otus_open_pipes(struct otus_softc *sc)
+{
+#if 0
+	int isize, error;
+	int i;
+#endif
+	int error;
+
+	OTUS_UNLOCK_ASSERT(sc);
+
+	if ((error = otus_alloc_tx_cmd_list(sc)) != 0) {
+		device_printf(sc->sc_dev,
+		    "%s: could not allocate command xfer\n",
+		    __func__);
+		goto fail;
+	}
+
+	if ((error = otus_alloc_tx_list(sc)) != 0) {
+		device_printf(sc->sc_dev, "%s: could not allocate Tx xfers\n",
+		    __func__);
+		goto fail;
+	}
+
+	if ((error = otus_alloc_rx_list(sc)) != 0) {
+		device_printf(sc->sc_dev, "%s: could not allocate Rx xfers\n",
+		    __func__);
+		goto fail;
+	}
+
+	/* Enable RX transfers; needed for initial firmware messages */
+	OTUS_LOCK(sc);
+	usbd_transfer_start(sc->sc_xfer[OTUS_BULK_RX]);
+	usbd_transfer_start(sc->sc_xfer[OTUS_BULK_IRQ]);
+	OTUS_UNLOCK(sc);
+	return 0;
+
+fail:	otus_close_pipes(sc);
+	return error;
+}
+
+void
+otus_close_pipes(struct otus_softc *sc)
+{
+
+	OTUS_LOCK(sc);
+	otus_free_tx_cmd_list(sc);
+	otus_free_tx_list(sc);
+	otus_free_rx_list(sc);
+	OTUS_UNLOCK(sc);
+
+	usbd_transfer_unsetup(sc->sc_xfer, OTUS_N_XFER);
+}
+
+static void
+otus_free_cmd_list(struct otus_softc *sc, struct otus_tx_cmd cmd[], int ndata)
+{
+	int i;
+
+	/* XXX TODO: someone has to have waken up waiters! */
+	for (i = 0; i < ndata; i++) {
+		struct otus_tx_cmd *dp = &cmd[i];
+
+		if (dp->buf != NULL) {
+			free(dp->buf, M_USBDEV);
+			dp->buf = NULL;
+		}
+	}
+}
+
+static int
+otus_alloc_cmd_list(struct otus_softc *sc, struct otus_tx_cmd cmd[],
+    int ndata, int maxsz)
+{
+	int i, error;
+
+	for (i = 0; i < ndata; i++) {
+		struct otus_tx_cmd *dp = &cmd[i];
+		dp->buf = malloc(maxsz, M_USBDEV, M_NOWAIT);
+		dp->odata = NULL;
+		if (dp->buf == NULL) {
+			device_printf(sc->sc_dev,
+			    "could not allocate buffer\n");
+			error = ENOMEM;
+			goto fail;
+		}
+	}
+
+	return (0);
+fail:
+	otus_free_cmd_list(sc, cmd, ndata);
+	return (error);
+}
+
+static int
+otus_alloc_tx_cmd_list(struct otus_softc *sc)
+{
+	int error, i;
+
+	error = otus_alloc_cmd_list(sc, sc->sc_cmd, OTUS_CMD_LIST_COUNT,
+	    OTUS_MAX_TXCMDSZ);
+	if (error != 0)
+		return (error);
+
+	STAILQ_INIT(&sc->sc_cmd_active);
+	STAILQ_INIT(&sc->sc_cmd_inactive);
+	STAILQ_INIT(&sc->sc_cmd_pending);
+	STAILQ_INIT(&sc->sc_cmd_waiting);
+
+	for (i = 0; i < OTUS_CMD_LIST_COUNT; i++)
+		STAILQ_INSERT_HEAD(&sc->sc_cmd_inactive, &sc->sc_cmd[i],
+		    next_cmd);
+
+	return (0);
+}
+
+static void
+otus_free_tx_cmd_list(struct otus_softc *sc)
+{
+
+	/*
+	 * XXX TODO: something needs to wake up any pending/sleeping
+	 * waiters!
+	 */
+	STAILQ_INIT(&sc->sc_cmd_active);
+	STAILQ_INIT(&sc->sc_cmd_inactive);
+	STAILQ_INIT(&sc->sc_cmd_pending);
+	STAILQ_INIT(&sc->sc_cmd_waiting);
+
+	otus_free_cmd_list(sc, sc->sc_cmd, OTUS_CMD_LIST_COUNT);
+}
+
+static int
+otus_alloc_list(struct otus_softc *sc, struct otus_data data[],
+    int ndata, int maxsz)
+{
+	int i, error;
+
+	for (i = 0; i < ndata; i++) {
+		struct otus_data *dp = &data[i];
+		dp->sc = sc;
+		dp->m = NULL;
+		dp->buf = malloc(maxsz, M_USBDEV, M_NOWAIT);
+		if (dp->buf == NULL) {
+			device_printf(sc->sc_dev,
+			    "could not allocate buffer\n");
+			error = ENOMEM;
+			goto fail;
+		}
+		dp->ni = NULL;
+	}
+
+	return (0);
+fail:
+	otus_free_list(sc, data, ndata);
+	return (error);
+}
+
+static int
+otus_alloc_rx_list(struct otus_softc *sc)
+{
+	int error, i;
+
+	error = otus_alloc_list(sc, sc->sc_rx, OTUS_RX_LIST_COUNT,
+	    OTUS_RXBUFSZ);
+	if (error != 0)
+		return (error);
+
+	STAILQ_INIT(&sc->sc_rx_active);
+	STAILQ_INIT(&sc->sc_rx_inactive);
+
+	for (i = 0; i < OTUS_RX_LIST_COUNT; i++)
+		STAILQ_INSERT_HEAD(&sc->sc_rx_inactive, &sc->sc_rx[i], next);
+
+	return (0);
+}
+
+static int
+otus_alloc_tx_list(struct otus_softc *sc)
+{
+	int error, i;
+
+	error = otus_alloc_list(sc, sc->sc_tx, OTUS_TX_LIST_COUNT,
+	    OTUS_TXBUFSZ);
+	if (error != 0)
+		return (error);
+
+	STAILQ_INIT(&sc->sc_tx_inactive);
+
+	for (i = 0; i != OTUS_N_XFER; i++) {
+		STAILQ_INIT(&sc->sc_tx_active[i]);
+		STAILQ_INIT(&sc->sc_tx_pending[i]);
+	}
+
+	for (i = 0; i < OTUS_TX_LIST_COUNT; i++) {
+		STAILQ_INSERT_HEAD(&sc->sc_tx_inactive, &sc->sc_tx[i], next);
+	}
+
+	return (0);
+}
+
+static void
+otus_free_tx_list(struct otus_softc *sc)
+{
+	int i;
+
+	/* prevent further allocations from TX list(s) */
+	STAILQ_INIT(&sc->sc_tx_inactive);
+
+	for (i = 0; i != OTUS_N_XFER; i++) {
+		STAILQ_INIT(&sc->sc_tx_active[i]);
+		STAILQ_INIT(&sc->sc_tx_pending[i]);
+	}
+
+	otus_free_list(sc, sc->sc_tx, OTUS_TX_LIST_COUNT);
+}
+
+static void
+otus_free_rx_list(struct otus_softc *sc)
+{
+	/* prevent further allocations from RX list(s) */
+	STAILQ_INIT(&sc->sc_rx_inactive);
+	STAILQ_INIT(&sc->sc_rx_active);
+
+	otus_free_list(sc, sc->sc_rx, OTUS_RX_LIST_COUNT);
+}
+
+static void
+otus_free_list(struct otus_softc *sc, struct otus_data data[], int ndata)
+{
+	int i;
+
+	for (i = 0; i < ndata; i++) {
+		struct otus_data *dp = &data[i];
+
+		if (dp->buf != NULL) {
+			free(dp->buf, M_USBDEV);
+			dp->buf = NULL;
+		}
+		if (dp->ni != NULL) {
+			ieee80211_free_node(dp->ni);
+			dp->ni = NULL;
+		}
+	}
+}
+
+static struct otus_data *
+_otus_getbuf(struct otus_softc *sc)
+{
+	struct otus_data *bf;
+
+	bf = STAILQ_FIRST(&sc->sc_tx_inactive);
+	if (bf != NULL)
+		STAILQ_REMOVE_HEAD(&sc->sc_tx_inactive, next);
+	else
+		bf = NULL;
+	return (bf);
+}
+
+static struct otus_data *
+otus_getbuf(struct otus_softc *sc)
+{
+	struct otus_data *bf;
+
+	OTUS_LOCK_ASSERT(sc);
+
+	bf = _otus_getbuf(sc);
+	return (bf);
+}
+
+static void
+otus_freebuf(struct otus_softc *sc, struct otus_data *bf)
+{
+
+	OTUS_LOCK_ASSERT(sc);
+	STAILQ_INSERT_TAIL(&sc->sc_tx_inactive, bf, next);
+}
+
+static struct otus_tx_cmd *
+_otus_get_txcmd(struct otus_softc *sc)
+{
+	struct otus_tx_cmd *bf;
+
+	bf = STAILQ_FIRST(&sc->sc_cmd_inactive);
+	if (bf != NULL)
+		STAILQ_REMOVE_HEAD(&sc->sc_cmd_inactive, next_cmd);
+	else
+		bf = NULL;
+	return (bf);
+}
+
+static struct otus_tx_cmd *
+otus_get_txcmd(struct otus_softc *sc)
+{
+	struct otus_tx_cmd *bf;
+
+	OTUS_LOCK_ASSERT(sc);
+
+	bf = _otus_get_txcmd(sc);
+	if (bf == NULL) {
+		device_printf(sc->sc_dev, "%s: no tx cmd buffers\n",
+		    __func__);
+	}
+	return (bf);
+}
+
+static void
+otus_free_txcmd(struct otus_softc *sc, struct otus_tx_cmd *bf)
+{
+
+	OTUS_LOCK_ASSERT(sc);
+	STAILQ_INSERT_TAIL(&sc->sc_cmd_inactive, bf, next_cmd);
+}
+
+void
+otus_next_scan(void *arg, int pending)
+{
+#if 0
+	struct otus_softc *sc = arg;
+
+	if (usbd_is_dying(sc->sc_udev))
+		return;
+
+	usbd_ref_incr(sc->sc_udev);
+
+	if (sc->sc_ic.ic_state == IEEE80211_S_SCAN)
+		ieee80211_next_scan(&sc->sc_ic.ic_if);
+
+	usbd_ref_decr(sc->sc_udev);
+#endif
+}
+
+int
+otus_newstate(struct ieee80211vap *vap, enum ieee80211_state nstate, int arg)
+{
+	struct otus_vap *uvp = OTUS_VAP(vap);
+	struct ieee80211com *ic = vap->iv_ic;
+	struct otus_softc *sc = ic->ic_softc;
+	struct ieee80211_node *ni;
+	enum ieee80211_state ostate;
+
+	ostate = vap->iv_state;
+	OTUS_DPRINTF(sc, OTUS_DEBUG_STATE, "%s: %s -> %s\n", __func__,
+	    ieee80211_state_name[ostate],
+	    ieee80211_state_name[nstate]);
+
+	IEEE80211_UNLOCK(ic);
+
+	OTUS_LOCK(sc);
+
+	/* XXX TODO: more fleshing out! */
+
+	switch (nstate) {
+	case IEEE80211_S_RUN:
+		ni = ieee80211_ref_node(vap->iv_bss);
+
+		if (ic->ic_opmode == IEEE80211_M_STA) {
+			otus_updateslot(sc);
+			otus_set_bssid(sc, ni->ni_bssid);
+
+			/* Start calibration timer. */
+			taskqueue_enqueue_timeout(taskqueue_thread,
+			    &sc->calib_to, hz);
+		}
+		break;
+	default:
+		break;
+	}
+
+	/* XXX TODO: calibration? */
+
+	sc->sc_led_newstate(sc);
+
+	OTUS_UNLOCK(sc);
+	IEEE80211_LOCK(ic);
+	return (uvp->newstate(vap, nstate, arg));
+}
+
+int
+otus_cmd(struct otus_softc *sc, uint8_t code, const void *idata, int ilen,
+    void *odata, int odatalen)
+{
+	struct otus_tx_cmd *cmd;
+	struct ar_cmd_hdr *hdr;
+	int xferlen, error;
+
+	OTUS_LOCK_ASSERT(sc);
+
+	/* Always bulk-out a multiple of 4 bytes. */
+	xferlen = (sizeof (*hdr) + ilen + 3) & ~3;
+	if (xferlen > OTUS_MAX_TXCMDSZ) {
+		device_printf(sc->sc_dev, "%s: command (0x%02x) size (%d) > %d\n",
+		    __func__,
+		    code,
+		    xferlen,
+		    OTUS_MAX_TXCMDSZ);
+		return (EIO);
+	}
+
+	cmd = otus_get_txcmd(sc);
+	if (cmd == NULL) {
+		device_printf(sc->sc_dev, "%s: failed to get buf\n",
+		    __func__);
+		return (EIO);
+	}
+
+	hdr = (struct ar_cmd_hdr *)cmd->buf;
+	hdr->code  = code;
+	hdr->len   = ilen;
+	hdr->token = ++sc->token;	/* Don't care about endianness. */
+	cmd->token = hdr->token;
+	/* XXX TODO: check max cmd length? */
+	memcpy((uint8_t *)&hdr[1], idata, ilen);
+
+	OTUS_DPRINTF(sc, OTUS_DEBUG_CMD,
+	    "%s: sending command code=0x%02x len=%d token=%d\n",
+	    __func__, code, ilen, hdr->token);
+
+	cmd->odata = odata;
+	cmd->odatalen = odatalen;
+	cmd->buflen = xferlen;
+
+	/* Queue the command to the endpoint */
+	STAILQ_INSERT_TAIL(&sc->sc_cmd_pending, cmd, next_cmd);
+	usbd_transfer_start(sc->sc_xfer[OTUS_BULK_CMD]);
+
+	/* Sleep on the command; wait for it to complete */
+	error = msleep(cmd, &sc->sc_mtx, PCATCH, "otuscmd", hz);
+
+	/*
+	 * At this point we don't own cmd any longer; it'll be
+	 * freed by the cmd bulk path or the RX notification
+	 * path.  If the data is made available then it'll be copied
+	 * to the caller.  All that is left to do is communicate
+	 * status back to the caller.
+	 */
+	if (error != 0) {
+		device_printf(sc->sc_dev,
+		    "%s: timeout waiting for command 0x%02x reply\n",
+		    __func__, code);
+	}
+	return error;
+}
+
+void
+otus_write(struct otus_softc *sc, uint32_t reg, uint32_t val)
+{
+
+	OTUS_LOCK_ASSERT(sc);
+
+	sc->write_buf[sc->write_idx].reg = htole32(reg);
+	sc->write_buf[sc->write_idx].val = htole32(val);
+
+	if (++sc->write_idx > (AR_MAX_WRITE_IDX-1))
+		(void)otus_write_barrier(sc);
+}
+
+int
+otus_write_barrier(struct otus_softc *sc)
+{
+	int error;
+
+	OTUS_LOCK_ASSERT(sc);
+
+	if (sc->write_idx == 0)
+		return 0;	/* Nothing to flush. */
+
+	OTUS_DPRINTF(sc, OTUS_DEBUG_REGIO, "%s: called; %d updates\n",
+	    __func__,
+	    sc->write_idx);
+
+	error = otus_cmd(sc, AR_CMD_WREG, sc->write_buf,
+	    sizeof (sc->write_buf[0]) * sc->write_idx, NULL, 0);
+	sc->write_idx = 0;
+	return error;
+}
+
+struct ieee80211_node *
+otus_node_alloc(struct ieee80211com *ic)
+{
+	return malloc(sizeof (struct otus_node), M_DEVBUF, M_NOWAIT | M_ZERO);
+}
+
+#if 0
+int
+otus_media_change(struct ifnet *ifp)
+{
+	struct otus_softc *sc = ifp->if_softc;
+	struct ieee80211com *ic = &sc->sc_ic;
+	uint8_t rate, ridx;
+	int error;
+
+	error = ieee80211_media_change(ifp);
+	if (error != ENETRESET)
+		return error;
+
+	if (ic->ic_fixed_rate != -1) {
+		rate = ic->ic_sup_rates[ic->ic_curmode].
+		    rs_rates[ic->ic_fixed_rate] & IEEE80211_RATE_VAL;
+		for (ridx = 0; ridx <= OTUS_RIDX_MAX; ridx++)
+			if (otus_rates[ridx].rate == rate)
+				break;
+		sc->fixed_ridx = ridx;
+	}
+
+	if ((ifp->if_flags & (IFF_UP | IFF_RUNNING)) == (IFF_UP | IFF_RUNNING))
+		error = otus_init(sc);
+
+	return error;
+}
+#endif
+
+int
+otus_read_eeprom(struct otus_softc *sc)
+{
+	uint32_t regs[8], reg;
+	uint8_t *eep;
+	int i, j, error;
+
+	OTUS_LOCK_ASSERT(sc);
+
+	/* Read EEPROM by blocks of 32 bytes. */
+	eep = (uint8_t *)&sc->eeprom;
+	reg = AR_EEPROM_OFFSET;
+	for (i = 0; i < sizeof (sc->eeprom) / 32; i++) {
+		for (j = 0; j < 8; j++, reg += 4)
+			regs[j] = htole32(reg);
+		error = otus_cmd(sc, AR_CMD_RREG, regs, sizeof regs, eep, 32);
+		if (error != 0)
+			break;
+		eep += 32;
+	}
+	return error;
+}
+
+void
+otus_newassoc(struct ieee80211_node *ni, int isnew)
+{
+	struct ieee80211com *ic = ni->ni_ic;
+	struct otus_softc *sc = ic->ic_softc;
+	struct otus_node *on = OTUS_NODE(ni);
+
+	OTUS_DPRINTF(sc, OTUS_DEBUG_STATE, "new assoc isnew=%d addr=%s\n",
+	    isnew, ether_sprintf(ni->ni_macaddr));
+
+	on->tx_done = 0;
+	on->tx_err = 0;
+	on->tx_retries = 0;
+}
+
+static void
+otus_cmd_handle_response(struct otus_softc *sc, struct ar_cmd_hdr *hdr)
+{
+	struct otus_tx_cmd *cmd;
+
+	OTUS_LOCK_ASSERT(sc);
+
+	OTUS_DPRINTF(sc, OTUS_DEBUG_CMDDONE,
+	    "%s: received reply code=0x%02x len=%d token=%d\n",
+	    __func__,
+	    hdr->code, hdr->len, hdr->token);
+
+	/*
+	 * Walk the list, freeing items that aren't ours,
+	 * stopping when we hit our token.
+	 */
+	while ((cmd = STAILQ_FIRST(&sc->sc_cmd_waiting)) != NULL) {
+		STAILQ_REMOVE_HEAD(&sc->sc_cmd_waiting, next_cmd);
+		OTUS_DPRINTF(sc, OTUS_DEBUG_CMDDONE,
+		    "%s: cmd=%p; hdr.token=%d, cmd.token=%d\n",
+		    __func__,
+		    cmd,
+		    (int) hdr->token,
+		    (int) cmd->token);
+		if (hdr->token == cmd->token) {
+			/* Copy answer into caller's supplied buffer. */
+			if (cmd->odata != NULL) {
+				if (hdr->len != cmd->odatalen) {
+					device_printf(sc->sc_dev,
+					    "%s: code 0x%02x, len=%d, olen=%d\n",
+					    __func__,
+					    (int) hdr->code,
+					    (int) hdr->len,
+					    (int) cmd->odatalen);
+				}
+				memcpy(cmd->odata, &hdr[1],
+				    MIN(cmd->odatalen, hdr->len));
+			}
+			wakeup(cmd);
+		}
+
+		STAILQ_INSERT_TAIL(&sc->sc_cmd_inactive, cmd, next_cmd);
+	}
+}
+
+void
+otus_cmd_rxeof(struct otus_softc *sc, uint8_t *buf, int len)
+{
+	struct ieee80211com *ic = &sc->sc_ic;
+	struct ar_cmd_hdr *hdr;
+
+	OTUS_LOCK_ASSERT(sc);
+
+	if (__predict_false(len < sizeof (*hdr))) {
+		OTUS_DPRINTF(sc, OTUS_DEBUG_CMDDONE,
+		    "cmd too small %d\n", len);
+		return;
+	}
+	hdr = (struct ar_cmd_hdr *)buf;
+	if (__predict_false(sizeof (*hdr) + hdr->len > len ||
+	    sizeof (*hdr) + hdr->len > 64)) {
+		OTUS_DPRINTF(sc, OTUS_DEBUG_CMDDONE,
+		    "cmd too large %d\n", hdr->len);
+		return;
+	}
+
+	OTUS_DPRINTF(sc, OTUS_DEBUG_RXDONE,
+	    "%s: code=%.02x\n",
+	    __func__,
+	    hdr->code);
+
+	/*
+	 * This has to reach into the cmd queue "waiting for
+	 * an RX response" list, grab the head entry and check
+	 * if we need to wake anyone up.
+	 */
+	if ((hdr->code & 0xc0) != 0xc0) {
+		otus_cmd_handle_response(sc, hdr);
+		return;
+	}
+
+	/* Received unsolicited notification. */
+	switch (hdr->code & 0x3f) {
+	case AR_EVT_BEACON:
+		break;
+	case AR_EVT_TX_COMP:
+	{
+		struct ar_evt_tx_comp *tx = (struct ar_evt_tx_comp *)&hdr[1];
+		struct ieee80211_node *ni;
+
+		ni = ieee80211_find_node(&ic->ic_sta, tx->macaddr);
+		if (ni == NULL) {
+			device_printf(sc->sc_dev,
+			    "%s: txcomp on unknown node (%s)\n",
+			    __func__,
+			    ether_sprintf(tx->macaddr));
+			break;
+		}
+
+		OTUS_DPRINTF(sc, OTUS_DEBUG_TXCOMP,
+		    "tx completed %s status=%d phy=0x%x\n",
+		    ether_sprintf(tx->macaddr), le16toh(tx->status),
+		    le32toh(tx->phy));
+
+		switch (le16toh(tx->status)) {
+		case AR_TX_STATUS_COMP:
+#if 0
+			ackfailcnt = 0;
+			ieee80211_ratectl_tx_complete(ni->ni_vap, ni,
+			    IEEE80211_RATECTL_TX_SUCCESS, &ackfailcnt, NULL);
+#endif
+			/*
+			 * We don't get the above; only error notifications.
+			 * Sigh.  So, don't worry about this.
+			 */
+			break;
+		case AR_TX_STATUS_RETRY_COMP:
+			OTUS_NODE(ni)->tx_retries++;
+			break;
+		case AR_TX_STATUS_FAILED:
+			OTUS_NODE(ni)->tx_err++;
+			break;
+		}
+		ieee80211_free_node(ni);
+		break;
+	}
+	case AR_EVT_TBTT:
+		break;
+	case AR_EVT_DO_BB_RESET:
+		/*
+		 * This is "tell driver to reset baseband" from ar9170-fw.
+		 *
+		 * I'm not sure what we should do here, so I'm going to
+		 * fall through; it gets generated when RTSRetryCnt internally
+		 * reaches '5' - I guess the firmware authors thought that
+		 * meant that the BB may have gone deaf or something.
+		 */
+	default:
+		device_printf(sc->sc_dev,
+		    "%s: received notification code=0x%02x len=%d\n",
+		    __func__,
+		    hdr->code, hdr->len);
+	}
+}
+
+void
+otus_sub_rxeof(struct otus_softc *sc, uint8_t *buf, int len, struct mbufq *rxq)
+{
+	struct ieee80211com *ic = &sc->sc_ic;
+	struct ieee80211_rx_stats rxs;
+#if 0
+	struct ieee80211_node *ni;
+#endif
+	struct ar_rx_tail *tail;
+	struct ieee80211_frame *wh;
+	struct mbuf *m;
+	uint8_t *plcp;
+//	int s;
+	int mlen;
+
+	if (__predict_false(len < AR_PLCP_HDR_LEN)) {
+		OTUS_DPRINTF(sc, OTUS_DEBUG_RXDONE,
+		    "sub-xfer too short %d\n", len);
+		return;
+	}
+	plcp = buf;
+
+	/* All bits in the PLCP header are set to 1 for non-MPDU. */
+	if (memcmp(plcp, AR_PLCP_HDR_INTR, AR_PLCP_HDR_LEN) == 0) {
+		otus_cmd_rxeof(sc, plcp + AR_PLCP_HDR_LEN,
+		    len - AR_PLCP_HDR_LEN);
+		return;
+	}
+
+	/* Received MPDU. */
+	if (__predict_false(len < AR_PLCP_HDR_LEN + sizeof (*tail))) {
+		OTUS_DPRINTF(sc, OTUS_DEBUG_RXDONE, "MPDU too short %d\n", len);
+		counter_u64_add(ic->ic_ierrors, 1);
+		return;
+	}
+	tail = (struct ar_rx_tail *)(plcp + len - sizeof (*tail));
+
+	/* Discard error frames. */
+	if (__predict_false(tail->error != 0)) {
+		OTUS_DPRINTF(sc, OTUS_DEBUG_RXDONE, "error frame 0x%02x\n", tail->error);
+		if (tail->error & AR_RX_ERROR_FCS) {
+			OTUS_DPRINTF(sc, OTUS_DEBUG_RXDONE, "bad FCS\n");
+		} else if (tail->error & AR_RX_ERROR_MMIC) {
+			/* Report Michael MIC failures to net80211. */
+#if 0
+			ieee80211_notify_michael_failure(ni->ni_vap, wh, keyidx);
+#endif
+			device_printf(sc->sc_dev, "%s: MIC failure\n", __func__);
+		}
+		counter_u64_add(ic->ic_ierrors, 1);
+		return;
+	}
+	/* Compute MPDU's length. */
+	mlen = len - AR_PLCP_HDR_LEN - sizeof (*tail);
+	/* Make sure there's room for an 802.11 header + FCS. */
+	if (__predict_false(mlen < IEEE80211_MIN_LEN)) {
+		counter_u64_add(ic->ic_ierrors, 1);
+		return;
+	}
+	mlen -= IEEE80211_CRC_LEN;	/* strip 802.11 FCS */
+
+	wh = (struct ieee80211_frame *)(plcp + AR_PLCP_HDR_LEN);
+
+	m = m_get2(mlen, M_NOWAIT, MT_DATA, M_PKTHDR);
+	if (m == NULL) {
+		device_printf(sc->sc_dev, "%s: failed m_get2()\n", __func__);
+		counter_u64_add(ic->ic_ierrors, 1);
+	}
+
+	/* Finalize mbuf. */
+	memcpy(mtod(m, uint8_t *), wh, mlen);
+	m->m_pkthdr.len = m->m_len = mlen;
+
+#if 0
+	if (__predict_false(sc->sc_drvbpf != NULL)) {
+		struct otus_rx_radiotap_header *tap = &sc->sc_rxtap;
+		struct mbuf mb;
+
+		tap->wr_flags = 0;
+		tap->wr_chan_freq = htole16(ic->ic_ibss_chan->ic_freq);
+		tap->wr_chan_flags = htole16(ic->ic_ibss_chan->ic_flags);
+		tap->wr_antsignal = tail->rssi;
+		tap->wr_rate = 2;	/* In case it can't be found below. */
+		switch (tail->status & AR_RX_STATUS_MT_MASK) {
+		case AR_RX_STATUS_MT_CCK:
+			switch (plcp[0]) {
+			case  10: tap->wr_rate =   2; break;
+			case  20: tap->wr_rate =   4; break;
+			case  55: tap->wr_rate =  11; break;
+			case 110: tap->wr_rate =  22; break;
+			}
+			if (tail->status & AR_RX_STATUS_SHPREAMBLE)
+				tap->wr_flags |= IEEE80211_RADIOTAP_F_SHORTPRE;
+			break;
+		case AR_RX_STATUS_MT_OFDM:
+			switch (plcp[0] & 0xf) {
+			case 0xb: tap->wr_rate =  12; break;
+			case 0xf: tap->wr_rate =  18; break;
+			case 0xa: tap->wr_rate =  24; break;
+			case 0xe: tap->wr_rate =  36; break;
+			case 0x9: tap->wr_rate =  48; break;
+			case 0xd: tap->wr_rate =  72; break;
+			case 0x8: tap->wr_rate =  96; break;
+			case 0xc: tap->wr_rate = 108; break;
+			}
+			break;
+		}
+		mb.m_data = (caddr_t)tap;
+		mb.m_len = sc->sc_rxtap_len;
+		mb.m_next = m;
+		mb.m_nextpkt = NULL;
+		mb.m_type = 0;
+		mb.m_flags = 0;
+		bpf_mtap(sc->sc_drvbpf, &mb, BPF_DIRECTION_IN);
+	}
+#endif
+
+	/* Add RSSI/NF to this mbuf */
+	bzero(&rxs, sizeof(rxs));
+	rxs.r_flags = IEEE80211_R_NF | IEEE80211_R_RSSI;
+	rxs.nf = sc->sc_nf[0];	/* XXX chain 0 != combined rssi/nf */
+	rxs.rssi = tail->rssi;
+	/* XXX TODO: add MIMO RSSI/NF as well */
+	ieee80211_add_rx_params(m, &rxs);
+
+	/* XXX make a method */
+	STAILQ_INSERT_TAIL(&rxq->mq_head, m, m_stailqpkt);
+
+#if 0
+	OTUS_UNLOCK(sc);
+	ni = ieee80211_find_rxnode(ic, wh);
+	rxi.rxi_flags = 0;
+	rxi.rxi_rssi = tail->rssi;
+	rxi.rxi_tstamp = 0;	/* unused */
+	ieee80211_input(ifp, m, ni, &rxi);
+
+	/* Node is no longer needed. */
+	ieee80211_release_node(ic, ni);
+	OTUS_LOCK(sc);
+#endif
+}
+
+static void
+otus_rxeof(struct usb_xfer *xfer, struct otus_data *data, struct mbufq *rxq)
+{
+	struct otus_softc *sc = usbd_xfer_softc(xfer);
+	caddr_t buf = data->buf;
+	struct ar_rx_head *head;
+	uint16_t hlen;
+	int len;
+
+	usbd_xfer_status(xfer, &len, NULL, NULL, NULL);
+
+	while (len >= sizeof (*head)) {
+		head = (struct ar_rx_head *)buf;
+		if (__predict_false(head->tag != htole16(AR_RX_HEAD_TAG))) {
+			OTUS_DPRINTF(sc, OTUS_DEBUG_RXDONE,
+			    "tag not valid 0x%x\n", le16toh(head->tag));
+			break;
+		}
+		hlen = le16toh(head->len);
+		if (__predict_false(sizeof (*head) + hlen > len)) {
+			OTUS_DPRINTF(sc, OTUS_DEBUG_RXDONE,
+			    "xfer too short %d/%d\n", len, hlen);
+			break;
+		}
+		/* Process sub-xfer. */
+		otus_sub_rxeof(sc, (uint8_t *)&head[1], hlen, rxq);
+
+		/* Next sub-xfer is aligned on a 32-bit boundary. */
+		hlen = (sizeof (*head) + hlen + 3) & ~3;
+		buf += hlen;
+		len -= hlen;
+	}
+}
+
+static void
+otus_bulk_rx_callback(struct usb_xfer *xfer, usb_error_t error)
+{
+	struct otus_softc *sc = usbd_xfer_softc(xfer);
+	struct ieee80211com *ic = &sc->sc_ic;
+	struct ieee80211_frame *wh;
+	struct ieee80211_node *ni;
+	struct mbuf *m;
+	struct mbufq scrx;
+	struct otus_data *data;
+
+	OTUS_LOCK_ASSERT(sc);
+
+	mbufq_init(&scrx, 1024);
+
+#if 0
+	device_printf(sc->sc_dev, "%s: called; state=%d; error=%d\n",
+	    __func__,
+	    USB_GET_STATE(xfer),
+	    error);
+#endif
+
+	switch (USB_GET_STATE(xfer)) {
+	case USB_ST_TRANSFERRED:
+		data = STAILQ_FIRST(&sc->sc_rx_active);
+		if (data == NULL)
+			goto tr_setup;
+		STAILQ_REMOVE_HEAD(&sc->sc_rx_active, next);
+		otus_rxeof(xfer, data, &scrx);
+		STAILQ_INSERT_TAIL(&sc->sc_rx_inactive, data, next);
+		/* FALLTHROUGH */
+	case USB_ST_SETUP:
+tr_setup:
+		/*
+		 * XXX TODO: what if sc_rx isn't empty, but data
+		 * is empty?  Then we leak mbufs.
+		 */
+		data = STAILQ_FIRST(&sc->sc_rx_inactive);
+		if (data == NULL) {
+			//KASSERT(m == NULL, ("mbuf isn't NULL"));
+			return;
+		}
+		STAILQ_REMOVE_HEAD(&sc->sc_rx_inactive, next);
+		STAILQ_INSERT_TAIL(&sc->sc_rx_active, data, next);
+		usbd_xfer_set_frame_data(xfer, 0, data->buf,
+		    usbd_xfer_max_len(xfer));
+		usbd_transfer_submit(xfer);
+		/*
+		 * To avoid LOR we should unlock our private mutex here to call
+		 * ieee80211_input() because here is at the end of a USB
+		 * callback and safe to unlock.
+		 */
+		OTUS_UNLOCK(sc);
+		while ((m = mbufq_dequeue(&scrx)) != NULL) {
+			wh = mtod(m, struct ieee80211_frame *);
+			ni = ieee80211_find_rxnode(ic,
+			    (struct ieee80211_frame_min *)wh);
+			if (ni != NULL) {
+				if (ni->ni_flags & IEEE80211_NODE_HT)
+					m->m_flags |= M_AMPDU;
+				(void)ieee80211_input_mimo(ni, m, NULL);
+				ieee80211_free_node(ni);
+			} else
+				(void)ieee80211_input_mimo_all(ic, m, NULL);
+		}
+#ifdef	IEEE80211_SUPPORT_SUPERG
+		ieee80211_ff_age_all(ic, 100);
+#endif
+		OTUS_LOCK(sc);
+		break;
+	default:
+		/* needs it to the inactive queue due to a error. */
+		data = STAILQ_FIRST(&sc->sc_rx_active);
+		if (data != NULL) {
+			STAILQ_REMOVE_HEAD(&sc->sc_rx_active, next);
+			STAILQ_INSERT_TAIL(&sc->sc_rx_inactive, data, next);
+		}
+		if (error != USB_ERR_CANCELLED) {
+			usbd_xfer_set_stall(xfer);
+			counter_u64_add(ic->ic_ierrors, 1);
+			goto tr_setup;
+		}
+		break;
+	}
+}
+
+static void
+otus_txeof(struct usb_xfer *xfer, struct otus_data *data)
+{
+	struct otus_softc *sc = usbd_xfer_softc(xfer);
+
+	OTUS_DPRINTF(sc, OTUS_DEBUG_TXDONE,
+	    "%s: called; data=%p\n", __func__, data);
+
+	OTUS_LOCK_ASSERT(sc);
+
+	if (sc->sc_tx_n_active == 0) {
+		device_printf(sc->sc_dev,
+		    "%s: completed but tx_active=0\n",
+		    __func__);
+	} else {
+		sc->sc_tx_n_active--;
+	}
+
+	if (data->m) {
+		/* XXX status? */
+		/* XXX we get TX status via the RX path.. */
+		ieee80211_tx_complete(data->ni, data->m, 0);
+		data->m = NULL;
+		data->ni = NULL;
+	}
+}
+
+static void
+otus_txcmdeof(struct usb_xfer *xfer, struct otus_tx_cmd *cmd)
+{
+	struct otus_softc *sc = usbd_xfer_softc(xfer);
+
+	OTUS_LOCK_ASSERT(sc);
+
+	OTUS_DPRINTF(sc, OTUS_DEBUG_CMDDONE,
+	    "%s: called; data=%p; odata=%p\n",
+	    __func__, cmd, cmd->odata);
+
+	/*
+	 * Non-response commands still need wakeup so the caller
+	 * knows it was submitted and completed OK; response commands should
+	 * wait until they're ACKed by the firmware with a response.
+	 */
+	if (cmd->odata) {
+		STAILQ_INSERT_TAIL(&sc->sc_cmd_waiting, cmd, next_cmd);
+	} else {
+		wakeup(cmd);
+		otus_free_txcmd(sc, cmd);
+	}
+}
+
+static void
+otus_bulk_tx_callback(struct usb_xfer *xfer, usb_error_t error)
+{
+	uint8_t which = OTUS_BULK_TX;
+	struct otus_softc *sc = usbd_xfer_softc(xfer);
+	struct ieee80211com *ic = &sc->sc_ic;
+	struct otus_data *data;
+
+	OTUS_LOCK_ASSERT(sc);
+
+	switch (USB_GET_STATE(xfer)) {
+	case USB_ST_TRANSFERRED:
+		data = STAILQ_FIRST(&sc->sc_tx_active[which]);
+		if (data == NULL)
+			goto tr_setup;
+		OTUS_DPRINTF(sc, OTUS_DEBUG_TXDONE,
+		    "%s: transfer done %p\n", __func__, data);
+		STAILQ_REMOVE_HEAD(&sc->sc_tx_active[which], next);
+		otus_txeof(xfer, data);
+		otus_freebuf(sc, data);
+		/* FALLTHROUGH */
+	case USB_ST_SETUP:
+tr_setup:
+		data = STAILQ_FIRST(&sc->sc_tx_pending[which]);
+		if (data == NULL) {
+			OTUS_DPRINTF(sc, OTUS_DEBUG_XMIT,
+			    "%s: empty pending queue sc %p\n", __func__, sc);
+			sc->sc_tx_n_active = 0;
+			goto finish;
+		}
+		STAILQ_REMOVE_HEAD(&sc->sc_tx_pending[which], next);
+		STAILQ_INSERT_TAIL(&sc->sc_tx_active[which], data, next);
+		usbd_xfer_set_frame_data(xfer, 0, data->buf, data->buflen);
+		OTUS_DPRINTF(sc, OTUS_DEBUG_XMIT,
+		    "%s: submitting transfer %p\n", __func__, data);
+		usbd_transfer_submit(xfer);
+		sc->sc_tx_n_active++;
+		break;
+	default:
+		data = STAILQ_FIRST(&sc->sc_tx_active[which]);
+		if (data != NULL) {
+			STAILQ_REMOVE_HEAD(&sc->sc_tx_active[which], next);
+			otus_txeof(xfer, data);
+			otus_freebuf(sc, data);
+		}
+		counter_u64_add(ic->ic_oerrors, 1);
+
+		if (error != USB_ERR_CANCELLED) {
+			usbd_xfer_set_stall(xfer);
+			goto tr_setup;
+		}
+		break;
+	}
+
+finish:
+#ifdef	IEEE80211_SUPPORT_SUPERG
+	/*
+	 * If the TX active queue drops below a certain
+	 * threshold, ensure we age fast-frames out so they're
+	 * transmitted.
+	 */
+	if (sc->sc_tx_n_active < 2) {
+		/* XXX ew - net80211 should defer this for us! */
+		OTUS_UNLOCK(sc);
+		ieee80211_ff_flush(ic, WME_AC_VO);
+		ieee80211_ff_flush(ic, WME_AC_VI);
+		ieee80211_ff_flush(ic, WME_AC_BE);
+		ieee80211_ff_flush(ic, WME_AC_BK);
+		OTUS_LOCK(sc);
+	}
+#endif
+	/* Kick TX */
+	otus_tx_start(sc);
+}
+
+static void
+otus_bulk_cmd_callback(struct usb_xfer *xfer, usb_error_t error)
+{
+	struct otus_softc *sc = usbd_xfer_softc(xfer);
+#if 0
+	struct ieee80211com *ic = &sc->sc_ic;
+#endif
+	struct otus_tx_cmd *cmd;
+
+	OTUS_LOCK_ASSERT(sc);
+
+	switch (USB_GET_STATE(xfer)) {
+	case USB_ST_TRANSFERRED:
+		cmd = STAILQ_FIRST(&sc->sc_cmd_active);
+		if (cmd == NULL)
+			goto tr_setup;
+		OTUS_DPRINTF(sc, OTUS_DEBUG_CMDDONE,
+		    "%s: transfer done %p\n", __func__, cmd);
+		STAILQ_REMOVE_HEAD(&sc->sc_cmd_active, next_cmd);
+		otus_txcmdeof(xfer, cmd);
+		/* FALLTHROUGH */
+	case USB_ST_SETUP:
+tr_setup:
+		cmd = STAILQ_FIRST(&sc->sc_cmd_pending);
+		if (cmd == NULL) {
+			OTUS_DPRINTF(sc, OTUS_DEBUG_CMD,
+			    "%s: empty pending queue sc %p\n", __func__, sc);
+			return;
+		}
+		STAILQ_REMOVE_HEAD(&sc->sc_cmd_pending, next_cmd);
+		STAILQ_INSERT_TAIL(&sc->sc_cmd_active, cmd, next_cmd);
+		usbd_xfer_set_frame_data(xfer, 0, cmd->buf, cmd->buflen);
+		OTUS_DPRINTF(sc, OTUS_DEBUG_CMD,
+		    "%s: submitting transfer %p; buf=%p, buflen=%d\n", __func__, cmd, cmd->buf, cmd->buflen);
+		usbd_transfer_submit(xfer);
+		break;
+	default:
+		cmd = STAILQ_FIRST(&sc->sc_cmd_active);
+		if (cmd != NULL) {
+			STAILQ_REMOVE_HEAD(&sc->sc_cmd_active, next_cmd);
+			otus_txcmdeof(xfer, cmd);
+		}
+
+		if (error != USB_ERR_CANCELLED) {
+			usbd_xfer_set_stall(xfer);
+			goto tr_setup;
+		}
+		break;
+	}
+}
+
+/*
+ * This isn't used by carl9170; it however may be used by the
+ * initial bootloader.
+ */
+static void
+otus_bulk_irq_callback(struct usb_xfer *xfer, usb_error_t error)
+{
+	struct otus_softc *sc = usbd_xfer_softc(xfer);
+	int actlen;
+	int sumlen;
+
+	usbd_xfer_status(xfer, &actlen, &sumlen, NULL, NULL);
+	OTUS_DPRINTF(sc, OTUS_DEBUG_IRQ,
+	    "%s: called; state=%d\n", __func__, USB_GET_STATE(xfer));
+
+	switch (USB_GET_STATE(xfer)) {
+	case USB_ST_TRANSFERRED:
+		/*
+		 * Read usb frame data, if any.
+		 * "actlen" has the total length for all frames
+		 * transferred.
+		 */
+		OTUS_DPRINTF(sc, OTUS_DEBUG_IRQ,
+		    "%s: comp; %d bytes\n",
+		    __func__,
+		    actlen);
+#if 0
+		pc = usbd_xfer_get_frame(xfer, 0);
+		otus_dump_usb_rx_page(sc, pc, actlen);
+#endif
+		/* XXX fallthrough */
+	case USB_ST_SETUP:
+		/*
+		 * Setup xfer frame lengths/count and data
+		 */
+		OTUS_DPRINTF(sc, OTUS_DEBUG_IRQ, "%s: setup\n", __func__);
+		usbd_xfer_set_frame_len(xfer, 0, usbd_xfer_max_len(xfer));
+		usbd_transfer_submit(xfer);
+		break;
+
+	default: /* Error */
+		/*
+		 * Print error message and clear stall
+		 * for example.
+		 */
+		OTUS_DPRINTF(sc, OTUS_DEBUG_IRQ, "%s: ERROR?\n", __func__);
+		break;
+	}
+}
+
+/*
+ * Map net80211 rate to hw rate for otus MAC/PHY.
+ */
+static uint8_t
+otus_rate_to_hw_rate(struct otus_softc *sc, uint8_t rate)
+{
+	int is_2ghz;
+
+	is_2ghz = !! (IEEE80211_IS_CHAN_2GHZ(sc->sc_ic.ic_curchan));
+
+	switch (rate) {
+	/* CCK */
+	case 2:
+		return (0x0);
+	case 4:
+		return (0x1);
+	case 11:
+		return (0x2);
+	case 22:
+		return (0x3);
+	/* OFDM */
+	case 12:
+		return (0xb);
+	case 18:
+		return (0xf);
+	case 24:
+		return (0xa);
+	case 36:
+		return (0xe);
+	case 48:
+		return (0x9);
+	case 72:
+		return (0xd);
+	case 96:
+		return (0x8);
+	case 108:
+		return (0xc);
+	default:
+		device_printf(sc->sc_dev, "%s: unknown rate '%d'\n",
+		    __func__, (int) rate);
+	case 0:
+		if (is_2ghz)
+			return (0x0);	/* 1MB CCK */
+		else
+			return (0xb);	/* 6MB OFDM */
+
+	/* XXX TODO: HT */
+	}
+}
+
+static int
+otus_hw_rate_is_ofdm(struct otus_softc *sc, uint8_t hw_rate)
+{
+
+	switch (hw_rate) {
+	case 0x0:
+	case 0x1:
+	case 0x2:
+	case 0x3:
+		return (0);
+	default:
+		return (1);
+	}
+}
+
+
+static void
+otus_tx_update_ratectl(struct otus_softc *sc, struct ieee80211_node *ni)
+{
+	int tx, tx_success, tx_retry;
+
+	tx = OTUS_NODE(ni)->tx_done;
+	tx_success = OTUS_NODE(ni)->tx_done - OTUS_NODE(ni)->tx_err;
+	tx_retry = OTUS_NODE(ni)->tx_retries;
+
+	ieee80211_ratectl_tx_update(ni->ni_vap, ni, &tx, &tx_success,
+	    &tx_retry);
+}
+
+/*
+ * XXX TODO: support tx bpf parameters for configuration!
+ */
+static int
+otus_tx(struct otus_softc *sc, struct ieee80211_node *ni, struct mbuf *m,
+    struct otus_data *data)
+{
+	struct ieee80211com *ic = &sc->sc_ic;
+	struct ieee80211vap *vap = ni->ni_vap;
+	struct ieee80211_frame *wh;
+	struct ieee80211_key *k;
+	struct ar_tx_head *head;
+	uint32_t phyctl;
+	uint16_t macctl, qos;
+	uint8_t qid, rate;
+	int hasqos, xferlen;
+
+	wh = mtod(m, struct ieee80211_frame *);
+	if (wh->i_fc[1] & IEEE80211_FC1_PROTECTED) {
+		k = ieee80211_crypto_encap(ni, m);
+		if (k == NULL) {
+			device_printf(sc->sc_dev,
+			    "%s: m=%p: ieee80211_crypto_encap returns NULL\n",
+			    __func__,
+			    m);
+			return (ENOBUFS);
+		}
+		wh = mtod(m, struct ieee80211_frame *);
+	}
+
+	/* Calculate transfer length; ensure data buffer is large enough */
+	xferlen = sizeof (*head) + m->m_pkthdr.len;
+	if (xferlen > OTUS_TXBUFSZ) {
+		device_printf(sc->sc_dev,
+		    "%s: 802.11 TX frame is %d bytes, max %d bytes\n",
+		    __func__,
+		    xferlen,
+		    OTUS_TXBUFSZ);
+		return (ENOBUFS);
+	}
+
+	hasqos = !! IEEE80211_QOS_HAS_SEQ(wh);
+
+	if (hasqos) {
+		uint8_t tid;
+		qos = ((const struct ieee80211_qosframe *)wh)->i_qos[0];
+		tid = qos & IEEE80211_QOS_TID;
+		qid = TID_TO_WME_AC(tid);
+	} else {
+		qos = 0;
+		qid = WME_AC_BE;
+	}
+
+	/* Pickup a rate index. */
+	if (IEEE80211_IS_MULTICAST(wh->i_addr1) ||
+	    (wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK) != IEEE80211_FC0_TYPE_DATA) {
+		/* Get lowest rate */
+		rate = otus_rate_to_hw_rate(sc, 0);
+	} else {
+		(void) ieee80211_ratectl_rate(ni, NULL, 0);
+		rate = otus_rate_to_hw_rate(sc, ni->ni_txrate);
+	}
+
+	phyctl = 0;
+	macctl = AR_TX_MAC_BACKOFF | AR_TX_MAC_HW_DUR | AR_TX_MAC_QID(qid);
+
+	if (IEEE80211_IS_MULTICAST(wh->i_addr1) ||
+	    (hasqos && ((qos & IEEE80211_QOS_ACKPOLICY) ==
+	     IEEE80211_QOS_ACKPOLICY_NOACK)))
+		macctl |= AR_TX_MAC_NOACK;
+
+	if (!IEEE80211_IS_MULTICAST(wh->i_addr1)) {
+		if (m->m_pkthdr.len + IEEE80211_CRC_LEN >= vap->iv_rtsthreshold)
+			macctl |= AR_TX_MAC_RTS;
+		else if (ic->ic_flags & IEEE80211_F_USEPROT) {
+			if (ic->ic_protmode == IEEE80211_PROT_CTSONLY)
+				macctl |= AR_TX_MAC_CTS;
+			else if (ic->ic_protmode == IEEE80211_PROT_RTSCTS)
+				macctl |= AR_TX_MAC_RTS;
+		}
+	}
+
+	phyctl |= AR_TX_PHY_MCS(rate);
+	if (otus_hw_rate_is_ofdm(sc, rate)) {
+		phyctl |= AR_TX_PHY_MT_OFDM;
+		/* Always use all tx antennas for now, just to be safe */
+		phyctl |= AR_TX_PHY_ANTMSK(sc->txmask);
+	} else {	/* CCK */
+		phyctl |= AR_TX_PHY_MT_CCK;
+		phyctl |= AR_TX_PHY_ANTMSK(sc->txmask);
+	}
+
+	/* Update net80211 with the current counters */
+	otus_tx_update_ratectl(sc, ni);
+
+	/* Update rate control stats for frames that are ACK'ed. */
+	if (!(macctl & AR_TX_MAC_NOACK))
+		OTUS_NODE(ni)->tx_done++;
+
+
+	/* Fill Tx descriptor. */
+	head = (struct ar_tx_head *)data->buf;
+	head->len = htole16(m->m_pkthdr.len + IEEE80211_CRC_LEN);
+	head->macctl = htole16(macctl);
+	head->phyctl = htole32(phyctl);
+
+	m_copydata(m, 0, m->m_pkthdr.len, (caddr_t)&head[1]);
+
+	data->buflen = xferlen;
+	data->ni = ni;
+	data->m = m;
+
+	OTUS_DPRINTF(sc, OTUS_DEBUG_XMIT,
+	    "%s: tx: m=%p; data=%p; len=%d mac=0x%04x phy=0x%08x rate=0x%02x, ni_txrate=%d\n",
+	    __func__, m, data, head->len, head->macctl, head->phyctl,
+	    (int) rate, (int) ni->ni_txrate);
+
+	/* Submit transfer */
+	STAILQ_INSERT_TAIL(&sc->sc_tx_pending[OTUS_BULK_TX], data, next);
+	usbd_transfer_start(sc->sc_xfer[OTUS_BULK_TX]);
+
+	return 0;
+}
+
+int
+otus_set_multi(struct otus_softc *sc)
+{
+	uint32_t lo, hi;
+	struct ieee80211com *ic = &sc->sc_ic;
+	int r;
+
+	if (ic->ic_allmulti > 0 || ic->ic_promisc > 0 ||
+	    ic->ic_opmode == IEEE80211_M_MONITOR) {
+		lo = 0xffffffff;
+		hi = 0xffffffff;
+	} else {
+		struct ieee80211vap *vap;
+		struct ifnet *ifp;
+		struct ifmultiaddr *ifma;
+
+		lo = hi = 0;
+		TAILQ_FOREACH(vap, &ic->ic_vaps, iv_next) {
+			ifp = vap->iv_ifp;
+			if_maddr_rlock(ifp);
+			TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
+				caddr_t dl;
+				uint32_t val;
+
+				dl = LLADDR((struct sockaddr_dl *) ifma->ifma_addr);
+				val = LE_READ_4(dl + 4);
+				/* Get address byte 5 */
+				val = val & 0x0000ff00;
+				val = val >> 8;
+
+				/* As per below, shift it >> 2 to get only 6 bits */
+				val = val >> 2;
+				if (val < 32)
+					lo |= 1 << val;
+				else
+					hi |= 1 << (val - 32);
+			}
+			if_maddr_runlock(ifp);
+		}
+	}
+#if 0
+	/* XXX openbsd code */
+	while (enm != NULL) {
+		bit = enm->enm_addrlo[5] >> 2;
+		if (bit < 32)
+			lo |= 1 << bit;
+		else
+			hi |= 1 << (bit - 32);
+		ETHER_NEXT_MULTI(step, enm);
+	}
+#endif
+
+	hi |= 1U << 31;	/* Make sure the broadcast bit is set. */
+
+	OTUS_LOCK(sc);
+	otus_write(sc, AR_MAC_REG_GROUP_HASH_TBL_L, lo);
+	otus_write(sc, AR_MAC_REG_GROUP_HASH_TBL_H, hi);
+	r = otus_write_barrier(sc);
+	OTUS_UNLOCK(sc);
+	return (r);
+}
+
+static void
+otus_updateedca(struct otus_softc *sc)
+{
+#define EXP2(val)	((1 << (val)) - 1)
+#define AIFS(val)	((val) * 9 + 10)
+	struct ieee80211com *ic = &sc->sc_ic;
+	const struct wmeParams *edca;
+
+	OTUS_LOCK_ASSERT(sc);
+
+	edca = ic->ic_wme.wme_chanParams.cap_wmeParams;
+
+	/* Set CWmin/CWmax values. */
+	otus_write(sc, AR_MAC_REG_AC0_CW,
+	    EXP2(edca[WME_AC_BE].wmep_logcwmax) << 16 |
+	    EXP2(edca[WME_AC_BE].wmep_logcwmin));
+	otus_write(sc, AR_MAC_REG_AC1_CW,
+	    EXP2(edca[WME_AC_BK].wmep_logcwmax) << 16 |
+	    EXP2(edca[WME_AC_BK].wmep_logcwmin));
+	otus_write(sc, AR_MAC_REG_AC2_CW,
+	    EXP2(edca[WME_AC_VI].wmep_logcwmax) << 16 |
+	    EXP2(edca[WME_AC_VI].wmep_logcwmin));
+	otus_write(sc, AR_MAC_REG_AC3_CW,
+	    EXP2(edca[WME_AC_VO].wmep_logcwmax) << 16 |
+	    EXP2(edca[WME_AC_VO].wmep_logcwmin));
+	otus_write(sc, AR_MAC_REG_AC4_CW,		/* Special TXQ. */
+	    EXP2(edca[WME_AC_VO].wmep_logcwmax) << 16 |
+	    EXP2(edca[WME_AC_VO].wmep_logcwmin));
+
+	/* Set AIFSN values. */
+	otus_write(sc, AR_MAC_REG_AC1_AC0_AIFS,
+	    AIFS(edca[WME_AC_VI].wmep_aifsn) << 24 |
+	    AIFS(edca[WME_AC_BK].wmep_aifsn) << 12 |
+	    AIFS(edca[WME_AC_BE].wmep_aifsn));
+	otus_write(sc, AR_MAC_REG_AC3_AC2_AIFS,
+	    AIFS(edca[WME_AC_VO].wmep_aifsn) << 16 |	/* Special TXQ. */
+	    AIFS(edca[WME_AC_VO].wmep_aifsn) <<  4 |
+	    AIFS(edca[WME_AC_VI].wmep_aifsn) >>  8);
+
+	/* Set TXOP limit. */
+	otus_write(sc, AR_MAC_REG_AC1_AC0_TXOP,
+	    edca[WME_AC_BK].wmep_txopLimit << 16 |
+	    edca[WME_AC_BE].wmep_txopLimit);
+	otus_write(sc, AR_MAC_REG_AC3_AC2_TXOP,
+	    edca[WME_AC_VO].wmep_txopLimit << 16 |
+	    edca[WME_AC_VI].wmep_txopLimit);
+
+	/* XXX ACK policy? */
+
+	(void)otus_write_barrier(sc);
+
+#undef AIFS
+#undef EXP2
+}
+
+static void
+otus_updateslot(struct otus_softc *sc)
+{
+	struct ieee80211com *ic = &sc->sc_ic;
+	uint32_t slottime;
+
+	OTUS_LOCK_ASSERT(sc);
+
+	slottime = (ic->ic_flags & IEEE80211_F_SHSLOT) ? 9 : 20;
+	otus_write(sc, AR_MAC_REG_SLOT_TIME, slottime << 10);
+	(void)otus_write_barrier(sc);
+}
+
+int
+otus_init_mac(struct otus_softc *sc)
+{
+	int error;
+
+	OTUS_LOCK_ASSERT(sc);
+
+	otus_write(sc, AR_MAC_REG_ACK_EXTENSION, 0x40);
+	otus_write(sc, AR_MAC_REG_RETRY_MAX, 0);
+	otus_write(sc, AR_MAC_REG_SNIFFER, 0x2000000);
+	otus_write(sc, AR_MAC_REG_RX_THRESHOLD, 0xc1f80);
+	otus_write(sc, AR_MAC_REG_RX_PE_DELAY, 0x70);
+	otus_write(sc, AR_MAC_REG_EIFS_AND_SIFS, 0xa144000);
+	otus_write(sc, AR_MAC_REG_SLOT_TIME, 9 << 10);
+	otus_write(sc, 0x1c3b2c, 0x19000000);
+	/* NAV protects ACK only (in TXOP). */
+	otus_write(sc, 0x1c3b38, 0x201);
+	/* Set beacon Tx power to 0x7. */
+	otus_write(sc, AR_MAC_REG_BCN_HT1, 0x8000170);
+	otus_write(sc, AR_MAC_REG_BACKOFF_PROTECT, 0x105);
+	otus_write(sc, 0x1c3b9c, 0x10000a);
+	/* Filter any control frames, BAR is bit 24. */
+	otus_write(sc, 0x1c368c, 0x0500ffff);
+	otus_write(sc, 0x1c3c40, 0x1);
+	otus_write(sc, AR_MAC_REG_BASIC_RATE, 0x150f);
+	otus_write(sc, AR_MAC_REG_MANDATORY_RATE, 0x150f);
+	otus_write(sc, AR_MAC_REG_RTS_CTS_RATE, 0x10b01bb);
+	otus_write(sc, 0x1c3694, 0x4003c1e);
+	/* Enable LED0 and LED1. */
+	otus_write(sc, 0x1d0100, 0x3);
+	otus_write(sc, 0x1d0104, 0x3);
+	/* Switch MAC to OTUS interface. */
+	otus_write(sc, 0x1c3600, 0x3);
+	otus_write(sc, 0x1c3c50, 0xffff);
+	otus_write(sc, 0x1c3680, 0xf00008);
+	/* Disable Rx timeout (workaround). */
+	otus_write(sc, 0x1c362c, 0);
+
+	/* Set USB Rx stream mode maximum frame number to 2. */
+	otus_write(sc, 0x1e1110, 0x4);
+	/* Set USB Rx stream mode timeout to 10us. */
+	otus_write(sc, 0x1e1114, 0x80);
+
+	/* Set clock frequency to 88/80MHz. */
+	otus_write(sc, 0x1d4008, 0x73);
+	/* Set WLAN DMA interrupt mode: generate intr per packet. */
+	otus_write(sc, 0x1c3d7c, 0x110011);
+	otus_write(sc, 0x1c3bb0, 0x4);
+	otus_write(sc, AR_MAC_REG_TXOP_NOT_ENOUGH_INDICATION, 0x141e0f48);
+
+	/* Disable HW decryption for now. */
+	otus_write(sc, 0x1c3678, 0x78);
+
+	if ((error = otus_write_barrier(sc)) != 0)
+		return error;
+
+	/* Set default EDCA parameters. */
+	otus_updateedca(sc);
+
+	return 0;
+}
+
+/*
+ * Return default value for PHY register based on current operating mode.
+ */
+uint32_t
+otus_phy_get_def(struct otus_softc *sc, uint32_t reg)
+{
+	int i;
+
+	for (i = 0; i < nitems(ar5416_phy_regs); i++)
+		if (AR_PHY(ar5416_phy_regs[i]) == reg)
+			return sc->phy_vals[i];
+	return 0;	/* Register not found. */
+}
+
+/*
+ * Update PHY's programming based on vendor-specific data stored in EEPROM.
+ * This is for FEM-type devices only.
+ */
+int
+otus_set_board_values(struct otus_softc *sc, struct ieee80211_channel *c)
+{
+	const struct ModalEepHeader *eep;
+	uint32_t tmp, offset;
+
+	if (IEEE80211_IS_CHAN_5GHZ(c))
+		eep = &sc->eeprom.modalHeader[0];
+	else
+		eep = &sc->eeprom.modalHeader[1];
+
+	/* Offset of chain 2. */
+	offset = 2 * 0x1000;
+
+	tmp = le32toh(eep->antCtrlCommon);
+	otus_write(sc, AR_PHY_SWITCH_COM, tmp);
+
+	tmp = le32toh(eep->antCtrlChain[0]);
+	otus_write(sc, AR_PHY_SWITCH_CHAIN_0, tmp);
+
+	tmp = le32toh(eep->antCtrlChain[1]);
+	otus_write(sc, AR_PHY_SWITCH_CHAIN_0 + offset, tmp);
+
+	if (1 /* sc->sc_sco == AR_SCO_SCN */) {
+		tmp = otus_phy_get_def(sc, AR_PHY_SETTLING);
+		tmp &= ~(0x7f << 7);
+		tmp |= (eep->switchSettling & 0x7f) << 7;
+		otus_write(sc, AR_PHY_SETTLING, tmp);
+	}
+
+	tmp = otus_phy_get_def(sc, AR_PHY_DESIRED_SZ);
+	tmp &= ~0xffff;
+	tmp |= eep->pgaDesiredSize << 8 | eep->adcDesiredSize;
+	otus_write(sc, AR_PHY_DESIRED_SZ, tmp);
+
+	tmp = eep->txEndToXpaOff << 24 | eep->txEndToXpaOff << 16 |
+	      eep->txFrameToXpaOn << 8 | eep->txFrameToXpaOn;
+	otus_write(sc, AR_PHY_RF_CTL4, tmp);
+
+	tmp = otus_phy_get_def(sc, AR_PHY_RF_CTL3);
+	tmp &= ~(0xff << 16);
+	tmp |= eep->txEndToRxOn << 16;
+	otus_write(sc, AR_PHY_RF_CTL3, tmp);
+
+	tmp = otus_phy_get_def(sc, AR_PHY_CCA);
+	tmp &= ~(0x7f << 12);
+	tmp |= (eep->thresh62 & 0x7f) << 12;
+	otus_write(sc, AR_PHY_CCA, tmp);
+
+	tmp = otus_phy_get_def(sc, AR_PHY_RXGAIN);
+	tmp &= ~(0x3f << 12);
+	tmp |= (eep->txRxAttenCh[0] & 0x3f) << 12;
+	otus_write(sc, AR_PHY_RXGAIN, tmp);
+
+	tmp = otus_phy_get_def(sc, AR_PHY_RXGAIN + offset);
+	tmp &= ~(0x3f << 12);
+	tmp |= (eep->txRxAttenCh[1] & 0x3f) << 12;
+	otus_write(sc, AR_PHY_RXGAIN + offset, tmp);
+
+	tmp = otus_phy_get_def(sc, AR_PHY_GAIN_2GHZ);
+	tmp &= ~(0x3f << 18);
+	tmp |= (eep->rxTxMarginCh[0] & 0x3f) << 18;
+	if (IEEE80211_IS_CHAN_5GHZ(c)) {
+		tmp &= ~(0xf << 10);
+		tmp |= (eep->bswMargin[0] & 0xf) << 10;
+	}
+	otus_write(sc, AR_PHY_GAIN_2GHZ, tmp);
+
+	tmp = otus_phy_get_def(sc, AR_PHY_GAIN_2GHZ + offset);
+	tmp &= ~(0x3f << 18);
+	tmp |= (eep->rxTxMarginCh[1] & 0x3f) << 18;
+	otus_write(sc, AR_PHY_GAIN_2GHZ + offset, tmp);
+
+	tmp = otus_phy_get_def(sc, AR_PHY_TIMING_CTRL4);
+	tmp &= ~(0x3f << 5 | 0x1f);
+	tmp |= (eep->iqCalICh[0] & 0x3f) << 5 | (eep->iqCalQCh[0] & 0x1f);
+	otus_write(sc, AR_PHY_TIMING_CTRL4, tmp);
+
+	tmp = otus_phy_get_def(sc, AR_PHY_TIMING_CTRL4 + offset);
+	tmp &= ~(0x3f << 5 | 0x1f);
+	tmp |= (eep->iqCalICh[1] & 0x3f) << 5 | (eep->iqCalQCh[1] & 0x1f);
+	otus_write(sc, AR_PHY_TIMING_CTRL4 + offset, tmp);
+
+	tmp = otus_phy_get_def(sc, AR_PHY_TPCRG1);
+	tmp &= ~(0xf << 16);
+	tmp |= (eep->xpd & 0xf) << 16;
+	otus_write(sc, AR_PHY_TPCRG1, tmp);
+
+	return otus_write_barrier(sc);
+}
+
+int
+otus_program_phy(struct otus_softc *sc, struct ieee80211_channel *c)
+{
+	const uint32_t *vals;
+	int error, i;
+
+	/* Select PHY programming based on band and bandwidth. */
+	if (IEEE80211_IS_CHAN_2GHZ(c))
+		vals = ar5416_phy_vals_2ghz_20mhz;
+	else
+		vals = ar5416_phy_vals_5ghz_20mhz;
+	for (i = 0; i < nitems(ar5416_phy_regs); i++)
+		otus_write(sc, AR_PHY(ar5416_phy_regs[i]), vals[i]);
+	sc->phy_vals = vals;
+
+	if (sc->eeprom.baseEepHeader.deviceType == 0x80)	/* FEM */
+		if ((error = otus_set_board_values(sc, c)) != 0)
+			return error;
+
+	/* Initial Tx power settings. */
+	otus_write(sc, AR_PHY_POWER_TX_RATE_MAX, 0x7f);
+	otus_write(sc, AR_PHY_POWER_TX_RATE1, 0x3f3f3f3f);
+	otus_write(sc, AR_PHY_POWER_TX_RATE2, 0x3f3f3f3f);
+	otus_write(sc, AR_PHY_POWER_TX_RATE3, 0x3f3f3f3f);
+	otus_write(sc, AR_PHY_POWER_TX_RATE4, 0x3f3f3f3f);
+	otus_write(sc, AR_PHY_POWER_TX_RATE5, 0x3f3f3f3f);
+	otus_write(sc, AR_PHY_POWER_TX_RATE6, 0x3f3f3f3f);
+	otus_write(sc, AR_PHY_POWER_TX_RATE7, 0x3f3f3f3f);
+	otus_write(sc, AR_PHY_POWER_TX_RATE8, 0x3f3f3f3f);
+	otus_write(sc, AR_PHY_POWER_TX_RATE9, 0x3f3f3f3f);
+
+	if (IEEE80211_IS_CHAN_2GHZ(c))
+		otus_write(sc, 0x1d4014, 0x5163);
+	else
+		otus_write(sc, 0x1d4014, 0x5143);
+
+	return otus_write_barrier(sc);
+}
+
+static __inline uint8_t
+otus_reverse_bits(uint8_t v)
+{
+	v = ((v >> 1) & 0x55) | ((v & 0x55) << 1);
+	v = ((v >> 2) & 0x33) | ((v & 0x33) << 2);
+	v = ((v >> 4) & 0x0f) | ((v & 0x0f) << 4);
+	return v;
+}
+
+int
+otus_set_rf_bank4(struct otus_softc *sc, struct ieee80211_channel *c)
+{
+	uint8_t chansel, d0, d1;
+	uint16_t data;
+	int error;
+
+	OTUS_LOCK_ASSERT(sc);
+
+	d0 = 0;
+	if (IEEE80211_IS_CHAN_5GHZ(c)) {
+		chansel = (c->ic_freq - 4800) / 5;
+		if (chansel & 1)
+			d0 |= AR_BANK4_AMODE_REFSEL(2);
+		else
+			d0 |= AR_BANK4_AMODE_REFSEL(1);
+	} else {
+		d0 |= AR_BANK4_AMODE_REFSEL(2);
+		if (c->ic_freq == 2484) {	/* CH 14 */
+			d0 |= AR_BANK4_BMODE_LF_SYNTH_FREQ;
+			chansel = 10 + (c->ic_freq - 2274) / 5;
+		} else
+			chansel = 16 + (c->ic_freq - 2272) / 5;
+		chansel <<= 2;
+	}
+	d0 |= AR_BANK4_ADDR(1) | AR_BANK4_CHUP;
+	d1 = otus_reverse_bits(chansel);
+
+	/* Write bits 0-4 of d0 and d1. */
+	data = (d1 & 0x1f) << 5 | (d0 & 0x1f);
+	otus_write(sc, AR_PHY(44), data);
+	/* Write bits 5-7 of d0 and d1. */
+	data = (d1 >> 5) << 5 | (d0 >> 5);
+	otus_write(sc, AR_PHY(58), data);
+
+	if ((error = otus_write_barrier(sc)) == 0)
+		otus_delay_ms(sc, 10);
+	return error;
+}
+
+void
+otus_get_delta_slope(uint32_t coeff, uint32_t *exponent, uint32_t *mantissa)
+{
+#define COEFF_SCALE_SHIFT	24
+	uint32_t exp, man;
+
+	/* exponent = 14 - floor(log2(coeff)) */
+	for (exp = 31; exp > 0; exp--)
+		if (coeff & (1 << exp))
+			break;
+	KASSERT(exp != 0, ("exp"));
+	exp = 14 - (exp - COEFF_SCALE_SHIFT);
+
+	/* mantissa = floor(coeff * 2^exponent + 0.5) */
+	man = coeff + (1 << (COEFF_SCALE_SHIFT - exp - 1));
+
+	*mantissa = man >> (COEFF_SCALE_SHIFT - exp);
+	*exponent = exp - 16;
+#undef COEFF_SCALE_SHIFT
+}
+
+static int
+otus_set_chan(struct otus_softc *sc, struct ieee80211_channel *c, int assoc)
+{
+	struct ieee80211com *ic = &sc->sc_ic;
+	struct ar_cmd_frequency cmd;
+	struct ar_rsp_frequency rsp;
+	const uint32_t *vals;
+	uint32_t coeff, exp, man, tmp;
+	uint8_t code;
+	int error, chan, i;
+
+	error = 0;
+	chan = ieee80211_chan2ieee(ic, c);
+
+	OTUS_DPRINTF(sc, OTUS_DEBUG_RESET,
+	    "setting channel %d (%dMHz)\n", chan, c->ic_freq);
+
+	tmp = IEEE80211_IS_CHAN_2GHZ(c) ? 0x105 : 0x104;
+	otus_write(sc, AR_MAC_REG_DYNAMIC_SIFS_ACK, tmp);
+	if ((error = otus_write_barrier(sc)) != 0)
+		goto finish;
+
+	/* Disable BB Heavy Clip. */
+	otus_write(sc, AR_PHY_HEAVY_CLIP_ENABLE, 0x200);
+	if ((error = otus_write_barrier(sc)) != 0)
+		goto finish;
+
+	/* XXX Is that FREQ_START ? */
+	error = otus_cmd(sc, AR_CMD_FREQ_STRAT, NULL, 0, NULL, 0);
+	if (error != 0)
+		goto finish;
+
+	/* Reprogram PHY and RF on channel band or bandwidth changes. */
+	if (sc->bb_reset || c->ic_flags != sc->sc_curchan->ic_flags) {
+		OTUS_DPRINTF(sc, OTUS_DEBUG_RESET, "band switch\n");
+
+		/* Cold/Warm reset BB/ADDA. */
+		otus_write(sc, 0x1d4004, sc->bb_reset ? 0x800 : 0x400);
+		if ((error = otus_write_barrier(sc)) != 0)
+			goto finish;
+		otus_write(sc, 0x1d4004, 0);
+		if ((error = otus_write_barrier(sc)) != 0)
+			goto finish;
+		sc->bb_reset = 0;
+
+		if ((error = otus_program_phy(sc, c)) != 0) {
+			device_printf(sc->sc_dev,
+			    "%s: could not program PHY\n",
+			    __func__);
+			goto finish;
+		}
+
+		/* Select RF programming based on band. */
+		if (IEEE80211_IS_CHAN_5GHZ(c))
+			vals = ar5416_banks_vals_5ghz;
+		else
+			vals = ar5416_banks_vals_2ghz;
+		for (i = 0; i < nitems(ar5416_banks_regs); i++)
+			otus_write(sc, AR_PHY(ar5416_banks_regs[i]), vals[i]);
+		if ((error = otus_write_barrier(sc)) != 0) {
+			device_printf(sc->sc_dev,
+			    "%s: could not program RF\n",
+			    __func__);
+			goto finish;
+		}
+		code = AR_CMD_RF_INIT;
+	} else {
+		code = AR_CMD_FREQUENCY;
+	}
+
+	if ((error = otus_set_rf_bank4(sc, c)) != 0)
+		goto finish;
+
+	tmp = (sc->txmask == 0x5) ? 0x340 : 0x240;
+	otus_write(sc, AR_PHY_TURBO, tmp);
+	if ((error = otus_write_barrier(sc)) != 0)
+		goto finish;
+
+	/* Send firmware command to set channel. */
+	cmd.freq = htole32((uint32_t)c->ic_freq * 1000);
+	cmd.dynht2040 = htole32(0);
+	cmd.htena = htole32(1);
+	/* Set Delta Slope (exponent and mantissa). */
+	coeff = (100 << 24) / c->ic_freq;
+	otus_get_delta_slope(coeff, &exp, &man);
+	cmd.dsc_exp = htole32(exp);
+	cmd.dsc_man = htole32(man);
+	OTUS_DPRINTF(sc, OTUS_DEBUG_RESET,
+	    "ds coeff=%u exp=%u man=%u\n", coeff, exp, man);
+	/* For Short GI, coeff is 9/10 that of normal coeff. */
+	coeff = (9 * coeff) / 10;
+	otus_get_delta_slope(coeff, &exp, &man);
+	cmd.dsc_shgi_exp = htole32(exp);
+	cmd.dsc_shgi_man = htole32(man);
+	OTUS_DPRINTF(sc, OTUS_DEBUG_RESET,
+	    "ds shgi coeff=%u exp=%u man=%u\n", coeff, exp, man);
+	/* Set wait time for AGC and noise calibration (100 or 200ms). */
+	cmd.check_loop_count = assoc ? htole32(2000) : htole32(1000);
+	OTUS_DPRINTF(sc, OTUS_DEBUG_RESET,
+	    "%s\n", (code == AR_CMD_RF_INIT) ? "RF_INIT" : "FREQUENCY");
+	error = otus_cmd(sc, code, &cmd, sizeof cmd, &rsp, sizeof(rsp));
+	if (error != 0)
+		goto finish;
+	if ((rsp.status & htole32(AR_CAL_ERR_AGC | AR_CAL_ERR_NF_VAL)) != 0) {
+		OTUS_DPRINTF(sc, OTUS_DEBUG_RESET,
+		    "status=0x%x\n", le32toh(rsp.status));
+		/* Force cold reset on next channel. */
+		sc->bb_reset = 1;
+	}
+#ifdef USB_DEBUG
+	if (otus_debug & OTUS_DEBUG_RESET) {
+		device_printf(sc->sc_dev, "calibration status=0x%x\n",
+		    le32toh(rsp.status));
+		for (i = 0; i < 2; i++) {	/* 2 Rx chains */
+			/* Sign-extend 9-bit NF values. */
+			device_printf(sc->sc_dev,
+			    "noisefloor chain %d=%d\n", i,
+			    (((int32_t)le32toh(rsp.nf[i])) << 4) >> 23);
+			device_printf(sc->sc_dev,
+			    "noisefloor ext chain %d=%d\n", i,
+			    ((int32_t)le32toh(rsp.nf_ext[i])) >> 23);
+		}
+	}
+#endif
+	for (i = 0; i < OTUS_NUM_CHAINS; i++) {
+		sc->sc_nf[i] = ((((int32_t)le32toh(rsp.nf[i])) << 4) >> 23);
+	}
+	sc->sc_curchan = c;
+finish:
+	return (error);
+}
+
+#ifdef notyet
+int
+otus_set_key(struct ieee80211com *ic, struct ieee80211_node *ni,
+    struct ieee80211_key *k)
+{
+	struct otus_softc *sc = ic->ic_softc;
+	struct otus_cmd_key cmd;
+
+	/* Defer setting of WEP keys until interface is brought up. */
+	if ((ic->ic_if.if_flags & (IFF_UP | IFF_RUNNING)) !=
+	    (IFF_UP | IFF_RUNNING))
+		return 0;
+
+	/* Do it in a process context. */
+	cmd.key = *k;
+	cmd.associd = (ni != NULL) ? ni->ni_associd : 0;
+	otus_do_async(sc, otus_set_key_cb, &cmd, sizeof cmd);
+	return 0;
+}
+
+void
+otus_set_key_cb(struct otus_softc *sc, void *arg)
+{
+	struct otus_cmd_key *cmd = arg;
+	struct ieee80211_key *k = &cmd->key;
+	struct ar_cmd_ekey key;
+	uint16_t cipher;
+	int error;
+
+	memset(&key, 0, sizeof key);
+	if (k->k_flags & IEEE80211_KEY_GROUP) {
+		key.uid = htole16(k->k_id);
+		IEEE80211_ADDR_COPY(key.macaddr, sc->sc_ic.ic_myaddr);
+		key.macaddr[0] |= 0x80;
+	} else {
+		key.uid = htole16(OTUS_UID(cmd->associd));
+		IEEE80211_ADDR_COPY(key.macaddr, ni->ni_macaddr);
+	}
+	key.kix = htole16(0);
+	/* Map net80211 cipher to hardware. */
+	switch (k->k_cipher) {
+	case IEEE80211_CIPHER_WEP40:
+		cipher = AR_CIPHER_WEP64;
+		break;
+	case IEEE80211_CIPHER_WEP104:
+		cipher = AR_CIPHER_WEP128;
+		break;
+	case IEEE80211_CIPHER_TKIP:
+		cipher = AR_CIPHER_TKIP;
+		break;
+	case IEEE80211_CIPHER_CCMP:
+		cipher = AR_CIPHER_AES;
+		break;
+	default:
+		return;
+	}
+	key.cipher = htole16(cipher);
+	memcpy(key.key, k->k_key, MIN(k->k_len, 16));
+	error = otus_cmd(sc, AR_CMD_EKEY, &key, sizeof key, NULL, 0);
+	if (error != 0 || k->k_cipher != IEEE80211_CIPHER_TKIP)
+		return;
+
+	/* TKIP: set Tx/Rx MIC Key. */
+	key.kix = htole16(1);
+	memcpy(key.key, k->k_key + 16, 16);
+	(void)otus_cmd(sc, AR_CMD_EKEY, &key, sizeof key, NULL, 0);
+}
+
+void
+otus_delete_key(struct ieee80211com *ic, struct ieee80211_node *ni,
+    struct ieee80211_key *k)
+{
+	struct otus_softc *sc = ic->ic_softc;
+	struct otus_cmd_key cmd;
+
+	if (!(ic->ic_if.if_flags & IFF_RUNNING) ||
+	    ic->ic_state != IEEE80211_S_RUN)
+		return;	/* Nothing to do. */
+
+	/* Do it in a process context. */
+	cmd.key = *k;
+	cmd.associd = (ni != NULL) ? ni->ni_associd : 0;
+	otus_do_async(sc, otus_delete_key_cb, &cmd, sizeof cmd);
+}
+
+void
+otus_delete_key_cb(struct otus_softc *sc, void *arg)
+{
+	struct otus_cmd_key *cmd = arg;
+	struct ieee80211_key *k = &cmd->key;
+	uint32_t uid;
+
+	if (k->k_flags & IEEE80211_KEY_GROUP)
+		uid = htole32(k->k_id);
+	else
+		uid = htole32(OTUS_UID(cmd->associd));
+	(void)otus_cmd(sc, AR_CMD_DKEY, &uid, sizeof uid, NULL, 0);
+}
+#endif
+
+/*
+ * XXX TODO: check if we have to be doing any calibration in the host
+ * or whether it's purely a firmware thing.
+ */
+void
+otus_calibrate_to(void *arg, int pending)
+{
+#if 0
+	struct otus_softc *sc = arg;
+
+	device_printf(sc->sc_dev, "%s: called\n", __func__);
+	struct ieee80211com *ic = &sc->sc_ic;
+	struct ieee80211_node *ni;
+	int s;
+
+	if (usbd_is_dying(sc->sc_udev))
+		return;
+
+	usbd_ref_incr(sc->sc_udev);
+
+	s = splnet();
+	ni = ic->ic_bss;
+	ieee80211_amrr_choose(&sc->amrr, ni, &((struct otus_node *)ni)->amn);
+	splx(s);
+
+	if (!usbd_is_dying(sc->sc_udev))
+		timeout_add_sec(&sc->calib_to, 1);
+
+	usbd_ref_decr(sc->sc_udev);
+#endif
+}
+
+int
+otus_set_bssid(struct otus_softc *sc, const uint8_t *bssid)
+{
+
+	OTUS_LOCK_ASSERT(sc);
+
+	otus_write(sc, AR_MAC_REG_BSSID_L,
+	    bssid[0] | bssid[1] << 8 | bssid[2] << 16 | bssid[3] << 24);
+	otus_write(sc, AR_MAC_REG_BSSID_H,
+	    bssid[4] | bssid[5] << 8);
+	return otus_write_barrier(sc);
+}
+
+int
+otus_set_macaddr(struct otus_softc *sc, const uint8_t *addr)
+{
+	OTUS_LOCK_ASSERT(sc);
+
+	otus_write(sc, AR_MAC_REG_MAC_ADDR_L,
+	    addr[0] | addr[1] << 8 | addr[2] << 16 | addr[3] << 24);
+	otus_write(sc, AR_MAC_REG_MAC_ADDR_H,
+	    addr[4] | addr[5] << 8);
+	return otus_write_barrier(sc);
+}
+
+/* Default single-LED. */
+void
+otus_led_newstate_type1(struct otus_softc *sc)
+{
+	/* TBD */
+	device_printf(sc->sc_dev, "%s: TODO\n", __func__);
+}
+
+/* NETGEAR, dual-LED. */
+void
+otus_led_newstate_type2(struct otus_softc *sc)
+{
+	/* TBD */
+	device_printf(sc->sc_dev, "%s: TODO\n", __func__);
+}
+
+/* NETGEAR, single-LED/3 colors (blue, red, purple.) */
+void
+otus_led_newstate_type3(struct otus_softc *sc)
+{
+#if 0
+	struct ieee80211com *ic = &sc->sc_ic;
+	struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
+
+	uint32_t state = sc->led_state;
+
+	OTUS_LOCK_ASSERT(sc);
+
+	if (!vap) {
+		state = 0;	/* led off */
+	} else if (vap->iv_state == IEEE80211_S_INIT) {
+		state = 0;	/* LED off. */
+	} else if (vap->iv_state == IEEE80211_S_RUN) {
+		/* Associated, LED always on. */
+		if (IEEE80211_IS_CHAN_2GHZ(sc->sc_curchan))
+			state = AR_LED0_ON;	/* 2GHz=>Red. */
+		else
+			state = AR_LED1_ON;	/* 5GHz=>Blue. */
+	} else {
+		/* Scanning, blink LED. */
+		state ^= AR_LED0_ON | AR_LED1_ON;
+		if (IEEE80211_IS_CHAN_2GHZ(sc->sc_curchan))
+			state &= ~AR_LED1_ON;
+		else
+			state &= ~AR_LED0_ON;
+	}
+	if (state != sc->led_state) {
+		otus_write(sc, 0x1d0104, state);
+		if (otus_write_barrier(sc) == 0)
+			sc->led_state = state;
+	}
+#endif
+}
+
+int
+otus_init(struct otus_softc *sc)
+{
+	struct ieee80211com *ic = &sc->sc_ic;
+	int error;
+
+	OTUS_UNLOCK_ASSERT(sc);
+
+	OTUS_LOCK(sc);
+
+	/* Drain any pending TX frames */
+	otus_drain_mbufq(sc);
+
+	/* Init MAC */
+	if ((error = otus_init_mac(sc)) != 0) {
+		OTUS_UNLOCK(sc);
+		device_printf(sc->sc_dev,
+		    "%s: could not initialize MAC\n", __func__);
+		return error;
+	}
+
+	(void) otus_set_macaddr(sc, ic->ic_macaddr);
+
+#if 0
+	switch (ic->ic_opmode) {
+#ifdef notyet
+#ifndef IEEE80211_STA_ONLY
+	case IEEE80211_M_HOSTAP:
+		otus_write(sc, 0x1c3700, 0x0f0000a1);
+		otus_write(sc, 0x1c3c40, 0x1);
+		break;
+	case IEEE80211_M_IBSS:
+		otus_write(sc, 0x1c3700, 0x0f000000);
+		otus_write(sc, 0x1c3c40, 0x1);
+		break;
+#endif
+#endif
+	case IEEE80211_M_STA:
+		otus_write(sc, 0x1c3700, 0x0f000002);
+		otus_write(sc, 0x1c3c40, 0x1);
+		break;
+	default:
+		break;
+	}
+#endif
+
+	/* Expect STA operation */
+	otus_write(sc, 0x1c3700, 0x0f000002);
+	otus_write(sc, 0x1c3c40, 0x1);
+
+	/* XXX ic_opmode? */
+	otus_write(sc, AR_MAC_REG_SNIFFER,
+	    (ic->ic_opmode == IEEE80211_M_MONITOR) ? 0x2000001 : 0x2000000);
+	(void)otus_write_barrier(sc);
+
+	sc->bb_reset = 1;	/* Force cold reset. */
+
+	if ((error = otus_set_chan(sc, ic->ic_curchan, 0)) != 0) {
+		OTUS_UNLOCK(sc);
+		device_printf(sc->sc_dev,
+		    "%s: could not set channel\n", __func__);
+		return error;
+	}
+
+	/* Start Rx. */
+	otus_write(sc, 0x1c3d30, 0x100);
+	(void)otus_write_barrier(sc);
+
+	sc->sc_running = 1;
+
+	OTUS_UNLOCK(sc);
+	return 0;
+}
+
+void
+otus_stop(struct otus_softc *sc)
+{
+#if 0
+	int s;
+#endif
+
+	OTUS_UNLOCK_ASSERT(sc);
+
+	OTUS_LOCK(sc);
+	sc->sc_running = 0;
+	sc->sc_tx_timer = 0;
+	OTUS_UNLOCK(sc);
+
+	taskqueue_drain_timeout(taskqueue_thread, &sc->scan_to);
+	taskqueue_drain_timeout(taskqueue_thread, &sc->calib_to);
+	taskqueue_drain(taskqueue_thread, &sc->tx_task);
+	taskqueue_drain(taskqueue_thread, &sc->wme_update_task);
+
+	OTUS_LOCK(sc);
+	sc->sc_running = 0;
+	/* Stop Rx. */
+	otus_write(sc, 0x1c3d30, 0);
+	(void)otus_write_barrier(sc);
+
+	/* Drain any pending TX frames */
+	otus_drain_mbufq(sc);
+
+	OTUS_UNLOCK(sc);
+}
diff --git a/sys/dev/otus/if_otusreg.h b/sys/dev/otus/if_otusreg.h
new file mode 100644
index 0000000000000..4f11b8dfd772c
--- /dev/null
+++ b/sys/dev/otus/if_otusreg.h
@@ -0,0 +1,1030 @@
+/*	$OpenBSD: if_otusreg.h,v 1.9 2013/11/26 20:33:18 deraadt Exp $	*/
+
+/*-
+ * Copyright (c) 2009 Damien Bergamini <damien.bergamini@free.fr>
+ * Copyright (c) 2007-2008 Atheros Communications, Inc.
+ * Copyright (c) 2015 Adrian Chadd <adrian@FreeBSD.org>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ *
+ * $FreeBSD$
+ */
+#ifndef	__IF_OTUSREG_H__
+#define	__IF_OTUSREG_H__
+
+/* USB Endpoints addresses. */
+#define AR_EPT_BULK_TX_NO	(UE_DIR_OUT | 1)
+#define AR_EPT_BULK_RX_NO	(UE_DIR_IN  | 2)
+#define AR_EPT_INTR_RX_NO	(UE_DIR_IN  | 3)
+#define AR_EPT_INTR_TX_NO	(UE_DIR_OUT | 4)
+
+/* USB Requests. */
+#define AR_FW_DOWNLOAD			0x30
+#define AR_FW_DOWNLOAD_COMPLETE		0x31
+
+/* Maximum number of writes that can fit in a single FW command is 7. */
+#define AR_MAX_WRITE_IDX	6	/* 56 bytes */
+
+#define AR_FW_INIT_ADDR			0x102800
+#define AR_FW_MAIN_ADDR			0x200000
+#define AR_USB_MODE_CTRL		0x1e1108
+
+/*
+ * AR9170 MAC registers.
+ */
+#define AR_MAC_REG_BASE			0x1c3000
+#define AR_MAC_REG_MAC_ADDR_L		(AR_MAC_REG_BASE + 0x610)
+#define AR_MAC_REG_MAC_ADDR_H		(AR_MAC_REG_BASE + 0x614)
+#define AR_MAC_REG_BSSID_L		(AR_MAC_REG_BASE + 0x618)
+#define AR_MAC_REG_BSSID_H		(AR_MAC_REG_BASE + 0x61c)
+#define AR_MAC_REG_GROUP_HASH_TBL_L	(AR_MAC_REG_BASE + 0x624)
+#define AR_MAC_REG_GROUP_HASH_TBL_H	(AR_MAC_REG_BASE + 0x628)
+#define AR_MAC_REG_BASIC_RATE		(AR_MAC_REG_BASE + 0x630)
+#define AR_MAC_REG_MANDATORY_RATE	(AR_MAC_REG_BASE + 0x634)
+#define AR_MAC_REG_RTS_CTS_RATE		(AR_MAC_REG_BASE + 0x638)
+#define AR_MAC_REG_BACKOFF_PROTECT	(AR_MAC_REG_BASE + 0x63c)
+#define AR_MAC_REG_RX_THRESHOLD		(AR_MAC_REG_BASE + 0x640)
+#define AR_MAC_REG_RX_PE_DELAY		(AR_MAC_REG_BASE + 0x64c)
+#define AR_MAC_REG_DYNAMIC_SIFS_ACK	(AR_MAC_REG_BASE + 0x658)
+#define AR_MAC_REG_SNIFFER		(AR_MAC_REG_BASE + 0x674)
+#define AR_MAC_REG_ACK_EXTENSION	(AR_MAC_REG_BASE + 0x690)
+#define AR_MAC_REG_EIFS_AND_SIFS	(AR_MAC_REG_BASE + 0x698)
+#define AR_MAC_REG_BUSY			(AR_MAC_REG_BASE + 0x6e8)
+#define AR_MAC_REG_BUSY_EXT		(AR_MAC_REG_BASE + 0x6ec)
+#define AR_MAC_REG_SLOT_TIME		(AR_MAC_REG_BASE + 0x6f0)
+#define AR_MAC_REG_AC0_CW		(AR_MAC_REG_BASE + 0xb00)
+#define AR_MAC_REG_AC1_CW		(AR_MAC_REG_BASE + 0xb04)
+#define AR_MAC_REG_AC2_CW		(AR_MAC_REG_BASE + 0xb08)
+#define AR_MAC_REG_AC3_CW		(AR_MAC_REG_BASE + 0xb0c)
+#define AR_MAC_REG_AC4_CW		(AR_MAC_REG_BASE + 0xb10)
+#define AR_MAC_REG_AC1_AC0_AIFS		(AR_MAC_REG_BASE + 0xb14)
+#define AR_MAC_REG_AC3_AC2_AIFS		(AR_MAC_REG_BASE + 0xb18)
+#define AR_MAC_REG_RETRY_MAX		(AR_MAC_REG_BASE + 0xb28)
+#define AR_MAC_REG_TXOP_NOT_ENOUGH_INDICATION	\
+					(AR_MAC_REG_BASE + 0xb30)
+#define AR_MAC_REG_AC1_AC0_TXOP		(AR_MAC_REG_BASE + 0xb44)
+#define AR_MAC_REG_AC3_AC2_TXOP		(AR_MAC_REG_BASE + 0xb48)
+#define AR_MAC_REG_OFDM_PHY_ERRORS	(AR_MAC_REG_BASE + 0xcb4)
+#define AR_MAC_REG_CCK_PHY_ERRORS	(AR_MAC_REG_BASE + 0xcb8)
+#define AR_MAC_REG_BCN_HT1		(AR_MAC_REG_BASE + 0xda0)
+
+/* Possible values for register AR_USB_MODE_CTRL. */
+#define AR_USB_DS_ENA		(1 << 0)
+#define AR_USB_US_ENA		(1 << 1)
+#define AR_USB_US_PACKET_MODE	(1 << 3)
+#define AR_USB_RX_STREAM_4K	(0 << 4)
+#define AR_USB_RX_STREAM_8K	(1 << 4)
+#define AR_USB_RX_STREAM_16K	(2 << 4)
+#define AR_USB_RX_STREAM_32K	(3 << 4)
+#define AR_USB_TX_STREAM_MODE	(1 << 6)
+
+#define AR_LED0_ON	(1 << 0)
+#define AR_LED1_ON	(1 << 1)
+
+/*
+ * PHY registers.
+ */
+#define AR_PHY_BASE			0x1c5800
+#define AR_PHY(reg)			(AR_PHY_BASE + (reg) * 4)
+#define AR_PHY_TURBO			(AR_PHY_BASE + 0x0004)
+#define AR_PHY_RF_CTL3			(AR_PHY_BASE + 0x0028)
+#define AR_PHY_RF_CTL4			(AR_PHY_BASE + 0x0034)
+#define AR_PHY_SETTLING			(AR_PHY_BASE + 0x0044)
+#define AR_PHY_RXGAIN			(AR_PHY_BASE + 0x0048)
+#define AR_PHY_DESIRED_SZ		(AR_PHY_BASE + 0x0050)
+#define AR_PHY_FIND_SIG			(AR_PHY_BASE + 0x0058)
+#define AR_PHY_AGC_CTL1			(AR_PHY_BASE + 0x005c)
+#define AR_PHY_SFCORR			(AR_PHY_BASE + 0x0068)
+#define AR_PHY_SFCORR_LOW		(AR_PHY_BASE + 0x006c)
+#define AR_PHY_TIMING_CTRL4		(AR_PHY_BASE + 0x0120)
+#define AR_PHY_TIMING5			(AR_PHY_BASE + 0x0124)
+#define AR_PHY_POWER_TX_RATE1		(AR_PHY_BASE + 0x0134)
+#define AR_PHY_POWER_TX_RATE2		(AR_PHY_BASE + 0x0138)
+#define AR_PHY_POWER_TX_RATE_MAX	(AR_PHY_BASE + 0x013c)
+#define AR_PHY_SWITCH_CHAIN_0		(AR_PHY_BASE + 0x0160)
+#define AR_PHY_SWITCH_COM		(AR_PHY_BASE + 0x0164)
+#define AR_PHY_HEAVY_CLIP_ENABLE	(AR_PHY_BASE + 0x01e0)
+#define AR_PHY_CCK_DETECT		(AR_PHY_BASE + 0x0a08)
+#define AR_PHY_GAIN_2GHZ		(AR_PHY_BASE + 0x0a0c)
+#define AR_PHY_POWER_TX_RATE3		(AR_PHY_BASE + 0x0a34)
+#define AR_PHY_POWER_TX_RATE4		(AR_PHY_BASE + 0x0a38)
+#define AR_PHY_TPCRG1			(AR_PHY_BASE + 0x0a58)
+#define AR_PHY_POWER_TX_RATE5		(AR_PHY_BASE + 0x0b8c)
+#define AR_PHY_POWER_TX_RATE6		(AR_PHY_BASE + 0x0b90)
+#define AR_PHY_POWER_TX_RATE7		(AR_PHY_BASE + 0x0bcc)
+#define AR_PHY_POWER_TX_RATE8		(AR_PHY_BASE + 0x0bd0)
+#define AR_PHY_POWER_TX_RATE9		(AR_PHY_BASE + 0x0bd4)
+#define AR_PHY_CCA			(AR_PHY_BASE + 0x3064)
+
+#define AR_SEEPROM_HW_TYPE_OFFSET	0x1374
+#define AR_EEPROM_OFFSET		0x1600
+
+#define AR_BANK4_CHUP			(1 << 0)
+#define AR_BANK4_BMODE_LF_SYNTH_FREQ	(1 << 1)
+#define AR_BANK4_AMODE_REFSEL(x)	((x) << 2)
+#define AR_BANK4_ADDR(x)		((x) << 5)
+
+/* Tx descriptor. */
+struct ar_tx_head {
+	uint16_t	len;
+	uint16_t	macctl;
+#define AR_TX_MAC_RTS		(1 <<  0)
+#define AR_TX_MAC_CTS		(1 <<  1)
+#define AR_TX_MAC_BACKOFF	(1 <<  3)
+#define AR_TX_MAC_NOACK		(1 <<  2)
+#define AR_TX_MAC_HW_DUR	(1 <<  9)
+#define AR_TX_MAC_QID(qid)	((qid) << 10)
+#define AR_TX_MAC_RATE_PROBING	(1 << 15)
+
+	uint32_t	phyctl;
+/* Modulation type. */
+#define AR_TX_PHY_MT_CCK	0
+#define AR_TX_PHY_MT_OFDM	1
+#define AR_TX_PHY_MT_HT		2
+#define AR_TX_PHY_GF		(1 << 2)
+#define AR_TX_PHY_BW_SHIFT	3
+#define AR_TX_PHY_TPC_SHIFT	9
+#define AR_TX_PHY_ANTMSK(msk)	((msk) << 15)
+#define AR_TX_PHY_MCS(mcs)	((mcs) << 18)
+#define AR_TX_PHY_SHGI		(1U << 31)
+} __packed;
+
+/* USB Rx stream mode header. */
+struct ar_rx_head {
+	uint16_t	len;
+	uint16_t	tag;
+#define AR_RX_HEAD_TAG	0x4e00
+} __packed;
+
+/* Rx descriptor. */
+struct ar_rx_tail {
+	uint8_t	rssi_ant[3];
+	uint8_t	rssi_ant_ext[3];
+	uint8_t	rssi;		/* Combined RSSI. */
+	uint8_t	evm[2][6];	/* Error Vector Magnitude. */
+	uint8_t	phy_err;
+	uint8_t	sa_idx;
+	uint8_t	da_idx;
+	uint8_t	error;
+#define AR_RX_ERROR_TIMEOUT	(1 << 0)
+#define AR_RX_ERROR_OVERRUN	(1 << 1)
+#define AR_RX_ERROR_DECRYPT	(1 << 2)
+#define AR_RX_ERROR_FCS		(1 << 3)
+#define AR_RX_ERROR_BAD_RA	(1 << 4)
+#define AR_RX_ERROR_PLCP	(1 << 5)
+#define AR_RX_ERROR_MMIC	(1 << 6)
+
+	uint8_t	status;
+/* Modulation type (same as AR_TX_PHY_MT). */
+#define AR_RX_STATUS_MT_MASK	0x3
+#define AR_RX_STATUS_MT_CCK	0
+#define AR_RX_STATUS_MT_OFDM	1
+#define AR_RX_STATUS_MT_HT	2
+#define AR_RX_STATUS_SHPREAMBLE	(1 << 3)
+} __packed;
+
+#define AR_PLCP_HDR_LEN	12
+/* Magic PLCP header for firmware notifications through Rx bulk pipe. */
+static uint8_t AR_PLCP_HDR_INTR[] = {
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+	0xff, 0xff, 0xff, 0xff, 0xff, 0xff
+};
+
+/* Firmware command/reply header. */
+struct ar_cmd_hdr {
+	uint8_t		len;
+	uint8_t		code;
+#define AR_CMD_RREG		0x00
+#define AR_CMD_WREG		0x01
+#define AR_CMD_RMEM		0x02
+#define AR_CMD_WMEM		0x03
+#define AR_CMD_BITAND		0x04
+#define AR_CMD_BITOR		0x05
+#define AR_CMD_EKEY		0x28
+#define AR_CMD_DKEY		0x29
+#define AR_CMD_FREQUENCY	0x30
+#define AR_CMD_RF_INIT		0x31
+#define AR_CMD_SYNTH		0x32
+#define AR_CMD_FREQ_STRAT	0x33
+#define AR_CMD_ECHO		0x80
+#define AR_CMD_TALLY		0x81
+#define AR_CMD_TALLY_APD	0x82
+#define AR_CMD_CONFIG		0x83
+#define AR_CMD_RESET		0x90
+#define AR_CMD_DKRESET		0x91
+#define AR_CMD_DKTX_STATUS	0x92
+#define AR_CMD_FDC		0xa0
+#define AR_CMD_WREEPROM		0xb0
+#define AR_CMD_WFLASH		AR_CMD_WREEPROM
+#define AR_CMD_FLASH_ERASE	0xb1
+#define AR_CMD_FLASH_PROG	0xb2
+#define AR_CMD_FLASH_CHKSUM	0xb3
+#define AR_CMD_FLASH_READ	0xb4
+#define AR_CMD_FW_DL_INIT	0xb5
+#define AR_CMD_MEM_WREEPROM	0xbb
+/* Those have the 2 MSB set to 1. */
+#define AR_EVT_BEACON		0x00
+#define AR_EVT_TX_COMP		0x01
+#define AR_EVT_TBTT		0x02
+#define AR_EVT_ATIM		0x03
+#define AR_EVT_DO_BB_RESET	0x09
+
+	uint16_t	token;	/* Driver private data. */
+} __packed;
+
+/* Structure for command AR_CMD_RF_INIT/AR_CMD_FREQUENCY. */
+struct ar_cmd_frequency {
+	uint32_t	freq;
+	uint32_t	dynht2040;
+	uint32_t	htena;
+	uint32_t	dsc_exp;
+	uint32_t	dsc_man;
+	uint32_t	dsc_shgi_exp;
+	uint32_t	dsc_shgi_man;
+	uint32_t	check_loop_count;
+} __packed;
+
+/* Firmware reply for command AR_CMD_FREQUENCY. */
+struct ar_rsp_frequency {
+	uint32_t	status;
+#define AR_CAL_ERR_AGC		(1 << 0)	/* AGC cal unfinished. */
+#define AR_CAL_ERR_NF		(1 << 1)	/* Noise cal unfinished. */
+#define AR_CAL_ERR_NF_VAL	(1 << 2)	/* NF value unexpected. */
+
+	uint32_t	nf[3];		/* Noisefloor. */
+	uint32_t	nf_ext[3];	/* Noisefloor ext. */
+} __packed;
+
+/* Structure for command AR_CMD_EKEY. */
+struct ar_cmd_ekey {
+	uint16_t	uid;	/* user ID */
+	uint16_t	kix;
+	uint16_t	cipher;
+#define AR_CIPHER_NONE		0
+#define AR_CIPHER_WEP64		1
+#define AR_CIPHER_TKIP		2
+#define AR_CIPHER_AES		4
+#define AR_CIPHER_WEP128	5
+#define AR_CIPHER_WEP256	6
+#define AR_CIPHER_CENC		7
+
+	uint8_t		macaddr[IEEE80211_ADDR_LEN];
+	uint8_t		key[16];
+} __packed;
+
+/* Structure for event AR_EVT_TX_COMP. */
+struct ar_evt_tx_comp {
+	uint8_t		macaddr[IEEE80211_ADDR_LEN];
+	uint32_t	phy;
+	uint16_t	status;
+#define AR_TX_STATUS_COMP	0
+#define AR_TX_STATUS_RETRY_COMP	1
+#define AR_TX_STATUS_FAILED	2
+} __packed;
+
+/* List of supported channels. */
+static const uint8_t ar_chans[] = {
+	1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14,
+	36, 40, 44, 48, 52, 56, 60, 64, 100, 104, 108, 112, 116, 120, 124,
+	128, 132, 136, 140, 149, 153, 157, 161, 165, 34, 38, 42, 46
+};
+
+/*
+ * This data is automatically generated from the "otus.ini" file.
+ * It is stored in a different way though, to reduce kernel's .rodata
+ * section overhead (5.1KB instead of 8.5KB).
+ */
+
+/* NB: apply AR_PHY(). */
+static const uint16_t ar5416_phy_regs[] = {
+	0x000, 0x001, 0x002, 0x003, 0x004, 0x005, 0x006, 0x007, 0x008,
+	0x009, 0x00a, 0x00b, 0x00c, 0x00d, 0x00e, 0x00f, 0x010, 0x011,
+	0x012, 0x013, 0x014, 0x015, 0x016, 0x017, 0x018, 0x01a, 0x01b,
+	0x040, 0x041, 0x042, 0x043, 0x045, 0x046, 0x047, 0x048, 0x049,
+	0x04a, 0x04b, 0x04d, 0x04e, 0x04f, 0x051, 0x052, 0x053, 0x055,
+	0x056, 0x058, 0x059, 0x05c, 0x05d, 0x05e, 0x05f, 0x060, 0x061,
+	0x062, 0x063, 0x064, 0x065, 0x066, 0x067, 0x068, 0x069, 0x06a,
+	0x06b, 0x06c, 0x06d, 0x070, 0x071, 0x072, 0x073, 0x074, 0x075,
+	0x076, 0x077, 0x078, 0x079, 0x07a, 0x07b, 0x07c, 0x07f, 0x080,
+	0x081, 0x082, 0x083, 0x084, 0x085, 0x086, 0x087, 0x088, 0x089,
+	0x08a, 0x08b, 0x08c, 0x08d, 0x08e, 0x08f, 0x090, 0x091, 0x092,
+	0x093, 0x094, 0x095, 0x096, 0x097, 0x098, 0x099, 0x09a, 0x09b,
+	0x09c, 0x09d, 0x09e, 0x09f, 0x0a0, 0x0a1, 0x0a2, 0x0a3, 0x0a4,
+	0x0a5, 0x0a6, 0x0a7, 0x0a8, 0x0a9, 0x0aa, 0x0ab, 0x0ac, 0x0ad,
+	0x0ae, 0x0af, 0x0b0, 0x0b1, 0x0b2, 0x0b3, 0x0b4, 0x0b5, 0x0b6,
+	0x0b7, 0x0b8, 0x0b9, 0x0ba, 0x0bb, 0x0bc, 0x0bd, 0x0be, 0x0bf,
+	0x0c0, 0x0c1, 0x0c2, 0x0c3, 0x0c4, 0x0c5, 0x0c6, 0x0c7, 0x0c8,
+	0x0c9, 0x0ca, 0x0cb, 0x0cc, 0x0cd, 0x0ce, 0x0cf, 0x0d0, 0x0d1,
+	0x0d2, 0x0d3, 0x0d4, 0x0d5, 0x0d6, 0x0d7, 0x0d8, 0x0d9, 0x0da,
+	0x0db, 0x0dc, 0x0dd, 0x0de, 0x0df, 0x0e0, 0x0e1, 0x0e2, 0x0e3,
+	0x0e4, 0x0e5, 0x0e6, 0x0e7, 0x0e8, 0x0e9, 0x0ea, 0x0eb, 0x0ec,
+	0x0ed, 0x0ee, 0x0ef, 0x0f0, 0x0f1, 0x0f2, 0x0f3, 0x0f4, 0x0f5,
+	0x0f6, 0x0f7, 0x0f8, 0x0f9, 0x0fa, 0x0fb, 0x0fc, 0x0fd, 0x0fe,
+	0x0ff, 0x100, 0x103, 0x104, 0x105, 0x106, 0x107, 0x108, 0x109,
+	0x10a, 0x10b, 0x10c, 0x10d, 0x10e, 0x10f, 0x13c, 0x13d, 0x13e,
+	0x13f, 0x280, 0x281, 0x282, 0x283, 0x284, 0x285, 0x286, 0x287,
+	0x288, 0x289, 0x28a, 0x28b, 0x28c, 0x28d, 0x28e, 0x28f, 0x290,
+	0x291, 0x292, 0x293, 0x294, 0x295, 0x296, 0x297, 0x298, 0x299,
+	0x29a, 0x29b, 0x29d, 0x29e, 0x29f, 0x2c0, 0x2c1, 0x2c2, 0x2c3,
+	0x2c4, 0x2c5, 0x2c6, 0x2c7, 0x2c8, 0x2c9, 0x2ca, 0x2cb, 0x2cc,
+	0x2cd, 0x2ce, 0x2cf, 0x2d0, 0x2d1, 0x2d2, 0x2d3, 0x2d4, 0x2d5,
+	0x2d6, 0x2e2, 0x2e3, 0x2e4, 0x2e5, 0x2e6, 0x2e7, 0x2e8, 0x2e9,
+	0x2ea, 0x2eb, 0x2ec, 0x2ed, 0x2ee, 0x2ef, 0x2f0, 0x2f1, 0x2f2,
+	0x2f3, 0x2f4, 0x2f5, 0x2f6, 0x2f7, 0x2f8, 0x412, 0x448, 0x458,
+	0x683, 0x69b, 0x812, 0x848, 0x858, 0xa83, 0xa9b, 0xc19, 0xc57,
+	0xc5a, 0xc6f, 0xe9c, 0xed7, 0xed8, 0xed9, 0xeda, 0xedb, 0xedc,
+	0xedd, 0xede, 0xedf, 0xee0, 0xee1
+};
+
+static const uint32_t ar5416_phy_vals_5ghz_20mhz[] = {
+	0x00000007, 0x00000300, 0x00000000, 0xad848e19, 0x7d14e000,
+	0x9c0a9f6b, 0x00000090, 0x00000000, 0x02020200, 0x00000e0e,
+	0x0a020001, 0x0000a000, 0x00000000, 0x00000e0e, 0x00000007,
+	0x00200400, 0x206a002e, 0x1372161e, 0x001a6a65, 0x1284233c,
+	0x6c48b4e4, 0x00000859, 0x7ec80d2e, 0x31395c5e, 0x0004dd10,
+	0x409a4190, 0x050cb081, 0x00000000, 0x00000000, 0x00000000,
+	0x00000000, 0x000007d0, 0x00000118, 0x10000fff, 0x0510081c,
+	0xd0058a15, 0x00000001, 0x00000004, 0x3f3f3f3f, 0x3f3f3f3f,
+	0x0000007f, 0xdfb81020, 0x9280b212, 0x00020028, 0x5d50e188,
+	0x00081fff, 0x00009b40, 0x00001120, 0x190fb515, 0x00000000,
+	0x00000001, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+	0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+	0x00000000, 0x00000007, 0x001fff00, 0x006f00c4, 0x03051000,
+	0x00000820, 0x038919be, 0x06336f77, 0x60f6532c, 0x08f186c8,
+	0x00046384, 0x00000000, 0x00000000, 0x00000000, 0x00000200,
+	0x64646464, 0x3c787878, 0x000000aa, 0x00000000, 0x00001042,
+	0x00000000, 0x00000040, 0x00000080, 0x000001a1, 0x000001e1,
+	0x00000021, 0x00000061, 0x00000168, 0x000001a8, 0x000001e8,
+	0x00000028, 0x00000068, 0x00000189, 0x000001c9, 0x00000009,
+	0x00000049, 0x00000089, 0x00000170, 0x000001b0, 0x000001f0,
+	0x00000030, 0x00000070, 0x00000191, 0x000001d1, 0x00000011,
+	0x00000051, 0x00000091, 0x000001b8, 0x000001f8, 0x00000038,
+	0x00000078, 0x00000199, 0x000001d9, 0x00000019, 0x00000059,
+	0x00000099, 0x000000d9, 0x000000f9, 0x000000f9, 0x000000f9,
+	0x000000f9, 0x000000f9, 0x000000f9, 0x000000f9, 0x000000f9,
+	0x000000f9, 0x000000f9, 0x000000f9, 0x000000f9, 0x000000f9,
+	0x000000f9, 0x000000f9, 0x000000f9, 0x000000f9, 0x000000f9,
+	0x000000f9, 0x000000f9, 0x000000f9, 0x000000f9, 0x000000f9,
+	0x000000f9, 0x000000f9, 0x000000f9, 0x000000f9, 0x00000000,
+	0x00000001, 0x00000002, 0x00000003, 0x00000004, 0x00000005,
+	0x00000008, 0x00000009, 0x0000000a, 0x0000000b, 0x0000000c,
+	0x0000000d, 0x00000010, 0x00000011, 0x00000012, 0x00000013,
+	0x00000014, 0x00000015, 0x00000018, 0x00000019, 0x0000001a,
+	0x0000001b, 0x0000001c, 0x0000001d, 0x00000020, 0x00000021,
+	0x00000022, 0x00000023, 0x00000024, 0x00000025, 0x00000028,
+	0x00000029, 0x0000002a, 0x0000002b, 0x0000002c, 0x0000002d,
+	0x00000030, 0x00000031, 0x00000032, 0x00000033, 0x00000034,
+	0x00000035, 0x00000035, 0x00000035, 0x00000035, 0x00000035,
+	0x00000035, 0x00000035, 0x00000035, 0x00000035, 0x00000035,
+	0x00000035, 0x00000035, 0x00000035, 0x00000035, 0x00000035,
+	0x00000035, 0x00000035, 0x00000035, 0x00000035, 0x00000035,
+	0x00000035, 0x00000010, 0x0000001a, 0x00000000, 0x00000000,
+	0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+	0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+	0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+	0x00000000, 0x00000008, 0x00000440, 0xd6be4788, 0x012e8160,
+	0x40806333, 0x00106c10, 0x009c4060, 0x1883800a, 0x018830c6,
+	0x00000400, 0x000009b5, 0x00000000, 0x00000108, 0x3f3f3f3f,
+	0x3f3f3f3f, 0x13c889af, 0x38490a20, 0x00007bb6, 0x0fff3ffc,
+	0x00000001, 0x0000a000, 0x00000000, 0x0cc75380, 0x0f0f0f01,
+	0xdfa91f01, 0x00418a11, 0x00000000, 0x09249126, 0x0a1a9caa,
+	0x1ce739ce, 0x051701ce, 0x18010000, 0x30032602, 0x48073e06,
+	0x560b4c0a, 0x641a600f, 0x7a4f6e1b, 0x8c5b7e5a, 0x9d0f96cf,
+	0xb51fa69f, 0xcb3fbd07, 0x0000d7bf, 0x00000000, 0x00000000,
+	0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+	0x3fffffff, 0x3fffffff, 0x3fffffff, 0x0003ffff, 0x79a8aa1f,
+	0x08000000, 0x3f3f3f3f, 0x3f3f3f3f, 0x1ce739ce, 0x000001ce,
+	0x00000007, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+	0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+	0x00000000, 0x00000000, 0x3f3f3f3f, 0x3f3f3f3f, 0x3f3f3f3f,
+	0x00000000, 0x1ce739ce, 0x000000c0, 0x00180a65, 0x0510001c,
+	0x00009b40, 0x012e8160, 0x09249126, 0x00180a65, 0x0510001c,
+	0x00009b40, 0x012e8160, 0x09249126, 0x0001c600, 0x004b6a8e,
+	0x000003ce, 0x00181400, 0x00820820, 0x066c420f, 0x0f282207,
+	0x17601685, 0x1f801104, 0x37a00c03, 0x3fc40883, 0x57c00803,
+	0x5fd80682, 0x7fe00482, 0x7f3c7bba, 0xf3307ff0
+};
+
+#ifdef notyet
+static const uint32_t ar5416_phy_vals_5ghz_40mhz[] = {
+	0x00000007, 0x000003c4, 0x00000000, 0xad848e19, 0x7d14e000,
+	0x9c0a9f6b, 0x00000090, 0x00000000, 0x02020200, 0x00000e0e,
+	0x0a020001, 0x0000a000, 0x00000000, 0x00000e0e, 0x00000007,
+	0x00200400, 0x206a002e, 0x13721c1e, 0x001a6a65, 0x1284233c,
+	0x6c48b4e4, 0x00000859, 0x7ec80d2e, 0x31395c5e, 0x0004dd10,
+	0x409a4190, 0x050cb081, 0x00000000, 0x00000000, 0x00000000,
+	0x00000000, 0x000007d0, 0x00000230, 0x10000fff, 0x0510081c,
+	0xd0058a15, 0x00000001, 0x00000004, 0x3f3f3f3f, 0x3f3f3f3f,
+	0x0000007f, 0xdfb81020, 0x9280b212, 0x00020028, 0x5d50e188,
+	0x00081fff, 0x00009b40, 0x00001120, 0x190fb515, 0x00000000,
+	0x00000001, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+	0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+	0x00000000, 0x00000007, 0x001fff00, 0x006f00c4, 0x03051000,
+	0x00000820, 0x038919be, 0x06336f77, 0x60f6532c, 0x08f186c8,
+	0x00046384, 0x00000000, 0x00000000, 0x00000000, 0x00000200,
+	0x64646464, 0x3c787878, 0x000000aa, 0x00000000, 0x00001042,
+	0x00000000, 0x00000040, 0x00000080, 0x000001a1, 0x000001e1,
+	0x00000021, 0x00000061, 0x00000168, 0x000001a8, 0x000001e8,
+	0x00000028, 0x00000068, 0x00000189, 0x000001c9, 0x00000009,
+	0x00000049, 0x00000089, 0x00000170, 0x000001b0, 0x000001f0,
+	0x00000030, 0x00000070, 0x00000191, 0x000001d1, 0x00000011,
+	0x00000051, 0x00000091, 0x000001b8, 0x000001f8, 0x00000038,
+	0x00000078, 0x00000199, 0x000001d9, 0x00000019, 0x00000059,
+	0x00000099, 0x000000d9, 0x000000f9, 0x000000f9, 0x000000f9,
+	0x000000f9, 0x000000f9, 0x000000f9, 0x000000f9, 0x000000f9,
+	0x000000f9, 0x000000f9, 0x000000f9, 0x000000f9, 0x000000f9,
+	0x000000f9, 0x000000f9, 0x000000f9, 0x000000f9, 0x000000f9,
+	0x000000f9, 0x000000f9, 0x000000f9, 0x000000f9, 0x000000f9,
+	0x000000f9, 0x000000f9, 0x000000f9, 0x000000f9, 0x00000000,
+	0x00000001, 0x00000002, 0x00000003, 0x00000004, 0x00000005,
+	0x00000008, 0x00000009, 0x0000000a, 0x0000000b, 0x0000000c,
+	0x0000000d, 0x00000010, 0x00000011, 0x00000012, 0x00000013,
+	0x00000014, 0x00000015, 0x00000018, 0x00000019, 0x0000001a,
+	0x0000001b, 0x0000001c, 0x0000001d, 0x00000020, 0x00000021,
+	0x00000022, 0x00000023, 0x00000024, 0x00000025, 0x00000028,
+	0x00000029, 0x0000002a, 0x0000002b, 0x0000002c, 0x0000002d,
+	0x00000030, 0x00000031, 0x00000032, 0x00000033, 0x00000034,
+	0x00000035, 0x00000035, 0x00000035, 0x00000035, 0x00000035,
+	0x00000035, 0x00000035, 0x00000035, 0x00000035, 0x00000035,
+	0x00000035, 0x00000035, 0x00000035, 0x00000035, 0x00000035,
+	0x00000035, 0x00000035, 0x00000035, 0x00000035, 0x00000035,
+	0x00000035, 0x00000010, 0x0000001a, 0x00000000, 0x00000000,
+	0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+	0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+	0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+	0x00000000, 0x00000008, 0x00000440, 0xd6be4788, 0x012e8160,
+	0x40806333, 0x00106c10, 0x009c4060, 0x1883800a, 0x018830c6,
+	0x00000400, 0x000009b5, 0x00000000, 0x00000210, 0x3f3f3f3f,
+	0x3f3f3f3f, 0x13c889af, 0x38490a20, 0x00007bb6, 0x0fff3ffc,
+	0x00000001, 0x0000a000, 0x00000000, 0x0cc75380, 0x0f0f0f01,
+	0xdfa91f01, 0x00418a11, 0x00000000, 0x09249126, 0x0a1a9caa,
+	0x1ce739ce, 0x051701ce, 0x18010000, 0x30032602, 0x48073e06,
+	0x560b4c0a, 0x641a600f, 0x7a4f6e1b, 0x8c5b7e5a, 0x9d0f96cf,
+	0xb51fa69f, 0xcb3fbcbf, 0x0000d7bf, 0x00000000, 0x00000000,
+	0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+	0x3fffffff, 0x3fffffff, 0x3fffffff, 0x0003ffff, 0x79a8aa1f,
+	0x08000000, 0x3f3f3f3f, 0x3f3f3f3f, 0x1ce739ce, 0x000001ce,
+	0x00000007, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+	0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+	0x00000000, 0x00000000, 0x3f3f3f3f, 0x3f3f3f3f, 0x3f3f3f3f,
+	0x00000000, 0x1ce739ce, 0x000000c0, 0x00180a65, 0x0510001c,
+	0x00009b40, 0x012e8160, 0x09249126, 0x00180a65, 0x0510001c,
+	0x00009b40, 0x012e8160, 0x09249126, 0x0001c600, 0x004b6a8e,
+	0x000003ce, 0x00181400, 0x00820820, 0x066c420f, 0x0f282207,
+	0x17601685, 0x1f801104, 0x37a00c03, 0x3fc40883, 0x57c00803,
+	0x5fd80682, 0x7fe00482, 0x7f3c7bba, 0xf3307ff0
+};
+#endif
+
+#ifdef notyet
+static const uint32_t ar5416_phy_vals_2ghz_40mhz[] = {
+	0x00000007, 0x000003c4, 0x00000000, 0xad848e19, 0x7d14e000,
+	0x9c0a9f6b, 0x00000090, 0x00000000, 0x02020200, 0x00000e0e,
+	0x0a020001, 0x0000a000, 0x00000000, 0x00000e0e, 0x00000007,
+	0x00200400, 0x206a002e, 0x13721c24, 0x00197a68, 0x1284233c,
+	0x6c48b0e4, 0x00000859, 0x7ec80d2e, 0x31395c5e, 0x0004dd20,
+	0x409a4190, 0x050cb081, 0x00000000, 0x00000000, 0x00000000,
+	0x00000000, 0x00000898, 0x00000268, 0x10000fff, 0x0510001c,
+	0xd0058a15, 0x00000001, 0x00000004, 0x3f3f3f3f, 0x3f3f3f3f,
+	0x0000007f, 0xdfb81020, 0x9280b212, 0x00020028, 0x5d50e188,
+	0x00081fff, 0x00009b40, 0x00001120, 0x190fb515, 0x00000000,
+	0x00000001, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+	0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+	0x00000000, 0x00000007, 0x001fff00, 0x006f00c4, 0x03051000,
+	0x00000820, 0x038919be, 0x06336f77, 0x60f6532c, 0x08f186c8,
+	0x00046384, 0x00000000, 0x00000000, 0x00000000, 0x00000200,
+	0x64646464, 0x3c787878, 0x000000aa, 0x00000000, 0x00001042,
+	0x00000000, 0x00000040, 0x00000080, 0x00000141, 0x00000181,
+	0x000001c1, 0x00000001, 0x00000041, 0x000001a8, 0x000001e8,
+	0x00000028, 0x00000068, 0x000000a8, 0x00000169, 0x000001a9,
+	0x000001e9, 0x00000029, 0x00000069, 0x00000190, 0x000001d0,
+	0x00000010, 0x00000050, 0x00000090, 0x00000151, 0x00000191,
+	0x000001d1, 0x00000011, 0x00000051, 0x00000198, 0x000001d8,
+	0x00000018, 0x00000058, 0x00000098, 0x00000159, 0x00000199,
+	0x000001d9, 0x00000019, 0x00000059, 0x00000099, 0x000000d9,
+	0x000000f9, 0x000000f9, 0x000000f9, 0x000000f9, 0x000000f9,
+	0x000000f9, 0x000000f9, 0x000000f9, 0x000000f9, 0x000000f9,
+	0x000000f9, 0x000000f9, 0x000000f9, 0x000000f9, 0x000000f9,
+	0x000000f9, 0x000000f9, 0x000000f9, 0x000000f9, 0x000000f9,
+	0x000000f9, 0x000000f9, 0x000000f9, 0x000000f9, 0x00000000,
+	0x00000001, 0x00000002, 0x00000003, 0x00000004, 0x00000005,
+	0x00000008, 0x00000009, 0x0000000a, 0x0000000b, 0x0000000c,
+	0x0000000d, 0x00000010, 0x00000011, 0x00000012, 0x00000013,
+	0x00000014, 0x00000015, 0x00000018, 0x00000019, 0x0000001a,
+	0x0000001b, 0x0000001c, 0x0000001d, 0x00000020, 0x00000021,
+	0x00000022, 0x00000023, 0x00000024, 0x00000025, 0x00000028,
+	0x00000029, 0x0000002a, 0x0000002b, 0x0000002c, 0x0000002d,
+	0x00000030, 0x00000031, 0x00000032, 0x00000033, 0x00000034,
+	0x00000035, 0x00000035, 0x00000035, 0x00000035, 0x00000035,
+	0x00000035, 0x00000035, 0x00000035, 0x00000035, 0x00000035,
+	0x00000035, 0x00000035, 0x00000035, 0x00000035, 0x00000035,
+	0x00000035, 0x00000035, 0x00000035, 0x00000035, 0x00000035,
+	0x00000035, 0x00000010, 0x0000001a, 0x00000000, 0x00000000,
+	0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+	0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+	0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+	0x00000000, 0x0000000e, 0x00000440, 0xd03e4788, 0x012a8160,
+	0x40806333, 0x00106c10, 0x009c4060, 0x1883800a, 0x018830c6,
+	0x00000400, 0x000009b5, 0x00000000, 0x00000210, 0x3f3f3f3f,
+	0x3f3f3f3f, 0x13c889af, 0x38490a20, 0x00007bb6, 0x0fff3ffc,
+	0x00000001, 0x0000a000, 0x00000000, 0x0cc75380, 0x0f0f0f01,
+	0xdfa91f01, 0x00418a11, 0x00000000, 0x09249126, 0x0a1a7caa,
+	0x1ce739ce, 0x051701ce, 0x18010000, 0x2e032402, 0x4a0a3c06,
+	0x621a540b, 0x764f6c1b, 0x845b7a5a, 0x950f8ccf, 0xa5cf9b4f,
+	0xbddfaf1f, 0xd1ffc93f, 0x00000000, 0x00000000, 0x00000000,
+	0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+	0x3fffffff, 0x3fffffff, 0x3fffffff, 0x0003ffff, 0x79a8aa1f,
+	0x08000000, 0x3f3f3f3f, 0x3f3f3f3f, 0x1ce739ce, 0x000001ce,
+	0x00000007, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+	0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+	0x00000000, 0x00000000, 0x3f3f3f3f, 0x3f3f3f3f, 0x3f3f3f3f,
+	0x00000000, 0x1ce739ce, 0x000000c0, 0x00180a68, 0x0510001c,
+	0x00009b40, 0x012a8160, 0x09249126, 0x00180a68, 0x0510001c,
+	0x00009b40, 0x012a8160, 0x09249126, 0x0001c600, 0x004b6a8e,
+	0x000003ce, 0x00181400, 0x00820820, 0x066c420f, 0x0f282207,
+	0x17601685, 0x1f801104, 0x37a00c03, 0x3fc40883, 0x57c00803,
+	0x5fd80682, 0x7fe00482, 0x7f3c7bba, 0xf3307ff0
+};
+#endif
+
+static const uint32_t ar5416_phy_vals_2ghz_20mhz[] = {
+	0x00000007, 0x00000300, 0x00000000, 0xad848e19, 0x7d14e000,
+	0x9c0a9f6b, 0x00000090, 0x00000000, 0x02020200, 0x00000e0e,
+	0x0a020001, 0x0000a000, 0x00000000, 0x00000e0e, 0x00000007,
+	0x00200400, 0x206a002e, 0x137216a4, 0x00197a68, 0x1284233c,
+	0x6c48b0e4, 0x00000859, 0x7ec80d2e, 0x31395c5e, 0x0004dd20,
+	0x409a4190, 0x050cb081, 0x00000000, 0x00000000, 0x00000000,
+	0x00000000, 0x00000898, 0x00000134, 0x10000fff, 0x0510001c,
+	0xd0058a15, 0x00000001, 0x00000004, 0x3f3f3f3f, 0x3f3f3f3f,
+	0x0000007f, 0xdfb81020, 0x9280b212, 0x00020028, 0x5d50e188,
+	0x00081fff, 0x00009b40, 0x00001120, 0x190fb515, 0x00000000,
+	0x00000001, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+	0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+	0x00000000, 0x00000007, 0x001fff00, 0x006f00c4, 0x03051000,
+	0x00000820, 0x038919be, 0x06336f77, 0x60f6532c, 0x08f186c8,
+	0x00046384, 0x00000000, 0x00000000, 0x00000000, 0x00000200,
+	0x64646464, 0x3c787878, 0x000000aa, 0x00000000, 0x00001042,
+	0x00000000, 0x00000040, 0x00000080, 0x00000141, 0x00000181,
+	0x000001c1, 0x00000001, 0x00000041, 0x000001a8, 0x000001e8,
+	0x00000028, 0x00000068, 0x000000a8, 0x00000169, 0x000001a9,
+	0x000001e9, 0x00000029, 0x00000069, 0x00000190, 0x000001d0,
+	0x00000010, 0x00000050, 0x00000090, 0x00000151, 0x00000191,
+	0x000001d1, 0x00000011, 0x00000051, 0x00000198, 0x000001d8,
+	0x00000018, 0x00000058, 0x00000098, 0x00000159, 0x00000199,
+	0x000001d9, 0x00000019, 0x00000059, 0x00000099, 0x000000d9,
+	0x000000f9, 0x000000f9, 0x000000f9, 0x000000f9, 0x000000f9,
+	0x000000f9, 0x000000f9, 0x000000f9, 0x000000f9, 0x000000f9,
+	0x000000f9, 0x000000f9, 0x000000f9, 0x000000f9, 0x000000f9,
+	0x000000f9, 0x000000f9, 0x000000f9, 0x000000f9, 0x000000f9,
+	0x000000f9, 0x000000f9, 0x000000f9, 0x000000f9, 0x00000000,
+	0x00000001, 0x00000002, 0x00000003, 0x00000004, 0x00000005,
+	0x00000008, 0x00000009, 0x0000000a, 0x0000000b, 0x0000000c,
+	0x0000000d, 0x00000010, 0x00000011, 0x00000012, 0x00000013,
+	0x00000014, 0x00000015, 0x00000018, 0x00000019, 0x0000001a,
+	0x0000001b, 0x0000001c, 0x0000001d, 0x00000020, 0x00000021,
+	0x00000022, 0x00000023, 0x00000024, 0x00000025, 0x00000028,
+	0x00000029, 0x0000002a, 0x0000002b, 0x0000002c, 0x0000002d,
+	0x00000030, 0x00000031, 0x00000032, 0x00000033, 0x00000034,
+	0x00000035, 0x00000035, 0x00000035, 0x00000035, 0x00000035,
+	0x00000035, 0x00000035, 0x00000035, 0x00000035, 0x00000035,
+	0x00000035, 0x00000035, 0x00000035, 0x00000035, 0x00000035,
+	0x00000035, 0x00000035, 0x00000035, 0x00000035, 0x00000035,
+	0x00000035, 0x00000010, 0x0000001a, 0x00000000, 0x00000000,
+	0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+	0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+	0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+	0x00000000, 0x0000000e, 0x00000440, 0xd03e4788, 0x012a8160,
+	0x40806333, 0x00106c10, 0x009c4060, 0x1883800a, 0x018830c6,
+	0x00000400, 0x000009b5, 0x00000000, 0x00000108, 0x3f3f3f3f,
+	0x3f3f3f3f, 0x13c889af, 0x38490a20, 0x00007bb6, 0x0fff3ffc,
+	0x00000001, 0x0000a000, 0x00000000, 0x0cc75380, 0x0f0f0f01,
+	0xdfa91f01, 0x00418a11, 0x00000000, 0x09249126, 0x0a1a7caa,
+	0x1ce739ce, 0x051701ce, 0x18010000, 0x2e032402, 0x4a0a3c06,
+	0x621a540b, 0x764f6c1b, 0x845b7a5a, 0x950f8ccf, 0xa5cf9b4f,
+	0xbddfaf1f, 0xd1ffc93f, 0x00000000, 0x00000000, 0x00000000,
+	0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+	0x3fffffff, 0x3fffffff, 0x3fffffff, 0x0003ffff, 0x79a8aa1f,
+	0x08000000, 0x3f3f3f3f, 0x3f3f3f3f, 0x1ce739ce, 0x000001ce,
+	0x00000007, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+	0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+	0x00000000, 0x00000000, 0x3f3f3f3f, 0x3f3f3f3f, 0x3f3f3f3f,
+	0x00000000, 0x1ce739ce, 0x000000c0, 0x00180a68, 0x0510001c,
+	0x00009b40, 0x012a8160, 0x09249126, 0x00180a68, 0x0510001c,
+	0x00009b40, 0x012a8160, 0x09249126, 0x0001c600, 0x004b6a8e,
+	0x000003ce, 0x00181400, 0x00820820, 0x066c420f, 0x0f282207,
+	0x17601685, 0x1f801104, 0x37a00c03, 0x3fc40883, 0x57c00803,
+	0x5fd80682, 0x7fe00482, 0x7f3c7bba, 0xf3307ff0
+};
+
+/* NB: apply AR_PHY(). */
+static const uint8_t ar5416_banks_regs[] = {
+	0x2c, 0x38, 0x2c, 0x3b, 0x2c, 0x38, 0x3c, 0x2c, 0x3a, 0x2c, 0x39,
+	0x2c, 0x2c, 0x2c, 0x2c, 0x2c, 0x2c, 0x2c, 0x2c, 0x2c, 0x2c, 0x2c,
+	0x2c, 0x2c, 0x2c, 0x2c, 0x2c, 0x2c, 0x2c, 0x2c, 0x2c, 0x2c, 0x2c,
+	0x2c, 0x2c, 0x2c, 0x2c, 0x2c, 0x2c, 0x2c, 0x2c, 0x2c, 0x2c, 0x2c,
+	0x2c, 0x2c, 0x2c, 0x2c, 0x2c, 0x2c, 0x2c, 0x2c, 0x2c, 0x2c, 0x2c,
+	0x2c, 0x2c, 0x2c, 0x2c, 0x2c, 0x2c, 0x2c, 0x2c, 0x38, 0x2c, 0x2c,
+	0x2c, 0x3c
+};
+
+static const uint32_t ar5416_banks_vals_5ghz[] = {
+	0x1e5795e5, 0x02008020, 0x02108421, 0x00000008, 0x0e73ff17,
+	0x00000420, 0x01400018, 0x000001a1, 0x00000001, 0x00000013,
+	0x00000002, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+	0x00000000, 0x00004000, 0x00006c00, 0x00002c00, 0x00004800,
+	0x00004000, 0x00006000, 0x00001000, 0x00004000, 0x00007c00,
+	0x00007c00, 0x00007c00, 0x00007c00, 0x00007c00, 0x00087c00,
+	0x00007c00, 0x00005400, 0x00000c00, 0x00001800, 0x00007c00,
+	0x00006c00, 0x00006c00, 0x00007c00, 0x00002c00, 0x00003c00,
+	0x00003800, 0x00001c00, 0x00000800, 0x00000408, 0x00004c15,
+	0x00004188, 0x0000201e, 0x00010408, 0x00000801, 0x00000c08,
+	0x0000181e, 0x00001016, 0x00002800, 0x00004010, 0x0000081c,
+	0x00000115, 0x00000015, 0x00000066, 0x0000001c, 0x00000000,
+	0x00000004, 0x00000015, 0x0000001f, 0x00000000, 0x000000a0,
+	0x00000000, 0x00000040, 0x0000001c
+};
+
+static const uint32_t ar5416_banks_vals_2ghz[] = {
+	0x1e5795e5, 0x02008020, 0x02108421, 0x00000008, 0x0e73ff17,
+	0x00000420, 0x01c00018, 0x000001a1, 0x00000001, 0x00000013,
+	0x00000002, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
+	0x00000000, 0x00004000, 0x00006c00, 0x00002c00, 0x00004800,
+	0x00004000, 0x00006000, 0x00001000, 0x00004000, 0x00007c00,
+	0x00007c00, 0x00007c00, 0x00007c00, 0x00007c00, 0x00087c00,
+	0x00007c00, 0x00005400, 0x00000c00, 0x00001800, 0x00007c00,
+	0x00006c00, 0x00006c00, 0x00007c00, 0x00002c00, 0x00003c00,
+	0x00003800, 0x00001c00, 0x00000800, 0x00000408, 0x00004c15,
+	0x00004188, 0x0000201e, 0x00010408, 0x00000801, 0x00000c08,
+	0x0000181e, 0x00001016, 0x00002800, 0x00004010, 0x0000081c,
+	0x00000115, 0x00000015, 0x00000066, 0x0000001c, 0x00000000,
+	0x00000004, 0x00000015, 0x0000001f, 0x00000400, 0x000000a0,
+	0x00000000, 0x00000040, 0x0000001c
+};
+
+/*
+ * EEPROM.
+ */
+/* Possible flags for opCapFlags. */
+#define AR5416_OPFLAGS_11A	0x01
+#define AR5416_OPFLAGS_11G	0x02
+#define AR5416_OPFLAGS_5G_HT40	0x04
+#define AR5416_OPFLAGS_2G_HT40	0x08
+#define AR5416_OPFLAGS_5G_HT20	0x10
+#define AR5416_OPFLAGS_2G_HT20	0x20
+
+#define AR5416_NUM_5G_CAL_PIERS		8
+#define AR5416_NUM_2G_CAL_PIERS		4
+#define AR5416_NUM_5G_20_TARGET_POWERS	8
+#define AR5416_NUM_5G_40_TARGET_POWERS	8
+#define AR5416_NUM_2G_CCK_TARGET_POWERS	3
+#define AR5416_NUM_2G_20_TARGET_POWERS	4
+#define AR5416_NUM_2G_40_TARGET_POWERS	4
+#define AR5416_NUM_CTLS			24
+#define AR5416_NUM_BAND_EDGES		8
+#define AR5416_NUM_PD_GAINS		4
+#define AR5416_PD_GAIN_ICEPTS		5
+#define AR5416_EEPROM_MODAL_SPURS	5
+#define AR5416_MAX_CHAINS		2
+
+struct BaseEepHeader {
+	uint16_t	length;
+	uint16_t	checksum;
+	uint16_t	version;
+	uint8_t		opCapFlags;
+	uint8_t		eepMisc;
+	uint16_t	regDmn[2];
+	uint8_t		macAddr[6];
+	uint8_t		rxMask;
+	uint8_t		txMask;
+	uint16_t	rfSilent;
+	uint16_t	blueToothOptions;
+	uint16_t	deviceCap;
+	uint32_t	binBuildNumber;
+	uint8_t		deviceType;
+	uint8_t		futureBase[33];
+} __packed;
+
+struct spurChanStruct {
+	uint16_t	spurChan;
+	uint8_t		spurRangeLow;
+	uint8_t		spurRangeHigh;
+} __packed;
+
+struct ModalEepHeader {
+	uint32_t	antCtrlChain[AR5416_MAX_CHAINS];
+	uint32_t	antCtrlCommon;
+	int8_t		antennaGainCh[AR5416_MAX_CHAINS];
+	uint8_t		switchSettling;
+	uint8_t		txRxAttenCh[AR5416_MAX_CHAINS];
+	uint8_t		rxTxMarginCh[AR5416_MAX_CHAINS];
+	uint8_t		adcDesiredSize;
+	int8_t		pgaDesiredSize;
+	uint8_t		xlnaGainCh[AR5416_MAX_CHAINS];
+	uint8_t		txEndToXpaOff;
+	uint8_t		txEndToRxOn;
+	uint8_t		txFrameToXpaOn;
+	uint8_t		thresh62;
+	uint8_t		noiseFloorThreshCh[AR5416_MAX_CHAINS];
+	uint8_t		xpdGain;
+	uint8_t		xpd;
+	int8_t		iqCalICh[AR5416_MAX_CHAINS];
+	int8_t		iqCalQCh[AR5416_MAX_CHAINS];
+	uint8_t		pdGainOverlap;
+	uint8_t		ob;
+	uint8_t		db;
+	uint8_t		xpaBiasLvl;
+	uint8_t		pwrDecreaseFor2Chain;
+	uint8_t		pwrDecreaseFor3Chain;
+	uint8_t		txFrameToDataStart;
+	uint8_t		txFrameToPaOn;
+	uint8_t		ht40PowerIncForPdadc;
+	uint8_t		bswAtten[AR5416_MAX_CHAINS];
+	uint8_t		bswMargin[AR5416_MAX_CHAINS];
+	uint8_t		swSettleHt40;
+	uint8_t		futureModal[22];
+	struct spurChanStruct spurChans[AR5416_EEPROM_MODAL_SPURS];
+} __packed;
+
+struct calDataPerFreq {
+	uint8_t		pwrPdg[AR5416_NUM_PD_GAINS][AR5416_PD_GAIN_ICEPTS];
+	uint8_t		vpdPdg[AR5416_NUM_PD_GAINS][AR5416_PD_GAIN_ICEPTS];
+} __packed;
+
+struct CalTargetPowerLegacy {
+	uint8_t		bChannel;
+	uint8_t		tPow2x[4];
+} __packed;
+
+struct CalTargetPowerHt {
+	uint8_t		bChannel;
+	uint8_t		tPow2x[8];
+} __packed;
+
+struct CalCtlEdges {
+	uint8_t		bChannel;
+	uint8_t		tPowerFlag;
+} __packed;
+
+struct CalCtlData {
+	struct CalCtlEdges ctlEdges[AR5416_MAX_CHAINS][AR5416_NUM_BAND_EDGES];
+} __packed;
+
+struct ar5416eeprom {
+	struct BaseEepHeader	baseEepHeader;
+	uint8_t			custData[64];
+	struct ModalEepHeader	modalHeader[2];
+	uint8_t			calFreqPier5G[AR5416_NUM_5G_CAL_PIERS];
+	uint8_t			calFreqPier2G[AR5416_NUM_2G_CAL_PIERS];
+	struct calDataPerFreq	calPierData5G[AR5416_MAX_CHAINS]
+					     [AR5416_NUM_5G_CAL_PIERS];
+	struct calDataPerFreq	calPierData2G[AR5416_MAX_CHAINS]
+					     [AR5416_NUM_2G_CAL_PIERS];
+	struct CalTargetPowerLegacy calTPow5G[AR5416_NUM_5G_20_TARGET_POWERS];
+	struct CalTargetPowerHt calTPow5GHT20[AR5416_NUM_5G_20_TARGET_POWERS];
+	struct CalTargetPowerHt calTPow5GHT40[AR5416_NUM_5G_40_TARGET_POWERS];
+	struct CalTargetPowerLegacy calTPowCck[AR5416_NUM_2G_CCK_TARGET_POWERS];
+	struct CalTargetPowerLegacy calTPow2G[AR5416_NUM_2G_20_TARGET_POWERS];
+	struct CalTargetPowerHt calTPow2GHT20[AR5416_NUM_2G_20_TARGET_POWERS];
+	struct CalTargetPowerHt calTPow2GHT40[AR5416_NUM_2G_40_TARGET_POWERS];
+	uint8_t			ctlIndex[AR5416_NUM_CTLS];
+	struct CalCtlData	ctlData[AR5416_NUM_CTLS];
+	uint8_t			padding;
+} __packed;
+
+#define	OTUS_NUM_CHAINS			2
+
+#define OTUS_UID(aid)		(IEEE80211_AID(aid) + 4)
+
+#define OTUS_MAX_TXCMDSZ	64
+#define OTUS_RXBUFSZ		(8 * 1024)
+/* Bumped for later A-MSDU and legacy fast-frames TX support */
+#define OTUS_TXBUFSZ		(8 * 1024)
+
+/* Default EDCA parameters for when QoS is disabled. */
+static const struct wmeParams otus_edca_def[WME_NUM_AC] = {
+	{ 4, 10, 3,  0 },
+	{ 4, 10, 7,  0 },
+	{ 3,  4, 2, 94 },
+	{ 2,  3, 2, 47 }
+};
+
+#define OTUS_RIDX_CCK1		 0
+#define OTUS_RIDX_OFDM6		 4
+#define OTUS_RIDX_OFDM24	 8
+#define OTUS_RIDX_MAX		11
+static const struct otus_rate {
+	uint8_t	rate;
+	uint8_t	mcs;
+} otus_rates[] = {
+	{   2, 0x0 },
+	{   4, 0x1 },
+	{  11, 0x2 },
+	{  22, 0x3 },
+	{  12, 0xb },
+	{  18, 0xf },
+	{  24, 0xa },
+	{  36, 0xe },
+	{  48, 0x9 },
+	{  72, 0xd },
+	{  96, 0x8 },
+	{ 108, 0xc }
+};
+
+struct otus_rx_radiotap_header {
+	struct ieee80211_radiotap_header wr_ihdr;
+	uint8_t		wr_flags;
+	uint8_t		wr_rate;
+	uint16_t	wr_chan_freq;
+	uint16_t	wr_chan_flags;
+	uint8_t		wr_antsignal;
+} __packed;
+
+#define OTUS_RX_RADIOTAP_PRESENT			\
+	(1 << IEEE80211_RADIOTAP_FLAGS |		\
+	 1 << IEEE80211_RADIOTAP_RATE |			\
+	 1 << IEEE80211_RADIOTAP_CHANNEL |		\
+	 1 << IEEE80211_RADIOTAP_DB_ANTSIGNAL)
+
+struct otus_tx_radiotap_header {
+	struct ieee80211_radiotap_header wt_ihdr;
+	uint8_t		wt_flags;
+	uint8_t		wt_rate;
+	uint16_t	wt_chan_freq;
+	uint16_t	wt_chan_flags;
+} __packed;
+
+#define OTUS_TX_RADIOTAP_PRESENT			\
+	(1 << IEEE80211_RADIOTAP_FLAGS |		\
+	 1 << IEEE80211_RADIOTAP_RATE |			\
+	 1 << IEEE80211_RADIOTAP_CHANNEL)
+
+struct otus_softc;
+
+/* Firmware commands */
+struct otus_tx_cmd {
+	uint8_t			*buf;
+	uint16_t		buflen;
+	void *			*odata;
+	uint16_t		odatalen;
+	uint16_t		token;
+	STAILQ_ENTRY(otus_tx_cmd)	next_cmd;
+};
+
+/* TX, RX buffers */
+struct otus_data {
+	struct otus_softc	*sc;
+	uint8_t			*buf;
+	uint16_t		buflen;
+	struct mbuf		*m;
+	struct ieee80211_node	*ni;
+	STAILQ_ENTRY(otus_data)	next;
+};
+
+struct otus_node {
+	struct ieee80211_node		ni;
+	uint64_t			tx_done;
+	uint64_t			tx_err;
+	uint64_t			tx_retries;
+};
+
+#define OTUS_CONFIG_INDEX               0
+#define OTUS_IFACE_INDEX                0
+
+/*
+ * The carl9170 firmware has the following specification:
+ *
+ * 0 - USB control
+ * 1 - TX
+ * 2 - RX
+ * 3 - IRQ
+ * 4 - CMD
+ * ..
+ * 10 - end
+ */
+enum {
+	OTUS_BULK_TX,
+	OTUS_BULK_RX,
+	OTUS_BULK_IRQ,
+	OTUS_BULK_CMD,
+	OTUS_N_XFER
+};
+
+struct otus_vap {
+	struct ieee80211vap	vap;
+	int			(*newstate)(struct ieee80211vap *,
+				    enum ieee80211_state, int);
+};
+#define	OTUS_VAP(vap)		((struct otus_vap *)(vap))
+#define	OTUS_NODE(ni)		((struct otus_node *)(ni))
+
+#define	OTUS_LOCK(sc)		mtx_lock(&(sc)->sc_mtx)
+#define	OTUS_UNLOCK(sc)		mtx_unlock(&(sc)->sc_mtx)
+#define	OTUS_LOCK_ASSERT(sc)	mtx_assert(&(sc)->sc_mtx, MA_OWNED)
+#define	OTUS_UNLOCK_ASSERT(sc)	mtx_assert(&(sc)->sc_mtx, MA_NOTOWNED)
+
+/* XXX the TX/RX endpoint dump says it's 0x200, (512)? */
+#define	OTUS_MAX_TXSZ		512
+#define	OTUS_MAX_RXSZ		512
+/* intr/cmd endpoint dump says 0x40 */
+#define	OTUS_MAX_CTRLSZ		64
+
+#define	OTUS_CMD_LIST_COUNT	32
+#define	OTUS_RX_LIST_COUNT	128
+#define	OTUS_TX_LIST_COUNT	32
+
+struct otus_softc {
+	struct ieee80211com		sc_ic;
+	struct mbufq			sc_snd;
+	device_t			sc_dev;
+	struct usb_device		*sc_udev;
+	int				(*sc_newstate)(struct ieee80211com *,
+					    enum ieee80211_state, int);
+	void				(*sc_led_newstate)(struct otus_softc *);
+	struct usbd_interface		*sc_iface;
+	struct mtx			sc_mtx;
+
+	struct ar5416eeprom		eeprom;
+	uint8_t				capflags;
+	uint8_t				rxmask;
+	uint8_t				txmask;
+	int				sc_running:1,
+					sc_calibrating:1,
+					sc_scanning:1;
+
+	int				sc_if_flags;
+	int				sc_tx_timer;
+	int				fixed_ridx;
+	int				bb_reset;
+
+	struct ieee80211_channel	*sc_curchan;
+
+	struct task			tx_task;
+	struct task			wme_update_task;
+	struct timeout_task		scan_to;
+	struct timeout_task		calib_to;
+
+	/* register batch writes */
+	int				write_idx;
+
+	uint32_t			led_state;
+
+	/* current firmware message serial / token number */
+	int				token;
+
+	/* current noisefloor, from SET_FREQUENCY */
+	int				sc_nf[OTUS_NUM_CHAINS];
+
+	/* How many pending, active transmit frames */
+	int				sc_tx_n_pending;
+	int				sc_tx_n_active;
+
+	const uint32_t			*phy_vals;
+
+	struct {
+		uint32_t	reg;
+		uint32_t	val;
+	} __packed			write_buf[AR_MAX_WRITE_IDX + 1];
+
+	struct otus_data		sc_rx[OTUS_RX_LIST_COUNT];
+	struct otus_data		sc_tx[OTUS_TX_LIST_COUNT];
+	struct otus_tx_cmd		sc_cmd[OTUS_CMD_LIST_COUNT];
+
+	struct usb_xfer			*sc_xfer[OTUS_N_XFER];
+
+	STAILQ_HEAD(, otus_data)	sc_rx_active;
+	STAILQ_HEAD(, otus_data)	sc_rx_inactive;
+	STAILQ_HEAD(, otus_data)	sc_tx_active[OTUS_N_XFER];
+	STAILQ_HEAD(, otus_data)	sc_tx_inactive;
+	STAILQ_HEAD(, otus_data)	sc_tx_pending[OTUS_N_XFER];
+
+	STAILQ_HEAD(, otus_tx_cmd)	sc_cmd_active;
+	STAILQ_HEAD(, otus_tx_cmd)	sc_cmd_inactive;
+	STAILQ_HEAD(, otus_tx_cmd)	sc_cmd_pending;
+	STAILQ_HEAD(, otus_tx_cmd)	sc_cmd_waiting;
+
+	union {
+		struct otus_rx_radiotap_header th;
+		uint8_t	pad[64];
+	}				sc_rxtapu;
+#define sc_rxtap	sc_rxtapu.th
+	int				sc_rxtap_len;
+
+	union {
+		struct otus_tx_radiotap_header th;
+		uint8_t	pad[64];
+	}				sc_txtapu;
+#define sc_txtap	sc_txtapu.th
+	int				sc_txtap_len;
+};
+
+#endif	/* __IF_OTUSREG_H__ */
diff --git a/sys/dev/pci/pci_pci.c b/sys/dev/pci/pci_pci.c
index 5b925c8a02e8e..11a4bcac7950f 100644
--- a/sys/dev/pci/pci_pci.c
+++ b/sys/dev/pci/pci_pci.c
@@ -1082,7 +1082,7 @@ pcib_attach(device_t dev)
     pcib_attach_common(dev);
     sc = device_get_softc(dev);
     if (sc->bus.sec != 0) {
-	child = device_add_child(dev, "pci", sc->bus.sec);
+	child = device_add_child(dev, "pci", -1);
 	if (child != NULL)
 	    return(bus_generic_attach(dev));
     }
diff --git a/sys/dev/ral/rt2560.c b/sys/dev/ral/rt2560.c
index ea9c4162b8f83..ef5a01c4e936e 100644
--- a/sys/dev/ral/rt2560.c
+++ b/sys/dev/ral/rt2560.c
@@ -768,7 +768,7 @@ rt2560_newstate(struct ieee80211vap *vap, enum ieee80211_state nstate, int arg)
 		if (vap->iv_opmode == IEEE80211_M_HOSTAP ||
 		    vap->iv_opmode == IEEE80211_M_IBSS ||
 		    vap->iv_opmode == IEEE80211_M_MBSS) {
-			m = ieee80211_beacon_alloc(ni, &rvp->ral_bo);
+			m = ieee80211_beacon_alloc(ni, &vap->iv_bcn_off);
 			if (m == NULL) {
 				device_printf(sc->sc_dev,
 				    "could not allocate beacon\n");
@@ -1273,8 +1273,7 @@ rt2560_rx_intr(struct rt2560_softc *sc)
 static void
 rt2560_beacon_update(struct ieee80211vap *vap, int item)
 {
-	struct rt2560_vap *rvp = RT2560_VAP(vap);
-	struct ieee80211_beacon_offsets *bo = &rvp->ral_bo;
+	struct ieee80211_beacon_offsets *bo = &vap->iv_bcn_off;
 
 	setbit(bo->bo_flags, item);
 }
@@ -1288,7 +1287,6 @@ rt2560_beacon_expire(struct rt2560_softc *sc)
 {
 	struct ieee80211com *ic = &sc->sc_ic;
 	struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
-	struct rt2560_vap *rvp = RT2560_VAP(vap);
 	struct rt2560_tx_data *data;
 
 	if (ic->ic_opmode != IEEE80211_M_IBSS &&
@@ -1307,7 +1305,7 @@ rt2560_beacon_expire(struct rt2560_softc *sc)
 	bus_dmamap_unload(sc->bcnq.data_dmat, data->map);
 
 	/* XXX 1 =>'s mcast frames which means all PS sta's will wakeup! */
-	ieee80211_beacon_update(data->ni, &rvp->ral_bo, data->m, 1);
+	ieee80211_beacon_update(data->ni, &vap->iv_bcn_off, data->m, 1);
 
 	rt2560_tx_bcn(sc, data->m, data->ni);
 
@@ -2301,7 +2299,6 @@ static void
 rt2560_set_basicrates(struct rt2560_softc *sc,
     const struct ieee80211_rateset *rs)
 {
-#define RV(r)	((r) & IEEE80211_RATE_VAL)
 	struct ieee80211com *ic = &sc->sc_ic;
 	uint32_t mask = 0;
 	uint8_t rate;
@@ -2313,13 +2310,13 @@ rt2560_set_basicrates(struct rt2560_softc *sc,
 		if (!(rate & IEEE80211_RATE_BASIC))
 			continue;
 
-		mask |= 1 << ieee80211_legacy_rate_lookup(ic->ic_rt, RV(rate));
+		mask |= 1 << ieee80211_legacy_rate_lookup(ic->ic_rt,
+		    IEEE80211_RV(rate));
 	}
 
 	RAL_WRITE(sc, RT2560_ARSP_PLCP_1, mask);
 
 	DPRINTF(sc, "Setting basic rate mask to 0x%x\n", mask);
-#undef RV
 }
 
 static void
@@ -2478,7 +2475,6 @@ rt2560_scan_end(struct ieee80211com *ic)
 static int
 rt2560_bbp_init(struct rt2560_softc *sc)
 {
-#define N(a)	(sizeof (a) / sizeof ((a)[0]))
 	int i, ntries;
 
 	/* wait for BBP to be ready */
@@ -2493,7 +2489,7 @@ rt2560_bbp_init(struct rt2560_softc *sc)
 	}
 
 	/* initialize BBP registers to default values */
-	for (i = 0; i < N(rt2560_def_bbp); i++) {
+	for (i = 0; i < nitems(rt2560_def_bbp); i++) {
 		rt2560_bbp_write(sc, rt2560_def_bbp[i].reg,
 		    rt2560_def_bbp[i].val);
 	}
@@ -2507,7 +2503,6 @@ rt2560_bbp_init(struct rt2560_softc *sc)
 	rt2560_bbp_write(sc, 17, 0x48);	/* XXX restore bbp17 */
 
 	return 0;
-#undef N
 }
 
 static void
@@ -2560,7 +2555,6 @@ rt2560_set_rxantenna(struct rt2560_softc *sc, int antenna)
 static void
 rt2560_init_locked(struct rt2560_softc *sc)
 {
-#define N(a)	(sizeof (a) / sizeof ((a)[0]))
 	struct ieee80211com *ic = &sc->sc_ic;
 	struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
 	uint32_t tmp;
@@ -2590,7 +2584,7 @@ rt2560_init_locked(struct rt2560_softc *sc)
 	RAL_WRITE(sc, RT2560_RXCSR2, sc->rxq.physaddr);
 
 	/* initialize MAC registers to default values */
-	for (i = 0; i < N(rt2560_def_mac); i++)
+	for (i = 0; i < nitems(rt2560_def_mac); i++)
 		RAL_WRITE(sc, rt2560_def_mac[i].reg, rt2560_def_mac[i].val);
 
 	rt2560_set_macaddr(sc, vap ? vap->iv_myaddr : ic->ic_macaddr);
@@ -2641,7 +2635,6 @@ rt2560_init_locked(struct rt2560_softc *sc)
 	sc->sc_flags |= RT2560_F_RUNNING;
 
 	callout_reset(&sc->watchdog_ch, hz, rt2560_watchdog, sc);
-#undef N
 }
 
 static void
diff --git a/sys/dev/ral/rt2560var.h b/sys/dev/ral/rt2560var.h
index 3a5fef981080f..12601d8ef38bc 100644
--- a/sys/dev/ral/rt2560var.h
+++ b/sys/dev/ral/rt2560var.h
@@ -97,7 +97,6 @@ struct rt2560_rx_ring {
 
 struct rt2560_vap {
 	struct ieee80211vap	ral_vap;
-	struct ieee80211_beacon_offsets	ral_bo;
 
 	int			(*ral_newstate)(struct ieee80211vap *,
 				    enum ieee80211_state, int);
diff --git a/sys/dev/ral/rt2661.c b/sys/dev/ral/rt2661.c
index c77d4f88df767..5c73ba02e0c5c 100644
--- a/sys/dev/ral/rt2661.c
+++ b/sys/dev/ral/rt2661.c
@@ -1859,7 +1859,6 @@ static void
 rt2661_set_basicrates(struct rt2661_softc *sc,
     const struct ieee80211_rateset *rs)
 {
-#define RV(r)	((r) & IEEE80211_RATE_VAL)
 	struct ieee80211com *ic = &sc->sc_ic;
 	uint32_t mask = 0;
 	uint8_t rate;
@@ -1871,13 +1870,13 @@ rt2661_set_basicrates(struct rt2661_softc *sc,
 		if (!(rate & IEEE80211_RATE_BASIC))
 			continue;
 
-		mask |= 1 << ieee80211_legacy_rate_lookup(ic->ic_rt, RV(rate));
+		mask |= 1 << ieee80211_legacy_rate_lookup(ic->ic_rt,
+		    IEEE80211_RV(rate));
 	}
 
 	RAL_WRITE(sc, RT2661_TXRX_CSR5, mask);
 
 	DPRINTF(sc, "Setting basic rate mask to 0x%x\n", mask);
-#undef RV
 }
 
 /*
@@ -2208,7 +2207,6 @@ rt2661_read_eeprom(struct rt2661_softc *sc, uint8_t macaddr[IEEE80211_ADDR_LEN])
 static int
 rt2661_bbp_init(struct rt2661_softc *sc)
 {
-#define N(a)	(sizeof (a) / sizeof ((a)[0]))
 	int i, ntries;
 	uint8_t val;
 
@@ -2225,7 +2223,7 @@ rt2661_bbp_init(struct rt2661_softc *sc)
 	}
 
 	/* initialize BBP registers to default values */
-	for (i = 0; i < N(rt2661_def_bbp); i++) {
+	for (i = 0; i < nitems(rt2661_def_bbp); i++) {
 		rt2661_bbp_write(sc, rt2661_def_bbp[i].reg,
 		    rt2661_def_bbp[i].val);
 	}
@@ -2238,13 +2236,11 @@ rt2661_bbp_init(struct rt2661_softc *sc)
 	}
 
 	return 0;
-#undef N
 }
 
 static void
 rt2661_init_locked(struct rt2661_softc *sc)
 {
-#define N(a)	(sizeof (a) / sizeof ((a)[0]))
 	struct ieee80211com *ic = &sc->sc_ic;
 	struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
 	uint32_t tmp, sta[3];
@@ -2305,7 +2301,7 @@ rt2661_init_locked(struct rt2661_softc *sc)
 	RAL_WRITE(sc, RT2661_RX_CNTL_CSR, 2);
 
 	/* initialize MAC registers to default values */
-	for (i = 0; i < N(rt2661_def_mac); i++)
+	for (i = 0; i < nitems(rt2661_def_mac); i++)
 		RAL_WRITE(sc, rt2661_def_mac[i].reg, rt2661_def_mac[i].val);
 
 	rt2661_set_macaddr(sc, vap ? vap->iv_myaddr : ic->ic_macaddr);
@@ -2354,7 +2350,7 @@ rt2661_init_locked(struct rt2661_softc *sc)
 	RAL_WRITE(sc, RT2661_TXRX_CSR0, tmp);
 
 	/* clear STA registers */
-	RAL_READ_REGION_4(sc, RT2661_STA_CSR0, sta, N(sta));
+	RAL_READ_REGION_4(sc, RT2661_STA_CSR0, sta, nitems(sta));
 
 	/* initialize ASIC */
 	RAL_WRITE(sc, RT2661_MAC_CSR1, 4);
@@ -2372,7 +2368,6 @@ rt2661_init_locked(struct rt2661_softc *sc)
 	sc->sc_flags |= RAL_RUNNING;
 
 	callout_reset(&sc->watchdog_ch, hz, rt2661_watchdog, sc);
-#undef N
 }
 
 static void
@@ -2631,12 +2626,12 @@ static int
 rt2661_prepare_beacon(struct rt2661_softc *sc, struct ieee80211vap *vap)
 {
 	struct ieee80211com *ic = vap->iv_ic;
-	struct ieee80211_beacon_offsets bo;
+	struct ieee80211_beacon_offsets *bo = &vap->iv_bcn_off;
 	struct rt2661_tx_desc desc;
 	struct mbuf *m0;
 	int rate;
 
-	m0 = ieee80211_beacon_alloc(vap->iv_bss, &bo);
+	m0 = ieee80211_beacon_alloc(vap->iv_bss, bo);
 	if (m0 == NULL) {
 		device_printf(sc->sc_dev, "could not allocate beacon frame\n");
 		return ENOBUFS;
diff --git a/sys/dev/ral/rt2860.c b/sys/dev/ral/rt2860.c
index b3c737d67c66b..a6b1e26a70695 100644
--- a/sys/dev/ral/rt2860.c
+++ b/sys/dev/ral/rt2860.c
@@ -2260,7 +2260,6 @@ void
 rt2860_set_basicrates(struct rt2860_softc *sc,
     const struct ieee80211_rateset *rs)
 {
-#define RV(r)	((r) & IEEE80211_RATE_VAL)
 	struct ieee80211com *ic = &sc->sc_ic;
 	uint32_t mask = 0;
 	uint8_t rate;
@@ -2272,11 +2271,11 @@ rt2860_set_basicrates(struct rt2860_softc *sc,
 		if (!(rate & IEEE80211_RATE_BASIC))
 			continue;
 
-		mask |= 1 << ieee80211_legacy_rate_lookup(ic->ic_rt, RV(rate));
+		mask |= 1 << ieee80211_legacy_rate_lookup(ic->ic_rt,
+		    IEEE80211_RV(rate));
 	}
 
 	RAL_WRITE(sc, RT2860_LEGACY_BASIC_RATE, mask);
-#undef RV
 }
 
 static void
@@ -3351,7 +3350,7 @@ rt2860_read_eeprom(struct rt2860_softc *sc, uint8_t macaddr[IEEE80211_ADDR_LEN])
 
 	/* read EEPROM version */
 	val = rt2860_srom_read(sc, RT2860_EEPROM_VERSION);
-	DPRINTF(("EEPROM rev=%d, FAE=%d\n", val & 0xff, val >> 8));
+	DPRINTF(("EEPROM rev=%d, FAE=%d\n", val >> 8, val & 0xff));
 
 	/* read MAC address */
 	val = rt2860_srom_read(sc, RT2860_EEPROM_MAC01);
@@ -4269,12 +4268,12 @@ static int
 rt2860_setup_beacon(struct rt2860_softc *sc, struct ieee80211vap *vap)
 {
 	struct ieee80211com *ic = vap->iv_ic;
-	struct ieee80211_beacon_offsets bo;
+	struct ieee80211_beacon_offsets *bo = &vap->iv_bcn_off;
 	struct rt2860_txwi txwi;
 	struct mbuf *m;
 	int ridx;
 
-	if ((m = ieee80211_beacon_alloc(vap->iv_bss, &bo)) == NULL)
+	if ((m = ieee80211_beacon_alloc(vap->iv_bss, bo)) == NULL)
 		return ENOBUFS;
 
 	memset(&txwi, 0, sizeof txwi);
diff --git a/sys/dev/re/if_re.c b/sys/dev/re/if_re.c
index 677d397a1c7a3..88fa5fc054efa 100644
--- a/sys/dev/re/if_re.c
+++ b/sys/dev/re/if_re.c
@@ -238,6 +238,7 @@ static const struct rl_hwrev re_hwrevs[] = {
 	{ RL_HWREV_8168F, RL_8169, "8168F/8111F", RL_JUMBO_MTU_9K},
 	{ RL_HWREV_8168G, RL_8169, "8168G/8111G", RL_JUMBO_MTU_9K},
 	{ RL_HWREV_8168GU, RL_8169, "8168GU/8111GU", RL_JUMBO_MTU_9K},
+	{ RL_HWREV_8168H, RL_8169, "8168H/8111H", RL_JUMBO_MTU_9K},
 	{ RL_HWREV_8411, RL_8169, "8411", RL_JUMBO_MTU_9K},
 	{ RL_HWREV_8411B, RL_8169, "8411B", RL_JUMBO_MTU_9K},
 	{ 0, 0, NULL, 0 }
@@ -1485,6 +1486,7 @@ re_attach(device_t dev)
 		break;
 	case RL_HWREV_8168EP:
 	case RL_HWREV_8168G:
+	case RL_HWREV_8168H:
 	case RL_HWREV_8411B:
 		sc->rl_flags |= RL_FLAG_PHYWAKE | RL_FLAG_PAR |
 		    RL_FLAG_DESCV2 | RL_FLAG_MACSTAT | RL_FLAG_CMDSTOP |
diff --git a/sys/dev/rl/if_rlreg.h b/sys/dev/rl/if_rlreg.h
index b0de60f49807f..4f02dcfac5dd4 100644
--- a/sys/dev/rl/if_rlreg.h
+++ b/sys/dev/rl/if_rlreg.h
@@ -195,6 +195,7 @@
 #define	RL_HWREV_8168G		0x4C000000
 #define	RL_HWREV_8168EP		0x50000000
 #define	RL_HWREV_8168GU		0x50800000
+#define	RL_HWREV_8168H		0x54000000
 #define	RL_HWREV_8411B		0x5C800000
 #define	RL_HWREV_8139		0x60000000
 #define	RL_HWREV_8139A		0x70000000
diff --git a/sys/dev/uart/uart_bus_pci.c b/sys/dev/uart/uart_bus_pci.c
index 48954a531058f..41662226f27fb 100644
--- a/sys/dev/uart/uart_bus_pci.c
+++ b/sys/dev/uart/uart_bus_pci.c
@@ -127,6 +127,7 @@ static const struct pci_id pci_ns8250_ids[] = {
 	24 * DEFAULT_RCLK, 2 },
 { 0x8086, 0x1c3d, 0xffff, 0, "Intel AMT - KT Controller", 0x10 },
 { 0x8086, 0x1d3d, 0xffff, 0, "Intel C600/X79 Series Chipset KT Controller", 0x10 },
+{ 0x8086, 0x1e3d, 0xffff, 0, "Intel Panther Point KT Controller", 0x10 },
 { 0x8086, 0x2a07, 0xffff, 0, "Intel AMT - PM965/GM965 KT Controller", 0x10 },
 { 0x8086, 0x2a47, 0xffff, 0, "Mobile 4 Series Chipset KT Controller", 0x10 },
 { 0x8086, 0x2e17, 0xffff, 0, "4 Series Chipset Serial KT Controller", 0x10 },
diff --git a/sys/dev/usb/controller/uhci_pci.c b/sys/dev/usb/controller/uhci_pci.c
index cd08e89a3da86..e7e4412de5efd 100644
--- a/sys/dev/usb/controller/uhci_pci.c
+++ b/sys/dev/usb/controller/uhci_pci.c
@@ -223,7 +223,7 @@ uhci_pci_match(device_t self)
 	case 0x76028086:
 		return ("Intel 82372FB/82468GX USB controller");
 
-	case 0x3309103c:
+	case 0x3300103c:
 		return ("HP iLO Standard Virtual USB controller");
 
 	case 0x30381106:
diff --git a/sys/dev/usb/controller/xhci_pci.c b/sys/dev/usb/controller/xhci_pci.c
index 0ad7357caf7e6..0df2e9fb60c33 100644
--- a/sys/dev/usb/controller/xhci_pci.c
+++ b/sys/dev/usb/controller/xhci_pci.c
@@ -113,6 +113,9 @@ xhci_pci_match(device_t self)
 	case 0x8cb18086:
 		return ("Intel Wildcat Point USB 3.0 controller");
 
+	case 0xa01b177d:
+		return ("Cavium ThunderX USB 3.0 controller");
+
 	default:
 		break;
 	}
diff --git a/sys/dev/usb/quirk/usb_quirk.c b/sys/dev/usb/quirk/usb_quirk.c
index 459dc2ab78009..927cec1524493 100644
--- a/sys/dev/usb/quirk/usb_quirk.c
+++ b/sys/dev/usb/quirk/usb_quirk.c
@@ -61,6 +61,7 @@ MODULE_VERSION(usb_quirk, 1);
 
 #define	USB_DEV_QUIRKS_MAX 384
 #define	USB_SUB_QUIRKS_MAX 8
+#define	USB_QUIRK_ENVROOT "hw.usb.quirk."
 
 struct usb_quirk_entry {
 	uint16_t vid;
@@ -608,8 +609,32 @@ static const char *usb_quirk_str[USB_QUIRK_MAX] = {
 static const char *
 usb_quirkstr(uint16_t quirk)
 {
-	return ((quirk < USB_QUIRK_MAX) ?
-	    usb_quirk_str[quirk] : "USB_QUIRK_UNKNOWN");
+	return ((quirk < USB_QUIRK_MAX && usb_quirk_str[quirk] != NULL) ?
+	    usb_quirk_str[quirk] : "UQ_UNKNOWN");
+}
+
+/*------------------------------------------------------------------------*
+ *	usb_strquirk
+ *
+ * This function converts a string into a USB quirk code.
+ *
+ * Returns:
+ * Less than USB_QUIRK_MAX: Quirk code
+ * Else: Quirk code not found
+ *------------------------------------------------------------------------*/
+static uint16_t
+usb_strquirk(const char *str, size_t len)
+{
+	const char *quirk;
+	uint16_t x;
+
+	for (x = 0; x != USB_QUIRK_MAX; x++) {
+		quirk = usb_quirkstr(x);
+		if (strncmp(str, quirk, len) == 0 &&
+		    quirk[len] == 0)
+			break;
+	}
+	return (x);
 }
 
 /*------------------------------------------------------------------------*
@@ -854,12 +879,122 @@ usb_quirk_ioctl(unsigned long cmd, caddr_t data,
 	return (ENOIOCTL);
 }
 
+/*------------------------------------------------------------------------*
+ *	usb_quirk_strtou16
+ *
+ * Helper function to scan a 16-bit integer.
+ *------------------------------------------------------------------------*/
+static uint16_t
+usb_quirk_strtou16(const char **pptr, const char *name, const char *what)
+{
+	unsigned long value;
+	char *end;
+
+	value = strtoul(*pptr, &end, 0);
+	if (value > 65535 || *pptr == end || (*end != ' ' && *end != '\t')) {
+		printf("%s: %s 16-bit %s value set to zero\n",
+		    name, what, *end == 0 ? "incomplete" : "invalid");
+		return (0);
+	}
+	*pptr = end + 1;
+	return ((uint16_t)value);
+}
+
+/*------------------------------------------------------------------------*
+ *	usb_quirk_add_entry_from_str
+ *
+ * Add a USB quirk entry from string.
+ *     "VENDOR PRODUCT LO_REV HI_REV QUIRK[,QUIRK[,...]]"
+ *------------------------------------------------------------------------*/
+static void
+usb_quirk_add_entry_from_str(const char *name, const char *env)
+{
+	struct usb_quirk_entry entry = { };
+	struct usb_quirk_entry *new;
+	uint16_t quirk_idx;
+	uint16_t quirk;
+	const char *end;
+
+	/* check for invalid environment variable */
+	if (name == NULL || env == NULL)
+		return;
+
+	if (bootverbose)
+		printf("Adding USB QUIRK '%s' = '%s'\n", name, env);
+
+	/* parse device information */
+	entry.vid = usb_quirk_strtou16(&env, name, "Vendor ID");
+	entry.pid = usb_quirk_strtou16(&env, name, "Product ID");
+	entry.lo_rev = usb_quirk_strtou16(&env, name, "Low revision");
+	entry.hi_rev = usb_quirk_strtou16(&env, name, "High revision");
+
+	/* parse quirk information */
+	quirk_idx = 0;
+	while (*env != 0 && quirk_idx != USB_SUB_QUIRKS_MAX) {
+		/* skip whitespace before quirks */
+		while (*env == ' ' || *env == '\t')
+			env++;
+
+		/* look for quirk separation character */
+		end = strchr(env, ',');
+		if (end == NULL)
+			end = env + strlen(env);
+
+		/* lookup quirk in string table */
+		quirk = usb_strquirk(env, end - env);
+		if (quirk < USB_QUIRK_MAX) {
+			entry.quirks[quirk_idx++] = quirk;
+		} else {
+			printf("%s: unknown USB quirk '%.*s' (skipped)\n",
+			    name, (int)(end - env), env);
+		}
+		env = end;
+
+		/* skip quirk delimiter, if any */
+		if (*env != 0)
+			env++;
+	}
+
+	/* register quirk */
+	if (quirk_idx != 0) {
+		if (*env != 0) {
+			printf("%s: Too many USB quirks, only %d allowed!\n",
+			    name, USB_SUB_QUIRKS_MAX);
+		}
+		mtx_lock(&usb_quirk_mtx);
+		new = usb_quirk_get_entry(entry.vid, entry.pid,
+		    entry.lo_rev, entry.hi_rev, 1);
+		if (new == NULL)
+			printf("%s: USB quirks table is full!\n", name);
+		else
+			memcpy(new->quirks, entry.quirks, sizeof(entry.quirks));
+		mtx_unlock(&usb_quirk_mtx);
+	} else {
+		printf("%s: No USB quirks found!\n", name);
+	}
+}
+
 static void
 usb_quirk_init(void *arg)
 {
+	char envkey[sizeof(USB_QUIRK_ENVROOT) + 2];	/* 2 digits max, 0 to 99 */
+	int i;
+  
 	/* initialize mutex */
 	mtx_init(&usb_quirk_mtx, "USB quirk", NULL, MTX_DEF);
 
+	/* look for quirks defined by the environment variable */
+	for (i = 0; i != 100; i++) {
+		snprintf(envkey, sizeof(envkey), USB_QUIRK_ENVROOT "%d", i);
+
+		/* Stop at first undefined var */
+		if (!testenv(envkey))
+			break;
+
+		/* parse environment variable */
+		usb_quirk_add_entry_from_str(envkey, kern_getenv(envkey));
+	}
+	
 	/* register our function */
 	usb_test_quirk_p = &usb_test_quirk_by_info;
 	usb_quirk_ioctl_p = &usb_quirk_ioctl;
diff --git a/sys/dev/usb/serial/u3g.c b/sys/dev/usb/serial/u3g.c
index ba825aa20fb17..065da281807cb 100644
--- a/sys/dev/usb/serial/u3g.c
+++ b/sys/dev/usb/serial/u3g.c
@@ -525,6 +525,7 @@ static const STRUCT_USB_HOST_ID u3g_devs[] = {
 	U3G_DEV(SIERRA, MC5727_2, 0),
 	U3G_DEV(SIERRA, MC5728, 0),
 	U3G_DEV(SIERRA, MC7354, 0),
+	U3G_DEV(SIERRA, MC7355, 0),
 	U3G_DEV(SIERRA, MC8700, 0),
 	U3G_DEV(SIERRA, MC8755, 0),
 	U3G_DEV(SIERRA, MC8755_2, 0),
diff --git a/sys/dev/usb/usbdevs b/sys/dev/usb/usbdevs
index 7b7373c553d42..42f2e855c8c80 100644
--- a/sys/dev/usb/usbdevs
+++ b/sys/dev/usb/usbdevs
@@ -776,6 +776,7 @@ vendor MOSCHIP		0x9710	MosChip Semiconductor
 vendor NETGEAR4		0x9846  Netgear
 vendor MARVELL		0x9e88	Marvell Technology Group Ltd.
 vendor 3COM3		0xa727	3Com
+vendor CACE		0xcace	CACE Technologies
 vendor EVOLUTION	0xdeee	Evolution Robotics products
 vendor DATAAPEX		0xdaae	DataApex
 vendor HP2		0xf003	Hewlett Packard
@@ -863,6 +864,7 @@ product	ACCTON RT3070_3		0xc522	RT3070
 product	ACCTON RT3070_5		0xd522	RT3070
 product ACCTON RTL8192SU        0xc512  RTL8192SU
 product ACCTON ZD1211B		0xe501	ZD1211B
+product ACCTON WN7512		0xf522	WN7512
 
 /* Aceeca products */
 product ACEECA MEZ1000		0x0001	MEZ1000 RDA
@@ -1223,6 +1225,11 @@ product ATHEROS2 AR5523_2	0x0003	AR5523
 product ATHEROS2 AR5523_2_NF	0x0004	AR5523 (no firmware)
 product ATHEROS2 AR5523_3	0x0005	AR5523
 product ATHEROS2 AR5523_3_NF	0x0006	AR5523 (no firmware)
+product ATHEROS2 TG121N		0x1001	TG121N
+product ATHEROS2 WN821NV2	0x1002	WN821NV2
+product ATHEROS2 3CRUSBN275	0x1010	3CRUSBN275
+product ATHEROS2 WN612		0x1011	WN612
+product ATHEROS2 AR9170		0x9170	AR9170
 
 /* Atmel Comp. products */
 product ATMEL STK541		0x2109	Zigbee Controller
@@ -1238,6 +1245,9 @@ product AUTHENTEC AES1610	0x1600	AES1610 Fingerprint Sensor
 /* Avision products */
 product AVISION 1200U		0x0268	1200U scanner
 
+/* AVM products */
+product AVM FRITZWLAN		0x8401	FRITZ!WLAN N
+
 /* Axesstel products */
 product AXESSTEL DATAMODEM	0x1000  Data Modem
 
@@ -1340,6 +1350,9 @@ product BROTHER MFC8600_9650	0x0100	MFC8600/9650 multifunction device
 product BTC BTC6100		0x5550	6100C Keyboard
 product BTC BTC7932		0x6782	Keyboard with mouse port
 
+/* CACE Technologies products */
+product CACE AIRPCAPNX		0x0300	AirPcap NX
+
 /* Canon, Inc. products */
 product CANON N656U		0x2206	CanoScan N656U
 product CANON N1220U		0x2207	CanoScan N1220U
@@ -1640,8 +1653,10 @@ product	DLINK DWA131B		0x330d	DWA-131 rev B
 product DLINK2 RTL8192SU_1	0x3300	RTL8192SU
 product DLINK2 RTL8192SU_2	0x3302	RTL8192SU
 product DLINK2 DWA131A1		0x3303	DWA-131 A1
+product DLINK2 DWA160A2		0x3a09	DWA-160 A2
 product DLINK2 DWA120		0x3a0c	DWA-120
 product DLINK2 DWA120_NF	0x3a0d	DWA-120 (no firmware)
+product DLINK2 DWA130D1		0x3a0f	DWA-130 D1
 product DLINK2 DWLG122C1	0x3c03	DWL-G122 c1
 product DLINK2 WUA1340		0x3c04	WUA-1340
 product DLINK2 DWA111		0x3c06	DWA-111
@@ -1652,6 +1667,7 @@ product DLINK2 RT3072_1		0x3c0b	RT3072
 product DLINK2 RT3070_1		0x3c0d	RT3070
 product DLINK2 RT3070_2		0x3c0e	RT3070
 product DLINK2 RT3070_3		0x3c0f	RT3070
+product DLINK2 DWA160A1		0x3c10	DWA-160 A1
 product DLINK2 RT2870_2		0x3c11	RT2870
 product DLINK2 DWA130		0x3c13	DWA-130
 product DLINK2 RT3070_4		0x3c15	RT3070
@@ -2469,6 +2485,7 @@ product IODATA USBETTXS		0x0913	USB ETTX
 product IODATA USBWNB11A	0x0919	USB WN-B11
 product IODATA USBWNB11		0x0922	USB Airport WN-B11
 product IODATA ETGUS2		0x0930	ETG-US2
+product IODATA WNGDNUS2		0x093f	WN-GDN/US2
 product IODATA RT3072_1		0x0944	RT3072
 product IODATA RT3072_2		0x0945	RT3072
 product IODATA RT3072_3		0x0947	RT3072
@@ -3209,6 +3226,7 @@ product NATIONAL BEARPAW2400	0x1001	BearPaw 2400
 /* NEC products */
 product NEC HUB_0050		0x0050	USB 2.0 7-Port Hub
 product NEC HUB_005A		0x005a	USB 2.0 4-Port Hub
+product NEC WL300NUG		0x0249	WL300NU-G
 product NEC HUB			0x55aa	hub
 product NEC HUB_B		0x55ab	hub
 
@@ -3241,7 +3259,11 @@ product NETGEAR WG111V3		0x4260	WG111v3
 product NETGEAR WG111U		0x4300	WG111U
 product NETGEAR WG111U_NF	0x4301	WG111U (no firmware)
 product NETGEAR WG111V2		0x6a00	WG111V2
+product NETGEAR WN111V2		0x9001	WN111V2
+product NETGEAR WNDA3100	0x9010	WNDA3100
+product NETGEAR WNDA3200	0x9018	WNDA3200
 product NETGEAR RTL8192CU	0x9021	RTL8192CU
+product NETGEAR WNA1000		0x9040	WNA1000
 product NETGEAR WNA1000M	0x9041	WNA1000M
 product NETGEAR2 MA101		0x4100	MA101
 product NETGEAR2 MA101B		0x4102	MA101 Rev B
@@ -3505,6 +3527,7 @@ product PLANEX GW_US11H		0x14ea	GW-US11H WLAN
 product PLANEX2 RTL8188CUS	0x1201	RTL8188CUS
 product PLANEX2 GW_US11S	0x3220	GW-US11S WLAN
 product PLANEX2 GW_US54GXS	0x5303	GW-US54GXS WLAN
+product PLANEX2 GW_US300	0x5304	GW-US300
 product PLANEX2 RTL8188CU_1	0xab2a	RTL8188CU
 product PLANEX2 RTL8188CU_2	0xed17	RTL8188CU
 product PLANEX2 RTL8188CU_3	0x4902	RTL8188CU
@@ -4044,6 +4067,7 @@ product SIERRA E6892		0x6892	E6892
 product SIERRA E6893		0x6893	E6893
 product SIERRA MC8700		0x68A3	MC8700
 product SIERRA MC7354		0x68C0	MC7354
+product SIERRA MC7355		0x9041	MC7355
 product SIERRA AC313U		0x68aa	Sierra Wireless AirCard 313U
 product SIERRA TRUINSTALL	0x0fff	Aircard Tru Installer
 
@@ -4576,8 +4600,10 @@ product WINBOND UH104		0x5518	4-port USB Hub
 product WINMAXGROUP FLASH64MC	0x6660	USB Flash Disk 64M-C
 
 /* Wistron NeWeb products */
+product WISTRONNEWEB WNC0600	0x0326	WNC-0600USB
 product WISTRONNEWEB UR045G	0x0427	PrismGT USB 2.0 WLAN
 product WISTRONNEWEB UR055G	0x0711	UR055G
+product WISTRONNEWEB O8494	0x0804	ORiNOCO 802.11n
 product WISTRONNEWEB AR5523_1	0x0826	AR5523
 product WISTRONNEWEB AR5523_1_NF	0x0827	AR5523 (no firmware)
 product WISTRONNEWEB AR5523_2	0x082a	AR5523
@@ -4626,7 +4652,9 @@ product ZCOM AR5523_NF		0x0013	AR5523 driver (no firmware)
 product ZCOM XM142		0x0015	XM-142
 product ZCOM ZD1211B		0x001a	ZD1211B
 product ZCOM RT2870_1		0x0022	RT2870
+product ZCOM UB81		0x0023	UB81
 product ZCOM RT2870_2		0x0025	RT2870
+product ZCOM UB82		0x0026	UB82
 
 /* Zinwell products */
 product ZINWELL RT2570		0x0260	RT2570
@@ -4645,6 +4673,7 @@ product ZORAN EX20DSC		0x4343	Digital Camera EX-20 DSC
 /* Zydas Technology Corporation products */
 product ZYDAS ZD1211		0x1211	ZD1211 WLAN abg
 product ZYDAS ZD1211B		0x1215	ZD1211B
+product ZYDAS ZD1221		0x1221	ZD1221
 
 /* ZyXEL Communication Co. products */
 product ZYXEL OMNI56K		0x1500	Omni 56K Plus
@@ -4656,6 +4685,8 @@ product ZYXEL M202		0x340a	M-202
 product ZYXEL G220V2		0x340f	G-220 v2
 product ZYXEL G202		0x3410	G-202
 product ZYXEL RT2870_1		0x3416	RT2870
+product ZYXEL NWD271N		0x3417	NWD-271N
+product ZYXEL NWD211AN		0x3418	NWD-211AN
 product ZYXEL RT2870_2		0x341a	RT2870
 product ZYXEL RT3070		0x341e	NWD2105
 product ZYXEL RTL8192CU		0x341f	RTL8192CU
diff --git a/sys/dev/usb/wlan/if_rsu.c b/sys/dev/usb/wlan/if_rsu.c
index 463a9bdc9a08f..4fb9d85bee9dd 100644
--- a/sys/dev/usb/wlan/if_rsu.c
+++ b/sys/dev/usb/wlan/if_rsu.c
@@ -22,10 +22,14 @@ __FBSDID("$FreeBSD$");
  * Driver for Realtek RTL8188SU/RTL8191SU/RTL8192SU.
  *
  * TODO:
- *   o 11n support
  *   o h/w crypto
  *   o hostap / ibss / mesh
+ *   o sensible RSSI levels
+ *   o power-save operation
  */
+
+#include "opt_wlan.h"
+
 #include <sys/param.h>
 #include <sys/endian.h>
 #include <sys/sockio.h>
@@ -74,8 +78,31 @@ static int rsu_debug = 0;
 SYSCTL_NODE(_hw_usb, OID_AUTO, rsu, CTLFLAG_RW, 0, "USB rsu");
 SYSCTL_INT(_hw_usb_rsu, OID_AUTO, debug, CTLFLAG_RWTUN, &rsu_debug, 0,
     "Debug level");
+#define	RSU_DPRINTF(_sc, _flg, ...)					\
+	do								\
+		if (((_flg) == (RSU_DEBUG_ANY)) || (rsu_debug & (_flg))) \
+			device_printf((_sc)->sc_dev, __VA_ARGS__);	\
+	while (0)
+#else
+#define	RSU_DPRINTF(_sc, _flg, ...)
 #endif
 
+static int rsu_enable_11n = 1;
+TUNABLE_INT("hw.usb.rsu.enable_11n", &rsu_enable_11n);
+
+#define	RSU_DEBUG_ANY		0xffffffff
+#define	RSU_DEBUG_TX		0x00000001
+#define	RSU_DEBUG_RX		0x00000002
+#define	RSU_DEBUG_RESET		0x00000004
+#define	RSU_DEBUG_CALIB		0x00000008
+#define	RSU_DEBUG_STATE		0x00000010
+#define	RSU_DEBUG_SCAN		0x00000020
+#define	RSU_DEBUG_FWCMD		0x00000040
+#define	RSU_DEBUG_TXDONE	0x00000080
+#define	RSU_DEBUG_FW		0x00000100
+#define	RSU_DEBUG_FWDBG		0x00000200
+#define	RSU_DEBUG_AMPDU		0x00000400
+
 static const STRUCT_USB_HOST_ID rsu_devs[] = {
 #define	RSU_HT_NOT_SUPPORTED 0
 #define	RSU_HT_SUPPORTED 1
@@ -131,6 +158,7 @@ static device_attach_t  rsu_attach;
 static device_detach_t  rsu_detach;
 static usb_callback_t   rsu_bulk_tx_callback_be_bk;
 static usb_callback_t   rsu_bulk_tx_callback_vi_vo;
+static usb_callback_t   rsu_bulk_tx_callback_h2c;
 static usb_callback_t   rsu_bulk_rx_callback;
 static usb_error_t	rsu_do_request(struct rsu_softc *,
 			    struct usb_device_request *, void *);
@@ -150,6 +178,7 @@ static void	rsu_free_tx_list(struct rsu_softc *);
 static void	rsu_free_list(struct rsu_softc *, struct rsu_data [], int);
 static struct rsu_data *_rsu_getbuf(struct rsu_softc *);
 static struct rsu_data *rsu_getbuf(struct rsu_softc *);
+static void	rsu_freebuf(struct rsu_softc *, struct rsu_data *);
 static int	rsu_write_region_1(struct rsu_softc *, uint16_t, uint8_t *,
 		    int);
 static void	rsu_write_1(struct rsu_softc *, uint16_t, uint8_t);
@@ -165,6 +194,7 @@ static uint8_t	rsu_efuse_read_1(struct rsu_softc *, uint16_t);
 static int	rsu_read_rom(struct rsu_softc *);
 static int	rsu_fw_cmd(struct rsu_softc *, uint8_t, void *, int);
 static void	rsu_calib_task(void *, int);
+static void	rsu_tx_task(void *, int);
 static int	rsu_newstate(struct ieee80211vap *, enum ieee80211_state, int);
 #ifdef notyet
 static void	rsu_set_key(struct rsu_softc *, const struct ieee80211_key *);
@@ -173,17 +203,18 @@ static void	rsu_delete_key(struct rsu_softc *, const struct ieee80211_key *);
 static int	rsu_site_survey(struct rsu_softc *, struct ieee80211vap *);
 static int	rsu_join_bss(struct rsu_softc *, struct ieee80211_node *);
 static int	rsu_disconnect(struct rsu_softc *);
+static int	rsu_hwrssi_to_rssi(struct rsu_softc *, int hw_rssi);
 static void	rsu_event_survey(struct rsu_softc *, uint8_t *, int);
 static void	rsu_event_join_bss(struct rsu_softc *, uint8_t *, int);
 static void	rsu_rx_event(struct rsu_softc *, uint8_t, uint8_t *, int);
 static void	rsu_rx_multi_event(struct rsu_softc *, uint8_t *, int);
+#if 0
 static int8_t	rsu_get_rssi(struct rsu_softc *, int, void *);
+#endif
+static struct mbuf * rsu_rx_frame(struct rsu_softc *, uint8_t *, int);
+static struct mbuf * rsu_rx_multi_frame(struct rsu_softc *, uint8_t *, int);
 static struct mbuf *
-		rsu_rx_frame(struct rsu_softc *, uint8_t *, int, int *);
-static struct mbuf *
-		rsu_rx_multi_frame(struct rsu_softc *, uint8_t *, int, int *);
-static struct mbuf *
-		rsu_rxeof(struct usb_xfer *, struct rsu_data *, int *);
+		rsu_rxeof(struct usb_xfer *, struct rsu_data *);
 static void	rsu_txeof(struct usb_xfer *, struct rsu_data *);
 static int	rsu_raw_xmit(struct ieee80211_node *, struct mbuf *, 
 		    const struct ieee80211_bpf_params *);
@@ -192,9 +223,10 @@ static int	rsu_tx_start(struct rsu_softc *, struct ieee80211_node *,
 		    struct mbuf *, struct rsu_data *);
 static int	rsu_transmit(struct ieee80211com *, struct mbuf *);
 static void	rsu_start(struct rsu_softc *);
+static void	_rsu_start(struct rsu_softc *);
 static void	rsu_parent(struct ieee80211com *);
 static void	rsu_stop(struct rsu_softc *);
-static void	rsu_ms_delay(struct rsu_softc *);
+static void	rsu_ms_delay(struct rsu_softc *, int);
 
 static device_method_t rsu_methods[] = {
 	DEVMETHOD(device_probe,		rsu_match),
@@ -225,6 +257,9 @@ static uint8_t rsu_wme_ac_xfer_map[4] = {
 	[WME_AC_VO] = RSU_BULK_TX_VI_VO,
 };
 
+/* XXX hard-coded */
+#define	RSU_H2C_ENDPOINT	3
+
 static const struct usb_config rsu_config[RSU_N_TRANSFER] = {
 	[RSU_BULK_RX] = {
 		.type = UE_BULK,
@@ -263,6 +298,19 @@ static const struct usb_config rsu_config[RSU_N_TRANSFER] = {
 		.callback = rsu_bulk_tx_callback_vi_vo,
 		.timeout = RSU_TX_TIMEOUT
 	},
+	[RSU_BULK_TX_H2C] = {
+		.type = UE_BULK,
+		.endpoint = 0x0d,
+		.direction = UE_DIR_OUT,
+		.bufsize = RSU_TXBUFSZ,
+		.flags = {
+			.ext_buffer = 1,
+			.pipe_bof = 1,
+			.short_xfer_ok = 1
+		},
+		.callback = rsu_bulk_tx_callback_h2c,
+		.timeout = RSU_TX_TIMEOUT
+	},
 };
 
 static int
@@ -279,6 +327,75 @@ rsu_match(device_t self)
 }
 
 static int
+rsu_send_mgmt(struct ieee80211_node *ni, int type, int arg)
+{
+
+	return (ENOTSUP);
+}
+
+static void
+rsu_update_chw(struct ieee80211com *ic)
+{
+
+}
+
+/*
+ * notification from net80211 that it'd like to do A-MPDU on the given TID.
+ *
+ * Note: this actually hangs traffic at the present moment, so don't use it.
+ * The firmware debug does indiciate it's sending and establishing a TX AMPDU
+ * session, but then no traffic flows.
+ */
+static int
+rsu_ampdu_enable(struct ieee80211_node *ni, struct ieee80211_tx_ampdu *tap)
+{
+#if 0
+	struct rsu_softc *sc = ni->ni_ic->ic_softc;
+	struct r92s_add_ba_req req;
+
+	/* Don't enable if it's requested or running */
+	if (IEEE80211_AMPDU_REQUESTED(tap))
+		return (0);
+	if (IEEE80211_AMPDU_RUNNING(tap))
+		return (0);
+
+	/* We've decided to send addba; so send it */
+	req.tid = htole32(tap->txa_tid);
+
+	/* Attempt net80211 state */
+	if (ieee80211_ampdu_tx_request_ext(ni, tap->txa_tid) != 1)
+		return (0);
+
+	/* Send the firmware command */
+	RSU_DPRINTF(sc, RSU_DEBUG_AMPDU, "%s: establishing AMPDU TX for TID %d\n",
+	    __func__,
+	    tap->txa_tid);
+
+	RSU_LOCK(sc);
+	if (rsu_fw_cmd(sc, R92S_CMD_ADDBA_REQ, &req, sizeof(req)) != 1) {
+		RSU_UNLOCK(sc);
+		/* Mark failure */
+		(void) ieee80211_ampdu_tx_request_active_ext(ni, tap->txa_tid, 0);
+		return (0);
+	}
+	RSU_UNLOCK(sc);
+
+	/* Mark success; we don't get any further notifications */
+	(void) ieee80211_ampdu_tx_request_active_ext(ni, tap->txa_tid, 1);
+#endif
+	/* Return 0, we're driving this ourselves */
+	return (0);
+}
+
+static int
+rsu_wme_update(struct ieee80211com *ic)
+{
+
+	/* Firmware handles this; not our problem */
+	return (0);
+}
+
+static int
 rsu_attach(device_t self)
 {
 	struct usb_attach_arg *uaa = device_get_ivars(self);
@@ -286,15 +403,31 @@ rsu_attach(device_t self)
 	struct ieee80211com *ic = &sc->sc_ic;
 	int error;
 	uint8_t iface_index, bands;
+	struct usb_interface *iface;
+	const char *rft;
 
 	device_set_usb_desc(self);
 	sc->sc_udev = uaa->device;
 	sc->sc_dev = self;
+	if (rsu_enable_11n)
+		sc->sc_ht = !! (USB_GET_DRIVER_INFO(uaa) & RSU_HT_SUPPORTED);
+
+	/* Get number of endpoints */
+	iface = usbd_get_iface(sc->sc_udev, 0);
+	sc->sc_nendpoints = iface->idesc->bNumEndpoints;
+
+	/* Endpoints are hard-coded for now, so enforce 4-endpoint only */
+	if (sc->sc_nendpoints != 4) {
+		device_printf(sc->sc_dev,
+		    "the driver currently only supports 4-endpoint devices\n");
+		return (ENXIO);
+	}
 
 	mtx_init(&sc->sc_mtx, device_get_nameunit(self), MTX_NETWORK_LOCK,
 	    MTX_DEF);
 	TIMEOUT_TASK_INIT(taskqueue_thread, &sc->calib_task, 0, 
 	    rsu_calib_task, sc);
+	TASK_INIT(&sc->tx_task, 0, rsu_tx_task, sc);
 	mbufq_init(&sc->sc_snd, ifqmaxlen);
 
 	/* Allocate Tx/Rx buffers. */
@@ -331,8 +464,37 @@ rsu_attach(device_t self)
 		device_printf(self, "could not read ROM\n");
 		goto fail_rom;
 	}
+
+	/* Figure out TX/RX streams */
+	switch (sc->rom[84]) {
+	case 0x0:
+		sc->sc_rftype = RTL8712_RFCONFIG_1T1R;
+		sc->sc_nrxstream = 1;
+		sc->sc_ntxstream = 1;
+		rft = "1T1R";
+		break;
+	case 0x1:
+		sc->sc_rftype = RTL8712_RFCONFIG_1T2R;
+		sc->sc_nrxstream = 2;
+		sc->sc_ntxstream = 1;
+		rft = "1T2R";
+		break;
+	case 0x2:
+		sc->sc_rftype = RTL8712_RFCONFIG_2T2R;
+		sc->sc_nrxstream = 2;
+		sc->sc_ntxstream = 2;
+		rft = "2T2R";
+		break;
+	default:
+		device_printf(sc->sc_dev,
+		    "%s: unknown board type (rfconfig=0x%02x)\n",
+		    __func__,
+		    sc->rom[84]);
+		goto fail_rom;
+	}
+
 	IEEE80211_ADDR_COPY(ic->ic_macaddr, &sc->rom[0x12]);
-	device_printf(self, "MAC/BB RTL8712 cut %d\n", sc->cut);
+	device_printf(self, "MAC/BB RTL8712 cut %d %s\n", sc->cut, rft);
 
 	ic->ic_softc = sc;
 	ic->ic_name = device_get_nameunit(self);
@@ -342,28 +504,37 @@ rsu_attach(device_t self)
 	/* Set device capabilities. */
 	ic->ic_caps =
 	    IEEE80211_C_STA |		/* station mode */
+#if 0
 	    IEEE80211_C_BGSCAN |	/* Background scan. */
+#endif
 	    IEEE80211_C_SHPREAMBLE |	/* Short preamble supported. */
+	    IEEE80211_C_WME |		/* WME/QoS */
 	    IEEE80211_C_SHSLOT |	/* Short slot time supported. */
 	    IEEE80211_C_WPA;		/* WPA/RSN. */
 
-#if 0
 	/* Check if HT support is present. */
-	if (usb_lookup(rsu_devs_noht, uaa->vendor, uaa->product) == NULL) {
-		/* Set HT capabilities. */
-		ic->ic_htcaps =
-		    IEEE80211_HTCAP_CBW20_40 |
-		    IEEE80211_HTCAP_DSSSCCK40;
-		/* Set supported HT rates. */
-		for (i = 0; i < 2; i++)
-			ic->ic_sup_mcs[i] = 0xff;
+	if (sc->sc_ht) {
+		device_printf(sc->sc_dev, "%s: enabling 11n\n", __func__);
+
+		/* Enable basic HT */
+		ic->ic_htcaps = IEEE80211_HTC_HT |
+		    IEEE80211_HTC_AMPDU |
+		    IEEE80211_HTC_AMSDU |
+		    IEEE80211_HTCAP_MAXAMSDU_3839 |
+		    IEEE80211_HTCAP_SMPS_OFF;
+		ic->ic_htcaps |= IEEE80211_HTCAP_CHWIDTH40;
+
+		/* set number of spatial streams */
+		ic->ic_txstream = sc->sc_ntxstream;
+		ic->ic_rxstream = sc->sc_nrxstream;
 	}
-#endif
 
 	/* Set supported .11b and .11g rates. */
 	bands = 0;
 	setbit(&bands, IEEE80211_MODE_11B);
 	setbit(&bands, IEEE80211_MODE_11G);
+	if (sc->sc_ht)
+		setbit(&bands, IEEE80211_MODE_11NG);
 	ieee80211_init_channels(ic, NULL, &bands);
 
 	ieee80211_ifattach(ic);
@@ -376,6 +547,10 @@ rsu_attach(device_t self)
 	ic->ic_update_mcast = rsu_update_mcast;
 	ic->ic_parent = rsu_parent;
 	ic->ic_transmit = rsu_transmit;
+	ic->ic_send_mgmt = rsu_send_mgmt;
+	ic->ic_update_chw = rsu_update_chw;
+	ic->ic_ampdu_enable = rsu_ampdu_enable;
+	ic->ic_wme.wme_update = rsu_wme_update;
 
 	ieee80211_radiotap_attach(ic, &sc->sc_txtap.wt_ihdr,
 	    sizeof(sc->sc_txtap), RSU_TX_RADIOTAP_PRESENT, 
@@ -403,16 +578,25 @@ rsu_detach(device_t self)
 	RSU_LOCK(sc);
 	rsu_stop(sc);
 	RSU_UNLOCK(sc);
-	usbd_transfer_unsetup(sc->sc_xfer, RSU_N_TRANSFER);
-	ieee80211_ifdetach(ic);
 
-	taskqueue_drain_timeout(taskqueue_thread, &sc->calib_task);
+	usbd_transfer_unsetup(sc->sc_xfer, RSU_N_TRANSFER);
 
+	/*
+	 * Free buffers /before/ we detach from net80211, else node
+	 * references to destroyed vaps will lead to a panic.
+	 */
 	/* Free Tx/Rx buffers. */
+	RSU_LOCK(sc);
 	rsu_free_tx_list(sc);
 	rsu_free_rx_list(sc);
+	RSU_UNLOCK(sc);
+
+	/* Frames are freed; detach from net80211 */
+	ieee80211_ifdetach(ic);
+
+	taskqueue_drain_timeout(taskqueue_thread, &sc->calib_task);
+	taskqueue_drain(taskqueue_thread, &sc->tx_task);
 
-	mbufq_drain(&sc->sc_snd);
 	mtx_destroy(&sc->sc_mtx);
 
 	return (0);
@@ -434,7 +618,7 @@ rsu_do_request(struct rsu_softc *sc, struct usb_device_request *req,
 			break;
 		DPRINTFN(1, "Control request failed, %s (retrying)\n",
 		    usbd_errstr(err));
-		usb_pause_mtx(&sc->sc_mtx, hz / 100);
+		rsu_ms_delay(sc, 10);
         }
 
         return (err);
@@ -466,6 +650,10 @@ rsu_vap_create(struct ieee80211com *ic, const char name[IFNAMSIZ], int unit,
 	uvp->newstate = vap->iv_newstate;
 	vap->iv_newstate = rsu_newstate;
 
+	/* Limits from the r92su driver */
+	vap->iv_ampdu_density = IEEE80211_HTCAP_MPDUDENSITY_16;
+	vap->iv_ampdu_rxmax = IEEE80211_HTCAP_MAXRXAMPDU_32K;
+
 	/* complete setup */
 	ieee80211_vap_attach(vap, ieee80211_media_change,
 	    ieee80211_media_status, mac);
@@ -491,6 +679,7 @@ rsu_scan_start(struct ieee80211com *ic)
 
 	/* Scanning is done by the firmware. */
 	RSU_LOCK(sc);
+	/* XXX TODO: force awake if in in network-sleep? */
 	error = rsu_site_survey(sc, TAILQ_FIRST(&ic->ic_vaps));
 	RSU_UNLOCK(sc);
 	if (error != 0)
@@ -640,9 +829,7 @@ _rsu_getbuf(struct rsu_softc *sc)
 		STAILQ_REMOVE_HEAD(&sc->sc_tx_inactive, next);
 	else
 		bf = NULL;
-	if (bf == NULL)
-		DPRINTF("out of xmit buffers\n");
-        return (bf);
+	return (bf);
 }
 
 static struct rsu_data *
@@ -653,11 +840,20 @@ rsu_getbuf(struct rsu_softc *sc)
 	RSU_ASSERT_LOCKED(sc);
 
 	bf = _rsu_getbuf(sc);
-	if (bf == NULL)
-		DPRINTF("stop queue\n");
+	if (bf == NULL) {
+		RSU_DPRINTF(sc, RSU_DEBUG_TX, "%s: no buffers\n", __func__);
+	}
 	return (bf);
 }
 
+static void
+rsu_freebuf(struct rsu_softc *sc, struct rsu_data *bf)
+{
+
+	RSU_ASSERT_LOCKED(sc);
+	STAILQ_INSERT_TAIL(&sc->sc_tx_inactive, bf, next);
+}
+
 static int
 rsu_write_region_1(struct rsu_softc *sc, uint16_t addr, uint8_t *buf,
     int len)
@@ -744,11 +940,11 @@ rsu_fw_iocmd(struct rsu_softc *sc, uint32_t iocmd)
 	int ntries;
 
 	rsu_write_4(sc, R92S_IOCMD_CTRL, iocmd);
-	rsu_ms_delay(sc);
+	rsu_ms_delay(sc, 1);
 	for (ntries = 0; ntries < 50; ntries++) {
 		if (rsu_read_4(sc, R92S_IOCMD_CTRL) == 0)
 			return (0);
-		rsu_ms_delay(sc);
+		rsu_ms_delay(sc, 1);
 	}
 	return (ETIMEDOUT);
 }
@@ -768,7 +964,7 @@ rsu_efuse_read_1(struct rsu_softc *sc, uint16_t addr)
 		reg = rsu_read_4(sc, R92S_EFUSE_CTRL);
 		if (reg & R92S_EFUSE_CTRL_VALID)
 			return (MS(reg, R92S_EFUSE_CTRL_DATA));
-		rsu_ms_delay(sc);
+		rsu_ms_delay(sc, 1);
 	}
 	device_printf(sc->sc_dev,
 	    "could not read efuse byte at address 0x%x\n", addr);
@@ -792,7 +988,7 @@ rsu_read_rom(struct rsu_softc *sc)
 	/* Turn on 2.5V to prevent eFuse leakage. */
 	reg = rsu_read_1(sc, R92S_EFUSE_TEST + 3);
 	rsu_write_1(sc, R92S_EFUSE_TEST + 3, reg | 0x80);
-	rsu_ms_delay(sc);
+	rsu_ms_delay(sc, 1);
 	rsu_write_1(sc, R92S_EFUSE_TEST + 3, reg & ~0x80);
 
 	/* Read full ROM image. */
@@ -830,18 +1026,24 @@ rsu_read_rom(struct rsu_softc *sc)
 static int
 rsu_fw_cmd(struct rsu_softc *sc, uint8_t code, void *buf, int len)
 {
-	const uint8_t which = rsu_wme_ac_xfer_map[WME_AC_VO];
+	const uint8_t which = RSU_H2C_ENDPOINT;
 	struct rsu_data *data;
 	struct r92s_tx_desc *txd;
 	struct r92s_fw_cmd_hdr *cmd;
 	int cmdsz;
 	int xferlen;
 
+	RSU_ASSERT_LOCKED(sc);
+
 	data = rsu_getbuf(sc);
 	if (data == NULL)
 		return (ENOMEM);
 
+	/* Blank the entire payload, just to be safe */
+	memset(data->buf, '\0', RSU_TXBUFSZ);
+
 	/* Round-up command length to a multiple of 8 bytes. */
+	/* XXX TODO: is this required? */
 	cmdsz = (len + 7) & ~7;
 
 	xferlen = sizeof(*txd) + sizeof(*cmd) + cmdsz;
@@ -866,7 +1068,9 @@ rsu_fw_cmd(struct rsu_softc *sc, uint8_t code, void *buf, int len)
 	/* Copy command payload. */
 	memcpy(&cmd[1], buf, len);
 
-	DPRINTFN(2, "Tx cmd code=0x%x len=0x%x\n", code, cmdsz);
+	RSU_DPRINTF(sc, RSU_DEBUG_TX | RSU_DEBUG_FWCMD,
+	    "%s: Tx cmd code=0x%x len=0x%x\n",
+	    __func__, code, cmdsz);
 	data->buflen = xferlen;
 	STAILQ_INSERT_TAIL(&sc->sc_tx_pending[which], data, next);
 	usbd_transfer_start(sc->sc_xfer[which]);
@@ -879,9 +1083,12 @@ static void
 rsu_calib_task(void *arg, int pending __unused)
 {
 	struct rsu_softc *sc = arg;
+#ifdef notyet
 	uint32_t reg;
+#endif
 
-	DPRINTFN(6, "running calibration task\n");
+	RSU_DPRINTF(sc, RSU_DEBUG_CALIB, "%s: running calibration task\n",
+	    __func__);
 
 	RSU_LOCK(sc);
 #ifdef notyet
@@ -896,14 +1103,95 @@ rsu_calib_task(void *arg, int pending __unused)
 #endif
 	/* Read current signal level. */
 	if (rsu_fw_iocmd(sc, 0xf4000001) == 0) {
-		reg = rsu_read_4(sc, R92S_IOCMD_DATA);
-		DPRINTFN(8, "RSSI=%d%%\n", reg >> 4);
+		sc->sc_currssi = rsu_read_4(sc, R92S_IOCMD_DATA);
+		RSU_DPRINTF(sc, RSU_DEBUG_CALIB, "%s: RSSI=%d (%d)\n",
+		    __func__, sc->sc_currssi,
+		    rsu_hwrssi_to_rssi(sc, sc->sc_currssi));
 	}
 	if (sc->sc_calibrating)
 		taskqueue_enqueue_timeout(taskqueue_thread, &sc->calib_task, hz);
 	RSU_UNLOCK(sc);
 }
 
+static void
+rsu_tx_task(void *arg, int pending __unused)
+{
+	struct rsu_softc *sc = arg;
+
+	RSU_LOCK(sc);
+	_rsu_start(sc);
+	RSU_UNLOCK(sc);
+}
+
+#define	RSU_PWR_UNKNOWN		0x0
+#define	RSU_PWR_ACTIVE		0x1
+#define	RSU_PWR_OFF		0x2
+#define	RSU_PWR_SLEEP		0x3
+
+/*
+ * Set the current power state.
+ *
+ * The rtlwifi code doesn't do this so aggressively; it
+ * waits for an idle period after association with
+ * no traffic before doing this.
+ *
+ * For now - it's on in all states except RUN, and
+ * in RUN it'll transition to allow sleep.
+ */
+
+struct r92s_pwr_cmd {
+	uint8_t mode;
+	uint8_t smart_ps;
+	uint8_t bcn_pass_time;
+};
+
+static int
+rsu_set_fw_power_state(struct rsu_softc *sc, int state)
+{
+	struct r92s_set_pwr_mode cmd;
+	//struct r92s_pwr_cmd cmd;
+	int error;
+
+	RSU_ASSERT_LOCKED(sc);
+
+	/* only change state if required */
+	if (sc->sc_curpwrstate == state)
+		return (0);
+
+	memset(&cmd, 0, sizeof(cmd));
+
+	switch (state) {
+	case RSU_PWR_ACTIVE:
+		/* Force the hardware awake */
+		rsu_write_1(sc, R92S_USB_HRPWM,
+		    R92S_USB_HRPWM_PS_ST_ACTIVE | R92S_USB_HRPWM_PS_ALL_ON);
+		cmd.mode = R92S_PS_MODE_ACTIVE;
+		break;
+	case RSU_PWR_SLEEP:
+		cmd.mode = R92S_PS_MODE_DTIM;	/* XXX configurable? */
+		cmd.smart_ps = 1; /* XXX 2 if doing p2p */
+		cmd.bcn_pass_time = 5; /* in 100mS usb.c, linux/rtlwifi */
+		break;
+	case RSU_PWR_OFF:
+		cmd.mode = R92S_PS_MODE_RADIOOFF;
+		break;
+	default:
+		device_printf(sc->sc_dev, "%s: unknown ps mode (%d)\n",
+		    __func__,
+		    state);
+		return (ENXIO);
+	}
+
+	RSU_DPRINTF(sc, RSU_DEBUG_RESET,
+	    "%s: setting ps mode to %d (mode %d)\n",
+	    __func__, state, cmd.mode);
+	error = rsu_fw_cmd(sc, R92S_CMD_SET_PWR_MODE, &cmd, sizeof(cmd));
+	if (error == 0)
+		sc->sc_curpwrstate = state;
+
+	return (error);
+}
+
 static int
 rsu_newstate(struct ieee80211vap *vap, enum ieee80211_state nstate, int arg)
 {
@@ -916,7 +1204,9 @@ rsu_newstate(struct ieee80211vap *vap, enum ieee80211_state nstate, int arg)
 	int error, startcal = 0;
 
 	ostate = vap->iv_state;
-	DPRINTF("%s -> %s\n", ieee80211_state_name[ostate],
+	RSU_DPRINTF(sc, RSU_DEBUG_STATE, "%s: %s -> %s\n",
+	    __func__,
+	    ieee80211_state_name[ostate],
 	    ieee80211_state_name[nstate]);
 
 	IEEE80211_UNLOCK(ic);
@@ -926,6 +1216,7 @@ rsu_newstate(struct ieee80211vap *vap, enum ieee80211_state nstate, int arg)
 		sc->sc_calibrating = 0;
 		RSU_UNLOCK(sc);
 		taskqueue_drain_timeout(taskqueue_thread, &sc->calib_task);
+		taskqueue_drain(taskqueue_thread, &sc->tx_task);
 		/* Disassociate from our current BSS. */
 		RSU_LOCK(sc);
 		rsu_disconnect(sc);
@@ -933,9 +1224,11 @@ rsu_newstate(struct ieee80211vap *vap, enum ieee80211_state nstate, int arg)
 		RSU_LOCK(sc);
 	switch (nstate) {
 	case IEEE80211_S_INIT:
+		(void) rsu_set_fw_power_state(sc, RSU_PWR_ACTIVE);
 		break;
 	case IEEE80211_S_AUTH:
 		ni = ieee80211_ref_node(vap->iv_bss);
+		(void) rsu_set_fw_power_state(sc, RSU_PWR_ACTIVE);
 		error = rsu_join_bss(sc, ni);
 		ieee80211_free_node(ni);
 		if (error != 0) {
@@ -948,6 +1241,7 @@ rsu_newstate(struct ieee80211vap *vap, enum ieee80211_state nstate, int arg)
 		rs = &ni->ni_rates;
 		/* Indicate highest supported rate. */
 		ni->ni_txrate = rs->rs_rates[rs->rs_nrates - 1];
+		(void) rsu_set_fw_power_state(sc, RSU_PWR_SLEEP);
 		ieee80211_free_node(ni);
 		startcal = 1;
 		break;
@@ -1008,6 +1302,9 @@ rsu_site_survey(struct rsu_softc *sc, struct ieee80211vap *vap)
 {
 	struct r92s_fw_cmd_sitesurvey cmd;
 	struct ieee80211com *ic = &sc->sc_ic;
+	int r;
+
+	RSU_ASSERT_LOCKED(sc);
 
 	memset(&cmd, 0, sizeof(cmd));
 	if ((ic->ic_flags & IEEE80211_F_ASCAN) || sc->sc_scan_pass == 1)
@@ -1021,7 +1318,11 @@ rsu_site_survey(struct rsu_softc *sc, struct ieee80211vap *vap)
 
 	}
 	DPRINTF("sending site survey command, pass=%d\n", sc->sc_scan_pass);
-	return (rsu_fw_cmd(sc, R92S_CMD_SITE_SURVEY, &cmd, sizeof(cmd)));
+	r = rsu_fw_cmd(sc, R92S_CMD_SITE_SURVEY, &cmd, sizeof(cmd));
+	if (r == 0) {
+		sc->sc_scanning = 1;
+	}
+	return (r);
 }
 
 static int
@@ -1036,10 +1337,33 @@ rsu_join_bss(struct rsu_softc *sc, struct ieee80211_node *ni)
 	uint8_t *frm;
 	uint8_t opmode;
 	int error;
+	int cnt;
+	char *msg = "rsujoin";
+
+	RSU_ASSERT_LOCKED(sc);
+
+	/*
+	 * Until net80211 scanning doesn't automatically finish
+	 * before we tell it to, let's just wait until any pending
+	 * scan is done.
+	 *
+	 * XXX TODO: yes, this releases and re-acquires the lock.
+	 * We should re-verify the state whenever we re-attempt this!
+	 */
+	cnt = 0;
+	while (sc->sc_scanning && cnt < 10) {
+		device_printf(sc->sc_dev,
+		    "%s: still scanning! (attempt %d)\n",
+		    __func__, cnt);
+		msleep(msg, &sc->sc_mtx, 0, msg, hz / 2);
+		cnt++;
+	}
 
 	/* Let the FW decide the opmode based on the capinfo field. */
 	opmode = NDIS802_11AUTOUNKNOWN;
-	DPRINTF("setting operating mode to %d\n", opmode);
+	RSU_DPRINTF(sc, RSU_DEBUG_RESET,
+	    "%s: setting operating mode to %d\n",
+	    __func__, opmode);
 	error = rsu_fw_cmd(sc, R92S_CMD_SET_OPMODE, &opmode, sizeof(opmode));
 	if (error != 0)
 		return (error);
@@ -1050,7 +1374,9 @@ rsu_join_bss(struct rsu_softc *sc, struct ieee80211_node *ni)
 		auth.dot1x = (ni->ni_authmode == IEEE80211_AUTH_8021X);
 	} else
 		auth.mode = R92S_AUTHMODE_OPEN;
-	DPRINTF("setting auth mode to %d\n", auth.mode);
+	RSU_DPRINTF(sc, RSU_DEBUG_RESET,
+	    "%s: setting auth mode to %d\n",
+	    __func__, auth.mode);
 	error = rsu_fw_cmd(sc, R92S_CMD_SET_AUTH, &auth, sizeof(auth));
 	if (error != 0)
 		return (error);
@@ -1071,6 +1397,7 @@ rsu_join_bss(struct rsu_softc *sc, struct ieee80211_node *ni)
 	bss->config.bintval = htole32(ni->ni_intval);
 	bss->config.dsconfig = htole32(ieee80211_chan2ieee(ic, ni->ni_chan));
 	bss->inframode = htole32(NDIS802_11INFRASTRUCTURE);
+	/* XXX verify how this is supposed to look! */
 	memcpy(bss->supprates, ni->ni_rates.rs_rates,
 	    ni->ni_rates.rs_nrates);
 	/* Write the fixed fields of the beacon frame. */
@@ -1083,11 +1410,18 @@ rsu_join_bss(struct rsu_softc *sc, struct ieee80211_node *ni)
 	frm = ieee80211_add_rsn(frm, vap);
 	frm = ieee80211_add_wpa(frm, vap);
 	frm = ieee80211_add_qos(frm, ni);
-	if (ni->ni_flags & IEEE80211_NODE_HT)
+	if ((ic->ic_flags & IEEE80211_F_WME) &&
+	    (ni->ni_ies.wme_ie != NULL))
+		frm = ieee80211_add_wme_info(frm, &ic->ic_wme);
+	if (ni->ni_flags & IEEE80211_NODE_HT) {
 		frm = ieee80211_add_htcap(frm, ni);
+		frm = ieee80211_add_htinfo(frm, ni);
+	}
 	bss->ieslen = htole32(frm - (uint8_t *)fixed);
 	bss->len = htole32(((frm - buf) + 3) & ~3);
-	DPRINTF("sending join bss command to %s chan %d\n",
+	RSU_DPRINTF(sc, RSU_DEBUG_RESET | RSU_DEBUG_FWCMD,
+	    "%s: sending join bss command to %s chan %d\n",
+	    __func__,
 	    ether_sprintf(bss->macaddr), le32toh(bss->config.dsconfig));
 	return (rsu_fw_cmd(sc, R92S_CMD_JOIN_BSS, buf, sizeof(buf)));
 }
@@ -1098,17 +1432,36 @@ rsu_disconnect(struct rsu_softc *sc)
 	uint32_t zero = 0;	/* :-) */
 
 	/* Disassociate from our current BSS. */
-	DPRINTF("sending disconnect command\n");
+	RSU_DPRINTF(sc, RSU_DEBUG_STATE | RSU_DEBUG_FWCMD,
+	    "%s: sending disconnect command\n", __func__);
 	return (rsu_fw_cmd(sc, R92S_CMD_DISCONNECT, &zero, sizeof(zero)));
 }
 
+/*
+ * Map the hardware provided RSSI value to a signal level.
+ * For the most part it's just something we divide by and cap
+ * so it doesn't overflow the representation by net80211.
+ */
+static int
+rsu_hwrssi_to_rssi(struct rsu_softc *sc, int hw_rssi)
+{
+	int v;
+
+	if (hw_rssi == 0)
+		return (0);
+	v = hw_rssi >> 4;
+	if (v > 80)
+		v = 80;
+	return (v);
+}
+
 static void
 rsu_event_survey(struct rsu_softc *sc, uint8_t *buf, int len)
 {
 	struct ieee80211com *ic = &sc->sc_ic;
 	struct ieee80211_frame *wh;
-	struct ieee80211_channel *c;
 	struct ndis_wlan_bssid_ex *bss;
+	struct ieee80211_rx_stats rxs;
 	struct mbuf *m;
 	int pktlen;
 
@@ -1118,13 +1471,17 @@ rsu_event_survey(struct rsu_softc *sc, uint8_t *buf, int len)
 	if (__predict_false(len < sizeof(*bss) + le32toh(bss->ieslen)))
 		return;
 
-	DPRINTFN(2, "found BSS %s: len=%d chan=%d inframode=%d "
-	    "networktype=%d privacy=%d\n",
+	RSU_DPRINTF(sc, RSU_DEBUG_SCAN,
+	    "%s: found BSS %s: len=%d chan=%d inframode=%d "
+	    "networktype=%d privacy=%d, RSSI=%d\n",
+	    __func__,
 	    ether_sprintf(bss->macaddr), le32toh(bss->len),
 	    le32toh(bss->config.dsconfig), le32toh(bss->inframode),
-	    le32toh(bss->networktype), le32toh(bss->privacy));
+	    le32toh(bss->networktype), le32toh(bss->privacy),
+	    le32toh(bss->rssi));
 
 	/* Build a fake beacon frame to let net80211 do all the parsing. */
+	/* XXX TODO: just call the new scan API methods! */
 	pktlen = sizeof(*wh) + le32toh(bss->ieslen);
 	if (__predict_false(pktlen > MCLBYTES))
 		return;
@@ -1144,17 +1501,20 @@ rsu_event_survey(struct rsu_softc *sc, uint8_t *buf, int len)
 
 	/* Finalize mbuf. */
 	m->m_pkthdr.len = m->m_len = pktlen;
-	/* Fix the channel. */
-	c = ieee80211_find_channel_byieee(ic, 
-	    le32toh(bss->config.dsconfig), 
-	    IEEE80211_CHAN_G);
-	if (c) {
-		ic->ic_curchan = c;
-		ieee80211_radiotap_chan_change(ic);
-	}
+
+	/* Set channel flags for input path */
+	bzero(&rxs, sizeof(rxs));
+	rxs.r_flags |= IEEE80211_R_IEEE | IEEE80211_R_FREQ;
+	rxs.r_flags |= IEEE80211_R_NF | IEEE80211_R_RSSI;
+	rxs.c_ieee = le32toh(bss->config.dsconfig);
+	rxs.c_freq = ieee80211_ieee2mhz(rxs.c_ieee, IEEE80211_CHAN_2GHZ);
+	/* This is a number from 0..100; so let's just divide it down a bit */
+	rxs.rssi = le32toh(bss->rssi) / 2;
+	rxs.nf = -96;
+
 	/* XXX avoid a LOR */
 	RSU_UNLOCK(sc);
-	ieee80211_input_all(ic, m, le32toh(bss->rssi), 0);
+	ieee80211_input_mimo_all(ic, m, &rxs);
 	RSU_LOCK(sc);
 }
 
@@ -1173,20 +1533,30 @@ rsu_event_join_bss(struct rsu_softc *sc, uint8_t *buf, int len)
 	rsp = (struct r92s_event_join_bss *)buf;
 	res = (int)le32toh(rsp->join_res);
 
-	DPRINTF("Rx join BSS event len=%d res=%d\n", len, res);
+	RSU_DPRINTF(sc, RSU_DEBUG_STATE | RSU_DEBUG_FWCMD,
+	    "%s: Rx join BSS event len=%d res=%d\n",
+	    __func__, len, res);
+
+	/*
+	 * XXX Don't do this; there's likely a better way to tell
+	 * the caller we failed.
+	 */
 	if (res <= 0) {
 		RSU_UNLOCK(sc);
 		ieee80211_new_state(vap, IEEE80211_S_SCAN, -1);
 		RSU_LOCK(sc);
 		return;
 	}
+
 	tmp = le32toh(rsp->associd);
 	if (tmp >= vap->iv_max_aid) {
 		DPRINTF("Assoc ID overflow\n");
 		tmp = 1;
 	}
-	DPRINTF("associated with %s associd=%d\n",
-	    ether_sprintf(rsp->bss.macaddr), tmp);
+	RSU_DPRINTF(sc, RSU_DEBUG_STATE | RSU_DEBUG_FWCMD,
+	    "%s: associated with %s associd=%d\n",
+	    __func__, ether_sprintf(rsp->bss.macaddr), tmp);
+	/* XXX is this required? What's the top two bits for again? */
 	ni->ni_associd = tmp | 0xc000;
 	RSU_UNLOCK(sc);
 	ieee80211_new_state(vap, IEEE80211_S_RUN,
@@ -1195,24 +1565,61 @@ rsu_event_join_bss(struct rsu_softc *sc, uint8_t *buf, int len)
 }
 
 static void
+rsu_event_addba_req_report(struct rsu_softc *sc, uint8_t *buf, int len)
+{
+	struct ieee80211com *ic = &sc->sc_ic;
+	struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
+	struct r92s_add_ba_event *ba = (void *) buf;
+	struct ieee80211_node *ni;
+
+	if (len < sizeof(*ba)) {
+		device_printf(sc->sc_dev, "%s: short read (%d)\n", __func__, len);
+		return;
+	}
+
+	if (vap == NULL)
+		return;
+
+	RSU_DPRINTF(sc, RSU_DEBUG_AMPDU, "%s: mac=%s, tid=%d, ssn=%d\n",
+	    __func__,
+	    ether_sprintf(ba->mac_addr),
+	    (int) ba->tid,
+	    (int) le16toh(ba->ssn));
+
+	/* XXX do node lookup; this is STA specific */
+
+	ni = ieee80211_ref_node(vap->iv_bss);
+	ieee80211_ampdu_rx_start_ext(ni, ba->tid, le16toh(ba->ssn) >> 4, 32);
+	ieee80211_free_node(ni);
+}
+
+static void
 rsu_rx_event(struct rsu_softc *sc, uint8_t code, uint8_t *buf, int len)
 {
 	struct ieee80211com *ic = &sc->sc_ic;
 	struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
 
-	DPRINTFN(4, "Rx event code=%d len=%d\n", code, len);
+	RSU_DPRINTF(sc, RSU_DEBUG_RX | RSU_DEBUG_FWCMD,
+	    "%s: Rx event code=%d len=%d\n", __func__, code, len);
 	switch (code) {
 	case R92S_EVT_SURVEY:
-		if (vap->iv_state == IEEE80211_S_SCAN)
-			rsu_event_survey(sc, buf, len);
+		rsu_event_survey(sc, buf, len);
 		break;
 	case R92S_EVT_SURVEY_DONE:
-		DPRINTF("site survey pass %d done, found %d BSS\n",
-		    sc->sc_scan_pass, le32toh(*(uint32_t *)buf));
+		RSU_DPRINTF(sc, RSU_DEBUG_SCAN,
+		    "%s: site survey pass %d done, found %d BSS\n",
+		    __func__, sc->sc_scan_pass, le32toh(*(uint32_t *)buf));
+		sc->sc_scanning = 0;
 		if (vap->iv_state != IEEE80211_S_SCAN)
 			break;	/* Ignore if not scanning. */
+
+		/*
+		 * XXX TODO: This needs to be done without a transition to
+		 * the SCAN state again.  Grr.
+		 */
 		if (sc->sc_scan_pass == 0 && vap->iv_des_nssid != 0) {
 			/* Schedule a directed scan for hidden APs. */
+			/* XXX bad! */
 			sc->sc_scan_pass = 1;
 			RSU_UNLOCK(sc);
 			ieee80211_new_state(vap, IEEE80211_S_SCAN, -1);
@@ -1225,11 +1632,10 @@ rsu_rx_event(struct rsu_softc *sc, uint8_t code, uint8_t *buf, int len)
 		if (vap->iv_state == IEEE80211_S_AUTH)
 			rsu_event_join_bss(sc, buf, len);
 		break;
-#if 0
-XXX This event is occurring regularly, possibly due to some power saving event
-XXX and disrupts the WLAN traffic. Disable for now.
 	case R92S_EVT_DEL_STA:
-		DPRINTF("disassociated from %s\n", ether_sprintf(buf));
+		RSU_DPRINTF(sc, RSU_DEBUG_FWCMD | RSU_DEBUG_STATE,
+		    "%s: disassociated from %s\n", __func__,
+		    ether_sprintf(buf));
 		if (vap->iv_state == IEEE80211_S_RUN &&
 		    IEEE80211_ADDR_EQ(vap->iv_bss->ni_bssid, buf)) {
 			RSU_UNLOCK(sc);
@@ -1237,17 +1643,19 @@ XXX and disrupts the WLAN traffic. Disable for now.
 			RSU_LOCK(sc);
 		}
 		break;
-#endif
 	case R92S_EVT_WPS_PBC:
-		DPRINTF("WPS PBC pushed.\n");
+		RSU_DPRINTF(sc, RSU_DEBUG_RX | RSU_DEBUG_FWCMD,
+		    "%s: WPS PBC pushed.\n", __func__);
 		break;
 	case R92S_EVT_FWDBG:
-		if (vap->iv_ifp->if_flags & IFF_DEBUG) {
-			buf[60] = '\0';
-			printf("FWDBG: %s\n", (char *)buf);
-		}
+		buf[60] = '\0';
+		RSU_DPRINTF(sc, RSU_DEBUG_FWDBG, "FWDBG: %s\n", (char *)buf);
+		break;
+	case R92S_EVT_ADDBA_REQ_REPORT:
+		rsu_event_addba_req_report(sc, buf, len);
 		break;
 	default:
+		device_printf(sc->sc_dev, "%s: unhandled code (%d)\n", __func__, code);
 		break;
 	}
 }
@@ -1258,7 +1666,7 @@ rsu_rx_multi_event(struct rsu_softc *sc, uint8_t *buf, int len)
 	struct r92s_fw_cmd_hdr *cmd;
 	int cmdsz;
 
-	DPRINTFN(6, "Rx events len=%d\n", len);
+	RSU_DPRINTF(sc, RSU_DEBUG_RX, "%s: Rx events len=%d\n", __func__, len);
 
 	/* Skip Rx status. */
 	buf += sizeof(struct r92s_rx_stat);
@@ -1285,6 +1693,7 @@ rsu_rx_multi_event(struct rsu_softc *sc, uint8_t *buf, int len)
 	}
 }
 
+#if 0
 static int8_t
 rsu_get_rssi(struct rsu_softc *sc, int rate, void *physt)
 {
@@ -1305,9 +1714,10 @@ rsu_get_rssi(struct rsu_softc *sc, int rate, void *physt)
 	}
 	return (rssi);
 }
+#endif
 
 static struct mbuf *
-rsu_rx_frame(struct rsu_softc *sc, uint8_t *buf, int pktlen, int *rssi)
+rsu_rx_frame(struct rsu_softc *sc, uint8_t *buf, int pktlen)
 {
 	struct ieee80211com *ic = &sc->sc_ic;
 	struct ieee80211_frame *wh;
@@ -1333,14 +1743,17 @@ rsu_rx_frame(struct rsu_softc *sc, uint8_t *buf, int pktlen, int *rssi)
 	rate = MS(rxdw3, R92S_RXDW3_RATE);
 	infosz = MS(rxdw0, R92S_RXDW0_INFOSZ) * 8;
 
+#if 0
 	/* Get RSSI from PHY status descriptor if present. */
 	if (infosz != 0)
 		*rssi = rsu_get_rssi(sc, rate, &stat[1]);
 	else
 		*rssi = 0;
+#endif
 
-	DPRINTFN(5, "Rx frame len=%d rate=%d infosz=%d rssi=%d\n",
-	    pktlen, rate, infosz, *rssi);
+	RSU_DPRINTF(sc, RSU_DEBUG_RX,
+	    "%s: Rx frame len=%d rate=%d infosz=%d\n",
+	    __func__, pktlen, rate, infosz);
 
 	m = m_get2(pktlen, M_NOWAIT, MT_DATA, M_PKTHDR);
 	if (__predict_false(m == NULL)) {
@@ -1382,7 +1795,11 @@ rsu_rx_frame(struct rsu_softc *sc, uint8_t *buf, int pktlen, int *rssi)
 			/* Bit 7 set means HT MCS instead of rate. */
 			tap->wr_rate = 0x80 | (rate - 12);
 		}
+#if 0
 		tap->wr_dbm_antsignal = *rssi;
+#endif
+		/* XXX not nice */
+		tap->wr_dbm_antsignal = rsu_hwrssi_to_rssi(sc, sc->sc_currssi);
 		tap->wr_chan_freq = htole16(ic->ic_curchan->ic_freq);
 		tap->wr_chan_flags = htole16(ic->ic_curchan->ic_flags);
 	}
@@ -1391,7 +1808,7 @@ rsu_rx_frame(struct rsu_softc *sc, uint8_t *buf, int pktlen, int *rssi)
 }
 
 static struct mbuf *
-rsu_rx_multi_frame(struct rsu_softc *sc, uint8_t *buf, int len, int *rssi)
+rsu_rx_multi_frame(struct rsu_softc *sc, uint8_t *buf, int len)
 {
 	struct r92s_rx_stat *stat;
 	uint32_t rxdw0;
@@ -1401,7 +1818,8 @@ rsu_rx_multi_frame(struct rsu_softc *sc, uint8_t *buf, int len, int *rssi)
 	/* Get the number of encapsulated frames. */
 	stat = (struct r92s_rx_stat *)buf;
 	npkts = MS(le32toh(stat->rxdw2), R92S_RXDW2_PKTCNT);
-	DPRINTFN(6, "Rx %d frames in one chunk\n", npkts);
+	RSU_DPRINTF(sc, RSU_DEBUG_RX,
+	    "%s: Rx %d frames in one chunk\n", __func__, npkts);
 
 	/* Process all of them. */
 	while (npkts-- > 0) {
@@ -1422,7 +1840,7 @@ rsu_rx_multi_frame(struct rsu_softc *sc, uint8_t *buf, int len, int *rssi)
 			break;
 
 		/* Process 802.11 frame. */
-		m = rsu_rx_frame(sc, buf, pktlen, rssi);
+		m = rsu_rx_frame(sc, buf, pktlen);
 		if (m0 == NULL)
 			m0 = m;
 		if (prevm == NULL)
@@ -1441,7 +1859,7 @@ rsu_rx_multi_frame(struct rsu_softc *sc, uint8_t *buf, int len, int *rssi)
 }
 
 static struct mbuf *
-rsu_rxeof(struct usb_xfer *xfer, struct rsu_data *data, int *rssi)
+rsu_rxeof(struct usb_xfer *xfer, struct rsu_data *data)
 {
 	struct rsu_softc *sc = data->sc;
 	struct ieee80211com *ic = &sc->sc_ic;
@@ -1462,7 +1880,7 @@ rsu_rxeof(struct usb_xfer *xfer, struct rsu_data *data, int *rssi)
 		/* No packets to process. */
 		return (NULL);
 	} else
-		return (rsu_rx_multi_frame(sc, data->buf, len, rssi));
+		return (rsu_rx_multi_frame(sc, data->buf, len));
 }
 
 static void
@@ -1474,7 +1892,6 @@ rsu_bulk_rx_callback(struct usb_xfer *xfer, usb_error_t error)
 	struct ieee80211_node *ni;
 	struct mbuf *m = NULL, *next;
 	struct rsu_data *data;
-	int rssi = 1;
 
 	RSU_ASSERT_LOCKED(sc);
 
@@ -1484,11 +1901,15 @@ rsu_bulk_rx_callback(struct usb_xfer *xfer, usb_error_t error)
 		if (data == NULL)
 			goto tr_setup;
 		STAILQ_REMOVE_HEAD(&sc->sc_rx_active, next);
-		m = rsu_rxeof(xfer, data, &rssi);
+		m = rsu_rxeof(xfer, data);
 		STAILQ_INSERT_TAIL(&sc->sc_rx_inactive, data, next);
 		/* FALLTHROUGH */
 	case USB_ST_SETUP:
 tr_setup:
+		/*
+		 * XXX TODO: if we have an mbuf list, but then
+		 * we hit data == NULL, what now?
+		 */
 		data = STAILQ_FIRST(&sc->sc_rx_inactive);
 		if (data == NULL) {
 			KASSERT(m == NULL, ("mbuf isn't NULL"));
@@ -1506,16 +1927,23 @@ tr_setup:
 		 */
 		RSU_UNLOCK(sc);
 		while (m != NULL) {
+			int rssi;
+
+			/* Cheat and get the last calibrated RSSI */
+			rssi = rsu_hwrssi_to_rssi(sc, sc->sc_currssi);
+
 			next = m->m_next;
 			m->m_next = NULL;
 			wh = mtod(m, struct ieee80211_frame *);
 			ni = ieee80211_find_rxnode(ic,
 			    (struct ieee80211_frame_min *)wh);
 			if (ni != NULL) {
-				(void)ieee80211_input(ni, m, rssi, 0);
+				if (ni->ni_flags & IEEE80211_NODE_HT)
+					m->m_flags |= M_AMPDU;
+				(void)ieee80211_input(ni, m, rssi, -96);
 				ieee80211_free_node(ni);
 			} else
-				(void)ieee80211_input_all(ic, m, rssi, 0);
+				(void)ieee80211_input_all(ic, m, rssi, -96);
 			m = next;
 		}
 		RSU_LOCK(sc);
@@ -1540,6 +1968,13 @@ tr_setup:
 static void
 rsu_txeof(struct usb_xfer *xfer, struct rsu_data *data)
 {
+#ifdef	USB_DEBUG
+	struct rsu_softc *sc = usbd_xfer_softc(xfer);
+#endif
+
+	RSU_DPRINTF(sc, RSU_DEBUG_TXDONE, "%s: called; data=%p\n",
+	    __func__,
+	    data);
 
 	if (data->m) {
 		/* XXX status? */
@@ -1564,22 +1999,27 @@ rsu_bulk_tx_callback_sub(struct usb_xfer *xfer, usb_error_t error,
 		data = STAILQ_FIRST(&sc->sc_tx_active[which]);
 		if (data == NULL)
 			goto tr_setup;
-		DPRINTF("transfer done %p\n", data);
+		RSU_DPRINTF(sc, RSU_DEBUG_TXDONE, "%s: transfer done %p\n",
+		    __func__, data);
 		STAILQ_REMOVE_HEAD(&sc->sc_tx_active[which], next);
 		rsu_txeof(xfer, data);
-		STAILQ_INSERT_TAIL(&sc->sc_tx_inactive, data, next);
+		rsu_freebuf(sc, data);
 		/* FALLTHROUGH */
 	case USB_ST_SETUP:
 tr_setup:
 		data = STAILQ_FIRST(&sc->sc_tx_pending[which]);
 		if (data == NULL) {
-			DPRINTF("empty pending queue sc %p\n", sc);
+			RSU_DPRINTF(sc, RSU_DEBUG_TXDONE,
+			    "%s: empty pending queue sc %p\n", __func__, sc);
 			return;
 		}
 		STAILQ_REMOVE_HEAD(&sc->sc_tx_pending[which], next);
 		STAILQ_INSERT_TAIL(&sc->sc_tx_active[which], data, next);
 		usbd_xfer_set_frame_data(xfer, 0, data->buf, data->buflen);
-		DPRINTF("submitting transfer %p\n", data);
+		RSU_DPRINTF(sc, RSU_DEBUG_TXDONE,
+		    "%s: submitting transfer %p\n",
+		    __func__,
+		    data);
 		usbd_transfer_submit(xfer);
 		break;
 	default:
@@ -1587,7 +2027,7 @@ tr_setup:
 		if (data != NULL) {
 			STAILQ_REMOVE_HEAD(&sc->sc_tx_active[which], next);
 			rsu_txeof(xfer, data);
-			STAILQ_INSERT_TAIL(&sc->sc_tx_inactive, data, next);
+			rsu_freebuf(sc, data);
 		}
 		counter_u64_add(ic->ic_oerrors, 1);
 
@@ -1597,20 +2037,52 @@ tr_setup:
 		}
 		break;
 	}
+
+	/*
+	 * XXX TODO: if the queue is low, flush out FF TX frames.
+	 * Remember to unlock the driver for now; net80211 doesn't
+	 * defer it for us.
+	 */
 }
 
 static void
 rsu_bulk_tx_callback_be_bk(struct usb_xfer *xfer, usb_error_t error)
 {
+	struct rsu_softc *sc = usbd_xfer_softc(xfer);
+
 	rsu_bulk_tx_callback_sub(xfer, error, RSU_BULK_TX_BE_BK);
+
+	/* This kicks the TX taskqueue */
+	rsu_start(sc);
 }
 
 static void
 rsu_bulk_tx_callback_vi_vo(struct usb_xfer *xfer, usb_error_t error)
 {
+	struct rsu_softc *sc = usbd_xfer_softc(xfer);
+
 	rsu_bulk_tx_callback_sub(xfer, error, RSU_BULK_TX_VI_VO);
+
+	/* This kicks the TX taskqueue */
+	rsu_start(sc);
+}
+
+static void
+rsu_bulk_tx_callback_h2c(struct usb_xfer *xfer, usb_error_t error)
+{
+	struct rsu_softc *sc = usbd_xfer_softc(xfer);
+
+	rsu_bulk_tx_callback_sub(xfer, error, RSU_BULK_TX_H2C);
+
+	/* This kicks the TX taskqueue */
+	rsu_start(sc);
 }
 
+/*
+ * Transmit the given frame.
+ *
+ * This doesn't free the node or mbuf upon failure.
+ */
 static int
 rsu_tx_start(struct rsu_softc *sc, struct ieee80211_node *ni, 
     struct mbuf *m0, struct rsu_data *data)
@@ -1621,27 +2093,47 @@ rsu_tx_start(struct rsu_softc *sc, struct ieee80211_node *ni,
 	struct ieee80211_key *k = NULL;
 	struct r92s_tx_desc *txd;
 	uint8_t type;
-	uint8_t tid = 0;
+	int prio = 0;
 	uint8_t which;
 	int hasqos;
 	int xferlen;
+	int qid;
 
 	RSU_ASSERT_LOCKED(sc);
 
 	wh = mtod(m0, struct ieee80211_frame *);
 	type = wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK;
 
+	RSU_DPRINTF(sc, RSU_DEBUG_TX, "%s: data=%p, m=%p\n",
+	    __func__, data, m0);
+
 	if (wh->i_fc[1] & IEEE80211_FC1_PROTECTED) {
 		k = ieee80211_crypto_encap(ni, m0);
 		if (k == NULL) {
 			device_printf(sc->sc_dev,
 			    "ieee80211_crypto_encap returns NULL.\n");
 			/* XXX we don't expect the fragmented frames */
-			m_freem(m0);
 			return (ENOBUFS);
 		}
 		wh = mtod(m0, struct ieee80211_frame *);
 	}
+	/* If we have QoS then use it */
+	/* XXX TODO: mbuf WME/PRI versus TID? */
+	if (IEEE80211_QOS_HAS_SEQ(wh)) {
+		/* Has QoS */
+		prio = M_WME_GETAC(m0);
+		which = rsu_wme_ac_xfer_map[prio];
+		hasqos = 1;
+	} else {
+		/* Non-QoS TID */
+		/* XXX TODO: tid=0 for non-qos TID? */
+		which = rsu_wme_ac_xfer_map[WME_AC_BE];
+		hasqos = 0;
+		prio = 0;
+	}
+
+	qid = rsu_ac2qid[prio];
+#if 0
 	switch (type) {
 	case IEEE80211_FC0_TYPE_CTL:
 	case IEEE80211_FC0_TYPE_MGT:
@@ -1652,6 +2144,13 @@ rsu_tx_start(struct rsu_softc *sc, struct ieee80211_node *ni,
 		break;
 	}
 	hasqos = 0;
+#endif
+
+	RSU_DPRINTF(sc, RSU_DEBUG_TX, "%s: pri=%d, which=%d, hasqos=%d\n",
+	    __func__,
+	    prio,
+	    which,
+	    hasqos);
 
 	/* Fill Tx descriptor. */
 	txd = (struct r92s_tx_desc *)data->buf;
@@ -1663,8 +2162,7 @@ rsu_tx_start(struct rsu_softc *sc, struct ieee80211_node *ni,
 	    R92S_TXDW0_OWN | R92S_TXDW0_FSG | R92S_TXDW0_LSG);
 
 	txd->txdw1 |= htole32(
-	    SM(R92S_TXDW1_MACID, R92S_MACID_BSS) |
-	    SM(R92S_TXDW1_QSEL, R92S_TXDW1_QSEL_BE));
+	    SM(R92S_TXDW1_MACID, R92S_MACID_BSS) | SM(R92S_TXDW1_QSEL, qid));
 	if (!hasqos)
 		txd->txdw1 |= htole32(R92S_TXDW1_NONQOS);
 #ifdef notyet
@@ -1687,14 +2185,15 @@ rsu_tx_start(struct rsu_softc *sc, struct ieee80211_node *ni,
 		    SM(R92S_TXDW1_KEYIDX, k->k_id));
 	}
 #endif
+	/* XXX todo: set AGGEN bit if appropriate? */
 	txd->txdw2 |= htole32(R92S_TXDW2_BK);
 	if (IEEE80211_IS_MULTICAST(wh->i_addr1))
 		txd->txdw2 |= htole32(R92S_TXDW2_BMCAST);
 	/*
 	 * Firmware will use and increment the sequence number for the
-	 * specified TID.
+	 * specified priority.
 	 */
-	txd->txdw3 |= htole32(SM(R92S_TXDW3_SEQ, tid));
+	txd->txdw3 |= htole32(SM(R92S_TXDW3_SEQ, prio));
 
 	if (ieee80211_radiotap_active_vap(vap)) {
 		struct rsu_tx_radiotap_header *tap = &sc->sc_txtap;
@@ -1704,6 +2203,7 @@ rsu_tx_start(struct rsu_softc *sc, struct ieee80211_node *ni,
 		tap->wt_chan_flags = htole16(ic->ic_curchan->ic_flags);
 		ieee80211_radiotap_tx(vap, m0);
 	}
+
 	xferlen = sizeof(*txd) + m0->m_pkthdr.len;
 	m_copydata(m0, 0, m0->m_pkthdr.len, (caddr_t)&txd[1]);
 
@@ -1728,19 +2228,45 @@ rsu_transmit(struct ieee80211com *ic, struct mbuf *m)
 		RSU_UNLOCK(sc);
 		return (ENXIO);
 	}
+
+	/*
+	 * XXX TODO: ensure that we treat 'm' as a list of frames
+	 * to transmit!
+	 */
 	error = mbufq_enqueue(&sc->sc_snd, m);
 	if (error) {
+		RSU_DPRINTF(sc, RSU_DEBUG_TX,
+		    "%s: mbufq_enable: failed (%d)\n",
+		    __func__,
+		    error);
 		RSU_UNLOCK(sc);
 		return (error);
 	}
-	rsu_start(sc);
 	RSU_UNLOCK(sc);
 
+	/* This kicks the TX taskqueue */
+	rsu_start(sc);
+
 	return (0);
 }
 
 static void
-rsu_start(struct rsu_softc *sc)
+rsu_drain_mbufq(struct rsu_softc *sc)
+{
+	struct mbuf *m;
+	struct ieee80211_node *ni;
+
+	RSU_ASSERT_LOCKED(sc);
+	while ((m = mbufq_dequeue(&sc->sc_snd)) != NULL) {
+		ni = (struct ieee80211_node *)m->m_pkthdr.rcvif;
+		m->m_pkthdr.rcvif = NULL;
+		ieee80211_free_node(ni);
+		m_freem(m);
+	}
+}
+
+static void
+_rsu_start(struct rsu_softc *sc)
 {
 	struct ieee80211_node *ni;
 	struct rsu_data *bf;
@@ -1751,6 +2277,8 @@ rsu_start(struct rsu_softc *sc)
 	while ((m = mbufq_dequeue(&sc->sc_snd)) != NULL) {
 		bf = rsu_getbuf(sc);
 		if (bf == NULL) {
+			RSU_DPRINTF(sc, RSU_DEBUG_TX,
+			    "%s: failed to get buffer\n", __func__);
 			mbufq_prepend(&sc->sc_snd, m);
 			break;
 		}
@@ -1759,16 +2287,26 @@ rsu_start(struct rsu_softc *sc)
 		m->m_pkthdr.rcvif = NULL;
 
 		if (rsu_tx_start(sc, ni, m, bf) != 0) {
+			RSU_DPRINTF(sc, RSU_DEBUG_TX,
+			    "%s: failed to transmit\n", __func__);
 			if_inc_counter(ni->ni_vap->iv_ifp,
 			    IFCOUNTER_OERRORS, 1);
-			STAILQ_INSERT_HEAD(&sc->sc_tx_inactive, bf, next);
+			rsu_freebuf(sc, bf);
 			ieee80211_free_node(ni);
+			m_freem(m);
 			break;
 		}
 	}
 }
 
 static void
+rsu_start(struct rsu_softc *sc)
+{
+
+	taskqueue_enqueue(taskqueue_thread, &sc->tx_task);
+}
+
+static void
 rsu_parent(struct ieee80211com *ic)
 {
 	struct rsu_softc *sc = ic->ic_softc;
@@ -1809,7 +2347,7 @@ rsu_power_on_acut(struct rsu_softc *sc)
 
 	rsu_write_1(sc, R92S_SPS1_CTRL,
 	    rsu_read_1(sc, R92S_SPS1_CTRL) | R92S_SPS1_LDEN);
-	usb_pause_mtx(&sc->sc_mtx, 2 * hz);
+	rsu_ms_delay(sc, 2000);
 	/* Enable switch regulator block. */
 	rsu_write_1(sc, R92S_SPS1_CTRL,
 	    rsu_read_1(sc, R92S_SPS1_CTRL) | R92S_SPS1_SWEN);
@@ -1881,7 +2419,7 @@ rsu_power_on_bcut(struct rsu_softc *sc)
 
 	/* Prevent eFuse leakage. */
 	rsu_write_1(sc, 0x37, 0xb0);
-	usb_pause_mtx(&sc->sc_mtx, hz / 100);
+	rsu_ms_delay(sc, 10);
 	rsu_write_1(sc, 0x37, 0x30);
 
 	/* Switch the control path to hardware. */
@@ -1892,7 +2430,7 @@ rsu_power_on_bcut(struct rsu_softc *sc)
 	}
 	rsu_write_1(sc, R92S_SYS_FUNC_EN + 1,
 	    rsu_read_1(sc, R92S_SYS_FUNC_EN + 1) & ~0x8c);
-	rsu_ms_delay(sc);
+	rsu_ms_delay(sc, 1);
 
 	rsu_write_1(sc, R92S_SPS0_CTRL + 1, 0x53);
 	rsu_write_1(sc, R92S_SPS0_CTRL + 0, 0x57);
@@ -1925,11 +2463,11 @@ rsu_power_on_bcut(struct rsu_softc *sc)
 	/* Enable AFE PLL macro block. */
 	reg = rsu_read_1(sc, R92S_AFE_PLL_CTRL);
 	rsu_write_1(sc, R92S_AFE_PLL_CTRL, reg | 0x11);
-	rsu_ms_delay(sc);
+	rsu_ms_delay(sc, 1);
 	rsu_write_1(sc, R92S_AFE_PLL_CTRL, reg | 0x51);
-	rsu_ms_delay(sc);
+	rsu_ms_delay(sc, 1);
 	rsu_write_1(sc, R92S_AFE_PLL_CTRL, reg | 0x11);
-	rsu_ms_delay(sc);
+	rsu_ms_delay(sc, 1);
 
 	/* Attach AFE PLL to MACTOP/BB. */
 	rsu_write_1(sc, R92S_SYS_ISO_CTRL,
@@ -1976,14 +2514,16 @@ rsu_power_on_bcut(struct rsu_softc *sc)
 		if ((reg & (R92S_TCR_IMEM_CHK_RPT | R92S_TCR_EMEM_CHK_RPT)) ==
 		    (R92S_TCR_IMEM_CHK_RPT | R92S_TCR_EMEM_CHK_RPT))
 			break;
-		rsu_ms_delay(sc);
+		rsu_ms_delay(sc, 1);
 	}
 	if (ntries == 20) {
-		DPRINTF("TxDMA is not ready\n");
+		RSU_DPRINTF(sc, RSU_DEBUG_RESET | RSU_DEBUG_TX,
+		    "%s: TxDMA is not ready\n",
+		    __func__);
 		/* Reset TxDMA. */
 		reg = rsu_read_1(sc, R92S_CR);
 		rsu_write_1(sc, R92S_CR, reg & ~R92S_CR_TXDMA_EN);
-		rsu_ms_delay(sc);
+		rsu_ms_delay(sc, 1);
 		rsu_write_1(sc, R92S_CR, reg | R92S_CR_TXDMA_EN);
 	}
 }
@@ -1993,7 +2533,7 @@ rsu_power_off(struct rsu_softc *sc)
 {
 	/* Turn RF off. */
 	rsu_write_1(sc, R92S_RF_CTRL, 0x00);
-	usb_pause_mtx(&sc->sc_mtx, hz / 200);
+	rsu_ms_delay(sc, 5);
 
 	/* Turn MAC off. */
 	/* Switch control path. */
@@ -2016,6 +2556,9 @@ rsu_power_off(struct rsu_softc *sc)
 	/* Disable 1.6V LDO. */
 	rsu_write_1(sc, R92S_SPS0_CTRL + 0, 0x56);
 	rsu_write_1(sc, R92S_SPS0_CTRL + 1, 0x43);
+
+	/* Firmware - tell it to switch things off */
+	(void) rsu_set_fw_power_state(sc, RSU_PWR_OFF);
 }
 
 static int
@@ -2041,7 +2584,9 @@ rsu_fw_loadsection(struct rsu_softc *sc, const uint8_t *buf, int len)
 		txd->txdw0 |= htole32(SM(R92S_TXDW0_PKTLEN, mlen));
 		memcpy(&txd[1], buf, mlen);
 		data->buflen = sizeof(*txd) + mlen;
-		DPRINTF("starting transfer %p\n", data);
+		RSU_DPRINTF(sc, RSU_DEBUG_TX | RSU_DEBUG_FW | RSU_DEBUG_RESET,
+		    "%s: starting transfer %p\n",
+		    __func__, data);
 		STAILQ_INSERT_TAIL(&sc->sc_tx_pending[which], data, next);
 		buf += mlen;
 		len -= mlen;
@@ -2055,6 +2600,7 @@ rsu_load_firmware(struct rsu_softc *sc)
 {
 	const struct r92s_fw_hdr *hdr;
 	struct r92s_fw_priv *dmem;
+	struct ieee80211com *ic = &sc->sc_ic;
 	const uint8_t *imem, *emem;
 	int imemsz, ememsz;
 	const struct firmware *fw;
@@ -2063,7 +2609,9 @@ rsu_load_firmware(struct rsu_softc *sc)
 	int ntries, error;
 
 	if (rsu_read_1(sc, R92S_TCR) & R92S_TCR_FWRDY) {
-		DPRINTF("Firmware already loaded\n");
+		RSU_DPRINTF(sc, RSU_DEBUG_ANY,
+		    "%s: Firmware already loaded\n",
+		    __func__);
 		return (0);
 	}
 
@@ -2122,7 +2670,7 @@ rsu_load_firmware(struct rsu_softc *sc)
 	}
 	/* Wait for load to complete. */
 	for (ntries = 0; ntries != 50; ntries++) {
-		usb_pause_mtx(&sc->sc_mtx, hz / 100);
+		rsu_ms_delay(sc, 10);
 		reg = rsu_read_1(sc, R92S_TCR);
 		if (reg & R92S_TCR_IMEM_CODE_DONE)
 			break;
@@ -2141,7 +2689,7 @@ rsu_load_firmware(struct rsu_softc *sc)
 	}
 	/* Wait for load to complete. */
 	for (ntries = 0; ntries != 50; ntries++) {
-		usb_pause_mtx(&sc->sc_mtx, hz / 100);
+		rsu_ms_delay(sc, 10);
 		reg = rsu_read_2(sc, R92S_TCR);
 		if (reg & R92S_TCR_EMEM_CODE_DONE)
 			break;
@@ -2171,7 +2719,7 @@ rsu_load_firmware(struct rsu_softc *sc)
 	for (ntries = 0; ntries < 100; ntries++) {
 		if (rsu_read_1(sc, R92S_TCR) & R92S_TCR_IMEM_RDY)
 			break;
-		rsu_ms_delay(sc);
+		rsu_ms_delay(sc, 1);
 	}
 	if (ntries == 100) {
 		device_printf(sc->sc_dev,
@@ -2184,14 +2732,19 @@ rsu_load_firmware(struct rsu_softc *sc)
 	dmem = __DECONST(struct r92s_fw_priv *, &hdr->priv);
 	memset(dmem, 0, sizeof(*dmem));
 	dmem->hci_sel = R92S_HCI_SEL_USB | R92S_HCI_SEL_8172;
-	dmem->nendpoints = 0;
-	dmem->rf_config = 0x12;	/* 1T2R */
+	dmem->nendpoints = sc->sc_nendpoints;
+	dmem->chip_version = sc->cut;
+	dmem->rf_config = sc->sc_rftype;
 	dmem->vcs_type = R92S_VCS_TYPE_AUTO;
 	dmem->vcs_mode = R92S_VCS_MODE_RTS_CTS;
-#ifdef notyet
-	dmem->bw40_en = (ic->ic_htcaps & IEEE80211_HTCAP_CBW20_40) != 0;
-#endif
-	dmem->turbo_mode = 1;
+	dmem->turbo_mode = 0;
+	dmem->bw40_en = !! (ic->ic_htcaps & IEEE80211_HTCAP_CHWIDTH40);
+	dmem->amsdu2ampdu_en = !! (sc->sc_ht);
+	dmem->ampdu_en = !! (sc->sc_ht);
+	dmem->agg_offload = !! (sc->sc_ht);
+	dmem->qos_en = 1;
+	dmem->ps_offload = 1;
+	dmem->lowpower_mode = 1;	/* XXX TODO: configurable? */
 	/* Load DMEM section. */
 	error = rsu_fw_loadsection(sc, (uint8_t *)dmem, sizeof(*dmem));
 	if (error != 0) {
@@ -2203,7 +2756,7 @@ rsu_load_firmware(struct rsu_softc *sc)
 	for (ntries = 0; ntries < 100; ntries++) {
 		if (rsu_read_1(sc, R92S_TCR) & R92S_TCR_DMEM_CODE_DONE)
 			break;
-		rsu_ms_delay(sc);
+		rsu_ms_delay(sc, 1);
 	}
 	if (ntries == 100) {
 		device_printf(sc->sc_dev, "timeout waiting for %s transfer\n",
@@ -2215,7 +2768,7 @@ rsu_load_firmware(struct rsu_softc *sc)
 	for (ntries = 0; ntries < 60; ntries++) {
 		if (!(rsu_read_1(sc, R92S_TCR) & R92S_TCR_FWRDY))
 			break;
-		rsu_ms_delay(sc);
+		rsu_ms_delay(sc, 1);
 	}
 	if (ntries == 60) {
 		device_printf(sc->sc_dev, 
@@ -2253,7 +2806,8 @@ rsu_raw_xmit(struct ieee80211_node *ni, struct mbuf *m,
 	}
 	if (rsu_tx_start(sc, ni, m, bf) != 0) {
 		ieee80211_free_node(ni);
-		STAILQ_INSERT_HEAD(&sc->sc_tx_inactive, bf, next);
+		m_freem(m);
+		rsu_freebuf(sc, bf);
 		RSU_UNLOCK(sc);
 		return (EIO);
 	}
@@ -2268,15 +2822,20 @@ rsu_init(struct rsu_softc *sc)
 	struct ieee80211com *ic = &sc->sc_ic;
 	struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
 	uint8_t macaddr[IEEE80211_ADDR_LEN];
-	struct r92s_set_pwr_mode cmd;
 	int error;
 	int i;
 
 	RSU_ASSERT_LOCKED(sc);
 
+	/* Ensure the mbuf queue is drained */
+	rsu_drain_mbufq(sc);
+
 	/* Init host async commands ring. */
 	sc->cmdq.cur = sc->cmdq.next = sc->cmdq.queued = 0;
 
+	/* Reset power management state. */
+	rsu_write_1(sc, R92S_USB_HRPWM, 0);
+
 	/* Power on adapter. */
 	if (sc->cut == 1)
 		rsu_power_on_acut(sc);
@@ -2317,9 +2876,11 @@ rsu_init(struct rsu_softc *sc)
 	rsu_write_region_1(sc, R92S_MACID, macaddr, IEEE80211_ADDR_LEN);
 
 	/* It really takes 1.5 seconds for the firmware to boot: */
-	usb_pause_mtx(&sc->sc_mtx, (3 * hz) / 2);
+	rsu_ms_delay(sc, 2000);
 
-	DPRINTF("setting MAC address to %s\n", ether_sprintf(macaddr));
+	RSU_DPRINTF(sc, RSU_DEBUG_RESET, "%s: setting MAC address to %s\n",
+	    __func__,
+	    ether_sprintf(macaddr));
 	error = rsu_fw_cmd(sc, R92S_CMD_SET_MAC_ADDRESS, macaddr,
 	    IEEE80211_ADDR_LEN);
 	if (error != 0) {
@@ -2327,40 +2888,20 @@ rsu_init(struct rsu_softc *sc)
 		goto fail;
 	}
 
-	rsu_write_1(sc, R92S_USB_HRPWM,
-	    R92S_USB_HRPWM_PS_ST_ACTIVE | R92S_USB_HRPWM_PS_ALL_ON);
+	/* Set PS mode fully active */
+	error = rsu_set_fw_power_state(sc, RSU_PWR_ACTIVE);
 
-	memset(&cmd, 0, sizeof(cmd));
-	cmd.mode = R92S_PS_MODE_ACTIVE;
-	DPRINTF("setting ps mode to %d\n", cmd.mode);
-	error = rsu_fw_cmd(sc, R92S_CMD_SET_PWR_MODE, &cmd, sizeof(cmd));
 	if (error != 0) {
 		device_printf(sc->sc_dev, "could not set PS mode\n");
 		goto fail;
 	}
 
-#if 0
-	if (ic->ic_htcaps & IEEE80211_HTCAP_CBW20_40) {
-		/* Enable 40MHz mode. */
-		error = rsu_fw_iocmd(sc,
-		    SM(R92S_IOCMD_CLASS, 0xf4) |
-		    SM(R92S_IOCMD_INDEX, 0x00) |
-		    SM(R92S_IOCMD_VALUE, 0x0007));
-		if (error != 0) {
-			device_printf(sc->sc_dev,
-			    "could not enable 40MHz mode\n");
-			goto fail;
-		}
-	}
-
-	/* Set default channel. */
-	ic->ic_bss->ni_chan = ic->ic_ibss_chan;
-#endif
 	sc->sc_scan_pass = 0;
 	usbd_transfer_start(sc->sc_xfer[RSU_BULK_RX]);
 
 	/* We're ready to go. */
 	sc->sc_running = 1;
+	sc->sc_scanning = 0;
 	return;
 fail:
 	/* Need to stop all failed transfers, if any */
@@ -2373,19 +2914,31 @@ rsu_stop(struct rsu_softc *sc)
 {
 	int i;
 
+	RSU_ASSERT_LOCKED(sc);
+
 	sc->sc_running = 0;
 	sc->sc_calibrating = 0;
 	taskqueue_cancel_timeout(taskqueue_thread, &sc->calib_task, NULL);
+	taskqueue_cancel(taskqueue_thread, &sc->tx_task, NULL);
 
 	/* Power off adapter. */
 	rsu_power_off(sc);
 
 	for (i = 0; i < RSU_N_TRANSFER; i++)
 		usbd_transfer_stop(sc->sc_xfer[i]);
+
+	/* Ensure the mbuf queue is drained */
+	rsu_drain_mbufq(sc);
 }
 
+/*
+ * Note: usb_pause_mtx() actually releases the mutex before calling pause(),
+ * which breaks any kind of driver serialisation.
+ */
 static void
-rsu_ms_delay(struct rsu_softc *sc)
+rsu_ms_delay(struct rsu_softc *sc, int ms)
 {
-	usb_pause_mtx(&sc->sc_mtx, hz / 1000);
+
+	//usb_pause_mtx(&sc->sc_mtx, hz / 1000);
+	DELAY(ms * 1000);
 }
diff --git a/sys/dev/usb/wlan/if_rsureg.h b/sys/dev/usb/wlan/if_rsureg.h
index 4e6cd884fab88..ddb29a36770ef 100644
--- a/sys/dev/usb/wlan/if_rsureg.h
+++ b/sys/dev/usb/wlan/if_rsureg.h
@@ -158,6 +158,20 @@
 	(((var) & ~field##_M) | SM(field, val))
 
 /*
+ * ROM field with RF config.
+ */
+enum {
+	RTL8712_RFCONFIG_1T = 0x10,
+	RTL8712_RFCONFIG_2T = 0x20,
+	RTL8712_RFCONFIG_1R = 0x01,
+	RTL8712_RFCONFIG_2R = 0x02,
+	RTL8712_RFCONFIG_1T1R = 0x11,
+	RTL8712_RFCONFIG_1T2R = 0x12,
+	RTL8712_RFCONFIG_TURBO = 0x92,
+	RTL8712_RFCONFIG_2T2R = 0x22
+};
+
+/*
  * Firmware image header.
  */
 struct r92s_fw_priv {
@@ -173,6 +187,7 @@ struct r92s_fw_priv {
 	uint8_t		chip_version;
 	uint16_t	custid;
 	uint8_t		rf_config;
+//0x11:  1T1R, 0x12: 1T2R, 0x92: 1T2R turbo, 0x22: 2T2R
 	uint8_t		nendpoints;
 	/* QWORD1 */
 	uint32_t	regulatory;
@@ -593,11 +608,20 @@ struct r92s_tx_desc {
 	uint16_t	reserved1;
 } __packed __aligned(4);
 
+struct r92s_add_ba_event {
+	uint8_t mac_addr[IEEE80211_ADDR_LEN];
+	uint16_t ssn;
+	uint8_t tid;
+};
+
+struct r92s_add_ba_req {
+	uint32_t tid;
+};
 
 /*
  * Driver definitions.
  */
-#define RSU_RX_LIST_COUNT	1
+#define RSU_RX_LIST_COUNT	100
 #define RSU_TX_LIST_COUNT	32
 
 #define RSU_HOST_CMD_RING_COUNT	32
@@ -700,6 +724,7 @@ enum {
 	RSU_BULK_RX,
 	RSU_BULK_TX_BE_BK,	/* = WME_AC_BE/BK */
 	RSU_BULK_TX_VI_VO,	/* = WME_AC_VI/VO */
+	RSU_BULK_TX_H2C,	/* H2C */
 	RSU_N_TRANSFER,
 };
 
@@ -714,7 +739,6 @@ struct rsu_data {
 
 struct rsu_vap {
 	struct ieee80211vap		vap;
-	struct ieee80211_beacon_offsets bo;
 
 	int				(*newstate)(struct ieee80211vap *,
 					    enum ieee80211_state, int);
@@ -734,13 +758,22 @@ struct rsu_softc {
 					    enum ieee80211_state, int);
 	struct usbd_interface		*sc_iface;
 	struct timeout_task		calib_task;
+	struct task			tx_task;
 	const uint8_t			*qid2idx;
 	struct mtx			sc_mtx;
+	int				sc_ht;
+	int				sc_nendpoints;
+	int				sc_curpwrstate;
+	int				sc_currssi;
 
 	u_int				sc_running:1,
 					sc_calibrating:1,
+					sc_scanning:1,
 					sc_scan_pass:1;
 	u_int				cut;
+	uint8_t				sc_rftype;
+	int8_t				sc_nrxstream;
+	int8_t				sc_ntxstream;
 	struct rsu_host_cmd_ring	cmdq;
 	struct rsu_data			sc_rx[RSU_RX_LIST_COUNT];
 	struct rsu_data			sc_tx[RSU_TX_LIST_COUNT];
diff --git a/sys/dev/usb/wlan/if_run.c b/sys/dev/usb/wlan/if_run.c
index b787c03429345..c8707c4b6427a 100644
--- a/sys/dev/usb/wlan/if_run.c
+++ b/sys/dev/usb/wlan/if_run.c
@@ -1730,7 +1730,7 @@ run_read_eeprom(struct run_softc *sc)
 
 	/* read ROM version */
 	run_srom_read(sc, RT2860_EEPROM_VERSION, &val);
-	DPRINTF("EEPROM rev=%d, FAE=%d\n", val & 0xff, val >> 8);
+	DPRINTF("EEPROM rev=%d, FAE=%d\n", val >> 8, val & 0xff);
 
 	/* read MAC address */
 	run_srom_read(sc, RT2860_EEPROM_MAC01, &val);
@@ -4811,13 +4811,13 @@ static void
 run_update_beacon(struct ieee80211vap *vap, int item)
 {
 	struct ieee80211com *ic = vap->iv_ic;
+	struct ieee80211_beacon_offsets *bo = &vap->iv_bcn_off;
+	struct ieee80211_node *ni = vap->iv_bss;
 	struct run_softc *sc = ic->ic_softc;
 	struct run_vap *rvp = RUN_VAP(vap);
 	int mcast = 0;
 	uint32_t i;
 
-	KASSERT(vap != NULL, ("no beacon"));
-
 	switch (item) {
 	case IEEE80211_BEACON_ERP:
 		run_updateslot(ic);
@@ -4832,14 +4832,13 @@ run_update_beacon(struct ieee80211vap *vap, int item)
 		break;
 	}
 
-	setbit(rvp->bo.bo_flags, item);
+	setbit(bo->bo_flags, item);
 	if (rvp->beacon_mbuf == NULL) {
-		rvp->beacon_mbuf = ieee80211_beacon_alloc(vap->iv_bss,
-		    &rvp->bo);
+		rvp->beacon_mbuf = ieee80211_beacon_alloc(ni, bo);
 		if (rvp->beacon_mbuf == NULL)
 			return;
 	}
-	ieee80211_beacon_update(vap->iv_bss, &rvp->bo, rvp->beacon_mbuf, mcast);
+	ieee80211_beacon_update(ni, bo, rvp->beacon_mbuf, mcast);
 
 	i = RUN_CMDQ_GET(&sc->cmdq_store);
 	DPRINTF("cmdq_store=%d\n", i);
@@ -4854,6 +4853,7 @@ static void
 run_update_beacon_cb(void *arg)
 {
 	struct ieee80211vap *vap = arg;
+	struct ieee80211_node *ni = vap->iv_bss;
 	struct run_vap *rvp = RUN_VAP(vap);
 	struct ieee80211com *ic = vap->iv_ic;
 	struct run_softc *sc = ic->ic_softc;
@@ -4862,7 +4862,7 @@ run_update_beacon_cb(void *arg)
 	uint16_t txwisize;
 	uint8_t ridx;
 
-	if (vap->iv_bss->ni_chan == IEEE80211_CHAN_ANYC)
+	if (ni->ni_chan == IEEE80211_CHAN_ANYC)
 		return;
 	if (ic->ic_bsschan == IEEE80211_CHAN_ANYC)
 		return;
@@ -4872,8 +4872,8 @@ run_update_beacon_cb(void *arg)
 	 * is taking care of apropriate calls.
 	 */
 	if (rvp->beacon_mbuf == NULL) {
-		rvp->beacon_mbuf = ieee80211_beacon_alloc(vap->iv_bss,
-		    &rvp->bo);
+		rvp->beacon_mbuf = ieee80211_beacon_alloc(ni,
+		    &vap->iv_bcn_off);
 		if (rvp->beacon_mbuf == NULL)
 			return;
 	}
diff --git a/sys/dev/usb/wlan/if_runvar.h b/sys/dev/usb/wlan/if_runvar.h
index 71e29ba1e65c3..bbefbb0bde396 100644
--- a/sys/dev/usb/wlan/if_runvar.h
+++ b/sys/dev/usb/wlan/if_runvar.h
@@ -119,7 +119,6 @@ struct run_cmdq {
 
 struct run_vap {
 	struct ieee80211vap             vap;
-	struct ieee80211_beacon_offsets bo;
 	struct mbuf			*beacon_mbuf;
 
 	int                             (*newstate)(struct ieee80211vap *,
diff --git a/sys/dev/usb/wlan/if_uath.c b/sys/dev/usb/wlan/if_uath.c
index 04e092040c145..39fa71b286388 100644
--- a/sys/dev/usb/wlan/if_uath.c
+++ b/sys/dev/usb/wlan/if_uath.c
@@ -102,6 +102,7 @@ __FBSDID("$FreeBSD$");
 #endif
 
 #include <net80211/ieee80211_var.h>
+#include <net80211/ieee80211_input.h>
 #include <net80211/ieee80211_regdomain.h>
 #include <net80211/ieee80211_radiotap.h>
 
@@ -156,15 +157,6 @@ enum {
 } while (0)
 #endif
 
-/* unaligned little endian access */
-#define LE_READ_2(p)							\
-	((u_int16_t)							\
-	 ((((u_int8_t *)(p))[0]      ) | (((u_int8_t *)(p))[1] <<  8)))
-#define LE_READ_4(p)							\
-	((u_int32_t)							\
-	 ((((u_int8_t *)(p))[0]      ) | (((u_int8_t *)(p))[1] <<  8) |	\
-	  (((u_int8_t *)(p))[2] << 16) | (((u_int8_t *)(p))[3] << 24)))
-
 /* recognized device vendors/products */
 static const STRUCT_USB_HOST_ID uath_devs[] = {
 #define	UATH_DEV(v,p) { USB_VP(USB_VENDOR_##v, USB_PRODUCT_##v##_##p) }
@@ -599,7 +591,6 @@ uath_dump_cmd(const uint8_t *buf, int len, char prefix)
 static const char *
 uath_codename(int code)
 {
-#define	N(a)	(sizeof(a)/sizeof(a[0]))
 	static const char *names[] = {
 	    "0x00",
 	    "HOST_AVAILABLE",
@@ -662,13 +653,12 @@ uath_codename(int code)
 	};
 	static char buf[8];
 
-	if (code < N(names))
+	if (code < nitems(names))
 		return names[code];
 	if (code == WDCMSG_SET_DEFAULT_KEY)
 		return "SET_DEFAULT_KEY";
 	snprintf(buf, sizeof(buf), "0x%02x", code);
 	return buf;
-#undef N
 }
 #endif
 
@@ -1486,7 +1476,7 @@ uath_wme_init(struct uath_softc *sc)
 		qinfo.attr.aifs		= htobe32(uath_wme_11g[ac].aifsn);
 		qinfo.attr.logcwmin	= htobe32(uath_wme_11g[ac].logcwmin);
 		qinfo.attr.logcwmax	= htobe32(uath_wme_11g[ac].logcwmax);
-		qinfo.attr.bursttime	= htobe32(UATH_TXOP_TO_US(
+		qinfo.attr.bursttime	= htobe32(IEEE80211_TXOP_TO_US(
 					    uath_wme_11g[ac].txop));
 		qinfo.attr.mode		= htobe32(uath_wme_11g[ac].acm);/*XXX? */
 		qinfo.attr.qflags	= htobe32(1);	/* XXX? */
diff --git a/sys/dev/usb/wlan/if_uathvar.h b/sys/dev/usb/wlan/if_uathvar.h
index 56b5ba35a2b66..4b4d9fc36dc4f 100644
--- a/sys/dev/usb/wlan/if_uathvar.h
+++ b/sys/dev/usb/wlan/if_uathvar.h
@@ -102,7 +102,6 @@ struct uath_wme_settings {
 	uint8_t				logcwmin;
 	uint8_t				logcwmax;
 	uint16_t			txop;
-#define	UATH_TXOP_TO_US(txop)		((txop) << 5)
 	uint8_t				acm;
 };
 
diff --git a/sys/dev/usb/wlan/if_ural.c b/sys/dev/usb/wlan/if_ural.c
index e8224ba3f1498..bff4de54d6671 100644
--- a/sys/dev/usb/wlan/if_ural.c
+++ b/sys/dev/usb/wlan/if_ural.c
@@ -566,10 +566,7 @@ ural_vap_create(struct ieee80211com *ic, const char name[IFNAMSIZ], int unit,
 
 	if (!TAILQ_EMPTY(&ic->ic_vaps))		/* only one at a time */
 		return NULL;
-	uvp = (struct ural_vap *) malloc(sizeof(struct ural_vap),
-	    M_80211_VAP, M_NOWAIT | M_ZERO);
-	if (uvp == NULL)
-		return NULL;
+	uvp = malloc(sizeof(struct ural_vap), M_80211_VAP, M_WAITOK | M_ZERO);
 	vap = &uvp->vap;
 	/* enable s/w bmiss handling for sta mode */
 
@@ -716,7 +713,7 @@ ural_newstate(struct ieee80211vap *vap, enum ieee80211_state nstate, int arg)
 
 		if (vap->iv_opmode == IEEE80211_M_HOSTAP ||
 		    vap->iv_opmode == IEEE80211_M_IBSS) {
-			m = ieee80211_beacon_alloc(ni, &uvp->bo);
+			m = ieee80211_beacon_alloc(ni, &vap->iv_bcn_off);
 			if (m == NULL) {
 				device_printf(sc->sc_dev,
 				    "could not allocate beacon\n");
@@ -1932,7 +1929,6 @@ ural_read_eeprom(struct ural_softc *sc)
 static int
 ural_bbp_init(struct ural_softc *sc)
 {
-#define N(a)	((int)(sizeof (a) / sizeof ((a)[0])))
 	int i, ntries;
 
 	/* wait for BBP to be ready */
@@ -1948,7 +1944,7 @@ ural_bbp_init(struct ural_softc *sc)
 	}
 
 	/* initialize BBP registers to default values */
-	for (i = 0; i < N(ural_def_bbp); i++)
+	for (i = 0; i < nitems(ural_def_bbp); i++)
 		ural_bbp_write(sc, ural_def_bbp[i].reg, ural_def_bbp[i].val);
 
 #if 0
@@ -1961,7 +1957,6 @@ ural_bbp_init(struct ural_softc *sc)
 #endif
 
 	return 0;
-#undef N
 }
 
 static void
@@ -2016,7 +2011,6 @@ ural_set_rxantenna(struct ural_softc *sc, int antenna)
 static void
 ural_init(struct ural_softc *sc)
 {
-#define N(a)	((int)(sizeof (a) / sizeof ((a)[0])))
 	struct ieee80211com *ic = &sc->sc_ic;
 	struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
 	uint16_t tmp;
@@ -2030,7 +2024,7 @@ ural_init(struct ural_softc *sc)
 	ural_stop(sc);
 
 	/* initialize MAC registers to default values */
-	for (i = 0; i < N(ural_def_mac); i++)
+	for (i = 0; i < nitems(ural_def_mac); i++)
 		ural_write(sc, ural_def_mac[i].reg, ural_def_mac[i].val);
 
 	/* wait for BBP and RF to wake up (this can take a long time!) */
@@ -2089,7 +2083,6 @@ ural_init(struct ural_softc *sc)
 	return;
 
 fail:	ural_stop(sc);
-#undef N
 }
 
 static void
diff --git a/sys/dev/usb/wlan/if_uralvar.h b/sys/dev/usb/wlan/if_uralvar.h
index f40f71d9de08f..5461e87a75c58 100644
--- a/sys/dev/usb/wlan/if_uralvar.h
+++ b/sys/dev/usb/wlan/if_uralvar.h
@@ -73,7 +73,7 @@ typedef STAILQ_HEAD(, ural_tx_data) ural_txdhead;
 
 struct ural_vap {
 	struct ieee80211vap		vap;
-	struct ieee80211_beacon_offsets	bo;
+
 	struct usb_callout		ratectl_ch;
 	struct task			ratectl_task;
 
diff --git a/sys/dev/usb/wlan/if_urtw.c b/sys/dev/usb/wlan/if_urtw.c
index f0d2674e56e73..040ba867df305 100644
--- a/sys/dev/usb/wlan/if_urtw.c
+++ b/sys/dev/usb/wlan/if_urtw.c
@@ -1093,7 +1093,6 @@ fail:
 static usb_error_t
 urtw_adapter_start_b(struct urtw_softc *sc)
 {
-#define N(a)	((int)(sizeof(a) / sizeof((a)[0])))
 	uint8_t data8;
 	usb_error_t error;
 
@@ -1176,7 +1175,6 @@ urtw_adapter_start_b(struct urtw_softc *sc)
 	urtw_write16_m(sc, 0x1ec, 0x800);	/* RX MAX SIZE */
 fail:
 	return (error);
-#undef N
 }
 
 static usb_error_t
@@ -1914,31 +1912,27 @@ fail:
 static uint16_t
 urtw_rate2rtl(uint32_t rate)
 {
-#define N(a)	((int)(sizeof(a) / sizeof((a)[0])))
 	int i;
 
-	for (i = 0; i < N(urtw_ratetable); i++) {
+	for (i = 0; i < nitems(urtw_ratetable); i++) {
 		if (rate == urtw_ratetable[i].reg)
 			return urtw_ratetable[i].val;
 	}
 
 	return (3);
-#undef N
 }
 
 static uint16_t
 urtw_rtl2rate(uint32_t rate)
 {
-#define N(a)	((int)(sizeof(a) / sizeof((a)[0])))
 	int i;
 
-	for (i = 0; i < N(urtw_ratetable); i++) {
+	for (i = 0; i < nitems(urtw_ratetable); i++) {
 		if (rate == urtw_ratetable[i].val)
 			return urtw_ratetable[i].reg;
 	}
 
 	return (0);
-#undef N
 }
 
 static usb_error_t
@@ -2458,7 +2452,6 @@ fail:
 static usb_error_t
 urtw_8225_rf_init(struct urtw_softc *sc)
 {
-#define N(a)	((int)(sizeof(a) / sizeof((a)[0])))
 	int i;
 	uint16_t data;
 	usb_error_t error;
@@ -2489,7 +2482,7 @@ urtw_8225_rf_init(struct urtw_softc *sc)
 		goto fail;
 	usb_pause_mtx(&sc->sc_mtx, 1000);
 
-	for (i = 0; i < N(urtw_8225_rf_part1); i++) {
+	for (i = 0; i < nitems(urtw_8225_rf_part1); i++) {
 		urtw_8225_write(sc, urtw_8225_rf_part1[i].reg,
 		    urtw_8225_rf_part1[i].val);
 		usb_pause_mtx(&sc->sc_mtx, 1);
@@ -2521,7 +2514,7 @@ urtw_8225_rf_init(struct urtw_softc *sc)
 		usb_pause_mtx(&sc->sc_mtx, 1);
 	}
 
-	for (i = 0; i < N(urtw_8225_rf_part2); i++) {
+	for (i = 0; i < nitems(urtw_8225_rf_part2); i++) {
 		urtw_8187_write_phy_ofdm(sc, urtw_8225_rf_part2[i].reg,
 		    urtw_8225_rf_part2[i].val);
 		usb_pause_mtx(&sc->sc_mtx, 1);
@@ -2531,7 +2524,7 @@ urtw_8225_rf_init(struct urtw_softc *sc)
 	if (error)
 		goto fail;
 
-	for (i = 0; i < N(urtw_8225_rf_part3); i++) {
+	for (i = 0; i < nitems(urtw_8225_rf_part3); i++) {
 		urtw_8187_write_phy_cck(sc, urtw_8225_rf_part3[i].reg,
 		    urtw_8225_rf_part3[i].val);
 		usb_pause_mtx(&sc->sc_mtx, 1);
@@ -2557,7 +2550,6 @@ urtw_8225_rf_init(struct urtw_softc *sc)
 	error = urtw_8225_rf_set_chan(sc, 1);
 fail:
 	return (error);
-#undef N
 }
 
 static usb_error_t
@@ -2841,7 +2833,6 @@ fail:
 static usb_error_t
 urtw_8225v2_rf_init(struct urtw_softc *sc)
 {
-#define N(a)	((int)(sizeof(a) / sizeof((a)[0])))
 	int i;
 	uint16_t data;
 	uint32_t data32;
@@ -2874,7 +2865,7 @@ urtw_8225v2_rf_init(struct urtw_softc *sc)
 
 	usb_pause_mtx(&sc->sc_mtx, 500);
 
-	for (i = 0; i < N(urtw_8225v2_rf_part1); i++) {
+	for (i = 0; i < nitems(urtw_8225v2_rf_part1); i++) {
 		urtw_8225_write(sc, urtw_8225v2_rf_part1[i].reg,
 		    urtw_8225v2_rf_part1[i].val);
 	}
@@ -2929,7 +2920,7 @@ urtw_8225v2_rf_init(struct urtw_softc *sc)
 		urtw_8187_write_phy_ofdm(sc, 0xa, (uint8_t)i + 0x80);
 	}
 
-	for (i = 0; i < N(urtw_8225v2_rf_part2); i++) {
+	for (i = 0; i < nitems(urtw_8225v2_rf_part2); i++) {
 		urtw_8187_write_phy_ofdm(sc, urtw_8225v2_rf_part2[i].reg,
 		    urtw_8225v2_rf_part2[i].val);
 	}
@@ -2938,7 +2929,7 @@ urtw_8225v2_rf_init(struct urtw_softc *sc)
 	if (error)
 		goto fail;
 
-	for (i = 0; i < N(urtw_8225v2_rf_part3); i++) {
+	for (i = 0; i < nitems(urtw_8225v2_rf_part3); i++) {
 		urtw_8187_write_phy_cck(sc, urtw_8225v2_rf_part3[i].reg,
 		    urtw_8225v2_rf_part3[i].val);
 	}
@@ -2961,7 +2952,6 @@ urtw_8225v2_rf_init(struct urtw_softc *sc)
 	error = urtw_8225_rf_set_chan(sc, 1);
 fail:
 	return (error);
-#undef N
 }
 
 static usb_error_t
@@ -3329,7 +3319,6 @@ urtw_8225v2b_rf_init(struct urtw_softc *sc)
 
 fail:
 	return (error);
-#undef N
 }
 
 static usb_error_t
diff --git a/sys/dev/usb/wlan/if_urtwn.c b/sys/dev/usb/wlan/if_urtwn.c
index 457a88526516b..b1e3804ea889c 100644
--- a/sys/dev/usb/wlan/if_urtwn.c
+++ b/sys/dev/usb/wlan/if_urtwn.c
@@ -24,6 +24,8 @@ __FBSDID("$FreeBSD$");
  * Driver for Realtek RTL8188CE-VAU/RTL8188CUS/RTL8188EU/RTL8188RU/RTL8192CU.
  */
 
+#include "opt_wlan.h"
+
 #include <sys/param.h>
 #include <sys/sockio.h>
 #include <sys/sysctl.h>
@@ -61,6 +63,7 @@ __FBSDID("$FreeBSD$");
 #include <netinet/ip.h>
 
 #include <net80211/ieee80211_var.h>
+#include <net80211/ieee80211_input.h>
 #include <net80211/ieee80211_regdomain.h>
 #include <net80211/ieee80211_radiotap.h>
 #include <net80211/ieee80211_ratectl.h>
@@ -83,8 +86,7 @@ SYSCTL_INT(_hw_usb_urtwn, OID_AUTO, debug, CTLFLAG_RWTUN, &urtwn_debug, 0,
 #endif
 
 #define	URTWN_RSSI(r)  (r) - 110
-#define	IEEE80211_HAS_ADDR4(wh)	\
-	(((wh)->i_fc[1] & IEEE80211_FC1_DIR_MASK) == IEEE80211_FC1_DIR_DSTODS)
+#define	IEEE80211_HAS_ADDR4(wh)	IEEE80211_IS_DSTODS(wh)
 
 /* various supported device vendors/products */
 static const STRUCT_USB_HOST_ID urtwn_devs[] = {
@@ -169,6 +171,7 @@ static device_detach_t	urtwn_detach;
 static usb_callback_t   urtwn_bulk_tx_callback;
 static usb_callback_t	urtwn_bulk_rx_callback;
 
+static void		urtwn_drain_mbufq(struct urtwn_softc *sc);
 static usb_error_t	urtwn_do_request(struct urtwn_softc *,
 			    struct usb_device_request *, void *);
 static struct ieee80211vap *urtwn_vap_create(struct ieee80211com *,
@@ -467,6 +470,20 @@ detach:
 	return (ENXIO);			/* failure */
 }
 
+static void
+urtwn_drain_mbufq(struct urtwn_softc *sc)
+{
+	struct mbuf *m;
+	struct ieee80211_node *ni;
+	URTWN_ASSERT_LOCKED(sc);
+	while ((m = mbufq_dequeue(&sc->sc_snd)) != NULL) {
+		ni = (struct ieee80211_node *)m->m_pkthdr.rcvif;
+		m->m_pkthdr.rcvif = NULL;
+		ieee80211_free_node(ni);
+		m_freem(m);
+	}
+}
+
 static int
 urtwn_detach(device_t self)
 {
@@ -482,6 +499,9 @@ urtwn_detach(device_t self)
 
 	callout_drain(&sc->sc_watchdog_ch);
 
+	/* stop all USB transfers */
+	usbd_transfer_unsetup(sc->sc_xfer, URTWN_N_TRANSFER);
+
 	/* Prevent further allocations from RX/TX data lists. */
 	URTWN_LOCK(sc);
 	STAILQ_INIT(&sc->sc_tx_active);
@@ -502,10 +522,7 @@ urtwn_detach(device_t self)
 	urtwn_free_rx_list(sc);
 	URTWN_UNLOCK(sc);
 
-	/* stop all USB transfers */
-	usbd_transfer_unsetup(sc->sc_xfer, URTWN_N_TRANSFER);
 	ieee80211_ifdetach(ic);
-	mbufq_drain(&sc->sc_snd);
 	mtx_destroy(&sc->sc_mtx);
 
 	return (0);
@@ -879,13 +896,12 @@ tr_setup:
 		data = STAILQ_FIRST(&sc->sc_tx_pending);
 		if (data == NULL) {
 			DPRINTF("%s: empty pending queue\n", __func__);
-			return;
+			goto finish;
 		}
 		STAILQ_REMOVE_HEAD(&sc->sc_tx_pending, next);
 		STAILQ_INSERT_TAIL(&sc->sc_tx_active, data, next);
 		usbd_xfer_set_frame_data(xfer, 0, data->buf, data->buflen);
 		usbd_transfer_submit(xfer);
-		urtwn_start(sc);
 		break;
 	default:
 		data = STAILQ_FIRST(&sc->sc_tx_active);
@@ -903,6 +919,9 @@ tr_setup:
 		}
 		break;
 	}
+finish:
+	/* Kick-start more transmit */
+	urtwn_start(sc);
 }
 
 static struct urtwn_data *
@@ -1778,7 +1797,6 @@ urtwn_tx_start(struct urtwn_softc *sc, struct ieee80211_node *ni,
 			device_printf(sc->sc_dev,
 			    "ieee80211_crypto_encap returns NULL.\n");
 			/* XXX we don't expect the fragmented frames */
-			m_freem(m0);
 			return (ENOBUFS);
 		}
 
@@ -1931,6 +1949,7 @@ urtwn_start(struct urtwn_softc *sc)
 			if_inc_counter(ni->ni_vap->iv_ifp,
 			    IFCOUNTER_OERRORS, 1);
 			STAILQ_INSERT_HEAD(&sc->sc_tx_inactive, bf, next);
+			m_freem(m);
 			ieee80211_free_node(ni);
 			break;
 		}
@@ -3417,6 +3436,8 @@ urtwn_stop(struct urtwn_softc *sc)
 	sc->sc_flags &= ~URTWN_RUNNING;
 	callout_stop(&sc->sc_watchdog_ch);
 	urtwn_abort_xfers(sc);
+
+	urtwn_drain_mbufq(sc);
 }
 
 static void
@@ -3455,14 +3476,15 @@ urtwn_raw_xmit(struct ieee80211_node *ni, struct mbuf *m,
 	}
 
 	if (urtwn_tx_start(sc, ni, m, bf) != 0) {
+		m_freem(m);
 		ieee80211_free_node(ni);
 		STAILQ_INSERT_HEAD(&sc->sc_tx_inactive, bf, next);
 		URTWN_UNLOCK(sc);
 		return (EIO);
 	}
+	sc->sc_txtimer = 5;
 	URTWN_UNLOCK(sc);
 
-	sc->sc_txtimer = 5;
 	return (0);
 }
 
diff --git a/sys/dev/usb/wlan/if_urtwnreg.h b/sys/dev/usb/wlan/if_urtwnreg.h
index 3c212794a763c..ece356da86aaf 100644
--- a/sys/dev/usb/wlan/if_urtwnreg.h
+++ b/sys/dev/usb/wlan/if_urtwnreg.h
@@ -801,10 +801,6 @@
 #define R92C_RAID_11B	6
 
 
-/* Macros to access unaligned little-endian memory. */
-#define LE_READ_2(x)	((x)[0] | (x)[1] << 8)
-#define LE_READ_4(x)	((x)[0] | (x)[1] << 8 | (x)[2] << 16 | (x)[3] << 24)
-
 /*
  * Macros to access subfields in registers.
  */
@@ -1138,7 +1134,6 @@ struct urtwn_fw_info {
 
 struct urtwn_vap {
 	struct ieee80211vap		vap;
-	struct ieee80211_beacon_offsets	bo;
 
 	int				(*newstate)(struct ieee80211vap *,
 					    enum ieee80211_state, int);
diff --git a/sys/dev/usb/wlan/if_zyd.c b/sys/dev/usb/wlan/if_zyd.c
index a86ab8a3f0619..5a59d37946922 100644
--- a/sys/dev/usb/wlan/if_zyd.c
+++ b/sys/dev/usb/wlan/if_zyd.c
@@ -930,25 +930,23 @@ fail:
 static int
 zyd_rfmd_init(struct zyd_rf *rf)
 {
-#define N(a)	((int)(sizeof(a) / sizeof((a)[0])))
 	struct zyd_softc *sc = rf->rf_sc;
 	static const struct zyd_phy_pair phyini[] = ZYD_RFMD_PHY;
 	static const uint32_t rfini[] = ZYD_RFMD_RF;
 	int i, error;
 
 	/* init RF-dependent PHY registers */
-	for (i = 0; i < N(phyini); i++) {
+	for (i = 0; i < nitems(phyini); i++) {
 		zyd_write16_m(sc, phyini[i].reg, phyini[i].val);
 	}
 
 	/* init RFMD radio */
-	for (i = 0; i < N(rfini); i++) {
+	for (i = 0; i < nitems(rfini); i++) {
 		if ((error = zyd_rfwrite(sc, rfini[i])) != 0)
 			return (error);
 	}
 fail:
 	return (error);
-#undef N
 }
 
 static int
@@ -989,7 +987,6 @@ fail:
 static int
 zyd_al2230_init(struct zyd_rf *rf)
 {
-#define N(a)	((int)(sizeof(a) / sizeof((a)[0])))
 	struct zyd_softc *sc = rf->rf_sc;
 	static const struct zyd_phy_pair phyini[] = ZYD_AL2230_PHY;
 	static const struct zyd_phy_pair phy2230s[] = ZYD_AL2230S_PHY_INIT;
@@ -1003,16 +1000,16 @@ zyd_al2230_init(struct zyd_rf *rf)
 	int i, error;
 
 	/* init RF-dependent PHY registers */
-	for (i = 0; i < N(phyini); i++)
+	for (i = 0; i < nitems(phyini); i++)
 		zyd_write16_m(sc, phyini[i].reg, phyini[i].val);
 
 	if (sc->sc_rfrev == ZYD_RF_AL2230S || sc->sc_al2230s != 0) {
-		for (i = 0; i < N(phy2230s); i++)
+		for (i = 0; i < nitems(phy2230s); i++)
 			zyd_write16_m(sc, phy2230s[i].reg, phy2230s[i].val);
 	}
 
 	/* init AL2230 radio */
-	for (i = 0; i < N(rfini1); i++) {
+	for (i = 0; i < nitems(rfini1); i++) {
 		error = zyd_rfwrite(sc, rfini1[i]);
 		if (error != 0)
 			goto fail;
@@ -1025,34 +1022,32 @@ zyd_al2230_init(struct zyd_rf *rf)
 	if (error != 0)
 		goto fail;
 
-	for (i = 0; i < N(rfini2); i++) {
+	for (i = 0; i < nitems(rfini2); i++) {
 		error = zyd_rfwrite(sc, rfini2[i]);
 		if (error != 0)
 			goto fail;
 	}
 
-	for (i = 0; i < N(phypll); i++)
+	for (i = 0; i < nitems(phypll); i++)
 		zyd_write16_m(sc, phypll[i].reg, phypll[i].val);
 
-	for (i = 0; i < N(rfini3); i++) {
+	for (i = 0; i < nitems(rfini3); i++) {
 		error = zyd_rfwrite(sc, rfini3[i]);
 		if (error != 0)
 			goto fail;
 	}
 fail:
 	return (error);
-#undef N
 }
 
 static int
 zyd_al2230_fini(struct zyd_rf *rf)
 {
-#define N(a)	((int)(sizeof(a) / sizeof((a)[0])))
 	int error, i;
 	struct zyd_softc *sc = rf->rf_sc;
 	static const struct zyd_phy_pair phy[] = ZYD_AL2230_PHY_FINI_PART1;
 
-	for (i = 0; i < N(phy); i++)
+	for (i = 0; i < nitems(phy); i++)
 		zyd_write16_m(sc, phy[i].reg, phy[i].val);
 
 	if (sc->sc_newphy != 0)
@@ -1061,13 +1056,11 @@ zyd_al2230_fini(struct zyd_rf *rf)
 	zyd_write16_m(sc, ZYD_CR203, 0x6);
 fail:
 	return (error);
-#undef N
 }
 
 static int
 zyd_al2230_init_b(struct zyd_rf *rf)
 {
-#define N(a)	((int)(sizeof(a) / sizeof((a)[0])))
 	struct zyd_softc *sc = rf->rf_sc;
 	static const struct zyd_phy_pair phy1[] = ZYD_AL2230_PHY_PART1;
 	static const struct zyd_phy_pair phy2[] = ZYD_AL2230_PHY_PART2;
@@ -1080,15 +1073,15 @@ zyd_al2230_init_b(struct zyd_rf *rf)
 	static const uint32_t zyd_al2230_chtable[][3] = ZYD_AL2230_CHANTABLE;
 	int i, error;
 
-	for (i = 0; i < N(phy1); i++)
+	for (i = 0; i < nitems(phy1); i++)
 		zyd_write16_m(sc, phy1[i].reg, phy1[i].val);
 
 	/* init RF-dependent PHY registers */
-	for (i = 0; i < N(phyini); i++)
+	for (i = 0; i < nitems(phyini); i++)
 		zyd_write16_m(sc, phyini[i].reg, phyini[i].val);
 
 	if (sc->sc_rfrev == ZYD_RF_AL2230S || sc->sc_al2230s != 0) {
-		for (i = 0; i < N(phy2230s); i++)
+		for (i = 0; i < nitems(phy2230s); i++)
 			zyd_write16_m(sc, phy2230s[i].reg, phy2230s[i].val);
 	}
 
@@ -1098,7 +1091,7 @@ zyd_al2230_init_b(struct zyd_rf *rf)
 			return (error);
 	}
 
-	for (i = 0; i < N(rfini_part1); i++) {
+	for (i = 0; i < nitems(rfini_part1); i++) {
 		error = zyd_rfwrite_cr(sc, rfini_part1[i]);
 		if (error != 0)
 			return (error);
@@ -1111,28 +1104,27 @@ zyd_al2230_init_b(struct zyd_rf *rf)
 	if (error != 0)
 		goto fail;
 
-	for (i = 0; i < N(rfini_part2); i++) {
+	for (i = 0; i < nitems(rfini_part2); i++) {
 		error = zyd_rfwrite_cr(sc, rfini_part2[i]);
 		if (error != 0)
 			return (error);
 	}
 
-	for (i = 0; i < N(phy2); i++)
+	for (i = 0; i < nitems(phy2); i++)
 		zyd_write16_m(sc, phy2[i].reg, phy2[i].val);
 
-	for (i = 0; i < N(rfini_part3); i++) {
+	for (i = 0; i < nitems(rfini_part3); i++) {
 		error = zyd_rfwrite_cr(sc, rfini_part3[i]);
 		if (error != 0)
 			return (error);
 	}
 
-	for (i = 0; i < N(phy3); i++)
+	for (i = 0; i < nitems(phy3); i++)
 		zyd_write16_m(sc, phy3[i].reg, phy3[i].val);
 
 	error = zyd_al2230_fini(rf);
 fail:
 	return (error);
-#undef N
 }
 
 static int
@@ -1150,7 +1142,6 @@ fail:
 static int
 zyd_al2230_set_channel(struct zyd_rf *rf, uint8_t chan)
 {
-#define N(a)	((int)(sizeof(a) / sizeof((a)[0])))
 	int error, i;
 	struct zyd_softc *sc = rf->rf_sc;
 	static const struct zyd_phy_pair phy1[] = {
@@ -1170,17 +1161,15 @@ zyd_al2230_set_channel(struct zyd_rf *rf, uint8_t chan)
 	if (error != 0)
 		goto fail;
 
-	for (i = 0; i < N(phy1); i++)
+	for (i = 0; i < nitems(phy1); i++)
 		zyd_write16_m(sc, phy1[i].reg, phy1[i].val);
 fail:
 	return (error);
-#undef N
 }
 
 static int
 zyd_al2230_set_channel_b(struct zyd_rf *rf, uint8_t chan)
 {
-#define N(a)	((int)(sizeof(a) / sizeof((a)[0])))
 	int error, i;
 	struct zyd_softc *sc = rf->rf_sc;
 	static const struct zyd_phy_pair phy1[] = ZYD_AL2230_PHY_PART1;
@@ -1188,7 +1177,7 @@ zyd_al2230_set_channel_b(struct zyd_rf *rf, uint8_t chan)
 		uint32_t	r1, r2, r3;
 	} rfprog[] = ZYD_AL2230_CHANTABLE_B;
 
-	for (i = 0; i < N(phy1); i++)
+	for (i = 0; i < nitems(phy1); i++)
 		zyd_write16_m(sc, phy1[i].reg, phy1[i].val);
 
 	error = zyd_rfwrite_cr(sc, rfprog[chan - 1].r1);
@@ -1203,7 +1192,6 @@ zyd_al2230_set_channel_b(struct zyd_rf *rf, uint8_t chan)
 	error = zyd_al2230_fini(rf);
 fail:
 	return (error);
-#undef N
 }
 
 #define	ZYD_AL2230_PHY_BANDEDGE6					\
@@ -1215,7 +1203,6 @@ fail:
 static int
 zyd_al2230_bandedge6(struct zyd_rf *rf, struct ieee80211_channel *c)
 {
-#define N(a)	((int)(sizeof(a) / sizeof((a)[0])))
 	int error = 0, i;
 	struct zyd_softc *sc = rf->rf_sc;
 	struct ieee80211com *ic = &sc->sc_ic;
@@ -1225,11 +1212,10 @@ zyd_al2230_bandedge6(struct zyd_rf *rf, struct ieee80211_channel *c)
 	if (chan == 1 || chan == 11)
 		r[0].val = 0x12;
 	
-	for (i = 0; i < N(r); i++)
+	for (i = 0; i < nitems(r); i++)
 		zyd_write16_m(sc, r[i].reg, r[i].val);
 fail:
 	return (error);
-#undef N
 }
 
 /*
@@ -1238,7 +1224,6 @@ fail:
 static int
 zyd_al7230B_init(struct zyd_rf *rf)
 {
-#define N(a)	((int)(sizeof(a) / sizeof((a)[0])))
 	struct zyd_softc *sc = rf->rf_sc;
 	static const struct zyd_phy_pair phyini_1[] = ZYD_AL7230B_PHY_1;
 	static const struct zyd_phy_pair phyini_2[] = ZYD_AL7230B_PHY_2;
@@ -1250,29 +1235,28 @@ zyd_al7230B_init(struct zyd_rf *rf)
 	/* for AL7230B, PHY and RF need to be initialized in "phases" */
 
 	/* init RF-dependent PHY registers, part one */
-	for (i = 0; i < N(phyini_1); i++)
+	for (i = 0; i < nitems(phyini_1); i++)
 		zyd_write16_m(sc, phyini_1[i].reg, phyini_1[i].val);
 
 	/* init AL7230B radio, part one */
-	for (i = 0; i < N(rfini_1); i++) {
+	for (i = 0; i < nitems(rfini_1); i++) {
 		if ((error = zyd_rfwrite(sc, rfini_1[i])) != 0)
 			return (error);
 	}
 	/* init RF-dependent PHY registers, part two */
-	for (i = 0; i < N(phyini_2); i++)
+	for (i = 0; i < nitems(phyini_2); i++)
 		zyd_write16_m(sc, phyini_2[i].reg, phyini_2[i].val);
 
 	/* init AL7230B radio, part two */
-	for (i = 0; i < N(rfini_2); i++) {
+	for (i = 0; i < nitems(rfini_2); i++) {
 		if ((error = zyd_rfwrite(sc, rfini_2[i])) != 0)
 			return (error);
 	}
 	/* init RF-dependent PHY registers, part three */
-	for (i = 0; i < N(phyini_3); i++)
+	for (i = 0; i < nitems(phyini_3); i++)
 		zyd_write16_m(sc, phyini_3[i].reg, phyini_3[i].val);
 fail:
 	return (error);
-#undef N
 }
 
 static int
@@ -1290,7 +1274,6 @@ fail:
 static int
 zyd_al7230B_set_channel(struct zyd_rf *rf, uint8_t chan)
 {
-#define N(a)	((int)(sizeof(a) / sizeof((a)[0])))
 	struct zyd_softc *sc = rf->rf_sc;
 	static const struct {
 		uint32_t	r1, r2;
@@ -1301,7 +1284,7 @@ zyd_al7230B_set_channel(struct zyd_rf *rf, uint8_t chan)
 	zyd_write16_m(sc, ZYD_CR240, 0x57);
 	zyd_write16_m(sc, ZYD_CR251, 0x2f);
 
-	for (i = 0; i < N(rfsc); i++) {
+	for (i = 0; i < nitems(rfsc); i++) {
 		if ((error = zyd_rfwrite(sc, rfsc[i])) != 0)
 			return (error);
 	}
@@ -1327,7 +1310,6 @@ zyd_al7230B_set_channel(struct zyd_rf *rf, uint8_t chan)
 	zyd_write16_m(sc, ZYD_CR240, 0x08);
 fail:
 	return (error);
-#undef N
 }
 
 /*
@@ -1336,7 +1318,6 @@ fail:
 static int
 zyd_al2210_init(struct zyd_rf *rf)
 {
-#define N(a)	((int)(sizeof(a) / sizeof((a)[0])))
 	struct zyd_softc *sc = rf->rf_sc;
 	static const struct zyd_phy_pair phyini[] = ZYD_AL2210_PHY;
 	static const uint32_t rfini[] = ZYD_AL2210_RF;
@@ -1346,11 +1327,11 @@ zyd_al2210_init(struct zyd_rf *rf)
 	zyd_write32_m(sc, ZYD_CR18, 2);
 
 	/* init RF-dependent PHY registers */
-	for (i = 0; i < N(phyini); i++)
+	for (i = 0; i < nitems(phyini); i++)
 		zyd_write16_m(sc, phyini[i].reg, phyini[i].val);
 
 	/* init AL2210 radio */
-	for (i = 0; i < N(rfini); i++) {
+	for (i = 0; i < nitems(rfini); i++) {
 		if ((error = zyd_rfwrite(sc, rfini[i])) != 0)
 			return (error);
 	}
@@ -1364,7 +1345,6 @@ zyd_al2210_init(struct zyd_rf *rf)
 	zyd_write32_m(sc, ZYD_CR18, 3);
 fail:
 	return (error);
-#undef N
 }
 
 static int
@@ -1409,7 +1389,6 @@ static int
 zyd_gct_init(struct zyd_rf *rf)
 {
 #define	ZYD_GCT_INTR_REG	0x85c1
-#define N(a)	((int)(sizeof(a) / sizeof((a)[0])))
 	struct zyd_softc *sc = rf->rf_sc;
 	static const struct zyd_phy_pair phyini[] = ZYD_GCT_PHY;
 	static const uint32_t rfini[] = ZYD_GCT_RF;
@@ -1418,11 +1397,11 @@ zyd_gct_init(struct zyd_rf *rf)
 	uint16_t data;
 
 	/* init RF-dependent PHY registers */
-	for (i = 0; i < N(phyini); i++)
+	for (i = 0; i < nitems(phyini); i++)
 		zyd_write16_m(sc, phyini[i].reg, phyini[i].val);
 
 	/* init cgt radio */
-	for (i = 0; i < N(rfini); i++) {
+	for (i = 0; i < nitems(rfini); i++) {
 		if ((error = zyd_rfwrite(sc, rfini[i])) != 0)
 			return (error);
 	}
@@ -1431,7 +1410,7 @@ zyd_gct_init(struct zyd_rf *rf)
 	if (error != 0)
 		return (error);
 
-	for (i = 0; i < (int)(N(vco) - 1); i++) {
+	for (i = 0; i < (int)(nitems(vco) - 1); i++) {
 		error = zyd_gct_set_channel_synth(rf, 1, 0);
 		if (error != 0)
 			goto fail;
@@ -1458,26 +1437,23 @@ zyd_gct_init(struct zyd_rf *rf)
 	zyd_write16_m(sc, ZYD_CR203, 0x6);
 fail:
 	return (error);
-#undef N
 #undef ZYD_GCT_INTR_REG
 }
 
 static int
 zyd_gct_mode(struct zyd_rf *rf)
 {
-#define N(a)	((int)(sizeof(a) / sizeof((a)[0])))
 	struct zyd_softc *sc = rf->rf_sc;
 	static const uint32_t mode[] = {
 		0x25f98, 0x25f9a, 0x25f94, 0x27fd4
 	};
 	int i, error;
 
-	for (i = 0; i < N(mode); i++) {
+	for (i = 0; i < nitems(mode); i++) {
 		if ((error = zyd_rfwrite(sc, mode[i])) != 0)
 			break;
 	}
 	return (error);
-#undef N
 }
 
 static int
@@ -1524,7 +1500,6 @@ fail:
 static int
 zyd_gct_set_channel(struct zyd_rf *rf, uint8_t chan)
 {
-#define N(a)	((int)(sizeof(a) / sizeof((a)[0])))
 	int error, i;
 	struct zyd_softc *sc = rf->rf_sc;
 	static const struct zyd_phy_pair cmd[] = {
@@ -1543,7 +1518,7 @@ zyd_gct_set_channel(struct zyd_rf *rf, uint8_t chan)
 	error = zyd_gct_mode(rf);
 	if (error != 0)
 		return (error);
-	for (i = 0; i < N(cmd); i++)
+	for (i = 0; i < nitems(cmd); i++)
 		zyd_write16_m(sc, cmd[i].reg, cmd[i].val);
 	error = zyd_gct_txgain(rf, chan);
 	if (error != 0)
@@ -1551,25 +1526,22 @@ zyd_gct_set_channel(struct zyd_rf *rf, uint8_t chan)
 	zyd_write16_m(sc, ZYD_CR203, 0x6);
 fail:
 	return (error);
-#undef N
 }
 
 static int
 zyd_gct_txgain(struct zyd_rf *rf, uint8_t chan)
 {
-#define N(a)	(sizeof(a) / sizeof((a)[0]))
 	struct zyd_softc *sc = rf->rf_sc;
 	static uint32_t txgain[] = ZYD_GCT_TXGAIN;
 	uint8_t idx = sc->sc_pwrint[chan - 1];
 
-	if (idx >= N(txgain)) {
+	if (idx >= nitems(txgain)) {
 		device_printf(sc->sc_dev, "could not set TX gain (%d %#x)\n",
 		    chan, idx);
 		return 0;
 	}
 
 	return zyd_rfwrite(sc, 0x700000 | txgain[idx]);
-#undef N
 }
 
 /*
@@ -1578,7 +1550,6 @@ zyd_gct_txgain(struct zyd_rf *rf, uint8_t chan)
 static int
 zyd_maxim2_init(struct zyd_rf *rf)
 {
-#define N(a)	((int)(sizeof(a) / sizeof((a)[0])))
 	struct zyd_softc *sc = rf->rf_sc;
 	static const struct zyd_phy_pair phyini[] = ZYD_MAXIM2_PHY;
 	static const uint32_t rfini[] = ZYD_MAXIM2_RF;
@@ -1586,14 +1557,14 @@ zyd_maxim2_init(struct zyd_rf *rf)
 	int i, error;
 
 	/* init RF-dependent PHY registers */
-	for (i = 0; i < N(phyini); i++)
+	for (i = 0; i < nitems(phyini); i++)
 		zyd_write16_m(sc, phyini[i].reg, phyini[i].val);
 
 	zyd_read16_m(sc, ZYD_CR203, &tmp);
 	zyd_write16_m(sc, ZYD_CR203, tmp & ~(1 << 4));
 
 	/* init maxim2 radio */
-	for (i = 0; i < N(rfini); i++) {
+	for (i = 0; i < nitems(rfini); i++) {
 		if ((error = zyd_rfwrite(sc, rfini[i])) != 0)
 			return (error);
 	}
@@ -1601,7 +1572,6 @@ zyd_maxim2_init(struct zyd_rf *rf)
 	zyd_write16_m(sc, ZYD_CR203, tmp | (1 << 4));
 fail:
 	return (error);
-#undef N
 }
 
 static int
@@ -1615,7 +1585,6 @@ zyd_maxim2_switch_radio(struct zyd_rf *rf, int on)
 static int
 zyd_maxim2_set_channel(struct zyd_rf *rf, uint8_t chan)
 {
-#define N(a)	((int)(sizeof(a) / sizeof((a)[0])))
 	struct zyd_softc *sc = rf->rf_sc;
 	static const struct zyd_phy_pair phyini[] = ZYD_MAXIM2_PHY;
 	static const uint32_t rfini[] = ZYD_MAXIM2_RF;
@@ -1631,7 +1600,7 @@ zyd_maxim2_set_channel(struct zyd_rf *rf, uint8_t chan)
 	 */
 
 	/* init RF-dependent PHY registers */
-	for (i = 0; i < N(phyini); i++)
+	for (i = 0; i < nitems(phyini); i++)
 		zyd_write16_m(sc, phyini[i].reg, phyini[i].val);
 
 	zyd_read16_m(sc, ZYD_CR203, &tmp);
@@ -1646,7 +1615,7 @@ zyd_maxim2_set_channel(struct zyd_rf *rf, uint8_t chan)
 		goto fail;
 
 	/* init maxim2 radio - skipping the two first values */
-	for (i = 2; i < N(rfini); i++) {
+	for (i = 2; i < nitems(rfini); i++) {
 		if ((error = zyd_rfwrite(sc, rfini[i])) != 0)
 			return (error);
 	}
@@ -1654,7 +1623,6 @@ zyd_maxim2_set_channel(struct zyd_rf *rf, uint8_t chan)
 	zyd_write16_m(sc, ZYD_CR203, tmp | (1 << 4));
 fail:
 	return (error);
-#undef N
 }
 
 static int
diff --git a/sys/dev/wi/if_wivar.h b/sys/dev/wi/if_wivar.h
index 9b668cc0e1149..8cc320d50b60f 100644
--- a/sys/dev/wi/if_wivar.h
+++ b/sys/dev/wi/if_wivar.h
@@ -58,7 +58,6 @@
 
 struct wi_vap {
 	struct ieee80211vap	wv_vap;
-	struct ieee80211_beacon_offsets	wv_bo;
 
 	void		(*wv_recv_mgmt)(struct ieee80211_node *, struct mbuf *,
 			    int, const struct ieee80211_rx_stats *rxs, int, int);
diff --git a/sys/dev/wpi/if_wpi.c b/sys/dev/wpi/if_wpi.c
index 01ce3e4676953..abb3bd5baef15 100644
--- a/sys/dev/wpi/if_wpi.c
+++ b/sys/dev/wpi/if_wpi.c
@@ -3954,14 +3954,6 @@ wpi_config(struct wpi_softc *sc)
 	sc->rxon.cck_mask  = 0x0f;	/* not yet negotiated */
 	sc->rxon.ofdm_mask = 0xff;	/* not yet negotiated */
 
-	/* XXX Current configuration may be unusable. */
-	if (IEEE80211_IS_CHAN_NOADHOC(c) && sc->rxon.mode == WPI_MODE_IBSS) {
-		device_printf(sc->sc_dev,
-		    "%s: invalid channel (%d) selected for IBSS mode\n",
-		    __func__, ieee80211_chan2ieee(ic, c));
-		return EINVAL;
-	}
-
 	if ((error = wpi_send_rxon(sc, 0, 0)) != 0) {
 		device_printf(sc->sc_dev, "%s: could not send RXON\n",
 		    __func__);
@@ -4309,8 +4301,9 @@ wpi_auth(struct wpi_softc *sc, struct ieee80211vap *vap)
 static int
 wpi_config_beacon(struct wpi_vap *wvp)
 {
-	struct ieee80211com *ic = wvp->wv_vap.iv_ic;
-	struct ieee80211_beacon_offsets *bo = &wvp->wv_boff;
+	struct ieee80211vap *vap = &wvp->wv_vap;
+	struct ieee80211com *ic = vap->iv_ic;
+	struct ieee80211_beacon_offsets *bo = &vap->iv_bcn_off;
 	struct wpi_buf *bcn = &wvp->wv_bcbuf;
 	struct wpi_softc *sc = ic->ic_softc;
 	struct wpi_cmd_beacon *cmd = (struct wpi_cmd_beacon *)&bcn->data;
@@ -4361,9 +4354,10 @@ end:	bcn->m = m;
 static int
 wpi_setup_beacon(struct wpi_softc *sc, struct ieee80211_node *ni)
 {
-	struct wpi_vap *wvp = WPI_VAP(ni->ni_vap);
+	struct ieee80211vap *vap = ni->ni_vap;
+	struct ieee80211_beacon_offsets *bo = &vap->iv_bcn_off;
+	struct wpi_vap *wvp = WPI_VAP(vap);
 	struct wpi_buf *bcn = &wvp->wv_bcbuf;
-	struct ieee80211_beacon_offsets *bo = &wvp->wv_boff;
 	struct mbuf *m;
 	int error;
 
@@ -4397,7 +4391,7 @@ wpi_update_beacon(struct ieee80211vap *vap, int item)
 	struct wpi_softc *sc = vap->iv_ic->ic_softc;
 	struct wpi_vap *wvp = WPI_VAP(vap);
 	struct wpi_buf *bcn = &wvp->wv_bcbuf;
-	struct ieee80211_beacon_offsets *bo = &wvp->wv_boff;
+	struct ieee80211_beacon_offsets *bo = &vap->iv_bcn_off;
 	struct ieee80211_node *ni = vap->iv_bss;
 	int mcast = 0;
 
diff --git a/sys/dev/wpi/if_wpivar.h b/sys/dev/wpi/if_wpivar.h
index 4e1baa5fdc50c..0934eaa68ace7 100644
--- a/sys/dev/wpi/if_wpivar.h
+++ b/sys/dev/wpi/if_wpivar.h
@@ -125,7 +125,6 @@ struct wpi_vap {
 	struct ieee80211vap	wv_vap;
 
 	struct wpi_buf		wv_bcbuf;
-	struct ieee80211_beacon_offsets wv_boff;
 	struct mtx		wv_mtx;
 
 	uint32_t		wv_gtk;
diff --git a/sys/dev/wtap/if_wtap.c b/sys/dev/wtap/if_wtap.c
index 526ce54928d47..7c15c358de83f 100644
--- a/sys/dev/wtap/if_wtap.c
+++ b/sys/dev/wtap/if_wtap.c
@@ -183,7 +183,7 @@ wtap_reset_vap(struct ieee80211vap *vap, u_long cmd)
 static void
 wtap_beacon_update(struct ieee80211vap *vap, int item)
 {
-	struct ieee80211_beacon_offsets *bo = &WTAP_VAP(vap)->av_boff;
+	struct ieee80211_beacon_offsets *bo = &vap->iv_bcn_off;
 
 	DWTAP_PRINTF("%s\n", __func__);
 	setbit(bo->bo_flags, item);
@@ -205,7 +205,7 @@ wtap_beacon_alloc(struct wtap_softc *sc, struct ieee80211_node *ni)
 	 * we assume the mbuf routines will return us something
 	 * with this alignment (perhaps should assert).
 	 */
-	avp->beacon = ieee80211_beacon_alloc(ni, &avp->av_boff);
+	avp->beacon = ieee80211_beacon_alloc(ni, &vap->iv_bcn_off);
 	if (avp->beacon == NULL) {
 		printf("%s: cannot get mbuf\n", __func__);
 		return ENOMEM;
@@ -242,7 +242,7 @@ wtap_beacon_intrp(void *arg)
 	 * of the TIM bitmap).
 	 */
 	m = m_dup(avp->beacon, M_NOWAIT);
-	if (ieee80211_beacon_update(avp->bf_node, &avp->av_boff, m, 0)) {
+	if (ieee80211_beacon_update(avp->bf_node, &vap->iv_bcn_off, m, 0)) {
 		printf("%s, need to remap the memory because the beacon frame"
 		    " changed size.\n",__func__);
 	}
diff --git a/sys/dev/wtap/if_wtapvar.h b/sys/dev/wtap/if_wtapvar.h
index 04a1818d5c415..5ba55cbb3080c 100644
--- a/sys/dev/wtap/if_wtapvar.h
+++ b/sys/dev/wtap/if_wtapvar.h
@@ -116,7 +116,6 @@ struct wtap_vap {
 	struct wtap_medium	*av_md;		/* back pointer */
 	struct mbuf *beacon;			/* beacon */
 	struct ieee80211_node	*bf_node;	/* pointer to the node */
-	struct ieee80211_beacon_offsets av_boff;/* dynamic update state */
 	struct callout		av_swba;	/* software beacon alert */
 	uint32_t		av_bcinterval;	/* beacon interval */
 	void (*av_recv_mgmt)(struct ieee80211_node *,
diff --git a/sys/dev/xen/pcifront/pcifront.c b/sys/dev/xen/pcifront/pcifront.c
index 1730bf8fc09f6..833e38bd7e2f9 100644
--- a/sys/dev/xen/pcifront/pcifront.c
+++ b/sys/dev/xen/pcifront/pcifront.c
@@ -559,7 +559,7 @@ xpcib_attach(device_t dev)
 
 	DPRINTF("xpcib attach (bus=%d)\n", sc->bus);
 
-	device_add_child(dev, "pci", sc->bus);
+	device_add_child(dev, "pci", -1);
 	return bus_generic_attach(dev);
 }
 
diff --git a/sys/fs/fifofs/fifo_vnops.c b/sys/fs/fifofs/fifo_vnops.c
index e544bba82f0de..716faa302bf58 100644
--- a/sys/fs/fifofs/fifo_vnops.c
+++ b/sys/fs/fifofs/fifo_vnops.c
@@ -64,6 +64,8 @@ struct fifoinfo {
 	struct pipe *fi_pipe;
 	long	fi_readers;
 	long	fi_writers;
+	u_int	fi_rgen;
+	u_int	fi_wgen;
 };
 
 static vop_print_t	fifo_print;
@@ -137,6 +139,7 @@ fifo_open(ap)
 	struct thread *td;
 	struct fifoinfo *fip;
 	struct pipe *fpipe;
+	u_int gen;
 	int error, stops_deferred;
 
 	vp = ap->a_vp;
@@ -164,6 +167,7 @@ fifo_open(ap)
 	PIPE_LOCK(fpipe);
 	if (ap->a_mode & FREAD) {
 		fip->fi_readers++;
+		fip->fi_rgen++;
 		if (fip->fi_readers == 1) {
 			fpipe->pipe_state &= ~PIPE_EOF;
 			if (fip->fi_writers > 0)
@@ -179,6 +183,7 @@ fifo_open(ap)
 			return (ENXIO);
 		}
 		fip->fi_writers++;
+		fip->fi_wgen++;
 		if (fip->fi_writers == 1) {
 			fpipe->pipe_state &= ~PIPE_EOF;
 			if (fip->fi_readers > 0)
@@ -187,6 +192,7 @@ fifo_open(ap)
 	}
 	if ((ap->a_mode & O_NONBLOCK) == 0) {
 		if ((ap->a_mode & FREAD) && fip->fi_writers == 0) {
+			gen = fip->fi_wgen;
 			VOP_UNLOCK(vp, 0);
 			stops_deferred = sigallowstop();
 			error = msleep(&fip->fi_readers, PIPE_MTX(fpipe),
@@ -194,7 +200,7 @@ fifo_open(ap)
 			if (stops_deferred)
 				sigdeferstop();
 			vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
-			if (error) {
+			if (error != 0 && gen == fip->fi_wgen) {
 				fip->fi_readers--;
 				if (fip->fi_readers == 0) {
 					PIPE_LOCK(fpipe);
@@ -214,6 +220,7 @@ fifo_open(ap)
 			 */
 		}
 		if ((ap->a_mode & FWRITE) && fip->fi_readers == 0) {
+			gen = fip->fi_rgen;
 			VOP_UNLOCK(vp, 0);
 			stops_deferred = sigallowstop();
 			error = msleep(&fip->fi_writers, PIPE_MTX(fpipe),
@@ -221,7 +228,7 @@ fifo_open(ap)
 			if (stops_deferred)
 				sigdeferstop();
 			vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
-			if (error) {
+			if (error != 0 && gen == fip->fi_rgen) {
 				fip->fi_writers--;
 				if (fip->fi_writers == 0) {
 					PIPE_LOCK(fpipe);
diff --git a/sys/geom/geom_dev.c b/sys/geom/geom_dev.c
index 0d34ef5a51d5b..789de4d3c4576 100644
--- a/sys/geom/geom_dev.c
+++ b/sys/geom/geom_dev.c
@@ -124,6 +124,7 @@ g_dev_fini(struct g_class *mp)
 {
 
 	freeenv(dumpdev);
+	dumpdev = NULL;
 }
 
 static int
@@ -152,10 +153,16 @@ g_dev_setdumpdev(struct cdev *dev, struct thread *td)
 static void
 init_dumpdev(struct cdev *dev)
 {
+	const char *devprefix = "/dev/", *devname;
+	size_t len;
 
 	if (dumpdev == NULL)
 		return;
-	if (strcmp(devtoname(dev), dumpdev) != 0)
+	len = strlen(devprefix);
+	devname = devtoname(dev);
+	if (strcmp(devname, dumpdev) != 0 &&
+	   (strncmp(dumpdev, devprefix, len) != 0 ||
+	    strcmp(devname, dumpdev + len) != 0))
 		return;
 	if (g_dev_setdumpdev(dev, curthread) == 0) {
 		freeenv(dumpdev);
diff --git a/sys/geom/nop/g_nop.c b/sys/geom/nop/g_nop.c
index bd72d780a4b8a..837f3f4e33945 100644
--- a/sys/geom/nop/g_nop.c
+++ b/sys/geom/nop/g_nop.c
@@ -162,7 +162,7 @@ g_nop_access(struct g_provider *pp, int dr, int dw, int de)
 static int
 g_nop_create(struct gctl_req *req, struct g_class *mp, struct g_provider *pp,
     int ioerror, u_int rfailprob, u_int wfailprob, off_t offset, off_t size,
-    u_int secsize)
+    u_int secsize, u_int stripesize, u_int stripeoffset)
 {
 	struct g_nop_softc *sc;
 	struct g_geom *gp;
@@ -208,6 +208,18 @@ g_nop_create(struct gctl_req *req, struct g_class *mp, struct g_provider *pp,
 		return (EINVAL);
 	}
 	size -= size % secsize;
+	if ((stripesize % pp->sectorsize) != 0) {
+		gctl_error(req, "Invalid stripesize for provider %s.", pp->name);
+		return (EINVAL);
+	}
+	if ((stripeoffset % pp->sectorsize) != 0) {
+		gctl_error(req, "Invalid stripeoffset for provider %s.", pp->name);
+		return (EINVAL);
+	}
+	if (stripesize != 0 && stripeoffset >= stripesize) {
+		gctl_error(req, "stripeoffset is too big.");
+		return (EINVAL);
+	}
 	snprintf(name, sizeof(name), "%s%s", pp->name, G_NOP_SUFFIX);
 	LIST_FOREACH(gp, &mp->geom, geom) {
 		if (strcmp(gp->name, name) == 0) {
@@ -219,6 +231,8 @@ g_nop_create(struct gctl_req *req, struct g_class *mp, struct g_provider *pp,
 	sc = g_malloc(sizeof(*sc), M_WAITOK | M_ZERO);
 	sc->sc_offset = offset;
 	sc->sc_explicitsize = explicitsize;
+	sc->sc_stripesize = stripesize;
+	sc->sc_stripeoffset = stripeoffset;
 	sc->sc_error = ioerror;
 	sc->sc_rfailprob = rfailprob;
 	sc->sc_wfailprob = wfailprob;
@@ -238,6 +252,8 @@ g_nop_create(struct gctl_req *req, struct g_class *mp, struct g_provider *pp,
 	newpp->flags |= G_PF_DIRECT_SEND | G_PF_DIRECT_RECEIVE;
 	newpp->mediasize = size;
 	newpp->sectorsize = secsize;
+	newpp->stripesize = stripesize;
+	newpp->stripeoffset = stripeoffset;
 
 	cp = g_new_consumer(gp);
 	cp->flags |= G_CF_DIRECT_SEND | G_CF_DIRECT_RECEIVE;
@@ -304,7 +320,8 @@ static void
 g_nop_ctl_create(struct gctl_req *req, struct g_class *mp)
 {
 	struct g_provider *pp;
-	intmax_t *error, *rfailprob, *wfailprob, *offset, *secsize, *size;
+	intmax_t *error, *rfailprob, *wfailprob, *offset, *secsize, *size,
+	    *stripesize, *stripeoffset;
 	const char *name;
 	char param[16];
 	int i, *nargs;
@@ -370,6 +387,24 @@ g_nop_ctl_create(struct gctl_req *req, struct g_class *mp)
 		gctl_error(req, "Invalid '%s' argument", "secsize");
 		return;
 	}
+	stripesize = gctl_get_paraml(req, "stripesize", sizeof(*stripesize));
+	if (stripesize == NULL) {
+		gctl_error(req, "No '%s' argument", "stripesize");
+		return;
+	}
+	if (*stripesize < 0) {
+		gctl_error(req, "Invalid '%s' argument", "stripesize");
+		return;
+	}
+	stripeoffset = gctl_get_paraml(req, "stripeoffset", sizeof(*stripeoffset));
+	if (stripeoffset == NULL) {
+		gctl_error(req, "No '%s' argument", "stripeoffset");
+		return;
+	}
+	if (*stripeoffset < 0) {
+		gctl_error(req, "Invalid '%s' argument", "stripeoffset");
+		return;
+	}
 
 	for (i = 0; i < *nargs; i++) {
 		snprintf(param, sizeof(param), "arg%d", i);
@@ -390,7 +425,8 @@ g_nop_ctl_create(struct gctl_req *req, struct g_class *mp)
 		    *error == -1 ? EIO : (int)*error,
 		    *rfailprob == -1 ? 0 : (u_int)*rfailprob,
 		    *wfailprob == -1 ? 0 : (u_int)*wfailprob,
-		    (off_t)*offset, (off_t)*size, (u_int)*secsize) != 0) {
+		    (off_t)*offset, (off_t)*size, (u_int)*secsize,
+		    (u_int)*stripesize, (u_int)*stripeoffset) != 0) {
 			return;
 		}
 	}
diff --git a/sys/geom/nop/g_nop.h b/sys/geom/nop/g_nop.h
index 3e37c05d93ede..b5e954ad6337d 100644
--- a/sys/geom/nop/g_nop.h
+++ b/sys/geom/nop/g_nop.h
@@ -59,6 +59,8 @@ struct g_nop_softc {
 	int		sc_error;
 	off_t		sc_offset;
 	off_t		sc_explicitsize;
+	off_t		sc_stripesize;
+	off_t		sc_stripeoffset;
 	u_int		sc_rfailprob;
 	u_int		sc_wfailprob;
 	uintmax_t	sc_reads;
diff --git a/sys/i386/i386/elf_machdep.c b/sys/i386/i386/elf_machdep.c
index 03ab10f17dc78..81d6e35b3a456 100644
--- a/sys/i386/i386/elf_machdep.c
+++ b/sys/i386/i386/elf_machdep.c
@@ -178,6 +178,7 @@ elf_reloc_internal(linker_file_t lf, Elf_Addr relocbase, const void *data,
 	Elf_Word rtype, symidx;
 	const Elf_Rel *rel;
 	const Elf_Rela *rela;
+	int error;
 
 	switch (type) {
 	case ELF_RELOC_REL:
@@ -213,8 +214,8 @@ elf_reloc_internal(linker_file_t lf, Elf_Addr relocbase, const void *data,
 			break;
 
 		case R_386_32:		/* S + A */
-			addr = lookup(lf, symidx, 1);
-			if (addr == 0)
+			error = lookup(lf, symidx, 1, &addr);
+			if (error != 0)
 				return -1;
 			addr += addend;
 			if (*where != addr)
@@ -222,8 +223,8 @@ elf_reloc_internal(linker_file_t lf, Elf_Addr relocbase, const void *data,
 			break;
 
 		case R_386_PC32:	/* S + A - P */
-			addr = lookup(lf, symidx, 1);
-			if (addr == 0)
+			error = lookup(lf, symidx, 1, &addr);
+			if (error != 0)
 				return -1;
 			addr += addend - (Elf_Addr)where;
 			if (*where != addr)
@@ -240,8 +241,8 @@ elf_reloc_internal(linker_file_t lf, Elf_Addr relocbase, const void *data,
 			break;
 
 		case R_386_GLOB_DAT:	/* S */
-			addr = lookup(lf, symidx, 1);
-			if (addr == 0)
+			error = lookup(lf, symidx, 1, &addr);
+			if (error != 0)
 				return -1;
 			if (*where != addr)
 				*where = addr;
diff --git a/sys/i386/i386/pmap.c b/sys/i386/i386/pmap.c
index 9b8b2ff9df2ba..270845b5516cb 100644
--- a/sys/i386/i386/pmap.c
+++ b/sys/i386/i386/pmap.c
@@ -2312,7 +2312,7 @@ free_pv_chunk(struct pv_chunk *pc)
 	/* entire chunk is free, return it */
 	m = PHYS_TO_VM_PAGE(pmap_kextract((vm_offset_t)pc));
 	pmap_qremove((vm_offset_t)pc, 1);
-	vm_page_unwire(m, PQ_INACTIVE);
+	vm_page_unwire(m, PQ_NONE);
 	vm_page_free(m);
 	pmap_ptelist_free(&pv_vafree, (vm_offset_t)pc);
 }
diff --git a/sys/kern/kern_event.c b/sys/kern/kern_event.c
index 0e26a7882c3b8..d41ac96a70af1 100644
--- a/sys/kern/kern_event.c
+++ b/sys/kern/kern_event.c
@@ -759,28 +759,25 @@ kern_kqueue(struct thread *td, int flags, struct filecaps *fcaps)
 	struct filedesc *fdp;
 	struct kqueue *kq;
 	struct file *fp;
-	struct proc *p;
 	struct ucred *cred;
 	int fd, error;
 
-	p = td->td_proc;
+	fdp = td->td_proc->p_fd;
 	cred = td->td_ucred;
-	crhold(cred);
-	if (!chgkqcnt(cred->cr_ruidinfo, 1, lim_cur(td, RLIMIT_KQUEUES))) {
-		crfree(cred);
+	if (!chgkqcnt(cred->cr_ruidinfo, 1, lim_cur(td, RLIMIT_KQUEUES)))
 		return (ENOMEM);
-	}
 
-	fdp = p->p_fd;
 	error = falloc_caps(td, &fp, &fd, flags, fcaps);
-	if (error)
-		goto done2;
+	if (error != 0) {
+		chgkqcnt(cred->cr_ruidinfo, -1, 0);
+		return (error);
+	}
 
 	/* An extra reference on `fp' has been held for us by falloc(). */
 	kq = malloc(sizeof *kq, M_KQUEUE, M_WAITOK | M_ZERO);
 	kqueue_init(kq);
 	kq->kq_fdp = fdp;
-	kq->kq_cred = cred;
+	kq->kq_cred = crhold(cred);
 
 	FILEDESC_XLOCK(fdp);
 	TAILQ_INSERT_HEAD(&fdp->fd_kqlist, kq, kq_list);
@@ -790,12 +787,7 @@ kern_kqueue(struct thread *td, int flags, struct filecaps *fcaps)
 	fdrop(fp, td);
 
 	td->td_retval[0] = fd;
-done2:
-	if (error != 0) {
-		chgkqcnt(cred->cr_ruidinfo, -1, 0);
-		crfree(cred);
-	}
-	return (error);
+	return (0);
 }
 
 #ifndef _SYS_SYSPROTO_H_
diff --git a/sys/kern/kern_exec.c b/sys/kern/kern_exec.c
index a3fd830bfceab..fb78adfa00d6f 100644
--- a/sys/kern/kern_exec.c
+++ b/sys/kern/kern_exec.c
@@ -418,7 +418,7 @@ do_execve(td, args, mac_p)
 		    | AUDITVNODE1, UIO_SYSSPACE, args->fname, td);
 	}
 
-	SDT_PROBE(proc, kernel, , exec, args->fname, 0, 0, 0, 0 );
+	SDT_PROBE1(proc, kernel, , exec, args->fname);
 
 interpret:
 	if (args->fname != NULL) {
@@ -846,7 +846,7 @@ interpret:
 
 	vfs_mark_atime(imgp->vp, td->td_ucred);
 
-	SDT_PROBE(proc, kernel, , exec__success, args->fname, 0, 0, 0, 0);
+	SDT_PROBE1(proc, kernel, , exec__success, args->fname);
 
 	VOP_UNLOCK(imgp->vp, 0);
 done1:
@@ -918,7 +918,7 @@ exec_fail:
 	p->p_flag &= ~P_INEXEC;
 	PROC_UNLOCK(p);
 
-	SDT_PROBE(proc, kernel, , exec__failure, error, 0, 0, 0, 0);
+	SDT_PROBE1(proc, kernel, , exec__failure, error);
 
 done2:
 #ifdef MAC
diff --git a/sys/kern/kern_exit.c b/sys/kern/kern_exit.c
index d84c26f7d4574..53ae14ec9010b 100644
--- a/sys/kern/kern_exit.c
+++ b/sys/kern/kern_exit.c
@@ -569,7 +569,7 @@ exit1(struct thread *td, int rval, int signo)
 		reason = CLD_DUMPED;
 	else if (WIFSIGNALED(signo))
 		reason = CLD_KILLED;
-	SDT_PROBE(proc, kernel, , exit, reason, 0, 0, 0, 0);
+	SDT_PROBE1(proc, kernel, , exit, reason);
 #endif
 
 	/*
diff --git a/sys/kern/kern_fork.c b/sys/kern/kern_fork.c
index 4aa531490fd15..b2727bc6a9918 100644
--- a/sys/kern/kern_fork.c
+++ b/sys/kern/kern_fork.c
@@ -753,7 +753,7 @@ do_fork(struct thread *td, int flags, struct proc *p2, struct thread *td2,
 	 * Tell any interested parties about the new process.
 	 */
 	knote_fork(&p1->p_klist, p2->p_pid);
-	SDT_PROBE(proc, kernel, , create, p2, p1, flags, 0, 0);
+	SDT_PROBE3(proc, kernel, , create, p2, p1, flags);
 
 	/*
 	 * Wait until debugger is attached to child.
diff --git a/sys/kern/kern_linker.c b/sys/kern/kern_linker.c
index a4fde06198f43..bb92e18db7904 100644
--- a/sys/kern/kern_linker.c
+++ b/sys/kern/kern_linker.c
@@ -292,10 +292,10 @@ linker_file_register_sysctls(linker_file_t lf)
 		return;
 
 	sx_xunlock(&kld_sx);
-	sysctl_xlock();
+	sysctl_wlock();
 	for (oidp = start; oidp < stop; oidp++)
 		sysctl_register_oid(*oidp);
-	sysctl_xunlock();
+	sysctl_wunlock();
 	sx_xlock(&kld_sx);
 }
 
@@ -313,10 +313,10 @@ linker_file_unregister_sysctls(linker_file_t lf)
 		return;
 
 	sx_xunlock(&kld_sx);
-	sysctl_xlock();
+	sysctl_wlock();
 	for (oidp = start; oidp < stop; oidp++)
 		sysctl_unregister_oid(*oidp);
-	sysctl_xunlock();
+	sysctl_wunlock();
 	sx_xlock(&kld_sx);
 }
 
diff --git a/sys/kern/kern_proc.c b/sys/kern/kern_proc.c
index aa4c9044c8d1a..8f533061a9e86 100644
--- a/sys/kern/kern_proc.c
+++ b/sys/kern/kern_proc.c
@@ -181,9 +181,9 @@ proc_ctor(void *mem, int size, void *arg, int flags)
 	struct proc *p;
 
 	p = (struct proc *)mem;
-	SDT_PROBE(proc, kernel, ctor , entry, p, size, arg, flags, 0);
+	SDT_PROBE4(proc, kernel, ctor , entry, p, size, arg, flags);
 	EVENTHANDLER_INVOKE(process_ctor, p);
-	SDT_PROBE(proc, kernel, ctor , return, p, size, arg, flags, 0);
+	SDT_PROBE4(proc, kernel, ctor , return, p, size, arg, flags);
 	return (0);
 }
 
@@ -199,7 +199,7 @@ proc_dtor(void *mem, int size, void *arg)
 	/* INVARIANTS checks go here */
 	p = (struct proc *)mem;
 	td = FIRST_THREAD_IN_PROC(p);
-	SDT_PROBE(proc, kernel, dtor, entry, p, size, arg, td, 0);
+	SDT_PROBE4(proc, kernel, dtor, entry, p, size, arg, td);
 	if (td != NULL) {
 #ifdef INVARIANTS
 		KASSERT((p->p_numthreads == 1),
@@ -212,7 +212,7 @@ proc_dtor(void *mem, int size, void *arg)
 	EVENTHANDLER_INVOKE(process_dtor, p);
 	if (p->p_ksi != NULL)
 		KASSERT(! KSI_ONQ(p->p_ksi), ("SIGCHLD queue"));
-	SDT_PROBE(proc, kernel, dtor, return, p, size, arg, 0, 0);
+	SDT_PROBE3(proc, kernel, dtor, return, p, size, arg);
 }
 
 /*
@@ -224,7 +224,7 @@ proc_init(void *mem, int size, int flags)
 	struct proc *p;
 
 	p = (struct proc *)mem;
-	SDT_PROBE(proc, kernel, init, entry, p, size, flags, 0, 0);
+	SDT_PROBE3(proc, kernel, init, entry, p, size, flags);
 	p->p_sched = (struct p_sched *)&p[1];
 	mtx_init(&p->p_mtx, "process lock", NULL, MTX_DEF | MTX_DUPOK | MTX_NEW);
 	mtx_init(&p->p_slock, "process slock", NULL, MTX_SPIN | MTX_NEW);
@@ -236,7 +236,7 @@ proc_init(void *mem, int size, int flags)
 	TAILQ_INIT(&p->p_threads);	     /* all threads in proc */
 	EVENTHANDLER_INVOKE(process_init, p);
 	p->p_stats = pstats_alloc();
-	SDT_PROBE(proc, kernel, init, return, p, size, flags, 0, 0);
+	SDT_PROBE3(proc, kernel, init, return, p, size, flags);
 	return (0);
 }
 
@@ -694,7 +694,7 @@ orphanpg(pg)
 
 	LIST_FOREACH(p, &pg->pg_members, p_pglist) {
 		PROC_LOCK(p);
-		if (P_SHOULDSTOP(p)) {
+		if (P_SHOULDSTOP(p) == P_STOPPED_SIG) {
 			PROC_UNLOCK(p);
 			LIST_FOREACH(p, &pg->pg_members, p_pglist) {
 				PROC_LOCK(p);
diff --git a/sys/kern/kern_racct.c b/sys/kern/kern_racct.c
index 2a1079084f39e..d32b4320649e2 100644
--- a/sys/kern/kern_racct.c
+++ b/sys/kern/kern_racct.c
@@ -445,7 +445,7 @@ racct_create(struct racct **racctp)
 	if (!racct_enable)
 		return;
 
-	SDT_PROBE(racct, kernel, racct, create, racctp, 0, 0, 0, 0);
+	SDT_PROBE1(racct, kernel, racct, create, racctp);
 
 	KASSERT(*racctp == NULL, ("racct already allocated"));
 
@@ -460,7 +460,7 @@ racct_destroy_locked(struct racct **racctp)
 
 	ASSERT_RACCT_ENABLED();
 
-	SDT_PROBE(racct, kernel, racct, destroy, racctp, 0, 0, 0, 0);
+	SDT_PROBE1(racct, kernel, racct, destroy, racctp);
 
 	mtx_assert(&racct_lock, MA_OWNED);
 	KASSERT(racctp != NULL, ("NULL racctp"));
@@ -538,7 +538,7 @@ racct_add_locked(struct proc *p, int resource, uint64_t amount)
 
 	ASSERT_RACCT_ENABLED();
 
-	SDT_PROBE(racct, kernel, rusage, add, p, resource, amount, 0, 0);
+	SDT_PROBE3(racct, kernel, rusage, add, p, resource, amount);
 
 	/*
 	 * We need proc lock to dereference p->p_ucred.
@@ -548,8 +548,8 @@ racct_add_locked(struct proc *p, int resource, uint64_t amount)
 #ifdef RCTL
 	error = rctl_enforce(p, resource, amount);
 	if (error && RACCT_IS_DENIABLE(resource)) {
-		SDT_PROBE(racct, kernel, rusage, add__failure, p, resource,
-		    amount, 0, 0);
+		SDT_PROBE3(racct, kernel, rusage, add__failure, p, resource,
+		    amount);
 		return (error);
 	}
 #endif
@@ -584,8 +584,7 @@ racct_add_cred_locked(struct ucred *cred, int resource, uint64_t amount)
 
 	ASSERT_RACCT_ENABLED();
 
-	SDT_PROBE(racct, kernel, rusage, add__cred, cred, resource, amount,
-	    0, 0);
+	SDT_PROBE3(racct, kernel, rusage, add__cred, cred, resource, amount);
 
 	racct_adjust_resource(cred->cr_ruidinfo->ui_racct, resource, amount);
 	for (pr = cred->cr_prison; pr != NULL; pr = pr->pr_parent)
@@ -623,7 +622,7 @@ racct_add_force(struct proc *p, int resource, uint64_t amount)
 	if (!racct_enable)
 		return;
 
-	SDT_PROBE(racct, kernel, rusage, add__force, p, resource, amount, 0, 0);
+	SDT_PROBE3(racct, kernel, rusage, add__force, p, resource, amount);
 
 	/*
 	 * We need proc lock to dereference p->p_ucred.
@@ -647,7 +646,7 @@ racct_set_locked(struct proc *p, int resource, uint64_t amount)
 
 	ASSERT_RACCT_ENABLED();
 
-	SDT_PROBE(racct, kernel, rusage, set, p, resource, amount, 0, 0);
+	SDT_PROBE3(racct, kernel, rusage, set, p, resource, amount);
 
 	/*
 	 * We need proc lock to dereference p->p_ucred.
@@ -679,8 +678,8 @@ racct_set_locked(struct proc *p, int resource, uint64_t amount)
 	if (diff_proc > 0) {
 		error = rctl_enforce(p, resource, diff_proc);
 		if (error && RACCT_IS_DENIABLE(resource)) {
-			SDT_PROBE(racct, kernel, rusage, set__failure, p,
-			    resource, amount, 0, 0);
+			SDT_PROBE3(racct, kernel, rusage, set__failure, p,
+			    resource, amount);
 			return (error);
 		}
 	}
@@ -723,7 +722,7 @@ racct_set_force_locked(struct proc *p, int resource, uint64_t amount)
 
 	ASSERT_RACCT_ENABLED();
 
-	SDT_PROBE(racct, kernel, rusage, set, p, resource, amount, 0, 0);
+	SDT_PROBE3(racct, kernel, rusage, set, p, resource, amount);
 
 	/*
 	 * We need proc lock to dereference p->p_ucred.
@@ -834,7 +833,7 @@ racct_sub(struct proc *p, int resource, uint64_t amount)
 	if (!racct_enable)
 		return;
 
-	SDT_PROBE(racct, kernel, rusage, sub, p, resource, amount, 0, 0);
+	SDT_PROBE3(racct, kernel, rusage, sub, p, resource, amount);
 
 	/*
 	 * We need proc lock to dereference p->p_ucred.
@@ -861,8 +860,7 @@ racct_sub_cred_locked(struct ucred *cred, int resource, uint64_t amount)
 
 	ASSERT_RACCT_ENABLED();
 
-	SDT_PROBE(racct, kernel, rusage, sub__cred, cred, resource, amount,
-	    0, 0);
+	SDT_PROBE3(racct, kernel, rusage, sub__cred, cred, resource, amount);
 
 #ifdef notyet
 	KASSERT(RACCT_CAN_DROP(resource),
diff --git a/sys/kern/kern_rmlock.c b/sys/kern/kern_rmlock.c
index 2667e6e3482c0..b6df53e5b3f75 100644
--- a/sys/kern/kern_rmlock.c
+++ b/sys/kern/kern_rmlock.c
@@ -407,9 +407,11 @@ _rm_rlock_hard(struct rmlock *rm, struct rm_priotracker *tracker, int trylock)
 				return (0);
 		}
 	} else {
-		if (rm->lock_object.lo_flags & LO_SLEEPABLE)
+		if (rm->lock_object.lo_flags & LO_SLEEPABLE) {
+			THREAD_SLEEPING_OK();
 			sx_xlock(&rm->rm_lock_sx);
-		else
+			THREAD_NO_SLEEPING();
+		} else
 			mtx_lock(&rm->rm_lock_mtx);
 	}
 
diff --git a/sys/kern/kern_shutdown.c b/sys/kern/kern_shutdown.c
index 3a91673bab115..29ea6931cad9b 100644
--- a/sys/kern/kern_shutdown.c
+++ b/sys/kern/kern_shutdown.c
@@ -50,6 +50,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/conf.h>
 #include <sys/cons.h>
 #include <sys/eventhandler.h>
+#include <sys/filedesc.h>
 #include <sys/jail.h>
 #include <sys/kdb.h>
 #include <sys/kernel.h>
@@ -150,10 +151,16 @@ static struct dumperinfo dumper;	/* our selected dumper */
 static struct pcb dumppcb;		/* Registers. */
 lwpid_t dumptid;			/* Thread ID. */
 
+static struct cdevsw reroot_cdevsw = {
+     .d_version = D_VERSION,
+     .d_name    = "reroot",
+};
+
 static void poweroff_wait(void *, int);
 static void shutdown_halt(void *junk, int howto);
 static void shutdown_panic(void *junk, int howto);
 static void shutdown_reset(void *junk, int howto);
+static int kern_reroot(void);
 
 /* register various local shutdown events */
 static void
@@ -173,6 +180,26 @@ shutdown_conf(void *unused)
 SYSINIT(shutdown_conf, SI_SUB_INTRINSIC, SI_ORDER_ANY, shutdown_conf, NULL);
 
 /*
+ * The only reason this exists is to create the /dev/reroot/ directory,
+ * used by reroot code in init(8) as a mountpoint for tmpfs.
+ */
+static void
+reroot_conf(void *unused)
+{
+	int error;
+	struct cdev *cdev;
+
+	error = make_dev_p(MAKEDEV_CHECKNAME | MAKEDEV_WAITOK, &cdev,
+	    &reroot_cdevsw, NULL, UID_ROOT, GID_WHEEL, 0600, "reroot/reroot");
+	if (error != 0) {
+		printf("%s: failed to create device node, error %d",
+		    __func__, error);
+	}
+}
+
+SYSINIT(reroot_conf, SI_SUB_DEVFS, SI_ORDER_ANY, reroot_conf, NULL);
+
+/*
  * The system call that results in a reboot.
  */
 /* ARGSUSED */
@@ -188,9 +215,13 @@ sys_reboot(struct thread *td, struct reboot_args *uap)
 	if (error == 0)
 		error = priv_check(td, PRIV_REBOOT);
 	if (error == 0) {
-		mtx_lock(&Giant);
-		kern_reboot(uap->opt);
-		mtx_unlock(&Giant);
+		if (uap->opt & RB_REROOT) {
+			error = kern_reroot();
+		} else {
+			mtx_lock(&Giant);
+			kern_reboot(uap->opt);
+			mtx_unlock(&Giant);
+		}
 	}
 	return (error);
 }
@@ -336,6 +367,102 @@ kern_reboot(int howto)
 }
 
 /*
+ * The system call that results in changing the rootfs.
+ */
+static int
+kern_reroot(void)
+{
+	struct vnode *oldrootvnode, *vp;
+	struct mount *mp, *devmp;
+	int error;
+
+	if (curproc != initproc)
+		return (EPERM);
+
+	/*
+	 * Mark the filesystem containing currently-running executable
+	 * (the temporary copy of init(8)) busy.
+	 */
+	vp = curproc->p_textvp;
+	error = vn_lock(vp, LK_SHARED);
+	if (error != 0)
+		return (error);
+	mp = vp->v_mount;
+	error = vfs_busy(mp, MBF_NOWAIT);
+	if (error != 0) {
+		vfs_ref(mp);
+		VOP_UNLOCK(vp, 0);
+		error = vfs_busy(mp, 0);
+		vn_lock(vp, LK_SHARED | LK_RETRY);
+		vfs_rel(mp);
+		if (error != 0) {
+			VOP_UNLOCK(vp, 0);
+			return (ENOENT);
+		}
+		if (vp->v_iflag & VI_DOOMED) {
+			VOP_UNLOCK(vp, 0);
+			vfs_unbusy(mp);
+			return (ENOENT);
+		}
+	}
+	VOP_UNLOCK(vp, 0);
+
+	/*
+	 * Remove the filesystem containing currently-running executable
+	 * from the mount list, to prevent it from being unmounted
+	 * by vfs_unmountall(), and to avoid confusing vfs_mountroot().
+	 *
+	 * Also preserve /dev - forcibly unmounting it could cause driver
+	 * reinitialization.
+	 */
+
+	vfs_ref(rootdevmp);
+	devmp = rootdevmp;
+	rootdevmp = NULL;
+
+	mtx_lock(&mountlist_mtx);
+	TAILQ_REMOVE(&mountlist, mp, mnt_list);
+	TAILQ_REMOVE(&mountlist, devmp, mnt_list);
+	mtx_unlock(&mountlist_mtx);
+
+	oldrootvnode = rootvnode;
+
+	/*
+	 * Unmount everything except for the two filesystems preserved above.
+	 */
+	vfs_unmountall();
+
+	/*
+	 * Add /dev back; vfs_mountroot() will move it into its new place.
+	 */
+	mtx_lock(&mountlist_mtx);
+	TAILQ_INSERT_HEAD(&mountlist, devmp, mnt_list);
+	mtx_unlock(&mountlist_mtx);
+	rootdevmp = devmp;
+	vfs_rel(rootdevmp);
+
+	/*
+	 * Mount the new rootfs.
+	 */
+	vfs_mountroot();
+
+	/*
+	 * Update all references to the old rootvnode.
+	 */
+	mountcheckdirs(oldrootvnode, rootvnode);
+
+	/*
+	 * Add the temporary filesystem back and unbusy it.
+	 */
+	mtx_lock(&mountlist_mtx);
+	TAILQ_INSERT_TAIL(&mountlist, mp, mnt_list);
+	mtx_unlock(&mountlist_mtx);
+	vfs_unbusy(mp);
+
+	return (0);
+}
+
+/*
  * If the shutdown was a clean halt, behave accordingly.
  */
 static void
diff --git a/sys/kern/kern_sig.c b/sys/kern/kern_sig.c
index b57f6b6e5b9d5..767eff0da7a6f 100644
--- a/sys/kern/kern_sig.c
+++ b/sys/kern/kern_sig.c
@@ -1308,7 +1308,7 @@ kern_sigtimedwait(struct thread *td, sigset_t waitset, ksiginfo_t *ksi,
 		reschedule_signals(p, new_block, 0);
 
 	if (error == 0) {
-		SDT_PROBE(proc, kernel, , signal__clear, sig, ksi, 0, 0, 0);
+		SDT_PROBE2(proc, kernel, , signal__clear, sig, ksi);
 
 		if (ksi->ksi_code == SI_TIMER)
 			itimer_accept(p, ksi->ksi_timerid, ksi);
@@ -2121,7 +2121,7 @@ tdsendsignal(struct proc *p, struct thread *td, int sig, ksiginfo_t *ksi)
 	} else
 		sigqueue = &td->td_sigqueue;
 
-	SDT_PROBE(proc, kernel, , signal__send, td, p, sig, 0, 0 );
+	SDT_PROBE3(proc, kernel, , signal__send, td, p, sig);
 
 	/*
 	 * If the signal is being ignored,
@@ -2132,7 +2132,7 @@ tdsendsignal(struct proc *p, struct thread *td, int sig, ksiginfo_t *ksi)
 	 */
 	mtx_lock(&ps->ps_mtx);
 	if (SIGISMEMBER(ps->ps_sigignore, sig)) {
-		SDT_PROBE(proc, kernel, , signal__discard, td, p, sig, 0, 0 );
+		SDT_PROBE3(proc, kernel, , signal__discard, td, p, sig);
 
 		mtx_unlock(&ps->ps_mtx);
 		if (ksi && (ksi->ksi_flags & KSI_INS))
diff --git a/sys/kern/kern_sysctl.c b/sys/kern/kern_sysctl.c
index b32198f184931..52075e47045d9 100644
--- a/sys/kern/kern_sysctl.c
+++ b/sys/kern/kern_sysctl.c
@@ -54,6 +54,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/jail.h>
 #include <sys/lock.h>
 #include <sys/mutex.h>
+#include <sys/rmlock.h>
 #include <sys/sbuf.h>
 #include <sys/sx.h>
 #include <sys/sysproto.h>
@@ -77,7 +78,7 @@ static MALLOC_DEFINE(M_SYSCTLTMP, "sysctltmp", "sysctl temp output buffer");
  * The sysctllock protects the MIB tree.  It also protects sysctl
  * contexts used with dynamic sysctls.  The sysctl_register_oid() and
  * sysctl_unregister_oid() routines require the sysctllock to already
- * be held, so the sysctl_xlock() and sysctl_xunlock() routines are
+ * be held, so the sysctl_wlock() and sysctl_wunlock() routines are
  * provided for the few places in the kernel which need to use that
  * API rather than using the dynamic API.  Use of the dynamic API is
  * strongly encouraged for most code.
@@ -86,20 +87,21 @@ static MALLOC_DEFINE(M_SYSCTLTMP, "sysctltmp", "sysctl temp output buffer");
  * sysctl requests.  This is implemented by serializing any userland
  * sysctl requests larger than a single page via an exclusive lock.
  */
-static struct sx sysctllock;
+static struct rmlock sysctllock;
 static struct sx sysctlmemlock;
 
-#define	SYSCTL_XLOCK()		sx_xlock(&sysctllock)
-#define	SYSCTL_XUNLOCK()	sx_xunlock(&sysctllock)
-#define	SYSCTL_SLOCK()		sx_slock(&sysctllock)
-#define	SYSCTL_SUNLOCK()	sx_sunlock(&sysctllock)
-#define	SYSCTL_XLOCKED()	sx_xlocked(&sysctllock)
-#define	SYSCTL_ASSERT_LOCKED()	sx_assert(&sysctllock, SA_LOCKED)
-#define	SYSCTL_ASSERT_XLOCKED()	sx_assert(&sysctllock, SA_XLOCKED)
-#define	SYSCTL_ASSERT_SLOCKED()	sx_assert(&sysctllock, SA_SLOCKED)
-#define	SYSCTL_INIT()		sx_init(&sysctllock, "sysctl lock")
+#define	SYSCTL_WLOCK()		rm_wlock(&sysctllock)
+#define	SYSCTL_WUNLOCK()	rm_wunlock(&sysctllock)
+#define	SYSCTL_RLOCK(tracker)	rm_rlock(&sysctllock, (tracker))
+#define	SYSCTL_RUNLOCK(tracker)	rm_runlock(&sysctllock, (tracker))
+#define	SYSCTL_WLOCKED()	rm_wowned(&sysctllock)
+#define	SYSCTL_ASSERT_LOCKED()	rm_assert(&sysctllock, RA_LOCKED)
+#define	SYSCTL_ASSERT_WLOCKED()	rm_assert(&sysctllock, RA_WLOCKED)
+#define	SYSCTL_ASSERT_RLOCKED()	rm_assert(&sysctllock, RA_RLOCKED)
+#define	SYSCTL_INIT()		rm_init_flags(&sysctllock, "sysctl lock", \
+				    RM_SLEEPABLE)
 #define	SYSCTL_SLEEP(ch, wmesg, timo)					\
-				sx_sleep(ch, &sysctllock, 0, wmesg, timo)
+				rm_sleep(ch, &sysctllock, 0, wmesg, timo)
 
 static int sysctl_root(SYSCTL_HANDLER_ARGS);
 
@@ -111,29 +113,6 @@ static int	sysctl_remove_oid_locked(struct sysctl_oid *oidp, int del,
 static int	sysctl_old_kernel(struct sysctl_req *, const void *, size_t);
 static int	sysctl_new_kernel(struct sysctl_req *, void *, size_t);
 
-static void
-sysctl_lock(bool xlock)
-{
-
-	if (xlock)
-		SYSCTL_XLOCK();
-	else
-		SYSCTL_SLOCK();
-}
-
-static bool
-sysctl_unlock(void)
-{
-	bool xlocked;
-
-	xlocked = SYSCTL_XLOCKED();
-	if (xlocked)
-		SYSCTL_XUNLOCK();
-	else
-		SYSCTL_SUNLOCK();
-	return (xlocked);
-}
-
 static struct sysctl_oid *
 sysctl_find_oidname(const char *name, struct sysctl_oid_list *list)
 {
@@ -154,29 +133,32 @@ sysctl_find_oidname(const char *name, struct sysctl_oid_list *list)
  * Order by number in each list.
  */
 void
-sysctl_xlock(void)
+sysctl_wlock(void)
 {
 
-	SYSCTL_XLOCK();
+	SYSCTL_WLOCK();
 }
 
 void
-sysctl_xunlock(void)
+sysctl_wunlock(void)
 {
 
-	SYSCTL_XUNLOCK();
+	SYSCTL_WUNLOCK();
 }
 
 static int
 sysctl_root_handler_locked(struct sysctl_oid *oid, void *arg1, intptr_t arg2,
-    struct sysctl_req *req)
+    struct sysctl_req *req, struct rm_priotracker *tracker)
 {
 	int error;
-	bool xlocked;
 
 	if (oid->oid_kind & CTLFLAG_DYN)
 		atomic_add_int(&oid->oid_running, 1);
-	xlocked = sysctl_unlock();
+
+	if (tracker != NULL)
+		SYSCTL_RUNLOCK(tracker);
+	else
+		SYSCTL_WUNLOCK();
 
 	if (!(oid->oid_kind & CTLFLAG_MPSAFE))
 		mtx_lock(&Giant);
@@ -184,7 +166,11 @@ sysctl_root_handler_locked(struct sysctl_oid *oid, void *arg1, intptr_t arg2,
 	if (!(oid->oid_kind & CTLFLAG_MPSAFE))
 		mtx_unlock(&Giant);
 
-	sysctl_lock(xlocked);
+	if (tracker != NULL)
+		SYSCTL_RLOCK(tracker);
+	else
+		SYSCTL_WLOCK();
+
 	if (oid->oid_kind & CTLFLAG_DYN) {
 		if (atomic_fetchadd_int(&oid->oid_running, -1) == 1 &&
 		    (oid->oid_kind & CTLFLAG_DYING) != 0)
@@ -283,7 +269,7 @@ sysctl_load_tunable_by_oid_locked(struct sysctl_oid *oidp)
 		return;
 	}
 	error = sysctl_root_handler_locked(oidp, oidp->oid_arg1,
-	    oidp->oid_arg2, &req);
+	    oidp->oid_arg2, &req, NULL);
 	if (error != 0)
 		printf("Setting sysctl %s failed: %d\n", path + rem, error);
 	if (penv != NULL)
@@ -303,7 +289,7 @@ sysctl_register_oid(struct sysctl_oid *oidp)
 	 * First check if another oid with the same name already
 	 * exists in the parent's list.
 	 */
-	SYSCTL_ASSERT_XLOCKED();
+	SYSCTL_ASSERT_WLOCKED();
 	p = sysctl_find_oidname(oidp->oid_name, parent);
 	if (p != NULL) {
 		if ((p->oid_kind & CTLTYPE) == CTLTYPE_NODE) {
@@ -397,7 +383,7 @@ sysctl_unregister_oid(struct sysctl_oid *oidp)
 	struct sysctl_oid *p;
 	int error;
 
-	SYSCTL_ASSERT_XLOCKED();
+	SYSCTL_ASSERT_WLOCKED();
 	error = ENOENT;
 	if (oidp->oid_number == OID_AUTO) {
 		error = EINVAL;
@@ -453,7 +439,7 @@ sysctl_ctx_free(struct sysctl_ctx_list *clist)
 	 * XXX This algorithm is a hack. But I don't know any
 	 * XXX better solution for now...
 	 */
-	SYSCTL_XLOCK();
+	SYSCTL_WLOCK();
 	TAILQ_FOREACH(e, clist, link) {
 		error = sysctl_remove_oid_locked(e->entry, 0, 0);
 		if (error)
@@ -473,7 +459,7 @@ sysctl_ctx_free(struct sysctl_ctx_list *clist)
 		e1 = TAILQ_PREV(e1, sysctl_ctx_list, link);
 	}
 	if (error) {
-		SYSCTL_XUNLOCK();
+		SYSCTL_WUNLOCK();
 		return(EBUSY);
 	}
 	/* Now really delete the entries */
@@ -487,7 +473,7 @@ sysctl_ctx_free(struct sysctl_ctx_list *clist)
 		free(e, M_SYSCTLOID);
 		e = e1;
 	}
-	SYSCTL_XUNLOCK();
+	SYSCTL_WUNLOCK();
 	return (error);
 }
 
@@ -497,7 +483,7 @@ sysctl_ctx_entry_add(struct sysctl_ctx_list *clist, struct sysctl_oid *oidp)
 {
 	struct sysctl_ctx_entry *e;
 
-	SYSCTL_ASSERT_XLOCKED();
+	SYSCTL_ASSERT_WLOCKED();
 	if (clist == NULL || oidp == NULL)
 		return(NULL);
 	e = malloc(sizeof(struct sysctl_ctx_entry), M_SYSCTLOID, M_WAITOK);
@@ -512,7 +498,7 @@ sysctl_ctx_entry_find(struct sysctl_ctx_list *clist, struct sysctl_oid *oidp)
 {
 	struct sysctl_ctx_entry *e;
 
-	SYSCTL_ASSERT_XLOCKED();
+	SYSCTL_ASSERT_WLOCKED();
 	if (clist == NULL || oidp == NULL)
 		return(NULL);
 	TAILQ_FOREACH(e, clist, link) {
@@ -534,15 +520,15 @@ sysctl_ctx_entry_del(struct sysctl_ctx_list *clist, struct sysctl_oid *oidp)
 
 	if (clist == NULL || oidp == NULL)
 		return (EINVAL);
-	SYSCTL_XLOCK();
+	SYSCTL_WLOCK();
 	e = sysctl_ctx_entry_find(clist, oidp);
 	if (e != NULL) {
 		TAILQ_REMOVE(clist, e, link);
-		SYSCTL_XUNLOCK();
+		SYSCTL_WUNLOCK();
 		free(e, M_SYSCTLOID);
 		return (0);
 	} else {
-		SYSCTL_XUNLOCK();
+		SYSCTL_WUNLOCK();
 		return (ENOENT);
 	}
 }
@@ -558,9 +544,9 @@ sysctl_remove_oid(struct sysctl_oid *oidp, int del, int recurse)
 {
 	int error;
 
-	SYSCTL_XLOCK();
+	SYSCTL_WLOCK();
 	error = sysctl_remove_oid_locked(oidp, del, recurse);
-	SYSCTL_XUNLOCK();
+	SYSCTL_WUNLOCK();
 	return (error);
 }
 
@@ -572,14 +558,14 @@ sysctl_remove_name(struct sysctl_oid *parent, const char *name,
 	int error;
 
 	error = ENOENT;
-	SYSCTL_XLOCK();
+	SYSCTL_WLOCK();
 	SLIST_FOREACH_SAFE(p, SYSCTL_CHILDREN(parent), oid_link, tmp) {
 		if (strcmp(p->oid_name, name) == 0) {
 			error = sysctl_remove_oid_locked(p, del, recurse);
 			break;
 		}
 	}
-	SYSCTL_XUNLOCK();
+	SYSCTL_WUNLOCK();
 
 	return (error);
 }
@@ -591,7 +577,7 @@ sysctl_remove_oid_locked(struct sysctl_oid *oidp, int del, int recurse)
 	struct sysctl_oid *p, *tmp;
 	int error;
 
-	SYSCTL_ASSERT_XLOCKED();
+	SYSCTL_ASSERT_WLOCKED();
 	if (oidp == NULL)
 		return(EINVAL);
 	if ((oidp->oid_kind & CTLFLAG_DYN) == 0) {
@@ -666,7 +652,7 @@ sysctl_add_oid(struct sysctl_ctx_list *clist, struct sysctl_oid_list *parent,
 	if (parent == NULL)
 		return(NULL);
 	/* Check if the node already exists, otherwise create it */
-	SYSCTL_XLOCK();
+	SYSCTL_WLOCK();
 	oidp = sysctl_find_oidname(name, parent);
 	if (oidp != NULL) {
 		if ((oidp->oid_kind & CTLTYPE) == CTLTYPE_NODE) {
@@ -674,10 +660,10 @@ sysctl_add_oid(struct sysctl_ctx_list *clist, struct sysctl_oid_list *parent,
 			/* Update the context */
 			if (clist != NULL)
 				sysctl_ctx_entry_add(clist, oidp);
-			SYSCTL_XUNLOCK();
+			SYSCTL_WUNLOCK();
 			return (oidp);
 		} else {
-			SYSCTL_XUNLOCK();
+			SYSCTL_WUNLOCK();
 			printf("can't re-use a leaf (%s)!\n", name);
 			return (NULL);
 		}
@@ -700,7 +686,7 @@ sysctl_add_oid(struct sysctl_ctx_list *clist, struct sysctl_oid_list *parent,
 		sysctl_ctx_entry_add(clist, oidp);
 	/* Register this oid */
 	sysctl_register_oid(oidp);
-	SYSCTL_XUNLOCK();
+	SYSCTL_WUNLOCK();
 	return (oidp);
 }
 
@@ -714,10 +700,10 @@ sysctl_rename_oid(struct sysctl_oid *oidp, const char *name)
 	char *oldname;
 
 	newname = strdup(name, M_SYSCTLOID);
-	SYSCTL_XLOCK();
+	SYSCTL_WLOCK();
 	oldname = __DECONST(char *, oidp->oid_name);
 	oidp->oid_name = newname;
-	SYSCTL_XUNLOCK();
+	SYSCTL_WUNLOCK();
 	free(oldname, M_SYSCTLOID);
 }
 
@@ -729,21 +715,21 @@ sysctl_move_oid(struct sysctl_oid *oid, struct sysctl_oid_list *parent)
 {
 	struct sysctl_oid *oidp;
 
-	SYSCTL_XLOCK();
+	SYSCTL_WLOCK();
 	if (oid->oid_parent == parent) {
-		SYSCTL_XUNLOCK();
+		SYSCTL_WUNLOCK();
 		return (0);
 	}
 	oidp = sysctl_find_oidname(oid->oid_name, parent);
 	if (oidp != NULL) {
-		SYSCTL_XUNLOCK();
+		SYSCTL_WUNLOCK();
 		return (EEXIST);
 	}
 	sysctl_unregister_oid(oid);
 	oid->oid_parent = parent;
 	oid->oid_number = OID_AUTO;
 	sysctl_register_oid(oid);
-	SYSCTL_XUNLOCK();
+	SYSCTL_WUNLOCK();
 	return (0);
 }
 
@@ -759,10 +745,10 @@ sysctl_register_all(void *arg)
 
 	sx_init(&sysctlmemlock, "sysctl mem");
 	SYSCTL_INIT();
-	SYSCTL_XLOCK();
+	SYSCTL_WLOCK();
 	SET_FOREACH(oidp, sysctl_set)
 		sysctl_register_oid(*oidp);
-	SYSCTL_XUNLOCK();
+	SYSCTL_WUNLOCK();
 }
 SYSINIT(sysctl, SI_SUB_KMEM, SI_ORDER_FIRST, sysctl_register_all, 0);
 
@@ -832,14 +818,15 @@ sysctl_sysctl_debug_dump_node(struct sysctl_oid_list *l, int i)
 static int
 sysctl_sysctl_debug(SYSCTL_HANDLER_ARGS)
 {
+	struct rm_priotracker tracker;
 	int error;
 
 	error = priv_check(req->td, PRIV_SYSCTL_DEBUG);
 	if (error)
 		return (error);
-	SYSCTL_SLOCK();
+	SYSCTL_RLOCK(&tracker);
 	sysctl_sysctl_debug_dump_node(&sysctl__children, 0);
-	SYSCTL_SUNLOCK();
+	SYSCTL_RUNLOCK(&tracker);
 	return (ENOENT);
 }
 
@@ -855,9 +842,10 @@ sysctl_sysctl_name(SYSCTL_HANDLER_ARGS)
 	int error = 0;
 	struct sysctl_oid *oid;
 	struct sysctl_oid_list *lsp = &sysctl__children, *lsp2;
+	struct rm_priotracker tracker;
 	char buf[10];
 
-	SYSCTL_SLOCK();
+	SYSCTL_RLOCK(&tracker);
 	while (namelen) {
 		if (!lsp) {
 			snprintf(buf,sizeof(buf),"%d",*name);
@@ -900,7 +888,7 @@ sysctl_sysctl_name(SYSCTL_HANDLER_ARGS)
 	}
 	error = SYSCTL_OUT(req, "", 1);
  out:
-	SYSCTL_SUNLOCK();
+	SYSCTL_RUNLOCK(&tracker);
 	return (error);
 }
 
@@ -979,11 +967,12 @@ sysctl_sysctl_next(SYSCTL_HANDLER_ARGS)
 	int i, j, error;
 	struct sysctl_oid *oid;
 	struct sysctl_oid_list *lsp = &sysctl__children;
+	struct rm_priotracker tracker;
 	int newoid[CTL_MAXNAME];
 
-	SYSCTL_SLOCK();
+	SYSCTL_RLOCK(&tracker);
 	i = sysctl_sysctl_next_ls(lsp, name, namelen, newoid, &j, 1, &oid);
-	SYSCTL_SUNLOCK();
+	SYSCTL_RUNLOCK(&tracker);
 	if (i)
 		return (ENOENT);
 	error = SYSCTL_OUT(req, newoid, j * sizeof (int));
@@ -1042,6 +1031,7 @@ sysctl_sysctl_name2oid(SYSCTL_HANDLER_ARGS)
 	char *p;
 	int error, oid[CTL_MAXNAME], len = 0;
 	struct sysctl_oid *op = 0;
+	struct rm_priotracker tracker;
 
 	if (!req->newlen) 
 		return (ENOENT);
@@ -1058,9 +1048,9 @@ sysctl_sysctl_name2oid(SYSCTL_HANDLER_ARGS)
 
 	p [req->newlen] = '\0';
 
-	SYSCTL_SLOCK();
+	SYSCTL_RLOCK(&tracker);
 	error = name2oid(p, oid, &len, &op);
-	SYSCTL_SUNLOCK();
+	SYSCTL_RUNLOCK(&tracker);
 
 	free(p, M_SYSCTL);
 
@@ -1083,9 +1073,10 @@ static int
 sysctl_sysctl_oidfmt(SYSCTL_HANDLER_ARGS)
 {
 	struct sysctl_oid *oid;
+	struct rm_priotracker tracker;
 	int error;
 
-	SYSCTL_SLOCK();
+	SYSCTL_RLOCK(&tracker);
 	error = sysctl_find_oid(arg1, arg2, &oid, NULL, req);
 	if (error)
 		goto out;
@@ -1099,7 +1090,7 @@ sysctl_sysctl_oidfmt(SYSCTL_HANDLER_ARGS)
 		goto out;
 	error = SYSCTL_OUT(req, oid->oid_fmt, strlen(oid->oid_fmt) + 1);
  out:
-	SYSCTL_SUNLOCK();
+	SYSCTL_RUNLOCK(&tracker);
 	return (error);
 }
 
@@ -1111,9 +1102,10 @@ static int
 sysctl_sysctl_oiddescr(SYSCTL_HANDLER_ARGS)
 {
 	struct sysctl_oid *oid;
+	struct rm_priotracker tracker;
 	int error;
 
-	SYSCTL_SLOCK();
+	SYSCTL_RLOCK(&tracker);
 	error = sysctl_find_oid(arg1, arg2, &oid, NULL, req);
 	if (error)
 		goto out;
@@ -1124,7 +1116,7 @@ sysctl_sysctl_oiddescr(SYSCTL_HANDLER_ARGS)
 	}
 	error = SYSCTL_OUT(req, oid->oid_descr, strlen(oid->oid_descr) + 1);
  out:
-	SYSCTL_SUNLOCK();
+	SYSCTL_RUNLOCK(&tracker);
 	return (error);
 }
 
@@ -1429,9 +1421,7 @@ kernel_sysctl(struct thread *td, int *name, u_int namelen, void *old,
 	req.newfunc = sysctl_new_kernel;
 	req.lock = REQ_UNWIRED;
 
-	SYSCTL_SLOCK();
 	error = sysctl_root(0, name, namelen, &req);
-	SYSCTL_SUNLOCK();
 
 	if (req.lock == REQ_WIRED && req.validlen > 0)
 		vsunlock(req.oldptr, req.validlen);
@@ -1608,13 +1598,14 @@ static int
 sysctl_root(SYSCTL_HANDLER_ARGS)
 {
 	struct sysctl_oid *oid;
+	struct rm_priotracker tracker;
 	int error, indx, lvl;
 
-	SYSCTL_ASSERT_SLOCKED();
+	SYSCTL_RLOCK(&tracker);
 
 	error = sysctl_find_oid(arg1, arg2, &oid, &indx, req);
 	if (error)
-		return (error);
+		goto out;
 
 	if ((oid->oid_kind & CTLTYPE) == CTLTYPE_NODE) {
 		/*
@@ -1622,13 +1613,17 @@ sysctl_root(SYSCTL_HANDLER_ARGS)
 		 * no handler.  Inform the user that it's a node.
 		 * The indx may or may not be the same as namelen.
 		 */
-		if (oid->oid_handler == NULL)
-			return (EISDIR);
+		if (oid->oid_handler == NULL) {
+			error = EISDIR;
+			goto out;
+		}
 	}
 
 	/* Is this sysctl writable? */
-	if (req->newptr && !(oid->oid_kind & CTLFLAG_WR))
-		return (EPERM);
+	if (req->newptr && !(oid->oid_kind & CTLFLAG_WR)) {
+		error = EPERM;
+		goto out;
+	}
 
 	KASSERT(req->td != NULL, ("sysctl_root(): req->td == NULL"));
 
@@ -1638,10 +1633,11 @@ sysctl_root(SYSCTL_HANDLER_ARGS)
 	 * writing unless specifically granted for the node.
 	 */
 	if (IN_CAPABILITY_MODE(req->td)) {
-		if (req->oldptr && !(oid->oid_kind & CTLFLAG_CAPRD))
-			return (EPERM);
-		if (req->newptr && !(oid->oid_kind & CTLFLAG_CAPWR))
-			return (EPERM);
+		if ((req->oldptr && !(oid->oid_kind & CTLFLAG_CAPRD)) ||
+		    (req->newptr && !(oid->oid_kind & CTLFLAG_CAPWR))) {
+			error = EPERM;
+			goto out;
+		}
 	}
 #endif
 
@@ -1650,7 +1646,7 @@ sysctl_root(SYSCTL_HANDLER_ARGS)
 		lvl = (oid->oid_kind & CTLMASK_SECURE) >> CTLSHIFT_SECURE;
 		error = securelevel_gt(req->td->td_ucred, lvl);
 		if (error)
-			return (error);
+			goto out;
 	}
 
 	/* Is this sysctl writable by only privileged users? */
@@ -1668,11 +1664,13 @@ sysctl_root(SYSCTL_HANDLER_ARGS)
 			priv = PRIV_SYSCTL_WRITE;
 		error = priv_check(req->td, priv);
 		if (error)
-			return (error);
+			goto out;
 	}
 
-	if (!oid->oid_handler)
-		return (EINVAL);
+	if (!oid->oid_handler) {
+		error = EINVAL;
+		goto out;
+	}
 
 	if ((oid->oid_kind & CTLTYPE) == CTLTYPE_NODE) {
 		arg1 = (int *)arg1 + indx;
@@ -1685,16 +1683,18 @@ sysctl_root(SYSCTL_HANDLER_ARGS)
 	error = mac_system_check_sysctl(req->td->td_ucred, oid, arg1, arg2,
 	    req);
 	if (error != 0)
-		return (error);
+		goto out;
 #endif
 #ifdef VIMAGE
 	if ((oid->oid_kind & CTLFLAG_VNET) && arg1 != NULL)
 		arg1 = (void *)(curvnet->vnet_data_base + (uintptr_t)arg1);
 #endif
-	error = sysctl_root_handler_locked(oid, arg1, arg2, req);
+	error = sysctl_root_handler_locked(oid, arg1, arg2, req, &tracker);
 
 	KFAIL_POINT_ERROR(_debug_fail_point, sysctl_running, error);
 
+out:
+	SYSCTL_RUNLOCK(&tracker);
 	return (error);
 }
 
@@ -1794,9 +1794,7 @@ userland_sysctl(struct thread *td, int *name, u_int namelen, void *old,
 	for (;;) {
 		req.oldidx = 0;
 		req.newidx = 0;
-		SYSCTL_SLOCK();
 		error = sysctl_root(0, name, namelen, &req);
-		SYSCTL_SUNLOCK();
 		if (error != EAGAIN)
 			break;
 		kern_yield(PRI_USER);
diff --git a/sys/kern/kern_tc.c b/sys/kern/kern_tc.c
index 432e38afb990f..d4a1a536be8e8 100644
--- a/sys/kern/kern_tc.c
+++ b/sys/kern/kern_tc.c
@@ -1924,20 +1924,27 @@ SYSINIT(timecounter, SI_SUB_CLOCKS, SI_ORDER_SECOND, inittimecounter, NULL);
 static int cpu_tick_variable;
 static uint64_t	cpu_tick_frequency;
 
+static DPCPU_DEFINE(uint64_t, tc_cpu_ticks_base);
+static DPCPU_DEFINE(unsigned, tc_cpu_ticks_last);
+
 static uint64_t
 tc_cpu_ticks(void)
 {
-	static uint64_t base;
-	static unsigned last;
-	unsigned u;
 	struct timecounter *tc;
+	uint64_t res, *base;
+	unsigned u, *last;
 
+	critical_enter();
+	base = DPCPU_PTR(tc_cpu_ticks_base);
+	last = DPCPU_PTR(tc_cpu_ticks_last);
 	tc = timehands->th_counter;
 	u = tc->tc_get_timecount(tc) & tc->tc_counter_mask;
-	if (u < last)
-		base += (uint64_t)tc->tc_counter_mask + 1;
-	last = u;
-	return (u + base);
+	if (u < *last)
+		*base += (uint64_t)tc->tc_counter_mask + 1;
+	*last = u;
+	res = u + *base;
+	critical_exit();
+	return (res);
 }
 
 void
diff --git a/sys/kern/kern_timeout.c b/sys/kern/kern_timeout.c
index 71c88e0290b97..01cf1769628ea 100644
--- a/sys/kern/kern_timeout.c
+++ b/sys/kern/kern_timeout.c
@@ -718,9 +718,9 @@ softclock_call_cc(struct callout *c, struct callout_cpu *cc,
 	sbt1 = sbinuptime();
 #endif
 	THREAD_NO_SLEEPING();
-	SDT_PROBE(callout_execute, kernel, , callout__start, c, 0, 0, 0, 0);
+	SDT_PROBE1(callout_execute, kernel, , callout__start, c);
 	c_func(c_arg);
-	SDT_PROBE(callout_execute, kernel, , callout__end, c, 0, 0, 0, 0);
+	SDT_PROBE1(callout_execute, kernel, , callout__end, c);
 	THREAD_SLEEPING_OK();
 #if defined(DIAGNOSTIC) || defined(CALLOUT_PROFILING)
 	sbt2 = sbinuptime();
diff --git a/sys/kern/link_elf.c b/sys/kern/link_elf.c
index 3b741cc38fbb5..c931ae927495e 100644
--- a/sys/kern/link_elf.c
+++ b/sys/kern/link_elf.c
@@ -158,7 +158,7 @@ static int	link_elf_each_function_nameval(linker_file_t,
 static void	link_elf_reloc_local(linker_file_t);
 static long	link_elf_symtab_get(linker_file_t, const Elf_Sym **);
 static long	link_elf_strtab_get(linker_file_t, caddr_t *);
-static Elf_Addr	elf_lookup(linker_file_t, Elf_Size, int);
+static int	elf_lookup(linker_file_t, Elf_Size, int, Elf_Addr *);
 
 static kobj_method_t link_elf_methods[] = {
 	KOBJMETHOD(linker_lookup_symbol,	link_elf_lookup_symbol),
@@ -1548,8 +1548,8 @@ elf_get_symname(linker_file_t lf, Elf_Size symidx)
  * This is not only more efficient, it's also more correct. It's not always
  * the case that the symbol can be found through the hash table.
  */
-static Elf_Addr
-elf_lookup(linker_file_t lf, Elf_Size symidx, int deps)
+static int
+elf_lookup(linker_file_t lf, Elf_Size symidx, int deps, Elf_Addr *res)
 {
 	elf_file_t ef = (elf_file_t)lf;
 	const Elf_Sym *sym;
@@ -1557,8 +1557,10 @@ elf_lookup(linker_file_t lf, Elf_Size symidx, int deps)
 	Elf_Addr addr, start, base;
 
 	/* Don't even try to lookup the symbol if the index is bogus. */
-	if (symidx >= ef->nchains)
-		return (0);
+	if (symidx >= ef->nchains) {
+		*res = 0;
+		return (EINVAL);
+	}
 
 	sym = ef->symtab + symidx;
 
@@ -1568,9 +1570,12 @@ elf_lookup(linker_file_t lf, Elf_Size symidx, int deps)
 	 */
 	if (ELF_ST_BIND(sym->st_info) == STB_LOCAL) {
 		/* Force lookup failure when we have an insanity. */
-		if (sym->st_shndx == SHN_UNDEF || sym->st_value == 0)
-			return (0);
-		return ((Elf_Addr)ef->address + sym->st_value);
+		if (sym->st_shndx == SHN_UNDEF || sym->st_value == 0) {
+			*res = 0;
+			return (EINVAL);
+		}
+		*res = ((Elf_Addr)ef->address + sym->st_value);
+		return (0);
 	}
 
 	/*
@@ -1583,8 +1588,10 @@ elf_lookup(linker_file_t lf, Elf_Size symidx, int deps)
 	symbol = ef->strtab + sym->st_name;
 
 	/* Force a lookup failure if the symbol name is bogus. */
-	if (*symbol == 0)
-		return (0);
+	if (*symbol == 0) {
+		*res = 0;
+		return (EINVAL);
+	}
 
 	addr = ((Elf_Addr)linker_file_lookup_symbol(lf, symbol, deps));
 
@@ -1594,7 +1601,8 @@ elf_lookup(linker_file_t lf, Elf_Size symidx, int deps)
 	else if (elf_set_find(&set_vnet_list, addr, &start, &base))
 		addr = addr - start + base;
 #endif
-	return addr;
+	*res = addr;
+	return (0);
 }
 
 static void
diff --git a/sys/kern/link_elf_obj.c b/sys/kern/link_elf_obj.c
index 021381db78d73..00fe1e418642f 100644
--- a/sys/kern/link_elf_obj.c
+++ b/sys/kern/link_elf_obj.c
@@ -144,7 +144,8 @@ static void	link_elf_reloc_local(linker_file_t);
 static long	link_elf_symtab_get(linker_file_t, const Elf_Sym **);
 static long	link_elf_strtab_get(linker_file_t, caddr_t *);
 
-static Elf_Addr elf_obj_lookup(linker_file_t lf, Elf_Size symidx, int deps);
+static int	elf_obj_lookup(linker_file_t lf, Elf_Size symidx, int deps,
+		    Elf_Addr *);
 
 static kobj_method_t link_elf_methods[] = {
 	KOBJMETHOD(linker_lookup_symbol,	link_elf_lookup_symbol),
@@ -1253,38 +1254,46 @@ elf_obj_cleanup_globals_cache(elf_file_t ef)
  * This is not only more efficient, it's also more correct. It's not always
  * the case that the symbol can be found through the hash table.
  */
-static Elf_Addr
-elf_obj_lookup(linker_file_t lf, Elf_Size symidx, int deps)
+static int
+elf_obj_lookup(linker_file_t lf, Elf_Size symidx, int deps, Elf_Addr *res)
 {
 	elf_file_t ef = (elf_file_t)lf;
 	Elf_Sym *sym;
 	const char *symbol;
-	Elf_Addr ret;
+	Elf_Addr res1;
 
 	/* Don't even try to lookup the symbol if the index is bogus. */
-	if (symidx >= ef->ddbsymcnt)
-		return (0);
+	if (symidx >= ef->ddbsymcnt) {
+		*res = 0;
+		return (EINVAL);
+	}
 
 	sym = ef->ddbsymtab + symidx;
 
 	/* Quick answer if there is a definition included. */
-	if (sym->st_shndx != SHN_UNDEF)
-		return (sym->st_value);
+	if (sym->st_shndx != SHN_UNDEF) {
+		*res = sym->st_value;
+		return (0);
+	}
 
 	/* If we get here, then it is undefined and needs a lookup. */
 	switch (ELF_ST_BIND(sym->st_info)) {
 	case STB_LOCAL:
 		/* Local, but undefined? huh? */
-		return (0);
+		*res = 0;
+		return (EINVAL);
 
 	case STB_GLOBAL:
+	case STB_WEAK:
 		/* Relative to Data or Function name */
 		symbol = ef->ddbstrtab + sym->st_name;
 
 		/* Force a lookup failure if the symbol name is bogus. */
-		if (*symbol == 0)
-			return (0);
-		ret = ((Elf_Addr)linker_file_lookup_symbol(lf, symbol, deps));
+		if (*symbol == 0) {
+			*res = 0;
+			return (EINVAL);
+		}
+		res1 = (Elf_Addr)linker_file_lookup_symbol(lf, symbol, deps);
 
 		/*
 		 * Cache global lookups during module relocation. The failure
@@ -1296,18 +1305,20 @@ elf_obj_lookup(linker_file_t lf, Elf_Size symidx, int deps)
 		 * restored to SHN_UNDEF in elf_obj_cleanup_globals_cache(),
 		 * above.
 		 */
-		if (ret != 0) {
+		if (res1 != 0) {
 			sym->st_shndx = SHN_FBSD_CACHED;
-			sym->st_value = ret;
+			sym->st_value = res1;
+			*res = res1;
+			return (0);
+		} else if (ELF_ST_BIND(sym->st_info) == STB_WEAK) {
+			sym->st_value = 0;
+			*res = 0;
+			return (0);
 		}
-		return (ret);
-
-	case STB_WEAK:
-		printf("link_elf_obj: Weak symbols not supported\n");
-		return (0);
+		return (EINVAL);
 
 	default:
-		return (0);
+		return (EINVAL);
 	}
 }
 
diff --git a/sys/kern/subr_param.c b/sys/kern/subr_param.c
index 52c6f167f753a..65fb3e78d322f 100644
--- a/sys/kern/subr_param.c
+++ b/sys/kern/subr_param.c
@@ -76,7 +76,7 @@ __FBSDID("$FreeBSD$");
 #define NBUF 0
 #endif
 #ifndef MAXFILES
-#define	MAXFILES (maxproc * 2)
+#define	MAXFILES (40 + 32 * maxusers)
 #endif
 
 static int sysctl_kern_vm_guest(SYSCTL_HANDLER_ARGS);
@@ -253,6 +253,8 @@ init_param2(long physpages)
 	TUNABLE_INT_FETCH("kern.maxproc", &maxproc);
 	if (maxproc > (physpages / 12))
 		maxproc = physpages / 12;
+	if (maxproc > pid_max)
+		maxproc = pid_max;
 	maxprocperuid = (maxproc * 9) / 10;
 
 	/*
diff --git a/sys/kern/subr_sbuf.c b/sys/kern/subr_sbuf.c
index 3b5660efef398..3c36b28628ef4 100644
--- a/sys/kern/subr_sbuf.c
+++ b/sys/kern/subr_sbuf.c
@@ -35,6 +35,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/ctype.h>
 #include <sys/errno.h>
 #include <sys/kernel.h>
+#include <sys/limits.h>
 #include <sys/malloc.h>
 #include <sys/systm.h>
 #include <sys/uio.h>
@@ -42,6 +43,7 @@ __FBSDID("$FreeBSD$");
 #else /* _KERNEL */
 #include <ctype.h>
 #include <errno.h>
+#include <limits.h>
 #include <stdarg.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -383,34 +385,51 @@ sbuf_drain(struct sbuf *s)
 }
 
 /*
- * Append a byte to an sbuf.  This is the core function for appending
+ * Append bytes to an sbuf.  This is the core function for appending
  * to an sbuf and is the main place that deals with extending the
  * buffer and marking overflow.
  */
 static void
-sbuf_put_byte(struct sbuf *s, int c)
+sbuf_put_bytes(struct sbuf *s, const char *buf, size_t len)
 {
+	size_t n;
 
 	assert_sbuf_integrity(s);
 	assert_sbuf_state(s, 0);
 
 	if (s->s_error != 0)
 		return;
-	if (SBUF_FREESPACE(s) <= 0) {
-		/*
-		 * If there is a drain, use it, otherwise extend the
-		 * buffer.
-		 */
-		if (s->s_drain_func != NULL)
-			(void)sbuf_drain(s);
-		else if (sbuf_extend(s, 1) < 0)
-			s->s_error = ENOMEM;
-		if (s->s_error != 0)
-			return;
+	while (len > 0) {
+		if (SBUF_FREESPACE(s) <= 0) {
+			/*
+			 * If there is a drain, use it, otherwise extend the
+			 * buffer.
+			 */
+			if (s->s_drain_func != NULL)
+				(void)sbuf_drain(s);
+			else if (sbuf_extend(s, len > INT_MAX ? INT_MAX : len)
+			    < 0)
+				s->s_error = ENOMEM;
+			if (s->s_error != 0)
+				return;
+		}
+		n = SBUF_FREESPACE(s);
+		if (len < n)
+			n = len;
+		memcpy(&s->s_buf[s->s_len], buf, n);
+		s->s_len += n;
+		if (SBUF_ISSECTION(s))
+			s->s_sect_len += n;
+		len -= n;
+		buf += n;
 	}
-	s->s_buf[s->s_len++] = c;
-	if (SBUF_ISSECTION(s))
-		s->s_sect_len++;
+}
+
+static void
+sbuf_put_byte(struct sbuf *s, char c)
+{
+
+	sbuf_put_bytes(s, &c, 1);
 }
 
 /*
@@ -419,19 +438,10 @@ sbuf_put_byte(struct sbuf *s, int c)
 int
 sbuf_bcat(struct sbuf *s, const void *buf, size_t len)
 {
-	const char *str = buf;
-	const char *end = str + len;
-
-	assert_sbuf_integrity(s);
-	assert_sbuf_state(s, 0);
 
+	sbuf_put_bytes(s, buf, len);
 	if (s->s_error != 0)
 		return (-1);
-	for (; str < end; str++) {
-		sbuf_put_byte(s, *str);
-		if (s->s_error != 0)
-			return (-1);
-	}
 	return (0);
 }
 
@@ -485,18 +495,12 @@ sbuf_bcpy(struct sbuf *s, const void *buf, size_t len)
 int
 sbuf_cat(struct sbuf *s, const char *str)
 {
+	size_t n;
 
-	assert_sbuf_integrity(s);
-	assert_sbuf_state(s, 0);
-
+	n = strlen(str);
+	sbuf_put_bytes(s, str, n);
 	if (s->s_error != 0)
 		return (-1);
-
-	while (*str != '\0') {
-		sbuf_put_byte(s, *str++);
-		if (s->s_error != 0)
-			return (-1);
-	}
 	return (0);
 }
 
diff --git a/sys/kern/subr_syscall.c b/sys/kern/subr_syscall.c
index 6d12b611b2dbb..af096138c208f 100644
--- a/sys/kern/subr_syscall.c
+++ b/sys/kern/subr_syscall.c
@@ -63,14 +63,14 @@ syscallenter(struct thread *td, struct syscall_args *sa)
 	td->td_pticks = 0;
 	if (td->td_cowgen != p->p_cowgen)
 		thread_cow_update(td);
-	if (p->p_flag & P_TRACED) {
-		traced = 1;
+	traced = (p->p_flag & P_TRACED) != 0;
+	if (traced || td->td_dbgflags & TDB_USERWR) {
 		PROC_LOCK(p);
 		td->td_dbgflags &= ~TDB_USERWR;
-		td->td_dbgflags |= TDB_SCE;
+		if (traced)
+			td->td_dbgflags |= TDB_SCE;
 		PROC_UNLOCK(p);
-	} else
-		traced = 0;
+	}
 	error = (p->p_sysent->sv_fetch_syscall_args)(td, sa);
 #ifdef KTRACE
 	if (KTRPOINT(td, KTR_SYSCALL))
diff --git a/sys/kern/vfs_bio.c b/sys/kern/vfs_bio.c
index ea7c0a4fd5ba7..0fab0009bc581 100644
--- a/sys/kern/vfs_bio.c
+++ b/sys/kern/vfs_bio.c
@@ -110,7 +110,9 @@ static void vfs_page_set_validclean(struct buf *bp, vm_ooffset_t off,
 		vm_page_t m);
 static void vfs_clean_pages_dirty_buf(struct buf *bp);
 static void vfs_setdirty_locked_object(struct buf *bp);
-static void vfs_vmio_release(struct buf *bp);
+static void vfs_vmio_invalidate(struct buf *bp);
+static void vfs_vmio_truncate(struct buf *bp, int npages);
+static void vfs_vmio_extend(struct buf *bp, int npages, int size);
 static int vfs_bio_clcheck(struct vnode *vp, int size,
 		daddr_t lblkno, daddr_t blkno);
 static int buf_flush(struct vnode *vp, int);
@@ -661,11 +663,9 @@ waitrunningbufspace(void)
  *	bit if the newly extended portion of the buffer does not contain
  *	valid data.
  */
-static __inline
-void
-vfs_buf_test_cache(struct buf *bp,
-		  vm_ooffset_t foff, vm_offset_t off, vm_offset_t size,
-		  vm_page_t m)
+static __inline void
+vfs_buf_test_cache(struct buf *bp, vm_ooffset_t foff, vm_offset_t off,
+    vm_offset_t size, vm_page_t m)
 {
 
 	VM_OBJECT_ASSERT_LOCKED(m->object);
@@ -1785,6 +1785,8 @@ brelse(struct buf *bp)
 	    bp, bp->b_vp, bp->b_flags);
 	KASSERT(!(bp->b_flags & (B_CLUSTER|B_PAGING)),
 	    ("brelse: inappropriate B_PAGING or B_CLUSTER bp %p", bp));
+	KASSERT((bp->b_flags & B_VMIO) != 0 || (bp->b_flags & B_NOREUSE) == 0,
+	    ("brelse: non-VMIO buffer marked NOREUSE"));
 
 	if (BUF_LOCKRECURSED(bp)) {
 		/*
@@ -1829,20 +1831,19 @@ brelse(struct buf *bp)
 			bdirtysub();
 		bp->b_flags &= ~(B_DELWRI | B_CACHE);
 		if ((bp->b_flags & B_VMIO) == 0) {
-			if (bp->b_bufsize)
-				allocbuf(bp, 0);
+			allocbuf(bp, 0);
 			if (bp->b_vp)
 				brelvp(bp);
 		}
 	}
 
 	/*
-	 * We must clear B_RELBUF if B_DELWRI is set.  If vfs_vmio_release() 
+	 * We must clear B_RELBUF if B_DELWRI is set.  If vfs_vmio_truncate() 
 	 * is called with B_DELWRI set, the underlying pages may wind up
 	 * getting freed causing a previous write (bdwrite()) to get 'lost'
 	 * because pages associated with a B_DELWRI bp are marked clean.
 	 * 
-	 * We still allow the B_INVAL case to call vfs_vmio_release(), even
+	 * We still allow the B_INVAL case to call vfs_vmio_truncate(), even
 	 * if B_DELWRI is set.
 	 */
 	if (bp->b_flags & B_DELWRI)
@@ -1865,107 +1866,19 @@ brelse(struct buf *bp)
 	 * around to prevent it from being reconstituted and starting a second
 	 * background write.
 	 */
-	if ((bp->b_flags & B_VMIO)
-	    && !(bp->b_vp->v_mount != NULL &&
-		 (bp->b_vp->v_mount->mnt_vfc->vfc_flags & VFCF_NETWORK) != 0 &&
-		 !vn_isdisk(bp->b_vp, NULL) &&
-		 (bp->b_flags & B_DELWRI))
-	    ) {
-
-		int i, j, resid;
-		vm_page_t m;
-		off_t foff;
-		vm_pindex_t poff;
-		vm_object_t obj;
-
-		obj = bp->b_bufobj->bo_object;
-
-		/*
-		 * Get the base offset and length of the buffer.  Note that 
-		 * in the VMIO case if the buffer block size is not
-		 * page-aligned then b_data pointer may not be page-aligned.
-		 * But our b_pages[] array *IS* page aligned.
-		 *
-		 * block sizes less then DEV_BSIZE (usually 512) are not 
-		 * supported due to the page granularity bits (m->valid,
-		 * m->dirty, etc...). 
-		 *
-		 * See man buf(9) for more information
-		 */
-		resid = bp->b_bufsize;
-		foff = bp->b_offset;
-		for (i = 0; i < bp->b_npages; i++) {
-			int had_bogus = 0;
-
-			m = bp->b_pages[i];
-
-			/*
-			 * If we hit a bogus page, fixup *all* the bogus pages
-			 * now.
-			 */
-			if (m == bogus_page) {
-				poff = OFF_TO_IDX(bp->b_offset);
-				had_bogus = 1;
-
-				VM_OBJECT_RLOCK(obj);
-				for (j = i; j < bp->b_npages; j++) {
-					vm_page_t mtmp;
-					mtmp = bp->b_pages[j];
-					if (mtmp == bogus_page) {
-						mtmp = vm_page_lookup(obj, poff + j);
-						if (!mtmp) {
-							panic("brelse: page missing\n");
-						}
-						bp->b_pages[j] = mtmp;
-					}
-				}
-				VM_OBJECT_RUNLOCK(obj);
-
-				if ((bp->b_flags & B_INVAL) == 0 &&
-				    buf_mapped(bp)) {
-					BUF_CHECK_MAPPED(bp);
-					pmap_qenter(
-					    trunc_page((vm_offset_t)bp->b_data),
-					    bp->b_pages, bp->b_npages);
-				}
-				m = bp->b_pages[i];
-			}
-			if ((bp->b_flags & B_NOCACHE) ||
-			    (bp->b_ioflags & BIO_ERROR &&
-			     bp->b_iocmd == BIO_READ)) {
-				int poffset = foff & PAGE_MASK;
-				int presid = resid > (PAGE_SIZE - poffset) ?
-					(PAGE_SIZE - poffset) : resid;
-
-				KASSERT(presid >= 0, ("brelse: extra page"));
-				VM_OBJECT_WLOCK(obj);
-				while (vm_page_xbusied(m)) {
-					vm_page_lock(m);
-					VM_OBJECT_WUNLOCK(obj);
-					vm_page_busy_sleep(m, "mbncsh");
-					VM_OBJECT_WLOCK(obj);
-				}
-				if (pmap_page_wired_mappings(m) == 0)
-					vm_page_set_invalid(m, poffset, presid);
-				VM_OBJECT_WUNLOCK(obj);
-				if (had_bogus)
-					printf("avoided corruption bug in bogus_page/brelse code\n");
-			}
-			resid -= PAGE_SIZE - (foff & PAGE_MASK);
-			foff = (foff + PAGE_SIZE) & ~(off_t)PAGE_MASK;
-		}
-		if (bp->b_flags & (B_INVAL | B_RELBUF))
-			vfs_vmio_release(bp);
-
-	} else if (bp->b_flags & B_VMIO) {
-
-		if (bp->b_flags & (B_INVAL | B_RELBUF)) {
-			vfs_vmio_release(bp);
-		}
+	if ((bp->b_flags & B_VMIO) && (bp->b_flags & B_NOCACHE ||
+	    (bp->b_ioflags & BIO_ERROR && bp->b_iocmd == BIO_READ)) &&
+	    !(bp->b_vp->v_mount != NULL &&
+	    (bp->b_vp->v_mount->mnt_vfc->vfc_flags & VFCF_NETWORK) != 0 &&
+	    !vn_isdisk(bp->b_vp, NULL) && (bp->b_flags & B_DELWRI))) {
+		vfs_vmio_invalidate(bp);
+		allocbuf(bp, 0);
+	}
 
-	} else if ((bp->b_flags & (B_INVAL | B_RELBUF)) != 0) {
-		if (bp->b_bufsize != 0)
-			allocbuf(bp, 0);
+	if ((bp->b_flags & (B_INVAL | B_RELBUF)) != 0 ||
+	    (bp->b_flags & (B_DELWRI | B_NOREUSE)) == B_NOREUSE) {
+		allocbuf(bp, 0);
+		bp->b_flags &= ~B_NOREUSE;
 		if (bp->b_vp != NULL)
 			brelvp(bp);
 	}
@@ -2060,6 +1973,10 @@ bqrelse(struct buf *bp)
 		if ((bp->b_flags & B_DELWRI) == 0 &&
 		    (bp->b_xflags & BX_VNDIRTY))
 			panic("bqrelse: not dirty");
+		if ((bp->b_flags & B_NOREUSE) != 0) {
+			brelse(bp);
+			return;
+		}
 		qindex = QUEUE_CLEAN;
 	}
 	binsfree(bp, qindex);
@@ -2069,53 +1986,290 @@ out:
 	BUF_UNLOCK(bp);
 }
 
-/* Give pages used by the bp back to the VM system (where possible) */
+/*
+ * Complete I/O to a VMIO backed page.  Validate the pages as appropriate,
+ * restore bogus pages.
+ */
 static void
-vfs_vmio_release(struct buf *bp)
+vfs_vmio_iodone(struct buf *bp)
+{
+	vm_ooffset_t foff;
+	vm_page_t m;
+	vm_object_t obj;
+	struct vnode *vp;
+	int bogus, i, iosize;
+
+	obj = bp->b_bufobj->bo_object;
+	KASSERT(obj->paging_in_progress >= bp->b_npages,
+	    ("vfs_vmio_iodone: paging in progress(%d) < b_npages(%d)",
+	    obj->paging_in_progress, bp->b_npages));
+
+	vp = bp->b_vp;
+	KASSERT(vp->v_holdcnt > 0,
+	    ("vfs_vmio_iodone: vnode %p has zero hold count", vp));
+	KASSERT(vp->v_object != NULL,
+	    ("vfs_vmio_iodone: vnode %p has no vm_object", vp));
+
+	foff = bp->b_offset;
+	KASSERT(bp->b_offset != NOOFFSET,
+	    ("vfs_vmio_iodone: bp %p has no buffer offset", bp));
+
+	bogus = 0;
+	iosize = bp->b_bcount - bp->b_resid;
+	VM_OBJECT_WLOCK(obj);
+	for (i = 0; i < bp->b_npages; i++) {
+		int resid;
+
+		resid = ((foff + PAGE_SIZE) & ~(off_t)PAGE_MASK) - foff;
+		if (resid > iosize)
+			resid = iosize;
+
+		/*
+		 * cleanup bogus pages, restoring the originals
+		 */
+		m = bp->b_pages[i];
+		if (m == bogus_page) {
+			bogus = 1;
+			m = vm_page_lookup(obj, OFF_TO_IDX(foff));
+			if (m == NULL)
+				panic("biodone: page disappeared!");
+			bp->b_pages[i] = m;
+		} else if ((bp->b_iocmd == BIO_READ) && resid > 0) {
+			/*
+			 * In the write case, the valid and clean bits are
+			 * already changed correctly ( see bdwrite() ), so we 
+			 * only need to do this here in the read case.
+			 */
+			KASSERT((m->dirty & vm_page_bits(foff & PAGE_MASK,
+			    resid)) == 0, ("vfs_vmio_iodone: page %p "
+			    "has unexpected dirty bits", m));
+			vfs_page_set_valid(bp, foff, m);
+		}
+		KASSERT(OFF_TO_IDX(foff) == m->pindex,
+		    ("vfs_vmio_iodone: foff(%jd)/pindex(%ju) mismatch",
+		    (intmax_t)foff, (uintmax_t)m->pindex));
+
+		vm_page_sunbusy(m);
+		vm_object_pip_subtract(obj, 1);
+		foff = (foff + PAGE_SIZE) & ~(off_t)PAGE_MASK;
+		iosize -= resid;
+	}
+	vm_object_pip_wakeupn(obj, 0);
+	VM_OBJECT_WUNLOCK(obj);
+	if (bogus && buf_mapped(bp)) {
+		BUF_CHECK_MAPPED(bp);
+		pmap_qenter(trunc_page((vm_offset_t)bp->b_data),
+		    bp->b_pages, bp->b_npages);
+	}
+}
+
+/*
+ * Unwire a page held by a buf and place it on the appropriate vm queue.
+ */
+static void
+vfs_vmio_unwire(struct buf *bp, vm_page_t m)
+{
+	bool freed;
+
+	vm_page_lock(m);
+	if (vm_page_unwire(m, PQ_NONE)) {
+		/*
+		 * Determine if the page should be freed before adding
+		 * it to the inactive queue.
+		 */
+		if (m->valid == 0) {
+			freed = !vm_page_busied(m);
+			if (freed)
+				vm_page_free(m);
+		} else if ((bp->b_flags & B_DIRECT) != 0)
+			freed = vm_page_try_to_free(m);
+		else
+			freed = false;
+		if (!freed) {
+			/*
+			 * If the page is unlikely to be reused, let the
+			 * VM know.  Otherwise, maintain LRU page
+			 * ordering and put the page at the tail of the
+			 * inactive queue.
+			 */
+			if ((bp->b_flags & B_NOREUSE) != 0)
+				vm_page_deactivate_noreuse(m);
+			else
+				vm_page_deactivate(m);
+		}
+	}
+	vm_page_unlock(m);
+}
+
+/*
+ * Perform page invalidation when a buffer is released.  The fully invalid
+ * pages will be reclaimed later in vfs_vmio_truncate().
+ */
+static void
+vfs_vmio_invalidate(struct buf *bp)
 {
 	vm_object_t obj;
 	vm_page_t m;
-	int i;
+	int i, resid, poffset, presid;
 
 	if (buf_mapped(bp)) {
 		BUF_CHECK_MAPPED(bp);
 		pmap_qremove(trunc_page((vm_offset_t)bp->b_data), bp->b_npages);
 	} else
 		BUF_CHECK_UNMAPPED(bp);
+	/*
+	 * Get the base offset and length of the buffer.  Note that 
+	 * in the VMIO case if the buffer block size is not
+	 * page-aligned then b_data pointer may not be page-aligned.
+	 * But our b_pages[] array *IS* page aligned.
+	 *
+	 * block sizes less then DEV_BSIZE (usually 512) are not 
+	 * supported due to the page granularity bits (m->valid,
+	 * m->dirty, etc...). 
+	 *
+	 * See man buf(9) for more information
+	 */
+	obj = bp->b_bufobj->bo_object;
+	resid = bp->b_bufsize;
+	poffset = bp->b_offset & PAGE_MASK;
+	VM_OBJECT_WLOCK(obj);
+	for (i = 0; i < bp->b_npages; i++) {
+		m = bp->b_pages[i];
+		if (m == bogus_page)
+			panic("vfs_vmio_invalidate: Unexpected bogus page.");
+		bp->b_pages[i] = NULL;
+
+		presid = resid > (PAGE_SIZE - poffset) ?
+		    (PAGE_SIZE - poffset) : resid;
+		KASSERT(presid >= 0, ("brelse: extra page"));
+		while (vm_page_xbusied(m)) {
+			vm_page_lock(m);
+			VM_OBJECT_WUNLOCK(obj);
+			vm_page_busy_sleep(m, "mbncsh");
+			VM_OBJECT_WLOCK(obj);
+		}
+		if (pmap_page_wired_mappings(m) == 0)
+			vm_page_set_invalid(m, poffset, presid);
+		vfs_vmio_unwire(bp, m);
+		resid -= presid;
+		poffset = 0;
+	}
+	VM_OBJECT_WUNLOCK(obj);
+	bp->b_npages = 0;
+}
+
+/*
+ * Page-granular truncation of an existing VMIO buffer.
+ */
+static void
+vfs_vmio_truncate(struct buf *bp, int desiredpages)
+{
+	vm_object_t obj;
+	vm_page_t m;
+	int i;
+
+	if (bp->b_npages == desiredpages)
+		return;
+
+	if (buf_mapped(bp)) {
+		BUF_CHECK_MAPPED(bp);
+		pmap_qremove((vm_offset_t)trunc_page((vm_offset_t)bp->b_data) +
+		    (desiredpages << PAGE_SHIFT), bp->b_npages - desiredpages);
+	} else
+		BUF_CHECK_UNMAPPED(bp);
 	obj = bp->b_bufobj->bo_object;
 	if (obj != NULL)
 		VM_OBJECT_WLOCK(obj);
-	for (i = 0; i < bp->b_npages; i++) {
+	for (i = desiredpages; i < bp->b_npages; i++) {
 		m = bp->b_pages[i];
+		KASSERT(m != bogus_page, ("allocbuf: bogus page found"));
 		bp->b_pages[i] = NULL;
-		/*
-		 * In order to keep page LRU ordering consistent, put
-		 * everything on the inactive queue.
-		 */
-		vm_page_lock(m);
-		vm_page_unwire(m, PQ_INACTIVE);
+		vfs_vmio_unwire(bp, m);
+	}
+	if (obj != NULL)
+		VM_OBJECT_WUNLOCK(obj);
+	bp->b_npages = desiredpages;
+}
+
+/*
+ * Byte granular extension of VMIO buffers.
+ */
+static void
+vfs_vmio_extend(struct buf *bp, int desiredpages, int size)
+{
+	/*
+	 * We are growing the buffer, possibly in a 
+	 * byte-granular fashion.
+	 */
+	vm_object_t obj;
+	vm_offset_t toff;
+	vm_offset_t tinc;
+	vm_page_t m;
 
+	/*
+	 * Step 1, bring in the VM pages from the object, allocating
+	 * them if necessary.  We must clear B_CACHE if these pages
+	 * are not valid for the range covered by the buffer.
+	 */
+	obj = bp->b_bufobj->bo_object;
+	VM_OBJECT_WLOCK(obj);
+	while (bp->b_npages < desiredpages) {
 		/*
-		 * Might as well free the page if we can and it has
-		 * no valid data.  We also free the page if the
-		 * buffer was used for direct I/O
+		 * We must allocate system pages since blocking
+		 * here could interfere with paging I/O, no
+		 * matter which process we are.
+		 *
+		 * Only exclusive busy can be tested here.
+		 * Blocking on shared busy might lead to
+		 * deadlocks once allocbuf() is called after
+		 * pages are vfs_busy_pages().
 		 */
-		if ((bp->b_flags & B_ASYNC) == 0 && !m->valid) {
-			if (m->wire_count == 0 && !vm_page_busied(m))
-				vm_page_free(m);
-		} else if (bp->b_flags & B_DIRECT)
-			vm_page_try_to_free(m);
-		vm_page_unlock(m);
+		m = vm_page_grab(obj, OFF_TO_IDX(bp->b_offset) + bp->b_npages,
+		    VM_ALLOC_NOBUSY | VM_ALLOC_SYSTEM |
+		    VM_ALLOC_WIRED | VM_ALLOC_IGN_SBUSY |
+		    VM_ALLOC_COUNT(desiredpages - bp->b_npages));
+		if (m->valid == 0)
+			bp->b_flags &= ~B_CACHE;
+		bp->b_pages[bp->b_npages] = m;
+		++bp->b_npages;
 	}
-	if (obj != NULL)
-		VM_OBJECT_WUNLOCK(obj);
-	
-	if (bp->b_bufsize)
-		bufspaceadjust(bp, 0);
-	bp->b_npages = 0;
-	bp->b_flags &= ~B_VMIO;
-	if (bp->b_vp)
-		brelvp(bp);
+
+	/*
+	 * Step 2.  We've loaded the pages into the buffer,
+	 * we have to figure out if we can still have B_CACHE
+	 * set.  Note that B_CACHE is set according to the
+	 * byte-granular range ( bcount and size ), not the
+	 * aligned range ( newbsize ).
+	 *
+	 * The VM test is against m->valid, which is DEV_BSIZE
+	 * aligned.  Needless to say, the validity of the data
+	 * needs to also be DEV_BSIZE aligned.  Note that this
+	 * fails with NFS if the server or some other client
+	 * extends the file's EOF.  If our buffer is resized, 
+	 * B_CACHE may remain set! XXX
+	 */
+	toff = bp->b_bcount;
+	tinc = PAGE_SIZE - ((bp->b_offset + toff) & PAGE_MASK);
+	while ((bp->b_flags & B_CACHE) && toff < size) {
+		vm_pindex_t pi;
+
+		if (tinc > (size - toff))
+			tinc = size - toff;
+		pi = ((bp->b_offset & PAGE_MASK) + toff) >> PAGE_SHIFT;
+		m = bp->b_pages[pi];
+		vfs_buf_test_cache(bp, bp->b_offset, toff, tinc, m);
+		toff += tinc;
+		tinc = PAGE_SIZE;
+	}
+	VM_OBJECT_WUNLOCK(obj);
+
+	/*
+	 * Step 3, fixup the KVA pmap.
+	 */
+	if (buf_mapped(bp))
+		bpmap_qenter(bp);
+	else
+		BUF_CHECK_UNMAPPED(bp);
 }
 
 /*
@@ -2315,14 +2469,16 @@ getnewbuf_reuse_bp(struct buf *bp, int qindex)
 	 * Note: we no longer distinguish between VMIO and non-VMIO
 	 * buffers.
 	 */
-	KASSERT((bp->b_flags & B_DELWRI) == 0,
-	    ("delwri buffer %p found in queue %d", bp, qindex));
+	KASSERT((bp->b_flags & (B_DELWRI | B_NOREUSE)) == 0,
+	    ("invalid buffer %p flags %#x found in queue %d", bp, bp->b_flags,
+	    qindex));
 
+	/*
+	 * When recycling a clean buffer we have to truncate it and
+	 * release the vnode.
+	 */
 	if (qindex == QUEUE_CLEAN) {
-		if (bp->b_flags & B_VMIO) {
-			bp->b_flags &= ~B_ASYNC;
-			vfs_vmio_release(bp);
-		}
+		allocbuf(bp, 0);
 		if (bp->b_vp != NULL)
 			brelvp(bp);
 	}
@@ -2331,7 +2487,6 @@ getnewbuf_reuse_bp(struct buf *bp, int qindex)
 	 * Get the rest of the buffer freed up.  b_kva* is still valid
 	 * after this operation.
 	 */
-
 	if (bp->b_rcred != NOCRED) {
 		crfree(bp->b_rcred);
 		bp->b_rcred = NOCRED;
@@ -2348,9 +2503,8 @@ getnewbuf_reuse_bp(struct buf *bp, int qindex)
 	    bp, bp->b_vp, qindex));
 	KASSERT((bp->b_xflags & (BX_VNCLEAN|BX_VNDIRTY)) == 0,
 	    ("bp: %p still on a buffer list. xflags %X", bp, bp->b_xflags));
-
-	if (bp->b_bufsize)
-		allocbuf(bp, 0);
+	KASSERT(bp->b_npages == 0,
+	    ("bp: %p still has %d vm pages\n", bp, bp->b_npages));
 
 	bp->b_flags = 0;
 	bp->b_ioflags = 0;
@@ -3266,8 +3420,7 @@ loop:
 		 * cleared.  If the size has not changed, B_CACHE remains
 		 * unchanged from its previous state.
 		 */
-		if (bp->b_bcount != size)
-			allocbuf(bp, size);
+		allocbuf(bp, size);
 
 		KASSERT(bp->b_offset != NOOFFSET, 
 		    ("getblk: no buffer offset"));
@@ -3424,6 +3577,80 @@ geteblk(int size, int flags)
 }
 
 /*
+ * Truncate the backing store for a non-vmio buffer.
+ */
+static void
+vfs_nonvmio_truncate(struct buf *bp, int newbsize)
+{
+
+	if (bp->b_flags & B_MALLOC) {
+		/*
+		 * malloced buffers are not shrunk
+		 */
+		if (newbsize == 0) {
+			bufmallocadjust(bp, 0);
+			free(bp->b_data, M_BIOBUF);
+			bp->b_data = bp->b_kvabase;
+			bp->b_flags &= ~B_MALLOC;
+		}
+		return;
+	}
+	vm_hold_free_pages(bp, newbsize);
+	bufspaceadjust(bp, newbsize);
+}
+
+/*
+ * Extend the backing for a non-VMIO buffer.
+ */
+static void
+vfs_nonvmio_extend(struct buf *bp, int newbsize)
+{
+	caddr_t origbuf;
+	int origbufsize;
+
+	/*
+	 * We only use malloced memory on the first allocation.
+	 * and revert to page-allocated memory when the buffer
+	 * grows.
+	 *
+	 * There is a potential smp race here that could lead
+	 * to bufmallocspace slightly passing the max.  It
+	 * is probably extremely rare and not worth worrying
+	 * over.
+	 */
+	if (bp->b_bufsize == 0 && newbsize <= PAGE_SIZE/2 &&
+	    bufmallocspace < maxbufmallocspace) {
+		bp->b_data = malloc(newbsize, M_BIOBUF, M_WAITOK);
+		bp->b_flags |= B_MALLOC;
+		bufmallocadjust(bp, newbsize);
+		return;
+	}
+
+	/*
+	 * If the buffer is growing on its other-than-first
+	 * allocation then we revert to the page-allocation
+	 * scheme.
+	 */
+	origbuf = NULL;
+	origbufsize = 0;
+	if (bp->b_flags & B_MALLOC) {
+		origbuf = bp->b_data;
+		origbufsize = bp->b_bufsize;
+		bp->b_data = bp->b_kvabase;
+		bufmallocadjust(bp, 0);
+		bp->b_flags &= ~B_MALLOC;
+		newbsize = round_page(newbsize);
+	}
+	vm_hold_load_pages(bp, (vm_offset_t) bp->b_data + bp->b_bufsize,
+	    (vm_offset_t) bp->b_data + newbsize);
+	if (origbuf != NULL) {
+		bcopy(origbuf, bp->b_data, origbufsize);
+		free(origbuf, M_BIOBUF);
+	}
+	bufspaceadjust(bp, newbsize);
+}
+
+/*
  * This code constitutes the buffer memory from either anonymous system
  * memory (in the case of non-VMIO operations) or from an associated
  * VM object (in the case of VMIO operations).  This code is able to
@@ -3437,100 +3664,36 @@ geteblk(int size, int flags)
  * allocbuf() only adjusts B_CACHE for VMIO buffers.  getblk() deals with
  * B_CACHE for the non-VMIO case.
  */
-
 int
 allocbuf(struct buf *bp, int size)
 {
-	int newbsize, mbsize;
-	int i;
+	int newbsize;
 
 	BUF_ASSERT_HELD(bp);
 
+	if (bp->b_bcount == size)
+		return (1);
+
 	if (bp->b_kvasize != 0 && bp->b_kvasize < size)
 		panic("allocbuf: buffer too small");
 
+	newbsize = (size + DEV_BSIZE - 1) & ~(DEV_BSIZE - 1);
 	if ((bp->b_flags & B_VMIO) == 0) {
-		caddr_t origbuf;
-		int origbufsize;
+		if ((bp->b_flags & B_MALLOC) == 0)
+			newbsize = round_page(newbsize);
 		/*
 		 * Just get anonymous memory from the kernel.  Don't
 		 * mess with B_CACHE.
 		 */
-		mbsize = (size + DEV_BSIZE - 1) & ~(DEV_BSIZE - 1);
-		if (bp->b_flags & B_MALLOC)
-			newbsize = mbsize;
-		else
-			newbsize = round_page(size);
-
-		if (newbsize < bp->b_bufsize) {
-			/*
-			 * malloced buffers are not shrunk
-			 */
-			if (bp->b_flags & B_MALLOC) {
-				if (newbsize) {
-					bp->b_bcount = size;
-				} else {
-					free(bp->b_data, M_BIOBUF);
-					bufmallocadjust(bp, 0);
-					bp->b_data = bp->b_kvabase;
-					bp->b_bcount = 0;
-					bp->b_flags &= ~B_MALLOC;
-				}
-				return 1;
-			}		
-			vm_hold_free_pages(bp, newbsize);
-		} else if (newbsize > bp->b_bufsize) {
-			/*
-			 * We only use malloced memory on the first allocation.
-			 * and revert to page-allocated memory when the buffer
-			 * grows.
-			 */
-			/*
-			 * There is a potential smp race here that could lead
-			 * to bufmallocspace slightly passing the max.  It
-			 * is probably extremely rare and not worth worrying
-			 * over.
-			 */
-			if ((bufmallocspace < maxbufmallocspace) &&
-				(bp->b_bufsize == 0) &&
-				(mbsize <= PAGE_SIZE/2)) {
-
-				bp->b_data = malloc(mbsize, M_BIOBUF, M_WAITOK);
-				bp->b_bcount = size;
-				bp->b_flags |= B_MALLOC;
-				bufmallocadjust(bp, mbsize);
-				return 1;
-			}
-			origbuf = NULL;
-			origbufsize = 0;
-			/*
-			 * If the buffer is growing on its other-than-first
-			 * allocation then we revert to the page-allocation
-			 * scheme.
-			 */
-			if (bp->b_flags & B_MALLOC) {
-				origbuf = bp->b_data;
-				origbufsize = bp->b_bufsize;
-				bp->b_data = bp->b_kvabase;
-				bufmallocadjust(bp, 0);
-				bp->b_flags &= ~B_MALLOC;
-				newbsize = round_page(newbsize);
-			}
-			vm_hold_load_pages(
-			    bp,
-			    (vm_offset_t) bp->b_data + bp->b_bufsize,
-			    (vm_offset_t) bp->b_data + newbsize);
-			if (origbuf) {
-				bcopy(origbuf, bp->b_data, origbufsize);
-				free(origbuf, M_BIOBUF);
-			}
-		}
+		if (newbsize < bp->b_bufsize)
+			vfs_nonvmio_truncate(bp, newbsize);
+		else if (newbsize > bp->b_bufsize)
+			vfs_nonvmio_extend(bp, newbsize);
 	} else {
 		int desiredpages;
 
-		newbsize = (size + DEV_BSIZE - 1) & ~(DEV_BSIZE - 1);
 		desiredpages = (size == 0) ? 0 :
-			num_pages((bp->b_offset & PAGE_MASK) + newbsize);
+		    num_pages((bp->b_offset & PAGE_MASK) + newbsize);
 
 		if (bp->b_flags & B_MALLOC)
 			panic("allocbuf: VMIO buffer can't be malloced");
@@ -3541,141 +3704,15 @@ allocbuf(struct buf *bp, int size)
 		if (size == 0 || bp->b_bufsize == 0)
 			bp->b_flags |= B_CACHE;
 
-		if (newbsize < bp->b_bufsize) {
-			/*
-			 * DEV_BSIZE aligned new buffer size is less then the
-			 * DEV_BSIZE aligned existing buffer size.  Figure out
-			 * if we have to remove any pages.
-			 */
-			if (desiredpages < bp->b_npages) {
-				vm_page_t m;
-
-				if (buf_mapped(bp)) {
-					BUF_CHECK_MAPPED(bp);
-					pmap_qremove((vm_offset_t)trunc_page(
-					    (vm_offset_t)bp->b_data) +
-					    (desiredpages << PAGE_SHIFT),
-					    (bp->b_npages - desiredpages));
-				} else
-					BUF_CHECK_UNMAPPED(bp);
-				VM_OBJECT_WLOCK(bp->b_bufobj->bo_object);
-				for (i = desiredpages; i < bp->b_npages; i++) {
-					/*
-					 * the page is not freed here -- it
-					 * is the responsibility of 
-					 * vnode_pager_setsize
-					 */
-					m = bp->b_pages[i];
-					KASSERT(m != bogus_page,
-					    ("allocbuf: bogus page found"));
-					while (vm_page_sleep_if_busy(m,
-					    "biodep"))
-						continue;
-
-					bp->b_pages[i] = NULL;
-					vm_page_lock(m);
-					vm_page_unwire(m, PQ_INACTIVE);
-					vm_page_unlock(m);
-				}
-				VM_OBJECT_WUNLOCK(bp->b_bufobj->bo_object);
-				bp->b_npages = desiredpages;
-			}
-		} else if (size > bp->b_bcount) {
-			/*
-			 * We are growing the buffer, possibly in a 
-			 * byte-granular fashion.
-			 */
-			vm_object_t obj;
-			vm_offset_t toff;
-			vm_offset_t tinc;
-
-			/*
-			 * Step 1, bring in the VM pages from the object, 
-			 * allocating them if necessary.  We must clear
-			 * B_CACHE if these pages are not valid for the 
-			 * range covered by the buffer.
-			 */
-
-			obj = bp->b_bufobj->bo_object;
-
-			VM_OBJECT_WLOCK(obj);
-			while (bp->b_npages < desiredpages) {
-				vm_page_t m;
-
-				/*
-				 * We must allocate system pages since blocking
-				 * here could interfere with paging I/O, no
-				 * matter which process we are.
-				 *
-				 * Only exclusive busy can be tested here.
-				 * Blocking on shared busy might lead to
-				 * deadlocks once allocbuf() is called after
-				 * pages are vfs_busy_pages().
-				 */
-				m = vm_page_grab(obj, OFF_TO_IDX(bp->b_offset) +
-				    bp->b_npages, VM_ALLOC_NOBUSY |
-				    VM_ALLOC_SYSTEM | VM_ALLOC_WIRED |
-				    VM_ALLOC_IGN_SBUSY |
-				    VM_ALLOC_COUNT(desiredpages - bp->b_npages));
-				if (m->valid == 0)
-					bp->b_flags &= ~B_CACHE;
-				bp->b_pages[bp->b_npages] = m;
-				++bp->b_npages;
-			}
-
-			/*
-			 * Step 2.  We've loaded the pages into the buffer,
-			 * we have to figure out if we can still have B_CACHE
-			 * set.  Note that B_CACHE is set according to the
-			 * byte-granular range ( bcount and size ), new the
-			 * aligned range ( newbsize ).
-			 *
-			 * The VM test is against m->valid, which is DEV_BSIZE
-			 * aligned.  Needless to say, the validity of the data
-			 * needs to also be DEV_BSIZE aligned.  Note that this
-			 * fails with NFS if the server or some other client
-			 * extends the file's EOF.  If our buffer is resized, 
-			 * B_CACHE may remain set! XXX
-			 */
-
-			toff = bp->b_bcount;
-			tinc = PAGE_SIZE - ((bp->b_offset + toff) & PAGE_MASK);
-
-			while ((bp->b_flags & B_CACHE) && toff < size) {
-				vm_pindex_t pi;
-
-				if (tinc > (size - toff))
-					tinc = size - toff;
-
-				pi = ((bp->b_offset & PAGE_MASK) + toff) >> 
-				    PAGE_SHIFT;
-
-				vfs_buf_test_cache(
-				    bp, 
-				    bp->b_offset,
-				    toff, 
-				    tinc, 
-				    bp->b_pages[pi]
-				);
-				toff += tinc;
-				tinc = PAGE_SIZE;
-			}
-			VM_OBJECT_WUNLOCK(obj);
-
-			/*
-			 * Step 3, fixup the KVA pmap.
-			 */
-			if (buf_mapped(bp))
-				bpmap_qenter(bp);
-			else
-				BUF_CHECK_UNMAPPED(bp);
-		}
-	}
-	/* Record changes in allocation size. */
-	if (bp->b_bufsize != newbsize)
+		if (newbsize < bp->b_bufsize)
+			vfs_vmio_truncate(bp, desiredpages);
+		/* XXX This looks as if it should be newbsize > b_bufsize */
+		else if (size > bp->b_bcount)
+			vfs_vmio_extend(bp, desiredpages, size);
 		bufspaceadjust(bp, newbsize);
+	}
 	bp->b_bcount = size;		/* requested buffer size. */
-	return 1;
+	return (1);
 }
 
 extern int inflight_transient_maps;
@@ -3827,87 +3864,16 @@ bufdone_finish(struct buf *bp)
 		buf_complete(bp);
 
 	if (bp->b_flags & B_VMIO) {
-		vm_ooffset_t foff;
-		vm_page_t m;
-		vm_object_t obj;
-		struct vnode *vp;
-		int bogus, i, iosize;
-
-		obj = bp->b_bufobj->bo_object;
-		KASSERT(obj->paging_in_progress >= bp->b_npages,
-		    ("biodone_finish: paging in progress(%d) < b_npages(%d)",
-		    obj->paging_in_progress, bp->b_npages));
-
-		vp = bp->b_vp;
-		KASSERT(vp->v_holdcnt > 0,
-		    ("biodone_finish: vnode %p has zero hold count", vp));
-		KASSERT(vp->v_object != NULL,
-		    ("biodone_finish: vnode %p has no vm_object", vp));
-
-		foff = bp->b_offset;
-		KASSERT(bp->b_offset != NOOFFSET,
-		    ("biodone_finish: bp %p has no buffer offset", bp));
-
 		/*
 		 * Set B_CACHE if the op was a normal read and no error
 		 * occured.  B_CACHE is set for writes in the b*write()
 		 * routines.
 		 */
-		iosize = bp->b_bcount - bp->b_resid;
 		if (bp->b_iocmd == BIO_READ &&
 		    !(bp->b_flags & (B_INVAL|B_NOCACHE)) &&
-		    !(bp->b_ioflags & BIO_ERROR)) {
+		    !(bp->b_ioflags & BIO_ERROR))
 			bp->b_flags |= B_CACHE;
-		}
-		bogus = 0;
-		VM_OBJECT_WLOCK(obj);
-		for (i = 0; i < bp->b_npages; i++) {
-			int bogusflag = 0;
-			int resid;
-
-			resid = ((foff + PAGE_SIZE) & ~(off_t)PAGE_MASK) - foff;
-			if (resid > iosize)
-				resid = iosize;
-
-			/*
-			 * cleanup bogus pages, restoring the originals
-			 */
-			m = bp->b_pages[i];
-			if (m == bogus_page) {
-				bogus = bogusflag = 1;
-				m = vm_page_lookup(obj, OFF_TO_IDX(foff));
-				if (m == NULL)
-					panic("biodone: page disappeared!");
-				bp->b_pages[i] = m;
-			}
-			KASSERT(OFF_TO_IDX(foff) == m->pindex,
-			    ("biodone_finish: foff(%jd)/pindex(%ju) mismatch",
-			    (intmax_t)foff, (uintmax_t)m->pindex));
-
-			/*
-			 * In the write case, the valid and clean bits are
-			 * already changed correctly ( see bdwrite() ), so we 
-			 * only need to do this here in the read case.
-			 */
-			if ((bp->b_iocmd == BIO_READ) && !bogusflag && resid > 0) {
-				KASSERT((m->dirty & vm_page_bits(foff &
-				    PAGE_MASK, resid)) == 0, ("bufdone_finish:"
-				    " page %p has unexpected dirty bits", m));
-				vfs_page_set_valid(bp, foff, m);
-			}
-
-			vm_page_sunbusy(m);
-			vm_object_pip_subtract(obj, 1);
-			foff = (foff + PAGE_SIZE) & ~(off_t)PAGE_MASK;
-			iosize -= resid;
-		}
-		vm_object_pip_wakeupn(obj, 0);
-		VM_OBJECT_WUNLOCK(obj);
-		if (bogus && buf_mapped(bp)) {
-			BUF_CHECK_MAPPED(bp);
-			pmap_qenter(trunc_page((vm_offset_t)bp->b_data),
-			    bp->b_pages, bp->b_npages);
-		}
+		vfs_vmio_iodone(bp);
 	}
 
 	/*
@@ -3915,9 +3881,9 @@ bufdone_finish(struct buf *bp)
 	 * will do a wakeup there if necessary - so no need to do a wakeup
 	 * here in the async case. The sync case always needs to do a wakeup.
 	 */
-
 	if (bp->b_flags & B_ASYNC) {
-		if ((bp->b_flags & (B_NOCACHE | B_INVAL | B_RELBUF)) || (bp->b_ioflags & BIO_ERROR))
+		if ((bp->b_flags & (B_NOCACHE | B_INVAL | B_RELBUF)) ||
+		    (bp->b_ioflags & BIO_ERROR))
 			brelse(bp);
 		else
 			bqrelse(bp);
diff --git a/sys/kern/vfs_cache.c b/sys/kern/vfs_cache.c
index 5fe038278f622..ca9f054b90d55 100644
--- a/sys/kern/vfs_cache.c
+++ b/sys/kern/vfs_cache.c
@@ -419,11 +419,11 @@ cache_zap(ncp)
 	CTR2(KTR_VFS, "cache_zap(%p) vp %p", ncp, ncp->nc_vp);
 #ifdef KDTRACE_HOOKS
 	if (ncp->nc_vp != NULL) {
-		SDT_PROBE(vfs, namecache, zap, done, ncp->nc_dvp,
-		    nc_get_name(ncp), ncp->nc_vp, 0, 0);
+		SDT_PROBE3(vfs, namecache, zap, done, ncp->nc_dvp,
+		    nc_get_name(ncp), ncp->nc_vp);
 	} else {
-		SDT_PROBE(vfs, namecache, zap_negative, done, ncp->nc_dvp,
-		    nc_get_name(ncp), 0, 0, 0);
+		SDT_PROBE2(vfs, namecache, zap_negative, done, ncp->nc_dvp,
+		    nc_get_name(ncp));
 	}
 #endif
 	vp = NULL;
@@ -498,8 +498,7 @@ retry_wlocked:
 			CTR2(KTR_VFS, "cache_lookup(%p, %s) found via .",
 			    dvp, cnp->cn_nameptr);
 			dothits++;
-			SDT_PROBE(vfs, namecache, lookup, hit, dvp, ".",
-			    *vpp, 0, 0);
+			SDT_PROBE3(vfs, namecache, lookup, hit, dvp, ".", *vpp);
 			if (tsp != NULL)
 				timespecclear(tsp);
 			if (ticksp != NULL)
@@ -509,8 +508,8 @@ retry_wlocked:
 		if (cnp->cn_namelen == 2 && cnp->cn_nameptr[1] == '.') {
 			dotdothits++;
 			if (dvp->v_cache_dd == NULL) {
-				SDT_PROBE(vfs, namecache, lookup, miss, dvp,
-				    "..", NULL, 0, 0);
+				SDT_PROBE3(vfs, namecache, lookup, miss, dvp,
+				    "..", NULL);
 				goto unlock;
 			}
 			if ((cnp->cn_flags & MAKEENTRY) == 0) {
@@ -532,8 +531,8 @@ retry_wlocked:
 				goto negative_success;
 			CTR3(KTR_VFS, "cache_lookup(%p, %s) found %p via ..",
 			    dvp, cnp->cn_nameptr, *vpp);
-			SDT_PROBE(vfs, namecache, lookup, hit, dvp, "..",
-			    *vpp, 0, 0);
+			SDT_PROBE3(vfs, namecache, lookup, hit, dvp, "..",
+			    *vpp);
 			cache_out_ts(ncp, tsp, ticksp);
 			if ((ncp->nc_flag & (NCF_ISDOTDOT | NCF_DTS)) ==
 			    NCF_DTS && tsp != NULL)
@@ -554,8 +553,8 @@ retry_wlocked:
 
 	/* We failed to find an entry */
 	if (ncp == NULL) {
-		SDT_PROBE(vfs, namecache, lookup, miss, dvp, cnp->cn_nameptr,
-		    NULL, 0, 0);
+		SDT_PROBE3(vfs, namecache, lookup, miss, dvp, cnp->cn_nameptr,
+		    NULL);
 		if ((cnp->cn_flags & MAKEENTRY) == 0) {
 			nummisszap++;
 		} else {
@@ -583,8 +582,8 @@ retry_wlocked:
 		*vpp = ncp->nc_vp;
 		CTR4(KTR_VFS, "cache_lookup(%p, %s) found %p via ncp %p",
 		    dvp, cnp->cn_nameptr, *vpp, ncp);
-		SDT_PROBE(vfs, namecache, lookup, hit, dvp, nc_get_name(ncp),
-		    *vpp, 0, 0);
+		SDT_PROBE3(vfs, namecache, lookup, hit, dvp, nc_get_name(ncp),
+		    *vpp);
 		cache_out_ts(ncp, tsp, ticksp);
 		goto success;
 	}
@@ -615,8 +614,8 @@ negative_success:
 	nchstats.ncs_neghits++;
 	if (ncp->nc_flag & NCF_WHITE)
 		cnp->cn_flags |= ISWHITEOUT;
-	SDT_PROBE(vfs, namecache, lookup, hit__negative, dvp, nc_get_name(ncp),
-	    0, 0, 0);
+	SDT_PROBE2(vfs, namecache, lookup, hit__negative, dvp,
+	    nc_get_name(ncp));
 	cache_out_ts(ncp, tsp, ticksp);
 	CACHE_WUNLOCK();
 	return (ENOENT);
@@ -769,8 +768,7 @@ cache_enter_time(dvp, vp, cnp, tsp, dtsp)
 				return;
 			}
 			dvp->v_cache_dd = NULL;
-			SDT_PROBE(vfs, namecache, enter, done, dvp, "..", vp,
-			    0, 0);
+			SDT_PROBE3(vfs, namecache, enter, done, dvp, "..", vp);
 			CACHE_WUNLOCK();
 			flag = NCF_ISDOTDOT;
 		}
@@ -890,12 +888,12 @@ cache_enter_time(dvp, vp, cnp, tsp, dtsp)
 	 */
 	if (vp) {
 		TAILQ_INSERT_HEAD(&vp->v_cache_dst, ncp, nc_dst);
-		SDT_PROBE(vfs, namecache, enter, done, dvp, nc_get_name(ncp),
-		    vp, 0, 0);
+		SDT_PROBE3(vfs, namecache, enter, done, dvp, nc_get_name(ncp),
+		    vp);
 	} else {
 		TAILQ_INSERT_TAIL(&ncneg, ncp, nc_dst);
-		SDT_PROBE(vfs, namecache, enter_negative, done, dvp,
-		    nc_get_name(ncp), 0, 0, 0);
+		SDT_PROBE2(vfs, namecache, enter_negative, done, dvp,
+		    nc_get_name(ncp));
 	}
 	if (numneg * ncnegfactor > numcache) {
 		ncp = TAILQ_FIRST(&ncneg);
@@ -984,7 +982,7 @@ cache_purge(vp)
 {
 
 	CTR1(KTR_VFS, "cache_purge(%p)", vp);
-	SDT_PROBE(vfs, namecache, purge, done, vp, 0, 0, 0, 0);
+	SDT_PROBE1(vfs, namecache, purge, done, vp);
 	CACHE_WLOCK();
 	while (!LIST_EMPTY(&vp->v_cache_src))
 		cache_zap(LIST_FIRST(&vp->v_cache_src));
@@ -1009,7 +1007,7 @@ cache_purge_negative(vp)
 	struct namecache *cp, *ncp;
 
 	CTR1(KTR_VFS, "cache_purge_negative(%p)", vp);
-	SDT_PROBE(vfs, namecache, purge_negative, done, vp, 0, 0, 0, 0);
+	SDT_PROBE1(vfs, namecache, purge_negative, done, vp);
 	CACHE_WLOCK();
 	LIST_FOREACH_SAFE(cp, &vp->v_cache_src, nc_src, ncp) {
 		if (cp->nc_vp == NULL)
@@ -1029,7 +1027,7 @@ cache_purgevfs(mp)
 	struct namecache *ncp, *nnp;
 
 	/* Scan hash tables for applicable entries */
-	SDT_PROBE(vfs, namecache, purgevfs, done, mp, 0, 0, 0, 0);
+	SDT_PROBE1(vfs, namecache, purgevfs, done, mp);
 	CACHE_WLOCK();
 	for (ncpp = &nchashtbl[nchash]; ncpp >= nchashtbl; ncpp--) {
 		LIST_FOREACH_SAFE(ncp, ncpp, nc_hash, nnp) {
@@ -1253,14 +1251,14 @@ vn_vptocnp_locked(struct vnode **vp, struct ucred *cred, char *buf,
 			vrele(*vp);
 			numfullpathfail4++;
 			error = ENOMEM;
-			SDT_PROBE(vfs, namecache, fullpath, return, error,
-			    vp, NULL, 0, 0);
+			SDT_PROBE3(vfs, namecache, fullpath, return, error,
+			    vp, NULL);
 			return (error);
 		}
 		*buflen -= ncp->nc_nlen;
 		memcpy(buf + *buflen, nc_get_name(ncp), ncp->nc_nlen);
-		SDT_PROBE(vfs, namecache, fullpath, hit, ncp->nc_dvp,
-		    nc_get_name(ncp), vp, 0, 0);
+		SDT_PROBE3(vfs, namecache, fullpath, hit, ncp->nc_dvp,
+		    nc_get_name(ncp), vp);
 		dvp = *vp;
 		*vp = ncp->nc_dvp;
 		vref(*vp);
@@ -1269,7 +1267,7 @@ vn_vptocnp_locked(struct vnode **vp, struct ucred *cred, char *buf,
 		CACHE_RLOCK();
 		return (0);
 	}
-	SDT_PROBE(vfs, namecache, fullpath, miss, vp, 0, 0, 0, 0);
+	SDT_PROBE1(vfs, namecache, fullpath, miss, vp);
 
 	CACHE_RUNLOCK();
 	vn_lock(*vp, LK_SHARED | LK_RETRY);
@@ -1277,8 +1275,7 @@ vn_vptocnp_locked(struct vnode **vp, struct ucred *cred, char *buf,
 	vput(*vp);
 	if (error) {
 		numfullpathfail2++;
-		SDT_PROBE(vfs, namecache, fullpath, return,  error, vp,
-		    NULL, 0, 0);
+		SDT_PROBE3(vfs, namecache, fullpath, return,  error, vp, NULL);
 		return (error);
 	}
 
@@ -1289,8 +1286,7 @@ vn_vptocnp_locked(struct vnode **vp, struct ucred *cred, char *buf,
 		CACHE_RUNLOCK();
 		vrele(dvp);
 		error = ENOENT;
-		SDT_PROBE(vfs, namecache, fullpath, return, error, vp,
-		    NULL, 0, 0);
+		SDT_PROBE3(vfs, namecache, fullpath, return, error, vp, NULL);
 		return (error);
 	}
 	/*
@@ -1318,7 +1314,7 @@ vn_fullpath1(struct thread *td, struct vnode *vp, struct vnode *rdir,
 	error = 0;
 	slash_prefixed = 0;
 
-	SDT_PROBE(vfs, namecache, fullpath, entry, vp, 0, 0, 0, 0);
+	SDT_PROBE1(vfs, namecache, fullpath, entry, vp);
 	numfullpathcalls++;
 	vref(vp);
 	CACHE_RLOCK();
@@ -1340,8 +1336,8 @@ vn_fullpath1(struct thread *td, struct vnode *vp, struct vnode *rdir,
 				CACHE_RUNLOCK();
 				vrele(vp);
 				error = ENOENT;
-				SDT_PROBE(vfs, namecache, fullpath, return,
-				    error, vp, NULL, 0, 0);
+				SDT_PROBE3(vfs, namecache, fullpath, return,
+				    error, vp, NULL);
 				break;
 			}
 			vp1 = vp->v_mount->mnt_vnodecovered;
@@ -1357,8 +1353,8 @@ vn_fullpath1(struct thread *td, struct vnode *vp, struct vnode *rdir,
 			vrele(vp);
 			numfullpathfail1++;
 			error = ENOTDIR;
-			SDT_PROBE(vfs, namecache, fullpath, return,
-			    error, vp, NULL, 0, 0);
+			SDT_PROBE3(vfs, namecache, fullpath, return,
+			    error, vp, NULL);
 			break;
 		}
 		error = vn_vptocnp_locked(&vp, td->td_ucred, buf, &buflen);
@@ -1368,8 +1364,8 @@ vn_fullpath1(struct thread *td, struct vnode *vp, struct vnode *rdir,
 			CACHE_RUNLOCK();
 			vrele(vp);
 			error = ENOMEM;
-			SDT_PROBE(vfs, namecache, fullpath, return, error,
-			    startvp, NULL, 0, 0);
+			SDT_PROBE3(vfs, namecache, fullpath, return, error,
+			    startvp, NULL);
 			break;
 		}
 		buf[--buflen] = '/';
@@ -1382,8 +1378,8 @@ vn_fullpath1(struct thread *td, struct vnode *vp, struct vnode *rdir,
 			CACHE_RUNLOCK();
 			vrele(vp);
 			numfullpathfail4++;
-			SDT_PROBE(vfs, namecache, fullpath, return, ENOMEM,
-			    startvp, NULL, 0, 0);
+			SDT_PROBE3(vfs, namecache, fullpath, return, ENOMEM,
+			    startvp, NULL);
 			return (ENOMEM);
 		}
 		buf[--buflen] = '/';
@@ -1392,8 +1388,7 @@ vn_fullpath1(struct thread *td, struct vnode *vp, struct vnode *rdir,
 	CACHE_RUNLOCK();
 	vrele(vp);
 
-	SDT_PROBE(vfs, namecache, fullpath, return, 0, startvp, buf + buflen,
-	    0, 0);
+	SDT_PROBE3(vfs, namecache, fullpath, return, 0, startvp, buf + buflen);
 	*retbuf = buf + buflen;
 	return (0);
 }
diff --git a/sys/kern/vfs_default.c b/sys/kern/vfs_default.c
index d70b68567dad0..d0074d3e34ed7 100644
--- a/sys/kern/vfs_default.c
+++ b/sys/kern/vfs_default.c
@@ -1034,9 +1034,12 @@ vop_stdallocate(struct vop_allocate_args *ap)
 int
 vop_stdadvise(struct vop_advise_args *ap)
 {
+	struct buf *bp;
+	struct buflists *bl;
 	struct vnode *vp;
+	daddr_t bn, startn, endn;
 	off_t start, end;
-	int error;
+	int bsize, error;
 
 	vp = ap->a_vp;
 	switch (ap->a_advice) {
@@ -1049,28 +1052,59 @@ vop_stdadvise(struct vop_advise_args *ap)
 		error = 0;
 		break;
 	case POSIX_FADV_DONTNEED:
-		/*
-		 * Flush any open FS buffers and then remove pages
-		 * from the backing VM object.  Using vinvalbuf() here
-		 * is a bit heavy-handed as it flushes all buffers for
-		 * the given vnode, not just the buffers covering the
-		 * requested range.
-		 */
 		error = 0;
 		vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
 		if (vp->v_iflag & VI_DOOMED) {
 			VOP_UNLOCK(vp, 0);
 			break;
 		}
-		vinvalbuf(vp, V_CLEANONLY, 0, 0);
+
+		/*
+		 * Deactivate pages in the specified range from the backing VM
+		 * object.  Pages that are resident in the buffer cache will
+		 * remain wired until their corresponding buffers are released
+		 * below.
+		 */
 		if (vp->v_object != NULL) {
 			start = trunc_page(ap->a_start);
 			end = round_page(ap->a_end);
 			VM_OBJECT_WLOCK(vp->v_object);
-			vm_object_page_cache(vp->v_object, OFF_TO_IDX(start),
+			vm_object_page_noreuse(vp->v_object, OFF_TO_IDX(start),
 			    OFF_TO_IDX(end));
 			VM_OBJECT_WUNLOCK(vp->v_object);
 		}
+
+		BO_RLOCK(&vp->v_bufobj);
+		bsize = vp->v_bufobj.bo_bsize;
+		startn = ap->a_start / bsize;
+		if (ap->a_end == OFF_MAX) {
+			endn = -1;
+			bl = &vp->v_bufobj.bo_clean.bv_hd;
+			if (!TAILQ_EMPTY(bl))
+				endn = TAILQ_LAST(bl, buflists)->b_lblkno;
+			bl = &vp->v_bufobj.bo_dirty.bv_hd;
+			if (!TAILQ_EMPTY(bl) &&
+			    endn < TAILQ_LAST(bl, buflists)->b_lblkno)
+				endn = TAILQ_LAST(bl, buflists)->b_lblkno;
+		} else
+			endn = ap->a_end / bsize;
+		BO_RUNLOCK(&vp->v_bufobj);
+		/*
+		 * In the VMIO case, use the B_NOREUSE flag to hint that the
+		 * pages backing each buffer in the range are unlikely to be
+		 * reused.  Dirty buffers will have the hint applied once
+		 * they've been written.
+		 */
+		for (bn = startn; bn <= endn; bn++) {
+			bp = getblk(vp, bn, bsize, 0, 0, GB_NOCREAT |
+			    GB_UNMAPPED);
+			if (bp == NULL)
+				continue;
+			bp->b_flags |= B_RELBUF;
+			if (vp->v_object != NULL)
+				bp->b_flags |= B_NOREUSE;
+			brelse(bp);
+		}
 		VOP_UNLOCK(vp, 0);
 		break;
 	default:
diff --git a/sys/kern/vfs_init.c b/sys/kern/vfs_init.c
index f935954fcef26..9ac193a9ebd5f 100644
--- a/sys/kern/vfs_init.c
+++ b/sys/kern/vfs_init.c
@@ -291,7 +291,7 @@ vfs_register(struct vfsconf *vfc)
 	 * preserved by re-registering the oid after modifying its
 	 * number.
 	 */
-	sysctl_xlock();
+	sysctl_wlock();
 	SLIST_FOREACH(oidp, SYSCTL_CHILDREN(&sysctl___vfs), oid_link) {
 		if (strcmp(oidp->oid_name, vfc->vfc_name) == 0) {
 			sysctl_unregister_oid(oidp);
@@ -300,7 +300,7 @@ vfs_register(struct vfsconf *vfc)
 			break;
 		}
 	}
-	sysctl_xunlock();
+	sysctl_wunlock();
 
 	return (0);
 }
diff --git a/sys/kern/vfs_lookup.c b/sys/kern/vfs_lookup.c
index d52d6229310cf..9a2a1db5bbb55 100644
--- a/sys/kern/vfs_lookup.c
+++ b/sys/kern/vfs_lookup.c
@@ -299,16 +299,15 @@ namei(struct nameidata *ndp)
 		namei_cleanup_cnp(cnp);
 		return (error);
 	}
-	SDT_PROBE(vfs, namei, lookup, entry, dp, cnp->cn_pnbuf,
-	    cnp->cn_flags, 0, 0);
+	SDT_PROBE3(vfs, namei, lookup, entry, dp, cnp->cn_pnbuf,
+	    cnp->cn_flags);
 	for (;;) {
 		ndp->ni_startdir = dp;
 		error = lookup(ndp);
 		if (error != 0) {
 			vrele(ndp->ni_rootdir);
 			namei_cleanup_cnp(cnp);
-			SDT_PROBE(vfs, namei, lookup, return, error, NULL, 0,
-			    0, 0);
+			SDT_PROBE2(vfs, namei, lookup, return, error, NULL);
 			return (error);
 		}
 		/*
@@ -321,8 +320,7 @@ namei(struct nameidata *ndp)
 			} else
 				cnp->cn_flags |= HASBUF;
 
-			SDT_PROBE(vfs, namei, lookup, return, 0, ndp->ni_vp,
-			    0, 0, 0);
+			SDT_PROBE2(vfs, namei, lookup, return, 0, ndp->ni_vp);
 			return (0);
 		}
 		if (ndp->ni_loopcnt++ >= MAXSYMLINKS) {
@@ -397,7 +395,7 @@ namei(struct nameidata *ndp)
 	vput(ndp->ni_vp);
 	ndp->ni_vp = NULL;
 	vrele(ndp->ni_dvp);
-	SDT_PROBE(vfs, namei, lookup, return, error, NULL, 0, 0, 0);
+	SDT_PROBE2(vfs, namei, lookup, return, error, NULL);
 	return (error);
 }
 
diff --git a/sys/kern/vfs_mountroot.c b/sys/kern/vfs_mountroot.c
index 8ebc3e4e42a5e..b6c295b44a09f 100644
--- a/sys/kern/vfs_mountroot.c
+++ b/sys/kern/vfs_mountroot.c
@@ -220,28 +220,39 @@ vfs_mountroot_devfs(struct thread *td, struct mount **mpp)
 
 	*mpp = NULL;
 
-	vfsp = vfs_byname("devfs");
-	KASSERT(vfsp != NULL, ("Could not find devfs by name"));
-	if (vfsp == NULL)
-		return (ENOENT);
+	if (rootdevmp != NULL) {
+		/*
+		 * Already have /dev; this happens during rerooting.
+		 */
+		error = vfs_busy(rootdevmp, 0);
+		if (error != 0)
+			return (error);
+		*mpp = rootdevmp;
+	} else {
+		vfsp = vfs_byname("devfs");
+		KASSERT(vfsp != NULL, ("Could not find devfs by name"));
+		if (vfsp == NULL)
+			return (ENOENT);
 
-	mp = vfs_mount_alloc(NULLVP, vfsp, "/dev", td->td_ucred);
+		mp = vfs_mount_alloc(NULLVP, vfsp, "/dev", td->td_ucred);
 
-	error = VFS_MOUNT(mp);
-	KASSERT(error == 0, ("VFS_MOUNT(devfs) failed %d", error));
-	if (error)
-		return (error);
+		error = VFS_MOUNT(mp);
+		KASSERT(error == 0, ("VFS_MOUNT(devfs) failed %d", error));
+		if (error)
+			return (error);
 
-	opts = malloc(sizeof(struct vfsoptlist), M_MOUNT, M_WAITOK);
-	TAILQ_INIT(opts);
-	mp->mnt_opt = opts;
+		opts = malloc(sizeof(struct vfsoptlist), M_MOUNT, M_WAITOK);
+		TAILQ_INIT(opts);
+		mp->mnt_opt = opts;
 
-	mtx_lock(&mountlist_mtx);
-	TAILQ_INSERT_HEAD(&mountlist, mp, mnt_list);
-	mtx_unlock(&mountlist_mtx);
+		mtx_lock(&mountlist_mtx);
+		TAILQ_INSERT_HEAD(&mountlist, mp, mnt_list);
+		mtx_unlock(&mountlist_mtx);
+
+		*mpp = mp;
+		rootdevmp = mp;
+	}
 
-	*mpp = mp;
-	rootdevmp = mp;
 	set_rootvnode();
 
 	error = kern_symlinkat(td, "/", AT_FDCWD, "dev", UIO_SYSSPACE);
@@ -251,7 +262,7 @@ vfs_mountroot_devfs(struct thread *td, struct mount **mpp)
 	return (error);
 }
 
-static int
+static void
 vfs_mountroot_shuffle(struct thread *td, struct mount *mpdevfs)
 {
 	struct nameidata nd;
@@ -362,8 +373,6 @@ vfs_mountroot_shuffle(struct thread *td, struct mount *mpdevfs)
 			printf("mountroot: unable to unlink /dev/dev "
 			    "(error %d)\n", error);
 	}
-
-	return (0);
 }
 
 /*
@@ -961,12 +970,10 @@ vfs_mountroot(void)
 	while (!error) {
 		error = vfs_mountroot_parse(sb, mp);
 		if (!error) {
-			error = vfs_mountroot_shuffle(td, mp);
-			if (!error) {
-				sbuf_clear(sb);
-				error = vfs_mountroot_readconf(td, sb);
-				sbuf_finish(sb);
-			}
+			vfs_mountroot_shuffle(td, mp);
+			sbuf_clear(sb);
+			error = vfs_mountroot_readconf(td, sb);
+			sbuf_finish(sb);
 		}
 	}
 
diff --git a/sys/kern/vfs_subr.c b/sys/kern/vfs_subr.c
index 410d1986b06f3..6976c3b2a06e9 100644
--- a/sys/kern/vfs_subr.c
+++ b/sys/kern/vfs_subr.c
@@ -1429,11 +1429,6 @@ flushbuflist(struct bufv *bufv, int flags, struct bufobj *bo, int slpflag,
 		KASSERT(bp->b_bufobj == bo,
 		    ("bp %p wrong b_bufobj %p should be %p",
 		    bp, bp->b_bufobj, bo));
-		if (bp->b_bufobj != bo) {	/* XXX: necessary ? */
-			BUF_UNLOCK(bp);
-			BO_LOCK(bo);
-			return (EAGAIN);
-		}
 		/*
 		 * XXX Since there are no node locks for NFS, I
 		 * believe there is a slight chance that a delayed
@@ -3071,8 +3066,8 @@ vn_printf(struct vnode *vp, const char *fmt, ...)
 		    "cleanbuf %d dirtybuf %d\n",
 		    vp->v_object, vp->v_object->ref_count,
 		    vp->v_object->resident_page_count,
-		    vp->v_bufobj.bo_dirty.bv_cnt,
-		    vp->v_bufobj.bo_clean.bv_cnt);
+		    vp->v_bufobj.bo_clean.bv_cnt,
+		    vp->v_bufobj.bo_dirty.bv_cnt);
 	printf("    ");
 	lockmgr_printinfo(vp->v_vnlock);
 	if (vp->v_data != NULL)
@@ -4345,6 +4340,15 @@ vop_mknod_post(void *ap, int rc)
 }
 
 void
+vop_reclaim_post(void *ap, int rc)
+{
+	struct vop_reclaim_args *a = ap;
+
+	if (!rc)
+		VFS_KNOTE_LOCKED(a->a_vp, NOTE_REVOKE);
+}
+
+void
 vop_remove_post(void *ap, int rc)
 {
 	struct vop_remove_args *a = ap;
@@ -4624,7 +4628,7 @@ filt_vfsread(struct knote *kn, long hint)
 	 * filesystem is gone, so set the EOF flag and schedule
 	 * the knote for deletion.
 	 */
-	if (hint == NOTE_REVOKE) {
+	if (hint == NOTE_REVOKE || (hint == 0 && vp->v_type == VBAD)) {
 		VI_LOCK(vp);
 		kn->kn_flags |= (EV_EOF | EV_ONESHOT);
 		VI_UNLOCK(vp);
@@ -4653,7 +4657,7 @@ filt_vfswrite(struct knote *kn, long hint)
 	 * filesystem is gone, so set the EOF flag and schedule
 	 * the knote for deletion.
 	 */
-	if (hint == NOTE_REVOKE)
+	if (hint == NOTE_REVOKE || (hint == 0 && vp->v_type == VBAD))
 		kn->kn_flags |= (EV_EOF | EV_ONESHOT);
 
 	kn->kn_data = 0;
@@ -4670,7 +4674,7 @@ filt_vfsvnode(struct knote *kn, long hint)
 	VI_LOCK(vp);
 	if (kn->kn_sfflags & hint)
 		kn->kn_fflags |= hint;
-	if (hint == NOTE_REVOKE) {
+	if (hint == NOTE_REVOKE || (hint == 0 && vp->v_type == VBAD)) {
 		kn->kn_flags |= EV_EOF;
 		VI_UNLOCK(vp);
 		return (1);
diff --git a/sys/kern/vfs_syscalls.c b/sys/kern/vfs_syscalls.c
index 76b83e3f47e9f..ce4436ad731ca 100644
--- a/sys/kern/vfs_syscalls.c
+++ b/sys/kern/vfs_syscalls.c
@@ -2160,9 +2160,9 @@ kern_statat(struct thread *td, int flag, int fd, char *path,
 		return (error);
 	error = vn_stat(nd.ni_vp, &sb, td->td_ucred, NOCRED, td);
 	if (error == 0) {
-		SDT_PROBE(vfs, , stat, mode, path, sb.st_mode, 0, 0, 0);
+		SDT_PROBE2(vfs, , stat, mode, path, sb.st_mode);
 		if (S_ISREG(sb.st_mode))
-			SDT_PROBE(vfs, , stat, reg, path, pathseg, 0, 0, 0);
+			SDT_PROBE2(vfs, , stat, reg, path, pathseg);
 		if (__predict_false(hook != NULL))
 			hook(nd.ni_vp, &sb);
 	}
@@ -4610,8 +4610,6 @@ kern_posix_fadvise(struct thread *td, int fd, off_t offset, off_t len,
 			new->fa_advice = advice;
 			new->fa_start = offset;
 			new->fa_end = end;
-			new->fa_prevstart = 0;
-			new->fa_prevend = 0;
 			fp->f_advice = new;
 			new = fa;
 		}
diff --git a/sys/kern/vfs_vnops.c b/sys/kern/vfs_vnops.c
index 18a9ac3110206..f07df31bbb2f2 100644
--- a/sys/kern/vfs_vnops.c
+++ b/sys/kern/vfs_vnops.c
@@ -770,10 +770,9 @@ vn_read(fp, uio, active_cred, flags, td)
 	struct thread *td;
 {
 	struct vnode *vp;
-	struct mtx *mtxp;
+	off_t orig_offset;
 	int error, ioflag;
 	int advice;
-	off_t offset, start, end;
 
 	KASSERT(uio->uio_td == td, ("uio_td %p is not td %p",
 	    uio->uio_td, td));
@@ -797,7 +796,7 @@ vn_read(fp, uio, active_cred, flags, td)
 		/* Disable read-ahead for random I/O. */
 		break;
 	}
-	offset = uio->uio_offset;
+	orig_offset = uio->uio_offset;
 
 #ifdef MAC
 	error = mac_vnode_check_read(active_cred, fp->f_cred, vp);
@@ -807,39 +806,14 @@ vn_read(fp, uio, active_cred, flags, td)
 	fp->f_nextoff = uio->uio_offset;
 	VOP_UNLOCK(vp, 0);
 	if (error == 0 && advice == POSIX_FADV_NOREUSE &&
-	    offset != uio->uio_offset) {
+	    orig_offset != uio->uio_offset)
 		/*
-		 * Use POSIX_FADV_DONTNEED to flush clean pages and
-		 * buffers for the backing file after a
-		 * POSIX_FADV_NOREUSE read(2).  To optimize the common
-		 * case of using POSIX_FADV_NOREUSE with sequential
-		 * access, track the previous implicit DONTNEED
-		 * request and grow this request to include the
-		 * current read(2) in addition to the previous
-		 * DONTNEED.  With purely sequential access this will
-		 * cause the DONTNEED requests to continously grow to
-		 * cover all of the previously read regions of the
-		 * file.  This allows filesystem blocks that are
-		 * accessed by multiple calls to read(2) to be flushed
-		 * once the last read(2) finishes.
+		 * Use POSIX_FADV_DONTNEED to flush pages and buffers
+		 * for the backing file after a POSIX_FADV_NOREUSE
+		 * read(2).
 		 */
-		start = offset;
-		end = uio->uio_offset - 1;
-		mtxp = mtx_pool_find(mtxpool_sleep, fp);
-		mtx_lock(mtxp);
-		if (fp->f_advice != NULL &&
-		    fp->f_advice->fa_advice == POSIX_FADV_NOREUSE) {
-			if (start != 0 && fp->f_advice->fa_prevend + 1 == start)
-				start = fp->f_advice->fa_prevstart;
-			else if (fp->f_advice->fa_prevstart != 0 &&
-			    fp->f_advice->fa_prevstart == end + 1)
-				end = fp->f_advice->fa_prevend;
-			fp->f_advice->fa_prevstart = start;
-			fp->f_advice->fa_prevend = end;
-		}
-		mtx_unlock(mtxp);
-		error = VOP_ADVISE(vp, start, end, POSIX_FADV_DONTNEED);
-	}
+		error = VOP_ADVISE(vp, orig_offset, uio->uio_offset - 1,
+		    POSIX_FADV_DONTNEED);
 	return (error);
 }
 
@@ -856,10 +830,9 @@ vn_write(fp, uio, active_cred, flags, td)
 {
 	struct vnode *vp;
 	struct mount *mp;
-	struct mtx *mtxp;
+	off_t orig_offset;
 	int error, ioflag, lock_flags;
 	int advice;
-	off_t offset, start, end;
 
 	KASSERT(uio->uio_td == td, ("uio_td %p is not td %p",
 	    uio->uio_td, td));
@@ -902,7 +875,7 @@ vn_write(fp, uio, active_cred, flags, td)
 		/* XXX: Is this correct? */
 		break;
 	}
-	offset = uio->uio_offset;
+	orig_offset = uio->uio_offset;
 
 #ifdef MAC
 	error = mac_vnode_check_write(active_cred, fp->f_cred, vp);
@@ -914,55 +887,14 @@ vn_write(fp, uio, active_cred, flags, td)
 	if (vp->v_type != VCHR)
 		vn_finished_write(mp);
 	if (error == 0 && advice == POSIX_FADV_NOREUSE &&
-	    offset != uio->uio_offset) {
+	    orig_offset != uio->uio_offset)
 		/*
-		 * Use POSIX_FADV_DONTNEED to flush clean pages and
-		 * buffers for the backing file after a
-		 * POSIX_FADV_NOREUSE write(2).  To optimize the
-		 * common case of using POSIX_FADV_NOREUSE with
-		 * sequential access, track the previous implicit
-		 * DONTNEED request and grow this request to include
-		 * the current write(2) in addition to the previous
-		 * DONTNEED.  With purely sequential access this will
-		 * cause the DONTNEED requests to continously grow to
-		 * cover all of the previously written regions of the
-		 * file.
-		 *
-		 * Note that the blocks just written are almost
-		 * certainly still dirty, so this only works when
-		 * VOP_ADVISE() calls from subsequent writes push out
-		 * the data written by this write(2) once the backing
-		 * buffers are clean.  However, as compared to forcing
-		 * IO_DIRECT, this gives much saner behavior.  Write
-		 * clustering is still allowed, and clean pages are
-		 * merely moved to the cache page queue rather than
-		 * outright thrown away.  This means a subsequent
-		 * read(2) can still avoid hitting the disk if the
-		 * pages have not been reclaimed.
-		 *
-		 * This does make POSIX_FADV_NOREUSE largely useless
-		 * with non-sequential access.  However, sequential
-		 * access is the more common use case and the flag is
-		 * merely advisory.
+		 * Use POSIX_FADV_DONTNEED to flush pages and buffers
+		 * for the backing file after a POSIX_FADV_NOREUSE
+		 * write(2).
 		 */
-		start = offset;
-		end = uio->uio_offset - 1;
-		mtxp = mtx_pool_find(mtxpool_sleep, fp);
-		mtx_lock(mtxp);
-		if (fp->f_advice != NULL &&
-		    fp->f_advice->fa_advice == POSIX_FADV_NOREUSE) {
-			if (start != 0 && fp->f_advice->fa_prevend + 1 == start)
-				start = fp->f_advice->fa_prevstart;
-			else if (fp->f_advice->fa_prevstart != 0 &&
-			    fp->f_advice->fa_prevstart == end + 1)
-				end = fp->f_advice->fa_prevend;
-			fp->f_advice->fa_prevstart = start;
-			fp->f_advice->fa_prevend = end;
-		}
-		mtx_unlock(mtxp);
-		error = VOP_ADVISE(vp, start, end, POSIX_FADV_DONTNEED);
-	}
-	
+		error = VOP_ADVISE(vp, orig_offset, uio->uio_offset - 1,
+		    POSIX_FADV_DONTNEED);
 unlock:
 	return (error);
 }
diff --git a/sys/kern/vnode_if.src b/sys/kern/vnode_if.src
index c3c8ed36adb24..63f8eb9237cf2 100644
--- a/sys/kern/vnode_if.src
+++ b/sys/kern/vnode_if.src
@@ -355,6 +355,7 @@ vop_inactive {
 
 
 %% reclaim	vp	E E E
+%! reclaim	post	vop_reclaim_post
 
 vop_reclaim {
 	IN struct vnode *vp;
diff --git a/sys/libkern/arm/memclr.S b/sys/libkern/arm/memclr.S
index b86b55853a9ec..2883d3f231040 100644
--- a/sys/libkern/arm/memclr.S
+++ b/sys/libkern/arm/memclr.S
@@ -35,6 +35,8 @@ __FBSDID("$FreeBSD$");
  *  void bzero(dest, len)
  */
 ENTRY_NP(__aeabi_memclr4)
+EENTRY_NP(__aeabi_memclr8)
 	b	bzero
+EEND(__aeabi_memclr8)
 END(__aeabi_memclr4)
 
diff --git a/sys/mips/adm5120/admpci.c b/sys/mips/adm5120/admpci.c
index 616d3bd683db2..e8a14a12c436c 100644
--- a/sys/mips/adm5120/admpci.c
+++ b/sys/mips/adm5120/admpci.c
@@ -181,7 +181,7 @@ admpci_attach(device_t dev)
 		panic("bus_space_map failed");
 	}
 
-	device_add_child(dev, "pci", busno);
+	device_add_child(dev, "pci", -1);
 	return (bus_generic_attach(dev));
 }
 
diff --git a/sys/mips/atheros/ar71xx_pci.c b/sys/mips/atheros/ar71xx_pci.c
index 7ef0e15531483..f745949d8d437 100644
--- a/sys/mips/atheros/ar71xx_pci.c
+++ b/sys/mips/atheros/ar71xx_pci.c
@@ -377,7 +377,6 @@ ar71xx_pci_probe(device_t dev)
 static int
 ar71xx_pci_attach(device_t dev)
 {
-	int busno = 0;
 	int rid = 0;
 	struct ar71xx_pci_softc *sc = device_get_softc(dev);
 
@@ -462,7 +461,7 @@ ar71xx_pci_attach(device_t dev)
 	ar71xx_pci_slot_fixup(dev, 0, 18, 0);
 #endif	/* AR71XX_ATH_EEPROM */
 
-	device_add_child(dev, "pci", busno);
+	device_add_child(dev, "pci", -1);
 	return (bus_generic_attach(dev));
 }
 
diff --git a/sys/mips/atheros/ar724x_pci.c b/sys/mips/atheros/ar724x_pci.c
index 283bd9b7efebf..ad8ec09640fda 100644
--- a/sys/mips/atheros/ar724x_pci.c
+++ b/sys/mips/atheros/ar724x_pci.c
@@ -369,7 +369,6 @@ static int
 ar724x_pci_attach(device_t dev)
 {
 	struct ar71xx_pci_softc *sc = device_get_softc(dev);
-	int busno = 0;
 	int rid = 0;
 
 	sc->sc_mem_rman.rm_type = RMAN_ARRAY;
@@ -429,7 +428,7 @@ ar724x_pci_attach(device_t dev)
 	    | PCIM_CMD_SERRESPEN | PCIM_CMD_BACKTOBACK
 	    | PCIM_CMD_PERRESPEN | PCIM_CMD_MWRICEN, 2);
 
-	device_add_child(dev, "pci", busno);
+	device_add_child(dev, "pci", -1);
 	return (bus_generic_attach(dev));
 }
 
diff --git a/sys/mips/atheros/qca955x_pci.c b/sys/mips/atheros/qca955x_pci.c
index 02322970e2032..0d78c1eb76b8f 100644
--- a/sys/mips/atheros/qca955x_pci.c
+++ b/sys/mips/atheros/qca955x_pci.c
@@ -272,7 +272,6 @@ qca955x_pci_attach(device_t dev)
 {
 	struct ar71xx_pci_softc *sc = device_get_softc(dev);
 	int unit = device_get_unit(dev);
-	int busno = 0;
 	int rid = 0;
 
 	/* Dirty; maybe these could all just be hints */
@@ -362,7 +361,7 @@ qca955x_pci_attach(device_t dev)
 	    | PCIM_CMD_SERRESPEN | PCIM_CMD_BACKTOBACK
 	    | PCIM_CMD_PERRESPEN | PCIM_CMD_MWRICEN, 2);
 
-	device_add_child(dev, "pci", busno);
+	device_add_child(dev, "pci", -1);
 	return (bus_generic_attach(dev));
 }
 
diff --git a/sys/mips/cavium/octopci.c b/sys/mips/cavium/octopci.c
index ae92063f9229c..f6d73bdc65cc1 100644
--- a/sys/mips/cavium/octopci.c
+++ b/sys/mips/cavium/octopci.c
@@ -205,7 +205,7 @@ octopci_attach(device_t dev)
 	subbus = octopci_init_bus(dev, sc->sc_bus);
 	octopci_write_config(dev, sc->sc_bus, 0, 0, PCIR_SUBBUS_1, subbus, 1);
 
-	device_add_child(dev, "pci", device_get_unit(dev));
+	device_add_child(dev, "pci", -1);
 
 	return (bus_generic_attach(dev));
 }
diff --git a/sys/mips/idt/idtpci.c b/sys/mips/idt/idtpci.c
index 7e003fd6feed8..d962e6872a36e 100644
--- a/sys/mips/idt/idtpci.c
+++ b/sys/mips/idt/idtpci.c
@@ -275,7 +275,7 @@ idtpci_attach(device_t dev)
 	        PCI_IRQ_END) != 0)
 		panic("idtpci_attach: failed to set up IRQ rman");
 
-	device_add_child(dev, "pci", busno);
+	device_add_child(dev, "pci", -1);
 	return (bus_generic_attach(dev));
 }
 
diff --git a/sys/mips/malta/gt_pci.c b/sys/mips/malta/gt_pci.c
index 3acc9b05bdf61..add9c07540f11 100644
--- a/sys/mips/malta/gt_pci.c
+++ b/sys/mips/malta/gt_pci.c
@@ -415,7 +415,7 @@ gt_pci_attach(device_t dev)
 	}
 
 	/* Initialize memory and i/o rmans. */
-	device_add_child(dev, "pci", busno);
+	device_add_child(dev, "pci", -1);
 	return (bus_generic_attach(dev));
 }
 
diff --git a/sys/mips/mips/elf_machdep.c b/sys/mips/mips/elf_machdep.c
index 8cbe8d3df6ccc..626b5f8a4c2f6 100644
--- a/sys/mips/mips/elf_machdep.c
+++ b/sys/mips/mips/elf_machdep.c
@@ -176,6 +176,7 @@ elf_reloc_internal(linker_file_t lf, Elf_Addr relocbase, const void *data,
 	Elf_Word rtype = (Elf_Word)0, symidx;
 	const Elf_Rel *rel = NULL;
 	const Elf_Rela *rela = NULL;
+	int error;
 
 	/*
 	 * Stash R_MIPS_HI16 info so we can use it when processing R_MIPS_LO16
@@ -215,8 +216,8 @@ elf_reloc_internal(linker_file_t lf, Elf_Addr relocbase, const void *data,
 		break;
 
 	case R_MIPS_32:		/* S + A */
-		addr = lookup(lf, symidx, 1);
-		if (addr == 0)
+		error = lookup(lf, symidx, 1, &addr);
+		if (error != 0)
 			return (-1);
 		addr += addend;
 		if (*where != addr)
@@ -224,8 +225,8 @@ elf_reloc_internal(linker_file_t lf, Elf_Addr relocbase, const void *data,
 		break;
 
 	case R_MIPS_26:		/* ((A << 2) | (P & 0xf0000000) + S) >> 2 */
-		addr = lookup(lf, symidx, 1);
-		if (addr == 0)
+		error = lookup(lf, symidx, 1, &addr);
+		if (error != 0)
 			return (-1);
 
 		addend &= 0x03ffffff;
@@ -243,8 +244,8 @@ elf_reloc_internal(linker_file_t lf, Elf_Addr relocbase, const void *data,
 		break;
 
 	case R_MIPS_64:		/* S + A */
-		addr = lookup(lf, symidx, 1);
-		if (addr == 0)
+		error = lookup(lf, symidx, 1, &addr);
+		if (error != 0)
 			return (-1);
 		addr += addend;
 		if (*(Elf64_Addr*)where != addr)
@@ -253,8 +254,8 @@ elf_reloc_internal(linker_file_t lf, Elf_Addr relocbase, const void *data,
 
 	case R_MIPS_HI16:	/* ((AHL + S) - ((short)(AHL + S)) >> 16 */
 		if (rela != NULL) {
-			addr = lookup(lf, symidx, 1);
-			if (addr == 0)
+			error = lookup(lf, symidx, 1, &addr);
+			if (error != 0)
 				return (-1);
 			addr += addend;
 			*where &= 0xffff0000;
@@ -268,8 +269,8 @@ elf_reloc_internal(linker_file_t lf, Elf_Addr relocbase, const void *data,
 
 	case R_MIPS_LO16:	/* AHL + S */
 		if (rela != NULL) {
-			addr = lookup(lf, symidx, 1);
-			if (addr == 0)
+			error = lookup(lf, symidx, 1, &addr);
+			if (error != 0)
 				return (-1);
 			addr += addend;
 			*where &= 0xffff0000;
@@ -277,8 +278,8 @@ elf_reloc_internal(linker_file_t lf, Elf_Addr relocbase, const void *data,
 		}
 		else {
 			ahl += (int16_t)addend;
-			addr = lookup(lf, symidx, 1);
-			if (addr == 0)
+			error = lookup(lf, symidx, 1, &addr);
+			if (error != 0)
 				return (-1);
 
 			addend &= 0xffff0000;
@@ -294,8 +295,8 @@ elf_reloc_internal(linker_file_t lf, Elf_Addr relocbase, const void *data,
 		break;
 
 	case R_MIPS_HIGHER:	/* %higher(A+S) */
-		addr = lookup(lf, symidx, 1);
-		if (addr == 0)
+		error = lookup(lf, symidx, 1, &addr);
+		if (error != 0)
 			return (-1);
 		addr += addend;
 		*where &= 0xffff0000;
@@ -303,8 +304,8 @@ elf_reloc_internal(linker_file_t lf, Elf_Addr relocbase, const void *data,
 		break;
 
 	case R_MIPS_HIGHEST:	/* %highest(A+S) */
-		addr = lookup(lf, symidx, 1);
-		if (addr == 0)
+		error = lookup(lf, symidx, 1, &addr);
+		if (error != 0)
 			return (-1);
 		addr += addend;
 		*where &= 0xffff0000;
diff --git a/sys/mips/mips/pmap.c b/sys/mips/mips/pmap.c
index 0d126e4b49620..292ce80307705 100644
--- a/sys/mips/mips/pmap.c
+++ b/sys/mips/mips/pmap.c
@@ -1527,7 +1527,7 @@ free_pv_chunk(struct pv_chunk *pc)
 	PV_STAT(pc_chunk_frees++);
 	/* entire chunk is free, return it */
 	m = PHYS_TO_VM_PAGE(MIPS_DIRECT_TO_PHYS((vm_offset_t)pc));
-	vm_page_unwire(m, PQ_INACTIVE);
+	vm_page_unwire(m, PQ_NONE);
 	vm_page_free(m);
 }
 
diff --git a/sys/mips/nlm/xlp_pci.c b/sys/mips/nlm/xlp_pci.c
index c06629fba085d..fee23cdd402d8 100644
--- a/sys/mips/nlm/xlp_pci.c
+++ b/sys/mips/nlm/xlp_pci.c
@@ -314,7 +314,7 @@ xlp_pcib_attach(device_t dev)
 		for (link = 0; link < 4; link++)
 			xlp_pcib_hardware_swap_enable(node, link);
 
-	device_add_child(dev, "pci", 0);
+	device_add_child(dev, "pci", -1);
 	bus_generic_attach(dev);
 	return (0);
 }
diff --git a/sys/mips/rmi/xlr_pci.c b/sys/mips/rmi/xlr_pci.c
index feea696d9b356..b41a5588ecf56 100644
--- a/sys/mips/rmi/xlr_pci.c
+++ b/sys/mips/rmi/xlr_pci.c
@@ -306,7 +306,7 @@ xlr_pcib_attach(device_t dev)
 		    0xff, 0x7fffffff, 0, NULL, NULL, &sc->sc_pci_dmat) != 0)
 			panic("%s: bus_dma_tag_create failed", __func__);
 	}
-	device_add_child(dev, "pci", 0);
+	device_add_child(dev, "pci", -1);
 	bus_generic_attach(dev);
 	return (0);
 }
diff --git a/sys/modules/Makefile b/sys/modules/Makefile
index df6b01a9b78c2..f5f9b4c3a3e1c 100644
--- a/sys/modules/Makefile
+++ b/sys/modules/Makefile
@@ -270,6 +270,8 @@ SUBDIR=	\
 	${_nxge} \
 	${_opensolaris} \
 	oce \
+	otus \
+	otusfw \
 	ow \
 	${_padlock} \
 	${_padlock_rng} \
diff --git a/sys/modules/netgraph/Makefile b/sys/modules/netgraph/Makefile
index c9b5748a7b4b6..1c787f64af398 100644
--- a/sys/modules/netgraph/Makefile
+++ b/sys/modules/netgraph/Makefile
@@ -62,4 +62,6 @@ _bluetooth=	bluetooth
 _mppc=		mppc
 .endif
 
+SUBDIR_PARALLEL=
+
 .include <bsd.subdir.mk>
diff --git a/sys/modules/otus/Makefile b/sys/modules/otus/Makefile
new file mode 100644
index 0000000000000..b263c06ab2df7
--- /dev/null
+++ b/sys/modules/otus/Makefile
@@ -0,0 +1,10 @@
+# $FreeBSD$
+
+.PATH: ${.CURDIR}/../../dev/otus
+
+KMOD    = if_otus
+SRCS    = if_otus.c if_otusreg.h	\
+	  bus_if.h device_if.h		\
+	  opt_bus.h opt_usb.h opt_wlan.h usb_if.h usbdevs.h
+
+.include <bsd.kmod.mk>
diff --git a/sys/modules/otusfw/Makefile b/sys/modules/otusfw/Makefile
new file mode 100644
index 0000000000000..0a47f66c9cab5
--- /dev/null
+++ b/sys/modules/otusfw/Makefile
@@ -0,0 +1,5 @@
+# $FreeBSD$
+
+SUBDIR=otusfw_init otusfw_main
+
+.include <bsd.subdir.mk>
diff --git a/sys/modules/otusfw/otusfw_init/Makefile b/sys/modules/otusfw/otusfw_init/Makefile
new file mode 100644
index 0000000000000..d188e9a83d945
--- /dev/null
+++ b/sys/modules/otusfw/otusfw_init/Makefile
@@ -0,0 +1,11 @@
+# $FreeBSD$
+
+KMOD=	otusfw_init
+FIRMWS=	otusfw_init:otusfw_init:1
+
+CLEANFILES=	otus_init
+
+otusfw_init: ${.CURDIR}/../../../contrib/dev/otus/otus-init
+	cp ${.CURDIR}/../../../contrib/dev/otus/otus-init ${.TARGET}
+
+.include <bsd.kmod.mk>
diff --git a/sys/modules/otusfw/otusfw_main/Makefile b/sys/modules/otusfw/otusfw_main/Makefile
new file mode 100644
index 0000000000000..94e1aeec1dafd
--- /dev/null
+++ b/sys/modules/otusfw/otusfw_main/Makefile
@@ -0,0 +1,11 @@
+# $FreeBSD$
+
+KMOD=	otusfw_main
+FIRMWS=	otusfw_main:otusfw_main:1
+
+CLEANFILES=	otusfw_main
+
+otusfw_main: ${.CURDIR}/../../../contrib/dev/otus/otus-main
+	cp ${.CURDIR}/../../../contrib/dev/otus/otus-main ${.TARGET}
+
+.include <bsd.kmod.mk>
diff --git a/sys/modules/usb/rsu/Makefile b/sys/modules/usb/rsu/Makefile
index 482071c217188..755f4b82a56d3 100644
--- a/sys/modules/usb/rsu/Makefile
+++ b/sys/modules/usb/rsu/Makefile
@@ -5,6 +5,6 @@
 KMOD    = if_rsu
 SRCS    = if_rsu.c if_rsureg.h \
 	  bus_if.h device_if.h \
-	  opt_bus.h opt_usb.h usb_if.h usbdevs.h
+	  opt_bus.h opt_usb.h opt_wlan.h usb_if.h usbdevs.h
 
 .include <bsd.kmod.mk>
diff --git a/sys/modules/usb/urtwn/Makefile b/sys/modules/usb/urtwn/Makefile
index 58c409afa2905..d965294940892 100644
--- a/sys/modules/usb/urtwn/Makefile
+++ b/sys/modules/usb/urtwn/Makefile
@@ -5,6 +5,6 @@
 KMOD    = if_urtwn
 SRCS    = if_urtwn.c if_urtwnreg.h \
 	  bus_if.h device_if.h \
-	  opt_bus.h opt_usb.h usb_if.h usbdevs.h
+	  opt_bus.h opt_usb.h opt_wlan.h usb_if.h usbdevs.h
 
 .include <bsd.kmod.mk>
diff --git a/sys/net/if.c b/sys/net/if.c
index 27fdbd9d31873..a68d54a7adfff 100644
--- a/sys/net/if.c
+++ b/sys/net/if.c
@@ -1545,76 +1545,53 @@ ifa_free(struct ifaddr *ifa)
 	}
 }
 
-int
-ifa_add_loopback_route(struct ifaddr *ifa, struct sockaddr *ia)
+static int
+ifa_maintain_loopback_route(int cmd, const char *otype, struct ifaddr *ifa,
+    struct sockaddr *ia)
 {
-	int error = 0;
-	struct rtentry *rt = NULL;
+	int error;
 	struct rt_addrinfo info;
-	static struct sockaddr_dl null_sdl = {sizeof(null_sdl), AF_LINK};
+	struct sockaddr_dl null_sdl;
+	struct ifnet *ifp;
+
+	ifp = ifa->ifa_ifp;
 
 	bzero(&info, sizeof(info));
-	info.rti_ifp = V_loif;
+	if (cmd != RTM_DELETE)
+		info.rti_ifp = V_loif;
 	info.rti_flags = ifa->ifa_flags | RTF_HOST | RTF_STATIC;
 	info.rti_info[RTAX_DST] = ia;
 	info.rti_info[RTAX_GATEWAY] = (struct sockaddr *)&null_sdl;
-	error = rtrequest1_fib(RTM_ADD, &info, &rt, ifa->ifa_ifp->if_fib);
+	link_init_sdl(ifp, (struct sockaddr *)&null_sdl, ifp->if_type);
+
+	error = rtrequest1_fib(cmd, &info, NULL, ifp->if_fib);
 
-	if (error == 0 && rt != NULL) {
-		RT_LOCK(rt);
-		((struct sockaddr_dl *)rt->rt_gateway)->sdl_type  =
-			ifa->ifa_ifp->if_type;
-		((struct sockaddr_dl *)rt->rt_gateway)->sdl_index =
-			ifa->ifa_ifp->if_index;
-		RT_REMREF(rt);
-		RT_UNLOCK(rt);
-	} else if (error != 0)
-		log(LOG_DEBUG, "%s: insertion failed: %u\n", __func__, error);
+	if (error != 0)
+		log(LOG_DEBUG, "%s: %s failed for interface %s: %u\n",
+		    __func__, otype, if_name(ifp), error);
 
 	return (error);
 }
 
 int
-ifa_del_loopback_route(struct ifaddr *ifa, struct sockaddr *ia)
+ifa_add_loopback_route(struct ifaddr *ifa, struct sockaddr *ia)
 {
-	int error = 0;
-	struct rt_addrinfo info;
-	struct sockaddr_dl null_sdl;
 
-	bzero(&null_sdl, sizeof(null_sdl));
-	null_sdl.sdl_len = sizeof(null_sdl);
-	null_sdl.sdl_family = AF_LINK;
-	null_sdl.sdl_type = ifa->ifa_ifp->if_type;
-	null_sdl.sdl_index = ifa->ifa_ifp->if_index;
-	bzero(&info, sizeof(info));
-	info.rti_flags = ifa->ifa_flags | RTF_HOST | RTF_STATIC;
-	info.rti_info[RTAX_DST] = ia;
-	info.rti_info[RTAX_GATEWAY] = (struct sockaddr *)&null_sdl;
-	error = rtrequest1_fib(RTM_DELETE, &info, NULL, ifa->ifa_ifp->if_fib);
+	return (ifa_maintain_loopback_route(RTM_ADD, "insertion", ifa, ia));
+}
 
-	if (error != 0)
-		log(LOG_DEBUG, "%s: deletion failed: %u\n", __func__, error);
+int
+ifa_del_loopback_route(struct ifaddr *ifa, struct sockaddr *ia)
+{
 
-	return (error);
+	return (ifa_maintain_loopback_route(RTM_DELETE, "deletion", ifa, ia));
 }
 
 int
-ifa_switch_loopback_route(struct ifaddr *ifa, struct sockaddr *sa, int fib)
+ifa_switch_loopback_route(struct ifaddr *ifa, struct sockaddr *ia)
 {
-	struct rtentry *rt;
 
-	rt = rtalloc1_fib(sa, 0, 0, fib);
-	if (rt == NULL) {
-		log(LOG_DEBUG, "%s: fail", __func__);
-		return (EHOSTUNREACH);
-	}
-	((struct sockaddr_dl *)rt->rt_gateway)->sdl_type =
-	    ifa->ifa_ifp->if_type;
-	((struct sockaddr_dl *)rt->rt_gateway)->sdl_index =
-	    ifa->ifa_ifp->if_index;
-	RTFREE_LOCKED(rt);
-
-	return (0);
+	return (ifa_maintain_loopback_route(RTM_CHANGE, "switch", ifa, ia));
 }
 
 /*
diff --git a/sys/net/if_arcsubr.c b/sys/net/if_arcsubr.c
index d6724cf65ea61..d59c4a04a58fd 100644
--- a/sys/net/if_arcsubr.c
+++ b/sys/net/if_arcsubr.c
@@ -103,8 +103,8 @@ arc_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst,
 	u_int8_t		atype, adst;
 	int			loop_copy = 0;
 	int			isphds;
-#ifdef INET
-	int			is_gw;
+#if defined(INET) || defined(INET6)
+	int			is_gw = 0;
 #endif
 
 	if (!((ifp->if_flags & IFF_UP) &&
@@ -112,6 +112,11 @@ arc_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst,
 		return(ENETDOWN); /* m, m1 aren't initialized yet */
 
 	error = 0;
+#if defined(INET) || defined(INET6)
+	if (ro != NULL && ro->ro_rt != NULL &&
+	    (ro->ro_rt->rt_flags & RTF_GATEWAY) != 0)
+		is_gw = 1;
+#endif
 
 	switch (dst->sa_family) {
 #ifdef INET
@@ -125,10 +130,6 @@ arc_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst,
 		else if (ifp->if_flags & IFF_NOARP)
 			adst = ntohl(SIN(dst)->sin_addr.s_addr) & 0xFF;
 		else {
-			is_gw = 0;
-			if (ro != NULL && ro->ro_rt != NULL &&
-			    (ro->ro_rt->rt_flags & RTF_GATEWAY) != 0)
-				is_gw = 1;
 			error = arpresolve(ifp, is_gw, m, dst, &adst, NULL);
 			if (error)
 				return (error == EWOULDBLOCK ? 0 : error);
@@ -169,10 +170,11 @@ arc_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst,
 	case AF_INET6:
 		if ((m->m_flags & M_MCAST) != 0)
 			adst = arcbroadcastaddr; /* ARCnet broadcast address */
-		else
-			error = nd6_storelladdr(ifp, m, dst, (u_char *)&adst, NULL);
-		if (error)
-			return (error);
+		else {
+			error = nd6_resolve(ifp, is_gw, m, dst, &adst, NULL);
+			if (error != 0)
+				return (error == EWOULDBLOCK ? 0 : error);
+		}
 		atype = ARCTYPE_INET6;
 		break;
 #endif
diff --git a/sys/net/if_ethersubr.c b/sys/net/if_ethersubr.c
index 18429303b10a6..7bb3d990f69c5 100644
--- a/sys/net/if_ethersubr.c
+++ b/sys/net/if_ethersubr.c
@@ -225,10 +225,10 @@ ether_output(struct ifnet *ifp, struct mbuf *m,
 		if (lle != NULL && (pflags & LLE_VALID))
 			memcpy(edst, &lle->ll_addr.mac16, sizeof(edst));
 		else
-			error = nd6_storelladdr(ifp, m, dst, (u_char *)edst,
+			error = nd6_resolve(ifp, is_gw, m, dst, (u_char *)edst,
 			    &pflags);
 		if (error)
-			return error;
+			return (error == EWOULDBLOCK ? 0 : error);
 		type = htons(ETHERTYPE_IPV6);
 		break;
 #endif
@@ -392,16 +392,6 @@ ether_input_internal(struct ifnet *ifp, struct mbuf *m)
 		return;
 	}
 #endif
-	/*
-	 * Do consistency checks to verify assumptions
-	 * made by code past this point.
-	 */
-	if ((m->m_flags & M_PKTHDR) == 0) {
-		if_printf(ifp, "discard frame w/o packet header\n");
-		if_inc_counter(ifp, IFCOUNTER_IERRORS, 1);
-		m_freem(m);
-		return;
-	}
 	if (m->m_len < ETHER_HDR_LEN) {
 		/* XXX maybe should pullup? */
 		if_printf(ifp, "discard frame w/o leading ethernet "
@@ -413,19 +403,6 @@ ether_input_internal(struct ifnet *ifp, struct mbuf *m)
 	}
 	eh = mtod(m, struct ether_header *);
 	etype = ntohs(eh->ether_type);
-	if (m->m_pkthdr.rcvif == NULL) {
-		if_printf(ifp, "discard frame w/o interface pointer\n");
-		if_inc_counter(ifp, IFCOUNTER_IERRORS, 1);
-		m_freem(m);
-		return;
-	}
-#ifdef DIAGNOSTIC
-	if (m->m_pkthdr.rcvif != ifp) {
-		if_printf(ifp, "Warning, frame marked as received on %s\n",
-			m->m_pkthdr.rcvif->if_xname);
-	}
-#endif
-
 	random_harvest_queue(m, sizeof(*m), 2, RANDOM_NET_ETHER);
 
 	CURVNET_SET_QUIET(ifp->if_vnet);
@@ -499,7 +476,6 @@ ether_input_internal(struct ifnet *ifp, struct mbuf *m)
 			if_printf(ifp, "cannot pullup VLAN header\n");
 #endif
 			if_inc_counter(ifp, IFCOUNTER_IERRORS, 1);
-			m_freem(m);
 			CURVNET_RESTORE();
 			return;
 		}
@@ -599,6 +575,9 @@ static void
 ether_nh_input(struct mbuf *m)
 {
 
+	M_ASSERTPKTHDR(m);
+	KASSERT(m->m_pkthdr.rcvif != NULL,
+	    ("%s: NULL interface pointer", __func__));
 	ether_input_internal(m->m_pkthdr.rcvif, m);
 }
 
diff --git a/sys/net/if_fddisubr.c b/sys/net/if_fddisubr.c
index 9066c0d3cd63f..4a09fcd9bab88 100644
--- a/sys/net/if_fddisubr.c
+++ b/sys/net/if_fddisubr.c
@@ -101,8 +101,8 @@ fddi_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst,
 	int loop_copy = 0, error = 0, hdrcmplt = 0;
  	u_char esrc[FDDI_ADDR_LEN], edst[FDDI_ADDR_LEN];
 	struct fddi_header *fh;
-#ifdef INET
-	int is_gw;
+#if defined(INET) || defined(INET6)
+	int is_gw = 0;
 #endif
 
 #ifdef MAC
@@ -118,13 +118,15 @@ fddi_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst,
 		senderr(ENETDOWN);
 	getmicrotime(&ifp->if_lastchange);
 
+#if defined(INET) || defined(INET6)
+	if (ro != NULL && ro->ro_rt != NULL &&
+	    (ro->ro_rt->rt_flags & RTF_GATEWAY) != 0)
+		is_gw = 1;
+#endif
+
 	switch (dst->sa_family) {
 #ifdef INET
 	case AF_INET: {
-		is_gw = 0;
-		if (ro != NULL && ro->ro_rt != NULL &&
-		    (ro->ro_rt->rt_flags & RTF_GATEWAY) != 0)
-			is_gw = 1;
 		error = arpresolve(ifp, is_gw, m, dst, edst, NULL);
 		if (error)
 			return (error == EWOULDBLOCK ? 0 : error);
@@ -161,9 +163,9 @@ fddi_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst,
 #endif /* INET */
 #ifdef INET6
 	case AF_INET6:
-		error = nd6_storelladdr(ifp, m, dst, (u_char *)edst, NULL);
+		error = nd6_resolve(ifp, is_gw, m, dst, edst, NULL);
 		if (error)
-			return (error); /* Something bad happened */
+			return (error == EWOULDBLOCK ? 0 : error);
 		type = htons(ETHERTYPE_IPV6);
 		break;
 #endif /* INET6 */
diff --git a/sys/net/if_fwsubr.c b/sys/net/if_fwsubr.c
index 5bb12f9ee3aab..f8ec120de8152 100644
--- a/sys/net/if_fwsubr.c
+++ b/sys/net/if_fwsubr.c
@@ -89,8 +89,8 @@ firewire_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst,
 	struct mbuf *mtail;
 	int unicast, dgl, foff;
 	static int next_dgl;
-#ifdef INET
-	int is_gw;
+#if defined(INET) || defined(INET6)
+	int is_gw = 0;
 #endif
 
 #ifdef MAC
@@ -105,6 +105,11 @@ firewire_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst,
 		goto bad;
 	}
 
+#if defined(INET) || defined(INET6)
+	if (ro != NULL && ro->ro_rt != NULL &&
+	    (ro->ro_rt->rt_flags & RTF_GATEWAY) != 0)
+		is_gw = 1;
+#endif
 	/*
 	 * For unicast, we make a tag to store the lladdr of the
 	 * destination. This might not be the first time we have seen
@@ -173,10 +178,10 @@ firewire_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst,
 #ifdef INET6
 	case AF_INET6:
 		if (unicast) {
-			error = nd6_storelladdr(fc->fc_ifp, m, dst,
+			error = nd6_resolve(fc->fc_ifp, is_gw, m, dst,
 			    (u_char *) destfw, NULL);
 			if (error)
-				return (error);
+				return (error == EWOULDBLOCK ? 0 : error);
 		}
 		type = ETHERTYPE_IPV6;
 		break;
diff --git a/sys/net/if_iso88025subr.c b/sys/net/if_iso88025subr.c
index 727df37f70db8..976e41033157a 100644
--- a/sys/net/if_iso88025subr.c
+++ b/sys/net/if_iso88025subr.c
@@ -293,9 +293,9 @@ iso88025_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst,
 #endif	/* INET */
 #ifdef INET6
 	case AF_INET6:
-		error = nd6_storelladdr(ifp, m, dst, (u_char *)edst, NULL);
+		error = nd6_resolve(ifp, is_gw, m, dst, edst, NULL);
 		if (error)
-			return (error);
+			return (error == EWOULDBLOCK ? 0 : error);
 		snap_type = ETHERTYPE_IPV6;
 		break;
 #endif	/* INET6 */
diff --git a/sys/net/if_llatbl.c b/sys/net/if_llatbl.c
index 1e246d4ab3b97..dbb96539b4e5a 100644
--- a/sys/net/if_llatbl.c
+++ b/sys/net/if_llatbl.c
@@ -186,7 +186,7 @@ htable_unlink_entry(struct llentry *lle)
 }
 
 struct prefix_match_data {
-	const struct sockaddr *prefix;
+	const struct sockaddr *addr;
 	const struct sockaddr *mask;
 	struct llentries dchain;
 	u_int flags;
@@ -199,7 +199,7 @@ htable_prefix_free_cb(struct lltable *llt, struct llentry *lle, void *farg)
 
 	pmd = (struct prefix_match_data *)farg;
 
-	if (llt->llt_match_prefix(pmd->prefix, pmd->mask, pmd->flags, lle)) {
+	if (llt->llt_match_prefix(pmd->addr, pmd->mask, pmd->flags, lle)) {
 		LLE_WLOCK(lle);
 		LIST_INSERT_HEAD(&pmd->dchain, lle, lle_chain);
 	}
@@ -208,14 +208,14 @@ htable_prefix_free_cb(struct lltable *llt, struct llentry *lle, void *farg)
 }
 
 static void
-htable_prefix_free(struct lltable *llt, const struct sockaddr *prefix,
+htable_prefix_free(struct lltable *llt, const struct sockaddr *addr,
     const struct sockaddr *mask, u_int flags)
 {
 	struct llentry *lle, *next;
 	struct prefix_match_data pmd;
 
 	bzero(&pmd, sizeof(pmd));
-	pmd.prefix = prefix;
+	pmd.addr = addr;
 	pmd.mask = mask;
 	pmd.flags = flags;
 	LIST_INIT(&pmd.dchain);
@@ -291,17 +291,11 @@ lltable_drop_entry_queue(struct llentry *lle)
 size_t
 llentry_free(struct llentry *lle)
 {
-	struct lltable *llt;
 	size_t pkts_dropped;
 
 	LLE_WLOCK_ASSERT(lle);
 
-	if ((lle->la_flags & LLE_LINKED) != 0) {
-		llt = lle->lle_tbl;
-
-		IF_AFDATA_WLOCK_ASSERT(llt->llt_ifp);
-		llt->llt_unlink_entry(lle);
-	}
+	KASSERT((lle->la_flags & LLE_LINKED) == 0, ("freeing linked lle"));
 
 	pkts_dropped = lltable_drop_entry_queue(lle);
 
@@ -427,8 +421,42 @@ lltable_drain(int af)
 }
 #endif
 
+/*
+ * Deletes an address from given lltable.
+ * Used for userland interaction to remove
+ * individual entries. Skips entries added by OS.
+ */
+int
+lltable_delete_addr(struct lltable *llt, u_int flags,
+    const struct sockaddr *l3addr)
+{
+	struct llentry *lle;
+	struct ifnet *ifp;
+
+	ifp = llt->llt_ifp;
+	IF_AFDATA_WLOCK(ifp);
+	lle = lla_lookup(llt, LLE_EXCLUSIVE, l3addr);
+
+	if (lle == NULL) {
+		IF_AFDATA_WUNLOCK(ifp);
+		return (ENOENT);
+	}
+	if ((lle->la_flags & LLE_IFADDR) != 0 && (flags & LLE_IFADDR) == 0) {
+		IF_AFDATA_WUNLOCK(ifp);
+		LLE_WUNLOCK(lle);
+		return (EPERM);
+	}
+
+	lltable_unlink_entry(llt, lle);
+	IF_AFDATA_WUNLOCK(ifp);
+
+	llt->llt_delete_entry(llt, lle);
+
+	return (0);
+}
+
 void
-lltable_prefix_free(int af, struct sockaddr *prefix, struct sockaddr *mask,
+lltable_prefix_free(int af, struct sockaddr *addr, struct sockaddr *mask,
     u_int flags)
 {
 	struct lltable *llt;
@@ -438,7 +466,7 @@ lltable_prefix_free(int af, struct sockaddr *prefix, struct sockaddr *mask,
 		if (llt->llt_af != af)
 			continue;
 
-		llt->llt_prefix_free(llt, prefix, mask, flags);
+		llt->llt_prefix_free(llt, addr, mask, flags);
 	}
 	LLTABLE_RUNLOCK();
 }
@@ -651,10 +679,7 @@ lla_rt_output(struct rt_msghdr *rtm, struct rt_addrinfo *info)
 		break;
 
 	case RTM_DELETE:
-		IF_AFDATA_WLOCK(ifp);
-		error = lla_delete(llt, 0, dst);
-		IF_AFDATA_WUNLOCK(ifp);
-		return (error == 0 ? 0 : ENOENT);
+		return (lltable_delete_addr(llt, 0, dst));
 
 	default:
 		error = EINVAL;
@@ -696,7 +721,6 @@ llatbl_lle_show(struct llentry_sa *la)
 	db_printf(" la_flags=0x%04x\n", lle->la_flags);
 	db_printf(" la_asked=%u\n", lle->la_asked);
 	db_printf(" la_preempt=%u\n", lle->la_preempt);
-	db_printf(" ln_byhint=%u\n", lle->ln_byhint);
 	db_printf(" ln_state=%d\n", lle->ln_state);
 	db_printf(" ln_router=%u\n", lle->ln_router);
 	db_printf(" ln_ntick=%ju\n", (uintmax_t)lle->ln_ntick);
diff --git a/sys/net/if_llatbl.h b/sys/net/if_llatbl.h
index 0571bf0c17898..1415ad440edaa 100644
--- a/sys/net/if_llatbl.h
+++ b/sys/net/if_llatbl.h
@@ -75,7 +75,6 @@ struct llentry {
 	uint16_t		 la_flags;
 	uint16_t		 la_asked;
 	uint16_t		 la_preempt;
-	uint16_t		 ln_byhint;
 	int16_t			 ln_state;	/* IPv6 has ND6_LLINFO_NOSTATE == -2 */
 	uint16_t		 ln_router;
 	time_t			 ln_ntick;
@@ -135,10 +134,9 @@ typedef	struct llentry *(llt_lookup_t)(struct lltable *, u_int flags,
     const struct sockaddr *l3addr);
 typedef	struct llentry *(llt_alloc_t)(struct lltable *, u_int flags,
     const struct sockaddr *l3addr);
-typedef	int (llt_delete_t)(struct lltable *, u_int flags,
-    const struct sockaddr *l3addr);
+typedef	void (llt_delete_t)(struct lltable *, struct llentry *);
 typedef void (llt_prefix_free_t)(struct lltable *,
-    const struct sockaddr *prefix, const struct sockaddr *mask, u_int flags);
+    const struct sockaddr *addr, const struct sockaddr *mask, u_int flags);
 typedef int (llt_dump_entry_t)(struct lltable *, struct llentry *,
     struct sysctl_req *);
 typedef uint32_t (llt_hash_t)(const struct llentry *, uint32_t);
@@ -162,7 +160,7 @@ struct lltable {
 
 	llt_lookup_t		*llt_lookup;
 	llt_alloc_t		*llt_alloc_entry;
-	llt_delete_t		*llt_delete;
+	llt_delete_t		*llt_delete_entry;
 	llt_prefix_free_t	*llt_prefix_free;
 	llt_dump_entry_t	*llt_dump_entry;
 	llt_hash_t		*llt_hash;
@@ -184,6 +182,7 @@ MALLOC_DECLARE(M_LLTABLE);
 #define	LLE_STATIC	0x0002	/* entry is static */
 #define	LLE_IFADDR	0x0004	/* entry is interface addr */
 #define	LLE_VALID	0x0008	/* ll_addr is valid */
+#define	LLE_REDIRECT	0x0010	/* installed by redirect; has host rtentry */
 #define	LLE_PUB		0x0020	/* publish entry ??? */
 #define	LLE_LINKED	0x0040	/* linked to lookup structure */
 /* LLE request flags */
@@ -212,6 +211,8 @@ size_t lltable_drop_entry_queue(struct llentry *);
 struct llentry *lltable_alloc_entry(struct lltable *llt, u_int flags,
     const struct sockaddr *l4addr);
 void lltable_free_entry(struct lltable *llt, struct llentry *lle);
+int lltable_delete_addr(struct lltable *llt, u_int flags,
+    const struct sockaddr *l3addr);
 void lltable_link_entry(struct lltable *llt, struct llentry *lle);
 void lltable_unlink_entry(struct lltable *llt, struct llentry *lle);
 void lltable_fill_sa_entry(const struct llentry *lle, struct sockaddr *sa);
@@ -230,14 +231,6 @@ lla_lookup(struct lltable *llt, u_int flags, const struct sockaddr *l3addr)
 	return (llt->llt_lookup(llt, flags, l3addr));
 }
 
-static __inline int
-lla_delete(struct lltable *llt, u_int flags, const struct sockaddr *l3addr)
-{
-
-	return (llt->llt_delete(llt, flags, l3addr));
-}
-
-
 int		lla_rt_output(struct rt_msghdr *, struct rt_addrinfo *);
 
 #include <sys/eventhandler.h>
diff --git a/sys/net/if_var.h b/sys/net/if_var.h
index 572e413a64cc4..9dfc459b954f7 100644
--- a/sys/net/if_var.h
+++ b/sys/net/if_var.h
@@ -243,11 +243,12 @@ struct ifnet {
 	 * count limit does not apply. If all three fields are zero,
 	 * there is no TSO limit.
 	 *
-	 * NOTE: The TSO limits only apply to the data payload part of
-	 * a TCP/IP packet. That means there is no need to subtract
-	 * space for ethernet-, vlan-, IP- or TCP- headers from the
-	 * TSO limits unless the hardware driver in question requires
-	 * so.
+	 * NOTE: The TSO limits should reflect the values used in the
+	 * BUSDMA tag a network adapter is using to load a mbuf chain
+	 * for transmission. The TCP/IP network stack will subtract
+	 * space for all linklevel and protocol level headers and
+	 * ensure that the full mbuf chain passed to the network
+	 * adapter fits within the given limits.
 	 */
 	u_int	if_hw_tsomax;		/* TSO maximum size in bytes */
 	u_int	if_hw_tsomaxsegcount;	/* TSO maximum segment count */
@@ -503,7 +504,7 @@ struct	ifnet *ifunit_ref(const char *);
 
 int	ifa_add_loopback_route(struct ifaddr *, struct sockaddr *);
 int	ifa_del_loopback_route(struct ifaddr *, struct sockaddr *);
-int	ifa_switch_loopback_route(struct ifaddr *, struct sockaddr *, int fib);
+int	ifa_switch_loopback_route(struct ifaddr *, struct sockaddr *);
 
 struct	ifaddr *ifa_ifwithaddr(const struct sockaddr *);
 int		ifa_ifwithaddr_check(const struct sockaddr *);
diff --git a/sys/net/route.c b/sys/net/route.c
index 1517dca1e22c9..4035b03db6234 100644
--- a/sys/net/route.c
+++ b/sys/net/route.c
@@ -584,13 +584,20 @@ rtredirect_fib(struct sockaddr *dst,
 	 * we have a routing loop, perhaps as a result of an interface
 	 * going down recently.
 	 */
-	if (!(flags & RTF_DONE) && rt &&
-	     (!sa_equal(src, rt->rt_gateway) || rt->rt_ifa != ifa))
-		error = EINVAL;
-	else if (ifa_ifwithaddr_check(gateway))
+	if (!(flags & RTF_DONE) && rt) {
+		if (!sa_equal(src, rt->rt_gateway)) {
+			error = EINVAL;
+			goto done;
+		}
+		if (rt->rt_ifa != ifa && ifa->ifa_addr->sa_family != AF_LINK) {
+			error = EINVAL;
+			goto done;
+		}
+	}
+	if ((flags & RTF_GATEWAY) && ifa_ifwithaddr_check(gateway)) {
 		error = EHOSTUNREACH;
-	if (error)
 		goto done;
+	}
 	/*
 	 * Create a new entry if we just got back a wildcard entry
 	 * or the lookup failed.  This is necessary for hosts
@@ -613,7 +620,7 @@ rtredirect_fib(struct sockaddr *dst,
 			rt0 = rt;
 			rt = NULL;
 		
-			flags |=  RTF_GATEWAY | RTF_DYNAMIC;
+			flags |= RTF_DYNAMIC;
 			bzero((caddr_t)&info, sizeof(info));
 			info.rti_info[RTAX_DST] = dst;
 			info.rti_info[RTAX_GATEWAY] = gateway;
@@ -640,6 +647,8 @@ rtredirect_fib(struct sockaddr *dst,
 			 * Smash the current notion of the gateway to
 			 * this destination.  Should check about netmask!!!
 			 */
+			if ((flags & RTF_GATEWAY) == 0)
+				rt->rt_flags &= ~RTF_GATEWAY;
 			rt->rt_flags |= RTF_MODIFIED;
 			flags |= RTF_MODIFIED;
 			stat = &V_rtstat.rts_newgateway;
@@ -653,7 +662,8 @@ rtredirect_fib(struct sockaddr *dst,
 			gwrt = rtalloc1(gateway, 1, RTF_RNH_LOCKED);
 			RADIX_NODE_HEAD_UNLOCK(rnh);
 			EVENTHANDLER_INVOKE(route_redirect_event, rt, gwrt, dst);
-			RTFREE_LOCKED(gwrt);
+			if (gwrt)
+				RTFREE_LOCKED(gwrt);
 		}
 	} else
 		error = EHOSTUNREACH;
diff --git a/sys/net80211/_ieee80211.h b/sys/net80211/_ieee80211.h
index 5120d413188b0..4ff395d41a408 100644
--- a/sys/net80211/_ieee80211.h
+++ b/sys/net80211/_ieee80211.h
@@ -390,15 +390,14 @@ struct ieee80211_regdomain {
  * MIMO antenna/radio state.
  */
 
-#define	IEEE80211_MAX_CHAINS		3
-#define	IEEE80211_MAX_EVM_PILOTS	6
-
 /*
  * XXX This doesn't yet export both ctl/ext chain details
+ * XXX TODO: IEEE80211_MAX_CHAINS is defined in _freebsd.h, not here;
+ * figure out how to pull it in!
  */
 struct ieee80211_mimo_info {
-	int8_t		rssi[IEEE80211_MAX_CHAINS];	/* per-antenna rssi */
-	int8_t		noise[IEEE80211_MAX_CHAINS];	/* per-antenna noise floor */
+	int8_t		rssi[3];	/* per-antenna rssi */
+	int8_t		noise[3];	/* per-antenna noise floor */
 	uint8_t		pad[2];
 	uint32_t	evm[3];		/* EVM data */
 };
diff --git a/sys/net80211/ieee80211.h b/sys/net80211/ieee80211.h
index 64d0754f358db..5bec9af3ea65a 100644
--- a/sys/net80211/ieee80211.h
+++ b/sys/net80211/ieee80211.h
@@ -815,6 +815,7 @@ struct ieee80211_csa_ie {
 /* rate set entries are in .5 Mb/s units, and potentially marked as basic */
 #define	IEEE80211_RATE_BASIC		0x80
 #define	IEEE80211_RATE_VAL		0x7f
+#define	IEEE80211_RV(v)			((v) & IEEE80211_RATE_VAL)
 
 /* ERP information element flags */
 #define	IEEE80211_ERP_NON_ERP_PRESENT	0x01
diff --git a/sys/net80211/ieee80211_freebsd.c b/sys/net80211/ieee80211_freebsd.c
index 9a8e941720303..6fdfb5cd31c8e 100644
--- a/sys/net80211/ieee80211_freebsd.c
+++ b/sys/net80211/ieee80211_freebsd.c
@@ -491,6 +491,43 @@ ieee80211_process_callback(struct ieee80211_node *ni,
 }
 
 /*
+ * Add RX parameters to the given mbuf.
+ *
+ * Returns 1 if OK, 0 on error.
+ */
+int
+ieee80211_add_rx_params(struct mbuf *m, const struct ieee80211_rx_stats *rxs)
+{
+	struct m_tag *mtag;
+	struct ieee80211_rx_params *rx;
+
+	mtag = m_tag_alloc(MTAG_ABI_NET80211, NET80211_TAG_RECV_PARAMS,
+	    sizeof(struct ieee80211_rx_stats), M_NOWAIT);
+	if (mtag == NULL)
+		return (0);
+
+	rx = (struct ieee80211_rx_params *)(mtag + 1);
+	memcpy(&rx->params, rxs, sizeof(*rxs));
+	m_tag_prepend(m, mtag);
+	return (1);
+}
+
+int
+ieee80211_get_rx_params(struct mbuf *m, struct ieee80211_rx_stats *rxs)
+{
+	struct m_tag *mtag;
+	struct ieee80211_rx_params *rx;
+
+	mtag = m_tag_locate(m, MTAG_ABI_NET80211, NET80211_TAG_RECV_PARAMS,
+	    NULL);
+	if (mtag == NULL)
+		return (-1);
+	rx = (struct ieee80211_rx_params *)(mtag + 1);
+	memcpy(rxs, &rx->params, sizeof(*rxs));
+	return (0);
+}
+
+/*
  * Transmit a frame to the parent interface.
  *
  * TODO: if the transmission fails, make sure the parent node is freed
diff --git a/sys/net80211/ieee80211_freebsd.h b/sys/net80211/ieee80211_freebsd.h
index 162cf4309d49a..f660c17abf511 100644
--- a/sys/net80211/ieee80211_freebsd.h
+++ b/sys/net80211/ieee80211_freebsd.h
@@ -330,6 +330,8 @@ void	ieee80211_process_callback(struct ieee80211_node *, struct mbuf *, int);
 #define	NET80211_TAG_XMIT_PARAMS	1
 /* See below; this is after the bpf_params definition */
 
+#define	NET80211_TAG_RECV_PARAMS	2
+
 struct ieee80211com;
 int	ieee80211_parent_xmitpkt(struct ieee80211com *, struct mbuf *);
 int	ieee80211_vap_xmitpkt(struct ieee80211vap *, struct mbuf *);
@@ -618,6 +620,43 @@ int	ieee80211_add_xmit_params(struct mbuf *m,
 	    const struct ieee80211_bpf_params *);
 int	ieee80211_get_xmit_params(struct mbuf *m,
 	    struct ieee80211_bpf_params *);
+
+#define	IEEE80211_MAX_CHAINS		3
+#define	IEEE80211_MAX_EVM_PILOTS	6
+
+#define	IEEE80211_R_NF		0x0000001	/* global NF value valid */
+#define	IEEE80211_R_RSSI	0x0000002	/* global RSSI value valid */
+#define	IEEE80211_R_C_CHAIN	0x0000004	/* RX chain count valid */
+#define	IEEE80211_R_C_NF	0x0000008	/* per-chain NF value valid */
+#define	IEEE80211_R_C_RSSI	0x0000010	/* per-chain RSSI value valid */
+#define	IEEE80211_R_C_EVM	0x0000020	/* per-chain EVM valid */
+#define	IEEE80211_R_C_HT40	0x0000040	/* RX'ed packet is 40mhz, pilots 4,5 valid */
+#define	IEEE80211_R_FREQ	0x0000080	/* Freq value populated, MHz */
+#define	IEEE80211_R_IEEE	0x0000100	/* IEEE value populated */
+#define	IEEE80211_R_BAND	0x0000200	/* Frequency band populated */
+
+struct ieee80211_rx_stats {
+	uint32_t r_flags;		/* IEEE80211_R_* flags */
+	uint8_t c_chain;		/* number of RX chains involved */
+	int16_t	c_nf_ctl[IEEE80211_MAX_CHAINS];	/* per-chain NF */
+	int16_t	c_nf_ext[IEEE80211_MAX_CHAINS];	/* per-chain NF */
+	int16_t	c_rssi_ctl[IEEE80211_MAX_CHAINS];	/* per-chain RSSI */
+	int16_t	c_rssi_ext[IEEE80211_MAX_CHAINS];	/* per-chain RSSI */
+	uint8_t nf;			/* global NF */
+	uint8_t rssi;			/* global RSSI */
+	uint8_t evm[IEEE80211_MAX_CHAINS][IEEE80211_MAX_EVM_PILOTS];
+					/* per-chain, per-pilot EVM values */
+	uint16_t c_freq;
+	uint8_t c_ieee;
+};
+
+struct ieee80211_rx_params {
+	struct ieee80211_rx_stats params;
+};
+int	ieee80211_add_rx_params(struct mbuf *m,
+	    const struct ieee80211_rx_stats *rxs);
+int	ieee80211_get_rx_params(struct mbuf *m,
+	    struct ieee80211_rx_stats *rxs);
 #endif /* _KERNEL */
 
 /*
diff --git a/sys/net80211/ieee80211_ht.c b/sys/net80211/ieee80211_ht.c
index d83cf6511f2ff..a850a334ea20b 100644
--- a/sys/net80211/ieee80211_ht.c
+++ b/sys/net80211/ieee80211_ht.c
@@ -558,6 +558,43 @@ ampdu_rx_start(struct ieee80211_node *ni, struct ieee80211_rx_ampdu *rap,
 }
 
 /*
+ * Public function; manually setup the RX ampdu state.
+ */
+int
+ieee80211_ampdu_rx_start_ext(struct ieee80211_node *ni, int tid, int seq, int baw)
+{
+	struct ieee80211_rx_ampdu *rap;
+
+	/* XXX TODO: sanity check tid, seq, baw */
+
+	rap = &ni->ni_rx_ampdu[tid];
+
+	if (rap->rxa_flags & IEEE80211_AGGR_RUNNING) {
+		/*
+		 * AMPDU previously setup and not terminated with a DELBA,
+		 * flush the reorder q's in case anything remains.
+		 */
+		ampdu_rx_purge(rap);
+	}
+
+	memset(rap, 0, sizeof(*rap));
+	rap->rxa_wnd = (baw== 0) ?
+	    IEEE80211_AGGR_BAWMAX : min(baw, IEEE80211_AGGR_BAWMAX);
+	rap->rxa_start = seq;
+	rap->rxa_flags |=  IEEE80211_AGGR_RUNNING | IEEE80211_AGGR_XCHGPEND;
+
+	IEEE80211_NOTE(ni->ni_vap, IEEE80211_MSG_11N, ni,
+	    "%s: tid=%d, start=%d, wnd=%d, flags=0x%08x\n",
+	    __func__,
+	    tid,
+	    seq,
+	    rap->rxa_wnd,
+	    rap->rxa_flags);
+
+	return 0;
+}
+
+/*
  * Stop A-MPDU rx processing for the specified TID.
  */
 static void
@@ -1044,7 +1081,7 @@ ieee80211_ht_node_init(struct ieee80211_node *ni)
 		tap = &ni->ni_tx_ampdu[tid];
 		tap->txa_tid = tid;
 		tap->txa_ni = ni;
-		tap->txa_lastsample = ticks;
+		ieee80211_txampdu_init_pps(tap);
 		/* NB: further initialization deferred */
 	}
 	ni->ni_flags |= IEEE80211_NODE_HT | IEEE80211_NODE_AMPDU;
@@ -1214,7 +1251,7 @@ ieee80211_ht_wds_init(struct ieee80211_node *ni)
 	for (tid = 0; tid < WME_NUM_TID; tid++) {
 		tap = &ni->ni_tx_ampdu[tid];
 		tap->txa_tid = tid;
-		tap->txa_lastsample = ticks;
+		ieee80211_txampdu_init_pps(tap);
 	}
 	/* NB: AMPDU tx/rx governed by IEEE80211_FHT_AMPDU_{TX,RX} */
 	ni->ni_flags |= IEEE80211_NODE_HT | IEEE80211_NODE_AMPDU;
@@ -1378,12 +1415,6 @@ ieee80211_ht_timeout(struct ieee80211com *ic)
 	}
 }
 
-/* unalligned little endian access */     
-#define LE_READ_2(p)					\
-	((uint16_t)					\
-	 ((((const uint8_t *)(p))[0]      ) |		\
-	  (((const uint8_t *)(p))[1] <<  8)))
-
 /*
  * Process an 802.11n HT capabilities ie.
  */
@@ -1721,8 +1752,7 @@ ampdu_tx_stop(struct ieee80211_tx_ampdu *tap)
 	/*
 	 * Reset packet estimate.
 	 */
-	tap->txa_lastsample = ticks;
-	tap->txa_avgpps = 0;
+	ieee80211_txampdu_init_pps(tap);
 
 	/* NB: clearing NAK means we may re-send ADDBA */ 
 	tap->txa_flags &= ~(IEEE80211_AGGR_SETUP | IEEE80211_AGGR_NAK);
@@ -1796,6 +1826,55 @@ ieee80211_addba_request(struct ieee80211_node *ni,
 }
 
 /*
+ * Called by drivers that wish to request an ADDBA session be
+ * setup.  This brings it up and starts the request timer.
+ */
+int
+ieee80211_ampdu_tx_request_ext(struct ieee80211_node *ni, int tid)
+{
+	struct ieee80211_tx_ampdu *tap;
+
+	if (tid < 0 || tid > 15)
+		return (0);
+	tap = &ni->ni_tx_ampdu[tid];
+
+	/* XXX locking */
+	if ((tap->txa_flags & IEEE80211_AGGR_SETUP) == 0) {
+		/* do deferred setup of state */
+		ampdu_tx_setup(tap);
+	}
+	/* XXX hack for not doing proper locking */
+	tap->txa_flags &= ~IEEE80211_AGGR_NAK;
+	addba_start_timeout(tap);
+	return (1);
+}
+
+/*
+ * Called by drivers that have marked a session as active.
+ */
+int
+ieee80211_ampdu_tx_request_active_ext(struct ieee80211_node *ni, int tid,
+    int status)
+{
+	struct ieee80211_tx_ampdu *tap;
+
+	if (tid < 0 || tid > 15)
+		return (0);
+	tap = &ni->ni_tx_ampdu[tid];
+
+	/* XXX locking */
+	addba_stop_timeout(tap);
+	if (status == 1) {
+		tap->txa_flags |= IEEE80211_AGGR_RUNNING;
+		tap->txa_attempts = 0;
+	} else {
+		/* mark tid so we don't try again */
+		tap->txa_flags |= IEEE80211_AGGR_NAK;
+	}
+	return (1);
+}
+
+/*
  * Default method for processing an A-MPDU tx aggregation
  * response.  We shutdown any pending timer and update the
  * state block according to the reply.
diff --git a/sys/net80211/ieee80211_ht.h b/sys/net80211/ieee80211_ht.h
index 3bda5fe4580f7..e2b331854dff9 100644
--- a/sys/net80211/ieee80211_ht.h
+++ b/sys/net80211/ieee80211_ht.h
@@ -84,8 +84,19 @@ struct ieee80211_tx_ampdu {
  */
 
 static __inline void
+ieee80211_txampdu_init_pps(struct ieee80211_tx_ampdu *tap)
+{
+	/*
+	 * Reset packet estimate.
+	 */
+	tap->txa_lastsample = ticks;
+	tap->txa_avgpps = 0;
+}
+
+static __inline void
 ieee80211_txampdu_update_pps(struct ieee80211_tx_ampdu *tap)
 {
+
 	/* NB: scale factor of 2 was picked heuristically */
 	tap->txa_avgpps = ((tap->txa_avgpps << 2) -
 	     tap->txa_avgpps + tap->txa_pkts) >> 2;
@@ -97,6 +108,7 @@ ieee80211_txampdu_update_pps(struct ieee80211_tx_ampdu *tap)
 static __inline void
 ieee80211_txampdu_count_packet(struct ieee80211_tx_ampdu *tap)
 {
+
 	/* XXX bound loop/do more crude estimate? */
 	while (ticks - tap->txa_lastsample >= hz) {
 		ieee80211_txampdu_update_pps(tap);
@@ -200,4 +212,10 @@ uint8_t	*ieee80211_add_htinfo_vendor(uint8_t *, struct ieee80211_node *);
 struct ieee80211_beacon_offsets;
 void	ieee80211_ht_update_beacon(struct ieee80211vap *,
 		struct ieee80211_beacon_offsets *);
+int	ieee80211_ampdu_rx_start_ext(struct ieee80211_node *ni, int tid,
+	    int seq, int baw);
+int	ieee80211_ampdu_tx_request_ext(struct ieee80211_node *ni, int tid);
+int	ieee80211_ampdu_tx_request_active_ext(struct ieee80211_node *ni,
+	    int tid, int status);
+
 #endif /* _NET80211_IEEE80211_HT_H_ */
diff --git a/sys/net80211/ieee80211_input.c b/sys/net80211/ieee80211_input.c
index b757ae635f9f4..0b9552012d3d3 100644
--- a/sys/net80211/ieee80211_input.c
+++ b/sys/net80211/ieee80211_input.c
@@ -86,10 +86,21 @@ int
 ieee80211_input_mimo(struct ieee80211_node *ni, struct mbuf *m,
     struct ieee80211_rx_stats *rx)
 {
+	struct ieee80211_rx_stats rxs;
+
+	if (rx) {
+		memcpy(&rxs, rx, sizeof(*rx));
+	} else {
+		/* try to read from mbuf */
+		bzero(&rxs, sizeof(rxs));
+		ieee80211_get_rx_params(m, &rxs);
+	}
+
 	/* XXX should assert IEEE80211_R_NF and IEEE80211_R_RSSI are set */
-	ieee80211_process_mimo(ni, rx);
+	ieee80211_process_mimo(ni, &rxs);
+
 	//return ieee80211_input(ni, m, rx->rssi, rx->nf);
-	return ni->ni_vap->iv_input(ni, m, rx, rx->rssi, rx->nf);
+	return ni->ni_vap->iv_input(ni, m, &rxs, rxs.rssi, rxs.nf);
 }
 
 int
@@ -107,11 +118,20 @@ int
 ieee80211_input_mimo_all(struct ieee80211com *ic, struct mbuf *m,
     struct ieee80211_rx_stats *rx)
 {
+	struct ieee80211_rx_stats rxs;
 	struct ieee80211vap *vap;
 	int type = -1;
 
 	m->m_flags |= M_BCAST;		/* NB: mark for bpf tap'ing */
 
+	if (rx) {
+		memcpy(&rxs, rx, sizeof(*rx));
+	} else {
+		/* try to read from mbuf */
+		bzero(&rxs, sizeof(rxs));
+		ieee80211_get_rx_params(m, &rxs);
+	}
+
 	/* XXX locking */
 	TAILQ_FOREACH(vap, &ic->ic_vaps, iv_next) {
 		struct ieee80211_node *ni;
@@ -143,7 +163,7 @@ ieee80211_input_mimo_all(struct ieee80211com *ic, struct mbuf *m,
 			m = NULL;
 		}
 		ni = ieee80211_ref_node(vap->iv_bss);
-		type = ieee80211_input_mimo(ni, mcopy, rx);
+		type = ieee80211_input_mimo(ni, mcopy, &rxs);
 		ieee80211_free_node(ni);
 	}
 	if (m != NULL)			/* no vaps, reclaim mbuf */
diff --git a/sys/net80211/ieee80211_mesh.c b/sys/net80211/ieee80211_mesh.c
index ef811845fc911..cb6ab81ff1388 100644
--- a/sys/net80211/ieee80211_mesh.c
+++ b/sys/net80211/ieee80211_mesh.c
@@ -434,13 +434,12 @@ mesh_rt_flush_invalid(struct ieee80211vap *vap)
 	MESH_RT_UNLOCK(ms);
 }
 
-#define	N(a)	(sizeof(a) / sizeof(a[0]))
 int
 ieee80211_mesh_register_proto_path(const struct ieee80211_mesh_proto_path *mpp)
 {
 	int i, firstempty = -1;
 
-	for (i = 0; i < N(mesh_proto_paths); i++) {
+	for (i = 0; i < nitems(mesh_proto_paths); i++) {
 		if (strncmp(mpp->mpp_descr, mesh_proto_paths[i].mpp_descr,
 		    IEEE80211_MESH_PROTO_DSZ) == 0)
 			return EEXIST;
@@ -460,7 +459,7 @@ ieee80211_mesh_register_proto_metric(const struct
 {
 	int i, firstempty = -1;
 
-	for (i = 0; i < N(mesh_proto_metrics); i++) {
+	for (i = 0; i < nitems(mesh_proto_metrics); i++) {
 		if (strncmp(mpm->mpm_descr, mesh_proto_metrics[i].mpm_descr,
 		    IEEE80211_MESH_PROTO_DSZ) == 0)
 			return EEXIST;
@@ -480,7 +479,7 @@ mesh_select_proto_path(struct ieee80211vap *vap, const char *name)
 	struct ieee80211_mesh_state *ms = vap->iv_mesh;
 	int i;
 
-	for (i = 0; i < N(mesh_proto_paths); i++) {
+	for (i = 0; i < nitems(mesh_proto_paths); i++) {
 		if (strcasecmp(mesh_proto_paths[i].mpp_descr, name) == 0) {
 			ms->ms_ppath = &mesh_proto_paths[i];
 			return 0;
@@ -495,7 +494,7 @@ mesh_select_proto_metric(struct ieee80211vap *vap, const char *name)
 	struct ieee80211_mesh_state *ms = vap->iv_mesh;
 	int i;
 
-	for (i = 0; i < N(mesh_proto_metrics); i++) {
+	for (i = 0; i < nitems(mesh_proto_metrics); i++) {
 		if (strcasecmp(mesh_proto_metrics[i].mpm_descr, name) == 0) {
 			ms->ms_pmetric = &mesh_proto_metrics[i];
 			return 0;
@@ -503,7 +502,6 @@ mesh_select_proto_metric(struct ieee80211vap *vap, const char *name)
 	}
 	return ENOENT;
 }
-#undef	N
 
 static void
 mesh_gatemode_setup(struct ieee80211vap *vap)
diff --git a/sys/net80211/ieee80211_node.h b/sys/net80211/ieee80211_node.h
index bb31136892d6f..e9b73933f66ed 100644
--- a/sys/net80211/ieee80211_node.h
+++ b/sys/net80211/ieee80211_node.h
@@ -219,6 +219,9 @@ struct ieee80211_node {
 	struct ieee80211_tx_ampdu ni_tx_ampdu[WME_NUM_TID];
 	struct ieee80211_rx_ampdu ni_rx_ampdu[WME_NUM_TID];
 
+	/* fast-frames state */
+	struct mbuf *		ni_tx_superg[WME_NUM_TID];
+
 	/* others */
 	short			ni_inact;	/* inactivity mark count */
 	short			ni_inact_reload;/* inactivity reload value */
diff --git a/sys/net80211/ieee80211_output.c b/sys/net80211/ieee80211_output.c
index eb333da139cb8..27fe929ac2887 100644
--- a/sys/net80211/ieee80211_output.c
+++ b/sys/net80211/ieee80211_output.c
@@ -213,6 +213,22 @@ ieee80211_vap_pkt_send_dest(struct ieee80211vap *vap, struct mbuf *m,
 		}
 	}
 
+	/*
+	 * XXX If we aren't doing AMPDU TX then we /could/ do
+	 * fast-frames encapsulation, however right now this
+	 * output logic doesn't handle that case.
+	 *
+	 * So we'll be limited to "fast-frames" xmit for non-11n STA
+	 * and "no fast frames" xmit for 11n STAs.
+	 * It'd be nice to eventually test fast-frames out by
+	 * gracefully falling from failing A-MPDU transmission
+	 * (driver says no, fail to negotiate it with peer) to
+	 * using fast-frames.
+	 *
+	 * Note: we can actually put A-MSDU's inside an A-MPDU,
+	 * so hopefully we can figure out how to make that particular
+	 * combination work right.
+	 */
 #ifdef IEEE80211_SUPPORT_SUPERG
 	else if (IEEE80211_ATH_CAP(vap, ni, IEEE80211_NODE_FF)) {
 		m = ieee80211_ff_check(ni, m);
@@ -230,6 +246,11 @@ ieee80211_vap_pkt_send_dest(struct ieee80211vap *vap, struct mbuf *m,
 	 */
 	IEEE80211_TX_LOCK(ic);
 
+	/*
+	 * XXX make the encap and transmit code a separate function
+	 * so things like the FF (and later A-MSDU) path can just call
+	 * it for flushed frames.
+	 */
 	if (__predict_true((vap->iv_caps & IEEE80211_C_8023ENCAP) == 0)) {
 		/*
 		 * Encapsulate the packet in prep for transmission.
@@ -1787,7 +1808,7 @@ add_ie(uint8_t *frm, const uint8_t *ie)
 /*
  * Add a WME information element to a frame.
  */
-static uint8_t *
+uint8_t *
 ieee80211_add_wme_info(uint8_t *frm, struct ieee80211_wme_state *wme)
 {
 	static const struct ieee80211_wme_info info = {
diff --git a/sys/net80211/ieee80211_proto.c b/sys/net80211/ieee80211_proto.c
index cbcf6b1a6ff88..52e92ba477766 100644
--- a/sys/net80211/ieee80211_proto.c
+++ b/sys/net80211/ieee80211_proto.c
@@ -107,6 +107,7 @@ static void update_mcast(void *, int);
 static void update_promisc(void *, int);
 static void update_channel(void *, int);
 static void update_chw(void *, int);
+static void update_wme(void *, int);
 static void ieee80211_newstate_cb(void *, int);
 
 static int
@@ -144,6 +145,7 @@ ieee80211_proto_attach(struct ieee80211com *ic)
 	TASK_INIT(&ic->ic_chan_task, 0, update_channel, ic);
 	TASK_INIT(&ic->ic_bmiss_task, 0, beacon_miss, ic);
 	TASK_INIT(&ic->ic_chw_task, 0, update_chw, ic);
+	TASK_INIT(&ic->ic_wme_task, 0, update_wme, ic);
 
 	ic->ic_wme.wme_hipri_switch_hysteresis =
 		AGGRESSIVE_MODE_SWITCH_HYSTERESIS;
@@ -491,7 +493,6 @@ int
 ieee80211_fix_rate(struct ieee80211_node *ni,
 	struct ieee80211_rateset *nrs, int flags)
 {
-#define	RV(v)	((v) & IEEE80211_RATE_VAL)
 	struct ieee80211vap *vap = ni->ni_vap;
 	struct ieee80211com *ic = ni->ni_ic;
 	int i, j, rix, error;
@@ -545,7 +546,8 @@ ieee80211_fix_rate(struct ieee80211_node *ni,
 			 * Sort rates.
 			 */
 			for (j = i + 1; j < nrs->rs_nrates; j++) {
-				if (RV(nrs->rs_rates[i]) > RV(nrs->rs_rates[j])) {
+				if (IEEE80211_RV(nrs->rs_rates[i]) >
+				    IEEE80211_RV(nrs->rs_rates[j])) {
 					r = nrs->rs_rates[i];
 					nrs->rs_rates[i] = nrs->rs_rates[j];
 					nrs->rs_rates[j] = r;
@@ -604,8 +606,7 @@ ieee80211_fix_rate(struct ieee80211_node *ni,
 		    "ucastrate %x\n", __func__, fixedrate, ucastrate, flags);
 		return badrate | IEEE80211_RATE_BASIC;
 	} else
-		return RV(okrate);
-#undef RV
+		return IEEE80211_RV(okrate);
 }
 
 /*
@@ -1134,7 +1135,8 @@ ieee80211_wme_updateparams_locked(struct ieee80211vap *vap)
 		ieee80211_beacon_notify(vap, IEEE80211_BEACON_WME);
 	}
 
-	wme->wme_update(ic);
+	/* schedule the deferred WME update */
+	ieee80211_runtask(ic, &ic->ic_wme_task);
 
 	IEEE80211_DPRINTF(vap, IEEE80211_MSG_WME,
 	    "%s: WME params updated, cap_info 0x%x\n", __func__,
@@ -1199,6 +1201,17 @@ update_chw(void *arg, int npending)
 	ic->ic_update_chw(ic);
 }
 
+static void
+update_wme(void *arg, int npending)
+{
+	struct ieee80211com *ic = arg;
+
+	/*
+	 * XXX should we defer the WME configuration update until now?
+	 */
+	ic->ic_wme.wme_update(ic);
+}
+
 /*
  * Block until the parent is in a known state.  This is
  * used after any operations that dispatch a task (e.g.
@@ -1214,6 +1227,7 @@ ieee80211_waitfor_parent(struct ieee80211com *ic)
 	ieee80211_draintask(ic, &ic->ic_chan_task);
 	ieee80211_draintask(ic, &ic->ic_bmiss_task);
 	ieee80211_draintask(ic, &ic->ic_chw_task);
+	ieee80211_draintask(ic, &ic->ic_wme_task);
 	taskqueue_unblock(ic->ic_tq);
 }
 
diff --git a/sys/net80211/ieee80211_proto.h b/sys/net80211/ieee80211_proto.h
index b5918b44a1cdc..273e75931e08c 100644
--- a/sys/net80211/ieee80211_proto.h
+++ b/sys/net80211/ieee80211_proto.h
@@ -62,32 +62,6 @@ void	ieee80211_syncflag(struct ieee80211vap *, int flag);
 void	ieee80211_syncflag_ht(struct ieee80211vap *, int flag);
 void	ieee80211_syncflag_ext(struct ieee80211vap *, int flag);
 
-#define	IEEE80211_R_NF		0x0000001	/* global NF value valid */
-#define	IEEE80211_R_RSSI	0x0000002	/* global RSSI value valid */
-#define	IEEE80211_R_C_CHAIN	0x0000004	/* RX chain count valid */
-#define	IEEE80211_R_C_NF	0x0000008	/* per-chain NF value valid */
-#define	IEEE80211_R_C_RSSI	0x0000010	/* per-chain RSSI value valid */
-#define	IEEE80211_R_C_EVM	0x0000020	/* per-chain EVM valid */
-#define	IEEE80211_R_C_HT40	0x0000040	/* RX'ed packet is 40mhz, pilots 4,5 valid */
-#define	IEEE80211_R_FREQ	0x0000080	/* Freq value populated, MHz */
-#define	IEEE80211_R_IEEE	0x0000100	/* IEEE value populated */
-#define	IEEE80211_R_BAND	0x0000200	/* Frequency band populated */
-
-struct ieee80211_rx_stats {
-	uint32_t r_flags;		/* IEEE80211_R_* flags */
-	uint8_t c_chain;		/* number of RX chains involved */
-	int16_t	c_nf_ctl[IEEE80211_MAX_CHAINS];	/* per-chain NF */
-	int16_t	c_nf_ext[IEEE80211_MAX_CHAINS];	/* per-chain NF */
-	int16_t	c_rssi_ctl[IEEE80211_MAX_CHAINS];	/* per-chain RSSI */
-	int16_t	c_rssi_ext[IEEE80211_MAX_CHAINS];	/* per-chain RSSI */
-	uint8_t nf;			/* global NF */
-	uint8_t rssi;			/* global RSSI */
-	uint8_t evm[IEEE80211_MAX_CHAINS][IEEE80211_MAX_EVM_PILOTS];
-					/* per-chain, per-pilot EVM values */
-	uint16_t c_freq;
-	uint8_t c_ieee;
-};
-
 #define	ieee80211_input(ni, m, rssi, nf) \
 	((ni)->ni_vap->iv_input(ni, m, NULL, rssi, nf))
 int	ieee80211_input_all(struct ieee80211com *, struct mbuf *, int, int);
@@ -156,6 +130,8 @@ uint8_t *ieee80211_add_rsn(uint8_t *, const struct ieee80211vap *);
 uint8_t *ieee80211_add_qos(uint8_t *, const struct ieee80211_node *);
 uint16_t ieee80211_getcapinfo(struct ieee80211vap *,
 		struct ieee80211_channel *);
+struct ieee80211_wme_state;
+uint8_t * ieee80211_add_wme_info(uint8_t *frm, struct ieee80211_wme_state *wme);
 
 void	ieee80211_reset_erp(struct ieee80211com *);
 void	ieee80211_set_shortslottime(struct ieee80211com *, int onoff);
diff --git a/sys/net80211/ieee80211_scan_sta.c b/sys/net80211/ieee80211_scan_sta.c
index 709710c1e9e40..ea922d7e82a33 100644
--- a/sys/net80211/ieee80211_scan_sta.c
+++ b/sys/net80211/ieee80211_scan_sta.c
@@ -737,12 +737,6 @@ sta_cancel(struct ieee80211_scan_state *ss, struct ieee80211vap *vap)
 	return 0;
 }
 
-/* unaligned little endian access */     
-#define LE_READ_2(p)					\
-	((uint16_t)					\
-	 ((((const uint8_t *)(p))[0]      ) |		\
-	  (((const uint8_t *)(p))[1] <<  8)))
- 
 /*
  * Demote any supplied 11g channel to 11b.  There should
  * always be an 11b channel but we check anyway...
@@ -876,7 +870,6 @@ static int
 check_rate(struct ieee80211vap *vap, const struct ieee80211_channel *chan,
     const struct ieee80211_scan_entry *se)
 {
-#define	RV(v)	((v) & IEEE80211_RATE_VAL)
 	const struct ieee80211_rateset *srs;
 	int i, j, nrs, r, okrate, badrate, fixedrate, ucastrate;
 	const uint8_t *rs;
@@ -891,7 +884,7 @@ check_rate(struct ieee80211vap *vap, const struct ieee80211_channel *chan,
 	fixedrate = IEEE80211_FIXED_RATE_NONE;
 again:
 	for (i = 0; i < nrs; i++) {
-		r = RV(rs[i]);
+		r = IEEE80211_RV(rs[i]);
 		badrate = r;
 		/*
 		 * Check any fixed rate is included. 
@@ -902,7 +895,7 @@ again:
 		 * Check against our supported rates.
 		 */
 		for (j = 0; j < srs->rs_nrates; j++)
-			if (r == RV(srs->rs_rates[j])) {
+			if (r == IEEE80211_RV(srs->rs_rates[j])) {
 				if (r > okrate)		/* NB: track max */
 					okrate = r;
 				break;
@@ -928,8 +921,7 @@ back:
 	if (okrate == 0 || ucastrate != fixedrate)
 		return badrate | IEEE80211_RATE_BASIC;
 	else
-		return RV(okrate);
-#undef RV
+		return IEEE80211_RV(okrate);
 }
 
 static __inline int
diff --git a/sys/net80211/ieee80211_superg.c b/sys/net80211/ieee80211_superg.c
index 2e2bbd18be846..e91da017c8a5a 100644
--- a/sys/net80211/ieee80211_superg.c
+++ b/sys/net80211/ieee80211_superg.c
@@ -530,7 +530,6 @@ ieee80211_ff_age(struct ieee80211com *ic, struct ieee80211_stageq *sq,
 {
 	struct mbuf *m, *head;
 	struct ieee80211_node *ni;
-	struct ieee80211_tx_ampdu *tap;
 
 #if 0
 	KASSERT(sq->head != NULL, ("stageq empty"));
@@ -541,11 +540,10 @@ ieee80211_ff_age(struct ieee80211com *ic, struct ieee80211_stageq *sq,
 	while ((m = sq->head) != NULL && M_AGE_GET(m) < quanta) {
 		int tid = WME_AC_TO_TID(M_WME_GETAC(m));
 
-		/* clear tap ref to frame */
+		/* clear staging ref to frame */
 		ni = (struct ieee80211_node *) m->m_pkthdr.rcvif;
-		tap = &ni->ni_tx_ampdu[tid];
-		KASSERT(tap->txa_private == m, ("staging queue empty"));
-		tap->txa_private = NULL;
+		KASSERT(ni->ni_tx_superg[tid] == m, ("staging queue empty"));
+		ni->ni_tx_superg[tid] = NULL;
 
 		sq->head = m->m_nextpkt;
 		sq->depth--;
@@ -658,7 +656,12 @@ ieee80211_ff_check(struct ieee80211_node *ni, struct mbuf *m)
 	 */
 	IEEE80211_LOCK(ic);
 	tap = &ni->ni_tx_ampdu[WME_AC_TO_TID(pri)];
-	mstaged = tap->txa_private;		/* NB: we reuse AMPDU state */
+	mstaged = ni->ni_tx_superg[WME_AC_TO_TID(pri)];
+	/* XXX NOTE: reusing packet counter state from A-MPDU */
+	/*
+	 * XXX NOTE: this means we're double-counting; it should just
+	 * be done in ieee80211_output.c once for both superg and A-MPDU.
+	 */
 	ieee80211_txampdu_count_packet(tap);
 
 	/*
@@ -698,7 +701,7 @@ ieee80211_ff_check(struct ieee80211_node *ni, struct mbuf *m)
 		    "%s: txtime %u exceeds txop limit %u\n",
 		    __func__, txtime, limit);
 
-		tap->txa_private = NULL;
+		ni->ni_tx_superg[WME_AC_TO_TID(pri)] = NULL;
 		if (mstaged != NULL)
 			stageq_remove(ic, sq, mstaged);
 		IEEE80211_UNLOCK(ic);
@@ -721,7 +724,7 @@ ieee80211_ff_check(struct ieee80211_node *ni, struct mbuf *m)
 	 * hold their node reference.
 	 */
 	if (mstaged != NULL) {
-		tap->txa_private = NULL;
+		ni->ni_tx_superg[WME_AC_TO_TID(pri)] = NULL;
 		stageq_remove(ic, sq, mstaged);
 		IEEE80211_UNLOCK(ic);
 
@@ -739,9 +742,10 @@ ieee80211_ff_check(struct ieee80211_node *ni, struct mbuf *m)
 		mstaged->m_nextpkt = m;
 		mstaged->m_flags |= M_FF; /* NB: mark for encap work */
 	} else {
-		KASSERT(tap->txa_private == NULL,
-		    ("txa_private %p", tap->txa_private));
-		tap->txa_private = m;
+		KASSERT(ni->ni_tx_superg[WME_AC_TO_TID(pri)]== NULL,
+		    ("ni_tx_superg[]: %p",
+		    ni->ni_tx_superg[WME_AC_TO_TID(pri)]));
+		ni->ni_tx_superg[WME_AC_TO_TID(pri)] = m;
 
 		stageq_add(ic, sq, m);
 		IEEE80211_UNLOCK(ic);
@@ -769,7 +773,6 @@ ieee80211_ff_node_cleanup(struct ieee80211_node *ni)
 {
 	struct ieee80211com *ic = ni->ni_ic;
 	struct ieee80211_superg *sg = ic->ic_superg;
-	struct ieee80211_tx_ampdu *tap;
 	struct mbuf *m, *next_m, *head;
 	int tid;
 
@@ -777,11 +780,16 @@ ieee80211_ff_node_cleanup(struct ieee80211_node *ni)
 	head = NULL;
 	for (tid = 0; tid < WME_NUM_TID; tid++) {
 		int ac = TID_TO_WME_AC(tid);
-
-		tap = &ni->ni_tx_ampdu[tid];
-		m = tap->txa_private;
+		/*
+		 * XXX Initialise the packet counter.
+		 *
+		 * This may be double-work for 11n stations;
+		 * but without it we never setup things.
+		 */
+		ieee80211_txampdu_init_pps(&ni->ni_tx_ampdu[tid]);
+		m = ni->ni_tx_superg[tid];
 		if (m != NULL) {
-			tap->txa_private = NULL;
+			ni->ni_tx_superg[tid] = NULL;
 			stageq_remove(ic, &sg->ff_stageq[ac], m);
 			m->m_nextpkt = head;
 			head = m;
diff --git a/sys/net80211/ieee80211_var.h b/sys/net80211/ieee80211_var.h
index a4507010f7fad..17f37d364c1dc 100644
--- a/sys/net80211/ieee80211_var.h
+++ b/sys/net80211/ieee80211_var.h
@@ -133,6 +133,7 @@ struct ieee80211com {
 	struct task		ic_chan_task;	/* deferred channel change */
 	struct task		ic_bmiss_task;	/* deferred beacon miss hndlr */
 	struct task		ic_chw_task;	/* deferred HT CHW update */
+	struct task		ic_wme_task;	/* deferred WME update */
 
 	counter_u64_t		ic_ierrors;	/* input errors */
 	counter_u64_t		ic_oerrors;	/* output errors */
@@ -417,6 +418,7 @@ struct ieee80211vap {
 	int			iv_amsdu_limit;	/* A-MSDU tx limit (bytes) */
 	u_int			iv_ampdu_mintraffic[WME_NUM_AC];
 
+	struct ieee80211_beacon_offsets iv_bcn_off;
 	uint32_t		*iv_aid_bitmap;	/* association id map */
 	uint16_t		iv_max_aid;
 	uint16_t		iv_sta_assoc;	/* stations associated */
diff --git a/sys/netinet/if_ether.c b/sys/netinet/if_ether.c
index ded65648420b4..ad07473f4725b 100644
--- a/sys/netinet/if_ether.c
+++ b/sys/netinet/if_ether.c
@@ -58,7 +58,6 @@ __FBSDID("$FreeBSD$");
 #include <net/if_dl.h>
 #include <net/if_types.h>
 #include <net/netisr.h>
-#include <net/if_llc.h>
 #include <net/ethernet.h>
 #include <net/route.h>
 #include <net/vnet.h>
@@ -71,13 +70,13 @@ __FBSDID("$FreeBSD$");
 #include <netinet/ip_carp.h>
 #endif
 
-#include <net/if_arc.h>
-#include <net/iso88025.h>
-
 #include <security/mac/mac_framework.h>
 
 #define SIN(s) ((const struct sockaddr_in *)(s))
-#define SDL(s) ((struct sockaddr_dl *)s)
+
+static struct timeval arp_lastlog;
+static int arp_curpps;
+static int arp_maxpps = 1;
 
 SYSCTL_DECL(_net_link_ether);
 static SYSCTL_NODE(_net_link_ether, PF_INET, inet, CTLFLAG_RW, 0, "");
@@ -122,6 +121,16 @@ SYSCTL_VNET_PCPUSTAT(_net_link_ether_arp, OID_AUTO, stats, struct arpstat,
 SYSCTL_INT(_net_link_ether_inet, OID_AUTO, maxhold, CTLFLAG_VNET | CTLFLAG_RW,
 	&VNET_NAME(arp_maxhold), 0,
 	"Number of packets to hold per ARP entry");
+SYSCTL_INT(_net_link_ether_inet, OID_AUTO, max_log_per_second,
+	CTLFLAG_RW, &arp_maxpps, 0,
+	"Maximum number of remotely triggered ARP messages that can be "
+	"logged per second");
+
+#define	ARP_LOG(pri, ...)	do {					\
+	if (ppsratecheck(&arp_lastlog, &arp_curpps, arp_maxpps))	\
+		log((pri), "arp: " __VA_ARGS__);			\
+} while (0)
+
 
 static void	arp_init(void);
 static void	arpintr(struct mbuf *);
@@ -144,26 +153,6 @@ static const struct netisr_handler arp_nh = {
 	.nh_policy = NETISR_POLICY_SOURCE,
 };
 
-#ifdef AF_INET
-/*
- * called by in_scrubprefix() to remove entry from the table when
- * the interface goes away
- */
-void
-arp_ifscrub(struct ifnet *ifp, uint32_t addr)
-{
-	struct sockaddr_in addr4;
-
-	bzero((void *)&addr4, sizeof(addr4));
-	addr4.sin_len    = sizeof(addr4);
-	addr4.sin_family = AF_INET;
-	addr4.sin_addr.s_addr = addr;
-	IF_AFDATA_WLOCK(ifp);
-	lla_delete(LLTABLE(ifp), LLE_IFADDR, (struct sockaddr *)&addr4);
-	IF_AFDATA_WUNLOCK(ifp);
-}
-#endif
-
 /*
  * Timeout routine.  Age arp_tab entries periodically.
  */
@@ -218,16 +207,14 @@ arptimer(void *arg)
 
 	/* Guard against race with other llentry_free(). */
 	if (lle->la_flags & LLE_LINKED) {
-
-		size_t pkts_dropped;
 		LLE_REMREF(lle);
-		pkts_dropped = llentry_free(lle);
-		ARPSTAT_ADD(dropped, pkts_dropped);
-	} else
-		LLE_FREE_LOCKED(lle);
-
+		lltable_unlink_entry(lle->lle_tbl, lle);
+	}
 	IF_AFDATA_UNLOCK(ifp);
 
+	size_t pkts_dropped = llentry_free(lle);
+
+	ARPSTAT_ADD(dropped, pkts_dropped);
 	ARPSTAT_INC(timeouts);
 
 	CURVNET_RESTORE();
@@ -529,34 +516,73 @@ static void
 arpintr(struct mbuf *m)
 {
 	struct arphdr *ar;
+	struct ifnet *ifp;
+	char *layer;
+	int hlen;
+
+	ifp = m->m_pkthdr.rcvif;
 
 	if (m->m_len < sizeof(struct arphdr) &&
 	    ((m = m_pullup(m, sizeof(struct arphdr))) == NULL)) {
-		log(LOG_NOTICE, "arp: runt packet -- m_pullup failed\n");
+		ARP_LOG(LOG_NOTICE, "packet with short header received on %s\n",
+		    if_name(ifp));
+		return;
+	}
+	ar = mtod(m, struct arphdr *);
+
+	/* Check if length is sufficient */
+	if ((m = m_pullup(m, arphdr_len(ar))) == NULL) {
+		ARP_LOG(LOG_NOTICE, "short packet received on %s\n",
+		    if_name(ifp));
 		return;
 	}
 	ar = mtod(m, struct arphdr *);
 
-	if (ntohs(ar->ar_hrd) != ARPHRD_ETHER &&
-	    ntohs(ar->ar_hrd) != ARPHRD_IEEE802 &&
-	    ntohs(ar->ar_hrd) != ARPHRD_ARCNET &&
-	    ntohs(ar->ar_hrd) != ARPHRD_IEEE1394 &&
-	    ntohs(ar->ar_hrd) != ARPHRD_INFINIBAND) {
-		log(LOG_NOTICE, "arp: unknown hardware address format (0x%2D)"
-		    " (from %*D to %*D)\n", (unsigned char *)&ar->ar_hrd, "",
-		    ETHER_ADDR_LEN, (u_char *)ar_sha(ar), ":",
-		    ETHER_ADDR_LEN, (u_char *)ar_tha(ar), ":");
+	hlen = 0;
+	layer = "";
+	switch (ntohs(ar->ar_hrd)) {
+	case ARPHRD_ETHER:
+		hlen = ETHER_ADDR_LEN; /* RFC 826 */
+		layer = "ethernet";
+		break;
+	case ARPHRD_IEEE802:
+		hlen = 6; /* RFC 1390, FDDI_ADDR_LEN */
+		layer = "fddi";
+		break;
+	case ARPHRD_ARCNET:
+		hlen = 1; /* RFC 1201, ARC_ADDR_LEN */
+		layer = "arcnet";
+		break;
+	case ARPHRD_INFINIBAND:
+		hlen = 20;	/* RFC 4391, INFINIBAND_ALEN */ 
+		layer = "infiniband";
+		break;
+	case ARPHRD_IEEE1394:
+		hlen = 0; /* SHALL be 16 */ /* RFC 2734 */
+		layer = "firewire";
+
+		/*
+		 * Restrict too long harware addresses.
+		 * Currently we are capable of handling 20-byte
+		 * addresses ( sizeof(lle->ll_addr) )
+		 */
+		if (ar->ar_hln >= 20)
+			hlen = 16;
+		break;
+	default:
+		ARP_LOG(LOG_NOTICE,
+		    "packet with unknown harware format 0x%02d received on %s\n",
+		    ntohs(ar->ar_hrd), if_name(ifp));
 		m_freem(m);
 		return;
 	}
 
-	if (m->m_len < arphdr_len(ar)) {
-		if ((m = m_pullup(m, arphdr_len(ar))) == NULL) {
-			log(LOG_NOTICE, "arp: runt packet\n");
-			m_freem(m);
-			return;
-		}
-		ar = mtod(m, struct arphdr *);
+	if (hlen != 0 && hlen != ar->ar_hln) {
+		ARP_LOG(LOG_NOTICE,
+		    "packet with invalid %s address length %d received on %s\n",
+		    layer, ar->ar_hln, if_name(ifp));
+		m_freem(m);
+		return;
 	}
 
 	ARPSTAT_INC(received);
@@ -589,9 +615,6 @@ static int log_arp_wrong_iface = 1;
 static int log_arp_movements = 1;
 static int log_arp_permanent_modify = 1;
 static int allow_multicast = 0;
-static struct timeval arp_lastlog;
-static int arp_curpps;
-static int arp_maxpps = 1;
 
 SYSCTL_INT(_net_link_ether_inet, OID_AUTO, log_arp_wrong_iface, CTLFLAG_RW,
 	&log_arp_wrong_iface, 0,
@@ -604,15 +627,6 @@ SYSCTL_INT(_net_link_ether_inet, OID_AUTO, log_arp_permanent_modify, CTLFLAG_RW,
 	"log arp replies from MACs different than the one in the permanent arp entry");
 SYSCTL_INT(_net_link_ether_inet, OID_AUTO, allow_multicast, CTLFLAG_RW,
 	&allow_multicast, 0, "accept multicast addresses");
-SYSCTL_INT(_net_link_ether_inet, OID_AUTO, max_log_per_second,
-	CTLFLAG_RW, &arp_maxpps, 0,
-	"Maximum number of remotely triggered ARP messages that can be "
-	"logged per second");
-
-#define	ARP_LOG(pri, ...)	do {					\
-	if (ppsratecheck(&arp_lastlog, &arp_curpps, arp_maxpps))	\
-		log((pri), "arp: " __VA_ARGS__);			\
-} while (0)
 
 static void
 in_arpinput(struct mbuf *m)
@@ -628,7 +642,6 @@ in_arpinput(struct mbuf *m)
 	struct in_addr isaddr, itaddr, myaddr;
 	u_int8_t *enaddr = NULL;
 	int op;
-	int req_len;
 	int bridged = 0, is_bridge = 0;
 	int carped;
 	struct sockaddr_in sin;
@@ -642,13 +655,12 @@ in_arpinput(struct mbuf *m)
 	if (ifp->if_type == IFT_BRIDGE)
 		is_bridge = 1;
 
-	req_len = arphdr_len2(ifp->if_addrlen, sizeof(struct in_addr));
-	if (m->m_len < req_len && (m = m_pullup(m, req_len)) == NULL) {
-		ARP_LOG(LOG_NOTICE, "runt packet -- m_pullup failed\n");
-		return;
-	}
-
+	/*
+	 * We already have checked that mbuf contains enough contiguous data
+	 * to hold entire arp message according to the arp header.
+	 */
 	ah = mtod(m, struct arphdr *);
+
 	/*
 	 * ARP is only for IPv4 so we can reject packets with
 	 * a protocol length not equal to an IPv4 address.
diff --git a/sys/netinet/if_ether.h b/sys/netinet/if_ether.h
index 39fc3868dc3c5..f23da5b2e5962 100644
--- a/sys/netinet/if_ether.h
+++ b/sys/netinet/if_ether.h
@@ -120,7 +120,6 @@ void	arprequest(struct ifnet *, const struct in_addr *,
 	    const struct in_addr *, u_char *);
 void	arp_ifinit(struct ifnet *, struct ifaddr *);
 void	arp_ifinit2(struct ifnet *, struct ifaddr *, u_char *);
-void	arp_ifscrub(struct ifnet *, uint32_t);
 #endif
 
 #endif
diff --git a/sys/netinet/in.c b/sys/netinet/in.c
index bc000126b5b3c..1441936c38085 100644
--- a/sys/netinet/in.c
+++ b/sys/netinet/in.c
@@ -724,6 +724,38 @@ in_addprefix(struct in_ifaddr *target, int flags)
 }
 
 /*
+ * Removes either all lle entries for given @ia, or lle
+ * corresponding to @ia address.
+ */
+static void
+in_scrubprefixlle(struct in_ifaddr *ia, int all, u_int flags)
+{
+	struct sockaddr_in addr, mask;
+	struct sockaddr *saddr, *smask;
+	struct ifnet *ifp;
+
+	/*
+	 * remove all L2 entries on the given prefix
+	 */
+	saddr = (struct sockaddr *)&addr;
+	bzero(&addr, sizeof(addr));
+	addr.sin_len = sizeof(addr);
+	addr.sin_family = AF_INET;
+	addr.sin_addr.s_addr = ntohl(ia->ia_addr.sin_addr.s_addr);
+	smask = (struct sockaddr *)&mask;
+	bzero(&mask, sizeof(mask));
+	mask.sin_len = sizeof(mask);
+	mask.sin_family = AF_INET;
+	mask.sin_addr.s_addr = ia->ia_subnetmask;
+	ifp = ia->ia_ifp;
+
+	if (all)
+		lltable_prefix_free(AF_INET, saddr, smask, flags);
+	else
+		lltable_delete_addr(LLTABLE(ifp), LLE_IFADDR, saddr);
+}
+
+/*
  * If there is no other address in the system that can serve a route to the
  * same prefix, remove the route.  Hand over the route to the new address
  * otherwise.
@@ -735,7 +767,6 @@ in_scrubprefix(struct in_ifaddr *target, u_int flags)
 	struct in_ifaddr *ia;
 	struct in_addr prefix, mask, p, m;
 	int error = 0;
-	struct sockaddr_in prefix0, mask0;
 
 	/*
 	 * Remove the loopback route to the interface address.
@@ -745,23 +776,21 @@ in_scrubprefix(struct in_ifaddr *target, u_int flags)
 	    (flags & LLE_STATIC)) {
 		struct in_ifaddr *eia;
 
+		/*
+		 * XXXME: add fib-aware in_localip.
+		 * We definitely don't want to switch between
+		 * prefixes in different fibs.
+		 */
 		eia = in_localip_more(target);
 
 		if (eia != NULL) {
-			int fibnum = target->ia_ifp->if_fib;
-
 			error = ifa_switch_loopback_route((struct ifaddr *)eia,
-			    (struct sockaddr *)&target->ia_addr, fibnum);
+			    (struct sockaddr *)&target->ia_addr);
 			ifa_free(&eia->ia_ifa);
 		} else {
 			error = ifa_del_loopback_route((struct ifaddr *)target,
 			    (struct sockaddr *)&target->ia_addr);
 		}
-
-		if (!(target->ia_ifp->if_flags & IFF_NOARP))
-			/* remove arp cache */
-			arp_ifscrub(target->ia_ifp,
-			    IA_SIN(target)->sin_addr.s_addr);
 	}
 
 	if (rtinitflags(target)) {
@@ -817,6 +846,9 @@ in_scrubprefix(struct in_ifaddr *target, u_int flags)
 			else
 				log(LOG_INFO, "in_scrubprefix: err=%d, old prefix delete failed\n",
 					error);
+			/* Scrub all entries IFF interface is different */
+			in_scrubprefixlle(target, target->ia_ifp != ia->ia_ifp,
+			    flags);
 			error = rtinit(&ia->ia_ifa, (int)RTM_ADD,
 			    rtinitflags(ia) | RTF_UP);
 			if (error == 0)
@@ -833,16 +865,7 @@ in_scrubprefix(struct in_ifaddr *target, u_int flags)
 	/*
 	 * remove all L2 entries on the given prefix
 	 */
-	bzero(&prefix0, sizeof(prefix0));
-	prefix0.sin_len = sizeof(prefix0);
-	prefix0.sin_family = AF_INET;
-	prefix0.sin_addr.s_addr = target->ia_subnet;
-	bzero(&mask0, sizeof(mask0));
-	mask0.sin_len = sizeof(mask0);
-	mask0.sin_family = AF_INET;
-	mask0.sin_addr.s_addr = target->ia_subnetmask;
-	lltable_prefix_free(AF_INET, (struct sockaddr *)&prefix0,
-	    (struct sockaddr *)&mask0, flags);
+	in_scrubprefixlle(target, 1, flags);
 
 	/*
 	 * As no-one seem to have this prefix, we can remove the route.
@@ -1001,22 +1024,38 @@ in_lltable_new(struct in_addr addr4, u_int flags)
 	return (&lle->base);
 }
 
-#define IN_ARE_MASKED_ADDR_EQUAL(d, a, m)	(			\
-	    (((ntohl((d).s_addr) ^ (a)->sin_addr.s_addr) & (m)->sin_addr.s_addr)) == 0 )
+#define IN_ARE_MASKED_ADDR_EQUAL(d, a, m)	(		\
+	((((d).s_addr ^ (a).s_addr) & (m).s_addr)) == 0 )
 
 static int
-in_lltable_match_prefix(const struct sockaddr *prefix,
-    const struct sockaddr *mask, u_int flags, struct llentry *lle)
+in_lltable_match_prefix(const struct sockaddr *saddr,
+    const struct sockaddr *smask, u_int flags, struct llentry *lle)
 {
-	const struct sockaddr_in *pfx = (const struct sockaddr_in *)prefix;
-	const struct sockaddr_in *msk = (const struct sockaddr_in *)mask;
+	struct in_addr addr, mask, lle_addr;
 
-	/*
-	 * (flags & LLE_STATIC) means deleting all entries
-	 * including static ARP entries.
-	 */
-	if (IN_ARE_MASKED_ADDR_EQUAL(lle->r_l3addr.addr4, pfx, msk) &&
-	    ((flags & LLE_STATIC) || !(lle->la_flags & LLE_STATIC)))
+	addr = ((const struct sockaddr_in *)saddr)->sin_addr;
+	mask = ((const struct sockaddr_in *)smask)->sin_addr;
+	lle_addr.s_addr = ntohl(lle->r_l3addr.addr4.s_addr);
+
+	if (IN_ARE_MASKED_ADDR_EQUAL(lle_addr, addr, mask) == 0)
+		return (0);
+
+	if (lle->la_flags & LLE_IFADDR) {
+
+		/*
+		 * Delete LLE_IFADDR records IFF address & flag matches.
+		 * Note that addr is the interface address within prefix
+		 * being matched.
+		 * Note also we should handle 'ifdown' cases without removing
+		 * ifaddr macs.
+		 */
+		if (addr.s_addr == lle_addr.s_addr && (flags & LLE_STATIC) != 0)
+			return (1);
+		return (0);
+	}
+
+	/* flags & LLE_STATIC means deleting both dynamic and static entries */
+	if ((flags & LLE_STATIC) || !(lle->la_flags & LLE_STATIC))
 		return (1);
 
 	return (0);
@@ -1166,39 +1205,16 @@ in_lltable_find_dst(struct lltable *llt, struct in_addr dst)
 	return (lle);
 }
 
-static int
-in_lltable_delete(struct lltable *llt, u_int flags,
-    const struct sockaddr *l3addr)
+static void
+in_lltable_delete_entry(struct lltable *llt, struct llentry *lle)
 {
-	const struct sockaddr_in *sin = (const struct sockaddr_in *)l3addr;
-	struct llentry *lle;
-
-	IF_AFDATA_WLOCK_ASSERT(llt->llt_ifp);
-	KASSERT(l3addr->sa_family == AF_INET,
-	    ("sin_family %d", l3addr->sa_family));
-
-	lle = in_lltable_find_dst(llt, sin->sin_addr);
-	if (lle == NULL) {
-#ifdef DIAGNOSTIC
-		log(LOG_INFO, "interface address is missing from cache = %p  in delete\n", lle);
-#endif
-		return (ENOENT);
-	}
 
-	if (!(lle->la_flags & LLE_IFADDR) || (flags & LLE_IFADDR)) {
-		LLE_WLOCK(lle);
-		lle->la_flags |= LLE_DELETED;
-		EVENTHANDLER_INVOKE(lle_event, lle, LLENTRY_DELETED);
+	lle->la_flags |= LLE_DELETED;
+	EVENTHANDLER_INVOKE(lle_event, lle, LLENTRY_DELETED);
 #ifdef DIAGNOSTIC
-		log(LOG_INFO, "ifaddr cache = %p is deleted\n", lle);
+	log(LOG_INFO, "ifaddr cache = %p is deleted\n", lle);
 #endif
-		if ((lle->la_flags & (LLE_STATIC | LLE_IFADDR)) == LLE_STATIC)
-			llentry_free(lle);
-		else
-			LLE_WUNLOCK(lle);
-	}
-
-	return (0);
+	llentry_free(lle);
 }
 
 static struct llentry *
@@ -1317,6 +1333,8 @@ in_lltable_dump_entry(struct lltable *llt, struct llentry *lle,
 			arpc.rtm.rtm_flags |= (RTF_HOST | RTF_LLDATA);
 			if (lle->la_flags & LLE_STATIC)
 				arpc.rtm.rtm_flags |= RTF_STATIC;
+			if (lle->la_flags & LLE_IFADDR)
+				arpc.rtm.rtm_flags |= RTF_PINNED;
 			arpc.rtm.rtm_index = ifp->if_index;
 			error = SYSCTL_OUT(wr, &arpc, sizeof(arpc));
 
@@ -1334,7 +1352,7 @@ in_lltattach(struct ifnet *ifp)
 
 	llt->llt_lookup = in_lltable_lookup;
 	llt->llt_alloc_entry = in_lltable_alloc;
-	llt->llt_delete = in_lltable_delete;
+	llt->llt_delete_entry = in_lltable_delete_entry;
 	llt->llt_dump_entry = in_lltable_dump_entry;
 	llt->llt_hash = in_lltable_hash;
 	llt->llt_fill_sa_entry = in_lltable_fill_sa_entry;
diff --git a/sys/netinet/in_kdtrace.c b/sys/netinet/in_kdtrace.c
index edcc85383a453..f39f0e88b1850 100644
--- a/sys/netinet/in_kdtrace.c
+++ b/sys/netinet/in_kdtrace.c
@@ -105,6 +105,25 @@ SDT_PROBE_DEFINE5_XLATE(tcp, , , send,
 SDT_PROBE_DEFINE1_XLATE(tcp, , , siftr,
     "struct pkt_node *", "siftrinfo_t *");
 
+SDT_PROBE_DEFINE3_XLATE(tcp, , , debug__input,
+    "struct tcpcb *", "tcpsinfo_t *" ,
+    "struct tcphdr *", "tcpinfo_t *",
+    "uint8_t *", "ipinfo_t *");
+
+SDT_PROBE_DEFINE3_XLATE(tcp, , , debug__output,
+    "struct tcpcb *", "tcpsinfo_t *" ,
+    "struct tcphdr *", "tcpinfo_t *",
+    "uint8_t *", "ipinfo_t *");
+
+SDT_PROBE_DEFINE2_XLATE(tcp, , , debug__user,
+    "struct tcpcb *", "tcpsinfo_t *" ,
+    "int", "int");
+
+SDT_PROBE_DEFINE3_XLATE(tcp, , , debug__drop,
+    "struct tcpcb *", "tcpsinfo_t *" ,
+    "struct tcphdr *", "tcpinfo_t *",
+    "uint8_t *", "ipinfo_t *")
+
 SDT_PROBE_DEFINE6_XLATE(tcp, , , state__change,
     "void *", "void *",
     "struct tcpcb *", "csinfo_t *",
diff --git a/sys/netinet/in_kdtrace.h b/sys/netinet/in_kdtrace.h
index c0511d084b775..ba8355a17c842 100644
--- a/sys/netinet/in_kdtrace.h
+++ b/sys/netinet/in_kdtrace.h
@@ -34,6 +34,12 @@
 	SDT_PROBE5(udp, , , probe, arg0, arg1, arg2, arg3, arg4)
 #define	TCP_PROBE1(probe, arg0) \
 	SDT_PROBE1(tcp, , , probe, arg0)
+#define	TCP_PROBE2(probe, arg0, arg1)		\
+	SDT_PROBE2(tcp, , , probe, arg0, arg1)
+#define	TCP_PROBE3(probe, arg0, arg1, arg2)		\
+	SDT_PROBE3(tcp, , , probe, arg0, arg1, arg2)
+#define	TCP_PROBE4(probe, arg0, arg1, arg2, arg3)	\
+	SDT_PROBE4(tcp, , , probe, arg0, arg1, arg2, arg3)
 #define	TCP_PROBE5(probe, arg0, arg1, arg2, arg3, arg4)			\
 	SDT_PROBE5(tcp, , , probe, arg0, arg1, arg2, arg3, arg4)
 #define	TCP_PROBE6(probe, arg0, arg1, arg2, arg3, arg4, arg5)		\
@@ -55,6 +61,10 @@ SDT_PROBE_DECLARE(tcp, , , receive);
 SDT_PROBE_DECLARE(tcp, , , send);
 SDT_PROBE_DECLARE(tcp, , , siftr);
 SDT_PROBE_DECLARE(tcp, , , state__change);
+SDT_PROBE_DECLARE(tcp, , , debug__input);
+SDT_PROBE_DECLARE(tcp, , , debug__output);
+SDT_PROBE_DECLARE(tcp, , , debug__user);
+SDT_PROBE_DECLARE(tcp, , , debug__drop);
 
 SDT_PROBE_DECLARE(udp, , , receive);
 SDT_PROBE_DECLARE(udp, , , send);
diff --git a/sys/netinet/ip_carp.c b/sys/netinet/ip_carp.c
index 2fa88205864d5..7e4506937680a 100644
--- a/sys/netinet/ip_carp.c
+++ b/sys/netinet/ip_carp.c
@@ -985,7 +985,7 @@ carp_ifa_delroute(struct ifaddr *ifa)
 	case AF_INET6:
 		ifa_del_loopback_route(ifa,
 		    (struct sockaddr *)&ifatoia6(ifa)->ia_addr);
-		nd6_rem_ifa_lle(ifatoia6(ifa));
+		nd6_rem_ifa_lle(ifatoia6(ifa), 1);
 		break;
 #endif
 	}
diff --git a/sys/netinet/ip_ipsec.c b/sys/netinet/ip_ipsec.c
index 47f683ddd71c3..546f802f96177 100644
--- a/sys/netinet/ip_ipsec.c
+++ b/sys/netinet/ip_ipsec.c
@@ -199,9 +199,7 @@ ip_ipsec_output(struct mbuf **m, struct inpcb *inp, int *error)
 
 		/* NB: callee frees mbuf */
 		*error = ipsec4_process_packet(*m, sp->req);
-		/* Release SP if an error occured */
-		if (*error != 0)
-			KEY_FREESP(&sp);
+		KEY_FREESP(&sp);
 		if (*error == EJUSTRETURN) {
 			/*
 			 * We had a SP with a level of 'use' and no SA. We
diff --git a/sys/netinet/sctp.h b/sys/netinet/sctp.h
index 8a033d8187c8a..784c1d47e9fb7 100644
--- a/sys/netinet/sctp.h
+++ b/sys/netinet/sctp.h
@@ -388,33 +388,32 @@ struct sctp_error_cause {
 }                SCTP_PACKED;
 
 struct sctp_error_invalid_stream {
-	struct sctp_error_cause cause;	/* code=SCTP_ERROR_INVALID_STREAM */
+	struct sctp_error_cause cause;	/* code=SCTP_CAUSE_INVALID_STREAM */
 	uint16_t stream_id;	/* stream id of the DATA in error */
 	uint16_t reserved;
 }                         SCTP_PACKED;
 
 struct sctp_error_missing_param {
-	struct sctp_error_cause cause;	/* code=SCTP_ERROR_MISSING_PARAM */
+	struct sctp_error_cause cause;	/* code=SCTP_CAUSE_MISSING_PARAM */
 	uint32_t num_missing_params;	/* number of missing parameters */
-	/* uint16_t param_type's follow */
+	uint16_t type[];
 }                        SCTP_PACKED;
 
 struct sctp_error_stale_cookie {
-	struct sctp_error_cause cause;	/* code=SCTP_ERROR_STALE_COOKIE */
+	struct sctp_error_cause cause;	/* code=SCTP_CAUSE_STALE_COOKIE */
 	uint32_t stale_time;	/* time in usec of staleness */
 }                       SCTP_PACKED;
 
 struct sctp_error_out_of_resource {
-	struct sctp_error_cause cause;	/* code=SCTP_ERROR_OUT_OF_RESOURCES */
+	struct sctp_error_cause cause;	/* code=SCTP_CAUSE_OUT_OF_RESOURCES */
 }                          SCTP_PACKED;
 
 struct sctp_error_unresolv_addr {
-	struct sctp_error_cause cause;	/* code=SCTP_ERROR_UNRESOLVABLE_ADDR */
-
+	struct sctp_error_cause cause;	/* code=SCTP_CAUSE_UNRESOLVABLE_ADDR */
 }                        SCTP_PACKED;
 
 struct sctp_error_unrecognized_chunk {
-	struct sctp_error_cause cause;	/* code=SCTP_ERROR_UNRECOG_CHUNK */
+	struct sctp_error_cause cause;	/* code=SCTP_CAUSE_UNRECOG_CHUNK */
 	struct sctp_chunkhdr ch;/* header from chunk in error */
 }                             SCTP_PACKED;
 
@@ -423,6 +422,11 @@ struct sctp_error_no_user_data {
 	uint32_t tsn;		/* TSN of the empty data chunk */
 }                       SCTP_PACKED;
 
+struct sctp_error_auth_invalid_hmac {
+	struct sctp_error_cause cause;	/* code=SCTP_CAUSE_UNSUPPORTED_HMACID */
+	uint16_t hmac_id;
+}                            SCTP_PACKED;
+
 /*
  * Main SCTP chunk types we place these here so natd and f/w's in user land
  * can find them.
diff --git a/sys/netinet/sctp_auth.c b/sys/netinet/sctp_auth.c
index 7c2e194e1d2fd..13454df92f159 100644
--- a/sys/netinet/sctp_auth.c
+++ b/sys/netinet/sctp_auth.c
@@ -1651,8 +1651,8 @@ sctp_handle_auth(struct sctp_tcb *stcb, struct sctp_auth_chunk *auth,
 
 	/* is the indicated HMAC supported? */
 	if (!sctp_auth_is_supported_hmac(stcb->asoc.local_hmacs, hmac_id)) {
-		struct mbuf *m_err;
-		struct sctp_auth_invalid_hmac *err;
+		struct mbuf *op_err;
+		struct sctp_error_auth_invalid_hmac *cause;
 
 		SCTP_STAT_INCR(sctps_recvivalhmacid);
 		SCTPDBG(SCTP_DEBUG_AUTH1,
@@ -1662,20 +1662,19 @@ sctp_handle_auth(struct sctp_tcb *stcb, struct sctp_auth_chunk *auth,
 		 * report this in an Error Chunk: Unsupported HMAC
 		 * Identifier
 		 */
-		m_err = sctp_get_mbuf_for_msg(sizeof(*err), 0, M_NOWAIT,
-		    1, MT_HEADER);
-		if (m_err != NULL) {
+		op_err = sctp_get_mbuf_for_msg(sizeof(struct sctp_error_auth_invalid_hmac),
+		    0, M_NOWAIT, 1, MT_HEADER);
+		if (op_err != NULL) {
 			/* pre-reserve some space */
-			SCTP_BUF_RESV_UF(m_err, sizeof(struct sctp_chunkhdr));
+			SCTP_BUF_RESV_UF(op_err, sizeof(struct sctp_chunkhdr));
 			/* fill in the error */
-			err = mtod(m_err, struct sctp_auth_invalid_hmac *);
-			bzero(err, sizeof(*err));
-			err->ph.param_type = htons(SCTP_CAUSE_UNSUPPORTED_HMACID);
-			err->ph.param_length = htons(sizeof(*err));
-			err->hmac_id = ntohs(hmac_id);
-			SCTP_BUF_LEN(m_err) = sizeof(*err);
+			cause = mtod(op_err, struct sctp_error_auth_invalid_hmac *);
+			cause->cause.code = htons(SCTP_CAUSE_UNSUPPORTED_HMACID);
+			cause->cause.length = htons(sizeof(struct sctp_error_auth_invalid_hmac));
+			cause->hmac_id = ntohs(hmac_id);
+			SCTP_BUF_LEN(op_err) = sizeof(struct sctp_error_auth_invalid_hmac);
 			/* queue it */
-			sctp_queue_op_err(stcb, m_err);
+			sctp_queue_op_err(stcb, op_err);
 		}
 		return (-1);
 	}
diff --git a/sys/netinet/sctp_header.h b/sys/netinet/sctp_header.h
index f322e04f4982c..dc05b3dc0a8d0 100644
--- a/sys/netinet/sctp_header.h
+++ b/sys/netinet/sctp_header.h
@@ -202,34 +202,6 @@ struct sctp_state_cookie {	/* this is our definition... */
 	 */
 }                 SCTP_PACKED;
 
-
-/* Used for NAT state error cause */
-struct sctp_missing_nat_state {
-	uint16_t cause;
-	uint16_t length;
-	uint8_t data[];
-}                      SCTP_PACKED;
-
-
-struct sctp_inv_mandatory_param {
-	uint16_t cause;
-	uint16_t length;
-	uint32_t num_param;
-	uint16_t param;
-	/*
-	 * We include this to 0 it since only a missing cookie will cause
-	 * this error.
-	 */
-	uint16_t resv;
-}                        SCTP_PACKED;
-
-struct sctp_unresolv_addr {
-	uint16_t cause;
-	uint16_t length;
-	uint16_t addr_type;
-	uint16_t reserved;	/* Only one invalid addr type */
-}                  SCTP_PACKED;
-
 /* state cookie parameter */
 struct sctp_state_cookie_param {
 	struct sctp_paramhdr ph;
@@ -370,28 +342,11 @@ struct sctp_shutdown_complete_chunk {
 	struct sctp_chunkhdr ch;
 }                            SCTP_PACKED;
 
-/* Oper error holding a stale cookie */
-struct sctp_stale_cookie_msg {
-	struct sctp_paramhdr ph;/* really an error cause */
-	uint32_t time_usec;
-}                     SCTP_PACKED;
-
 struct sctp_adaptation_layer_indication {
 	struct sctp_paramhdr ph;
 	uint32_t indication;
 }                                SCTP_PACKED;
 
-struct sctp_cookie_while_shutting_down {
-	struct sctphdr sh;
-	struct sctp_chunkhdr ch;
-	struct sctp_paramhdr ph;/* really an error cause */
-}                               SCTP_PACKED;
-
-struct sctp_shutdown_complete_msg {
-	struct sctphdr sh;
-	struct sctp_shutdown_complete_chunk shut_cmp;
-}                          SCTP_PACKED;
-
 /*
  * draft-ietf-tsvwg-addip-sctp
  */
@@ -554,12 +509,6 @@ struct sctp_auth_chunk {
 	uint8_t hmac[];
 }               SCTP_PACKED;
 
-struct sctp_auth_invalid_hmac {
-	struct sctp_paramhdr ph;
-	uint16_t hmac_id;
-	uint16_t padding;
-}                      SCTP_PACKED;
-
 /*
  * we pre-reserve enough room for a ECNE or CWR AND a SACK with no missing
  * pieces. If ENCE is missing we could have a couple of blocks. This way we
diff --git a/sys/netinet/sctp_indata.c b/sys/netinet/sctp_indata.c
index d9ca669467895..34f4bfd9d2dc3 100644
--- a/sys/netinet/sctp_indata.c
+++ b/sys/netinet/sctp_indata.c
@@ -1426,30 +1426,25 @@ sctp_process_a_data_chunk(struct sctp_tcb *stcb, struct sctp_association *asoc,
 	}
 	strmno = ntohs(ch->dp.stream_id);
 	if (strmno >= asoc->streamincnt) {
-		struct sctp_paramhdr *phdr;
-		struct mbuf *mb;
+		struct sctp_error_invalid_stream *cause;
 
-		mb = sctp_get_mbuf_for_msg((sizeof(struct sctp_paramhdr) * 2),
+		op_err = sctp_get_mbuf_for_msg(sizeof(struct sctp_error_invalid_stream),
 		    0, M_NOWAIT, 1, MT_DATA);
-		if (mb != NULL) {
+		if (op_err != NULL) {
 			/* add some space up front so prepend will work well */
-			SCTP_BUF_RESV_UF(mb, sizeof(struct sctp_chunkhdr));
-			phdr = mtod(mb, struct sctp_paramhdr *);
+			SCTP_BUF_RESV_UF(op_err, sizeof(struct sctp_chunkhdr));
+			cause = mtod(op_err, struct sctp_error_invalid_stream *);
 			/*
 			 * Error causes are just param's and this one has
 			 * two back to back phdr, one with the error type
 			 * and size, the other with the streamid and a rsvd
 			 */
-			SCTP_BUF_LEN(mb) = (sizeof(struct sctp_paramhdr) * 2);
-			phdr->param_type = htons(SCTP_CAUSE_INVALID_STREAM);
-			phdr->param_length =
-			    htons(sizeof(struct sctp_paramhdr) * 2);
-			phdr++;
-			/* We insert the stream in the type field */
-			phdr->param_type = ch->dp.stream_id;
-			/* And set the length to 0 for the rsvd field */
-			phdr->param_length = 0;
-			sctp_queue_op_err(stcb, mb);
+			SCTP_BUF_LEN(op_err) = sizeof(struct sctp_error_invalid_stream);
+			cause->cause.code = htons(SCTP_CAUSE_INVALID_STREAM);
+			cause->cause.length = htons(sizeof(struct sctp_error_invalid_stream));
+			cause->stream_id = ch->dp.stream_id;
+			cause->reserved = htons(0);
+			sctp_queue_op_err(stcb, op_err);
 		}
 		SCTP_STAT_INCR(sctps_badsid);
 		SCTP_TCB_LOCK_ASSERT(stcb);
@@ -2492,30 +2487,21 @@ sctp_process_data(struct mbuf **mm, int iphlen, int *offset, int length,
 				/* unknown chunk type, use bit rules */
 				if (ch->ch.chunk_type & 0x40) {
 					/* Add a error report to the queue */
-					struct mbuf *merr;
-					struct sctp_paramhdr *phd;
+					struct mbuf *op_err;
+					struct sctp_gen_error_cause *cause;
 
-					merr = sctp_get_mbuf_for_msg(sizeof(*phd), 0, M_NOWAIT, 1, MT_DATA);
-					if (merr) {
-						phd = mtod(merr, struct sctp_paramhdr *);
-						/*
-						 * We cheat and use param
-						 * type since we did not
-						 * bother to define a error
-						 * cause struct. They are
-						 * the same basic format
-						 * with different names.
-						 */
-						phd->param_type =
-						    htons(SCTP_CAUSE_UNRECOG_CHUNK);
-						phd->param_length =
-						    htons(chk_length + sizeof(*phd));
-						SCTP_BUF_LEN(merr) = sizeof(*phd);
-						SCTP_BUF_NEXT(merr) = SCTP_M_COPYM(m, *offset, chk_length, M_NOWAIT);
-						if (SCTP_BUF_NEXT(merr)) {
-							sctp_queue_op_err(stcb, merr);
+					op_err = sctp_get_mbuf_for_msg(sizeof(struct sctp_gen_error_cause),
+					    0, M_NOWAIT, 1, MT_DATA);
+					if (op_err != NULL) {
+						cause = mtod(op_err, struct sctp_gen_error_cause *);
+						cause->code = htons(SCTP_CAUSE_UNRECOG_CHUNK);
+						cause->length = htons(chk_length + sizeof(struct sctp_gen_error_cause));
+						SCTP_BUF_LEN(op_err) = sizeof(struct sctp_gen_error_cause);
+						SCTP_BUF_NEXT(op_err) = SCTP_M_COPYM(m, *offset, chk_length, M_NOWAIT);
+						if (SCTP_BUF_NEXT(op_err) != NULL) {
+							sctp_queue_op_err(stcb, op_err);
 						} else {
-							sctp_m_freem(merr);
+							sctp_m_freem(op_err);
 						}
 					}
 				}
diff --git a/sys/netinet/sctp_input.c b/sys/netinet/sctp_input.c
index 9f8d032888bfc..37d9e6051ea56 100644
--- a/sys/netinet/sctp_input.c
+++ b/sys/netinet/sctp_input.c
@@ -530,25 +530,21 @@ sctp_process_init_ack(struct mbuf *m, int iphlen, int offset,
 		 * abandon the peer, its broke.
 		 */
 		if (retval == -3) {
+			uint16_t len;
+
+			len = (uint16_t) (sizeof(struct sctp_error_missing_param) + sizeof(uint16_t));
 			/* We abort with an error of missing mandatory param */
-			op_err = sctp_generate_cause(SCTP_CAUSE_MISSING_PARAM, "");
-			if (op_err) {
-				/*
-				 * Expand beyond to include the mandatory
-				 * param cookie
-				 */
-				struct sctp_inv_mandatory_param *mp;
+			op_err = sctp_get_mbuf_for_msg(len, 0, M_NOWAIT, 1, MT_DATA);
+			if (op_err != NULL) {
+				struct sctp_error_missing_param *cause;
 
-				SCTP_BUF_LEN(op_err) =
-				    sizeof(struct sctp_inv_mandatory_param);
-				mp = mtod(op_err,
-				    struct sctp_inv_mandatory_param *);
+				SCTP_BUF_LEN(op_err) = len;
+				cause = mtod(op_err, struct sctp_error_missing_param *);
 				/* Subtract the reserved param */
-				mp->length =
-				    htons(sizeof(struct sctp_inv_mandatory_param) - 2);
-				mp->num_param = htonl(1);
-				mp->param = htons(SCTP_STATE_COOKIE);
-				mp->resv = 0;
+				cause->cause.code = htons(SCTP_CAUSE_MISSING_PARAM);
+				cause->cause.length = htons(len);
+				cause->num_missing_params = htonl(1);
+				cause->type[0] = htons(SCTP_STATE_COOKIE);
 			}
 			sctp_abort_association(stcb->sctp_ep, stcb, m, iphlen,
 			    src, dst, sh, op_err,
@@ -781,10 +777,10 @@ sctp_handle_abort(struct sctp_abort_chunk *abort,
 		 * Need to check the cause codes for our two magic nat
 		 * aborts which don't kill the assoc necessarily.
 		 */
-		struct sctp_missing_nat_state *natc;
+		struct sctp_gen_error_cause *cause;
 
-		natc = (struct sctp_missing_nat_state *)(abort + 1);
-		error = ntohs(natc->cause);
+		cause = (struct sctp_gen_error_cause *)(abort + 1);
+		error = ntohs(cause->code);
 		if (error == SCTP_CAUSE_NAT_COLLIDING_STATE) {
 			SCTPDBG(SCTP_DEBUG_INPUT2, "Received Colliding state abort flags:%x\n",
 			    abort->ch.chunk_flags);
@@ -2558,27 +2554,27 @@ sctp_handle_cookie_echo(struct mbuf *m, int iphlen, int offset,
 	if (timevalcmp(&now, &time_expires, >)) {
 		/* cookie is stale! */
 		struct mbuf *op_err;
-		struct sctp_stale_cookie_msg *scm;
+		struct sctp_error_stale_cookie *cause;
 		uint32_t tim;
 
-		op_err = sctp_get_mbuf_for_msg(sizeof(struct sctp_stale_cookie_msg),
+		op_err = sctp_get_mbuf_for_msg(sizeof(struct sctp_error_stale_cookie),
 		    0, M_NOWAIT, 1, MT_DATA);
 		if (op_err == NULL) {
 			/* FOOBAR */
 			return (NULL);
 		}
 		/* Set the len */
-		SCTP_BUF_LEN(op_err) = sizeof(struct sctp_stale_cookie_msg);
-		scm = mtod(op_err, struct sctp_stale_cookie_msg *);
-		scm->ph.param_type = htons(SCTP_CAUSE_STALE_COOKIE);
-		scm->ph.param_length = htons((sizeof(struct sctp_paramhdr) +
+		SCTP_BUF_LEN(op_err) = sizeof(struct sctp_error_stale_cookie);
+		cause = mtod(op_err, struct sctp_error_stale_cookie *);
+		cause->cause.code = htons(SCTP_CAUSE_STALE_COOKIE);
+		cause->cause.length = htons((sizeof(struct sctp_paramhdr) +
 		    (sizeof(uint32_t))));
 		/* seconds to usec */
 		tim = (now.tv_sec - time_expires.tv_sec) * 1000000;
 		/* add in usec */
 		if (tim == 0)
 			tim = now.tv_usec - cookie->time_entered.tv_usec;
-		scm->time_usec = htonl(tim);
+		cause->stale_time = htonl(tim);
 		sctp_send_operr_to(src, dst, sh, cookie->peers_vtag, op_err,
 		    mflowtype, mflowid, l_inp->fibnum,
 		    vrf_id, port);
@@ -5581,35 +5577,27 @@ process_control_chunks:
 	unknown_chunk:
 			/* it's an unknown chunk! */
 			if ((ch->chunk_type & 0x40) && (stcb != NULL)) {
-				struct mbuf *mm;
-				struct sctp_paramhdr *phd;
+				struct sctp_gen_error_cause *cause;
 				int len;
 
-				mm = sctp_get_mbuf_for_msg(sizeof(struct sctp_paramhdr),
+				op_err = sctp_get_mbuf_for_msg(sizeof(struct sctp_gen_error_cause),
 				    0, M_NOWAIT, 1, MT_DATA);
-				if (mm) {
+				if (op_err != NULL) {
 					len = min(SCTP_SIZE32(chk_length), (uint32_t) (length - *offset));
-					phd = mtod(mm, struct sctp_paramhdr *);
-					/*
-					 * We cheat and use param type since
-					 * we did not bother to define a
-					 * error cause struct. They are the
-					 * same basic format with different
-					 * names.
-					 */
-					phd->param_type = htons(SCTP_CAUSE_UNRECOG_CHUNK);
-					phd->param_length = htons(len + sizeof(*phd));
-					SCTP_BUF_LEN(mm) = sizeof(*phd);
-					SCTP_BUF_NEXT(mm) = SCTP_M_COPYM(m, *offset, len, M_NOWAIT);
-					if (SCTP_BUF_NEXT(mm)) {
+					cause = mtod(op_err, struct sctp_gen_error_cause *);
+					cause->code = htons(SCTP_CAUSE_UNRECOG_CHUNK);
+					cause->length = htons(len + sizeof(struct sctp_gen_error_cause));
+					SCTP_BUF_LEN(op_err) = sizeof(struct sctp_gen_error_cause);
+					SCTP_BUF_NEXT(op_err) = SCTP_M_COPYM(m, *offset, len, M_NOWAIT);
+					if (SCTP_BUF_NEXT(op_err) != NULL) {
 #ifdef SCTP_MBUF_LOGGING
 						if (SCTP_BASE_SYSCTL(sctp_logging_level) & SCTP_MBUF_LOGGING_ENABLE) {
-							sctp_log_mbc(SCTP_BUF_NEXT(mm), SCTP_MBUF_ICOPY);
+							sctp_log_mbc(SCTP_BUF_NEXT(op_err), SCTP_MBUF_ICOPY);
 						}
 #endif
-						sctp_queue_op_err(stcb, mm);
+						sctp_queue_op_err(stcb, op_err);
 					} else {
-						sctp_m_freem(mm);
+						sctp_m_freem(op_err);
 					}
 				}
 			}
diff --git a/sys/netinet/sctp_sysctl.h b/sys/netinet/sctp_sysctl.h
index 5055110cd57e0..38494add3d83f 100644
--- a/sys/netinet/sctp_sysctl.h
+++ b/sys/netinet/sctp_sysctl.h
@@ -545,7 +545,7 @@ struct sctp_sysctl {
 #define SCTPCTL_RTTVAR_DCCCECN_MAX	1
 #define SCTPCTL_RTTVAR_DCCCECN_DEFAULT	1	/* 0 means disable feature */
 
-#define SCTPCTL_BLACKHOLE_DESC		"Enable SCTP blackholing. See blackhole(4) man page for more details."
+#define SCTPCTL_BLACKHOLE_DESC		"Enable SCTP blackholing. See blackhole(4) for more details."
 #define SCTPCTL_BLACKHOLE_MIN		0
 #define SCTPCTL_BLACKHOLE_MAX		2
 #define SCTPCTL_BLACKHOLE_DEFAULT	SCTPCTL_BLACKHOLE_MIN
diff --git a/sys/netinet/tcp_input.c b/sys/netinet/tcp_input.c
index e7ef6dbb80d86..5d2cac493973f 100644
--- a/sys/netinet/tcp_input.c
+++ b/sys/netinet/tcp_input.c
@@ -469,18 +469,6 @@ tcp_signature_verify_input(struct mbuf *m, int off0, int tlen, int optlen,
 }
 #endif
 
-/* Neighbor Discovery, Neighbor Unreachability Detection Upper layer hint. */
-#ifdef INET6
-#define ND6_HINT(tp) \
-do { \
-	if ((tp) && (tp)->t_inpcb && \
-	    ((tp)->t_inpcb->inp_vflag & INP_IPV6) != 0) \
-		nd6_nud_hint(NULL, NULL, 0); \
-} while (0)
-#else
-#define ND6_HINT(tp)
-#endif
-
 /*
  * Indicate whether this ack should be delayed.  We can delay the ack if
  * following conditions are met:
@@ -1377,6 +1365,7 @@ relocked:
 			tcp_trace(TA_INPUT, ostate, tp,
 			    (void *)tcp_saveipgen, &tcp_savetcp, 0);
 #endif
+		TCP_PROBE3(debug__input, tp, th, mtod(m, const char *));
 		tcp_dooptions(&to, optp, optlen, TO_SYN);
 		syncache_add(&inc, &to, th, inp, &so, m, NULL, NULL);
 		/*
@@ -1762,7 +1751,6 @@ tcp_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so,
 				tp->snd_wl2 = th->th_ack;
 				tp->t_dupacks = 0;
 				m_freem(m);
-				ND6_HINT(tp); /* Some progress has been made. */
 
 				/*
 				 * If all outstanding data are acked, stop
@@ -1779,6 +1767,8 @@ tcp_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so,
 					    (void *)tcp_saveipgen,
 					    &tcp_savetcp, 0);
 #endif
+				TCP_PROBE3(debug__input, tp, th,
+					mtod(m, const char *));
 				if (tp->snd_una == tp->snd_max)
 					tcp_timer_activate(tp, TT_REXMT, 0);
 				else if (!tcp_timer_active(tp, TT_PERSIST))
@@ -1819,12 +1809,13 @@ tcp_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so,
 			tp->rcv_up = tp->rcv_nxt;
 			TCPSTAT_INC(tcps_rcvpack);
 			TCPSTAT_ADD(tcps_rcvbyte, tlen);
-			ND6_HINT(tp);	/* Some progress has been made */
 #ifdef TCPDEBUG
 			if (so->so_options & SO_DEBUG)
 				tcp_trace(TA_INPUT, ostate, tp,
 				    (void *)tcp_saveipgen, &tcp_savetcp, 0);
 #endif
+			TCP_PROBE3(debug__input, tp, th, mtod(m, const char *));
+
 		/*
 		 * Automatic sizing of receive socket buffer.  Often the send
 		 * buffer size is not optimally adjusted to the actual network
@@ -2919,7 +2910,6 @@ dodata:							/* XXX */
 			thflags = th->th_flags & TH_FIN;
 			TCPSTAT_INC(tcps_rcvpack);
 			TCPSTAT_ADD(tcps_rcvbyte, tlen);
-			ND6_HINT(tp);
 			SOCKBUF_LOCK(&so->so_rcv);
 			if (so->so_rcv.sb_state & SBS_CANTRCVMORE)
 				m_freem(m);
@@ -3022,6 +3012,7 @@ dodata:							/* XXX */
 		tcp_trace(TA_INPUT, ostate, tp, (void *)tcp_saveipgen,
 			  &tcp_savetcp, 0);
 #endif
+	TCP_PROBE3(debug__input, tp, th, mtod(m, const char *));
 
 	/*
 	 * Return any desired output.
@@ -3069,6 +3060,7 @@ dropafterack:
 		tcp_trace(TA_DROP, ostate, tp, (void *)tcp_saveipgen,
 			  &tcp_savetcp, 0);
 #endif
+	TCP_PROBE3(debug__input, tp, th, mtod(m, const char *));
 	if (ti_locked == TI_RLOCKED)
 		INP_INFO_RUNLOCK(&V_tcbinfo);
 	ti_locked = TI_UNLOCKED;
@@ -3109,6 +3101,7 @@ drop:
 		tcp_trace(TA_DROP, ostate, tp, (void *)tcp_saveipgen,
 			  &tcp_savetcp, 0);
 #endif
+	TCP_PROBE3(debug__input, tp, th, mtod(m, const char *));
 	if (tp != NULL)
 		INP_WUNLOCK(tp->t_inpcb);
 	m_freem(m);
diff --git a/sys/netinet/tcp_output.c b/sys/netinet/tcp_output.c
index 342990e22b70b..fc90b069b9d4b 100644
--- a/sys/netinet/tcp_output.c
+++ b/sys/netinet/tcp_output.c
@@ -811,7 +811,8 @@ send:
 			 */
 			if (if_hw_tsomax != 0) {
 				/* compute maximum TSO length */
-				max_len = (if_hw_tsomax - hdrlen);
+				max_len = (if_hw_tsomax - hdrlen -
+				    max_linkhdr);
 				if (max_len <= 0) {
 					len = 0;
 				} else if (len > max_len) {
@@ -826,6 +827,15 @@ send:
 			 */
 			if (if_hw_tsomaxsegcount != 0 &&
 			    if_hw_tsomaxsegsize != 0) {
+				/*
+				 * Subtract one segment for the LINK
+				 * and TCP/IP headers mbuf that will
+				 * be prepended to this mbuf chain
+				 * after the code in this section
+				 * limits the number of mbufs in the
+				 * chain to if_hw_tsomaxsegcount.
+				 */
+				if_hw_tsomaxsegcount -= 1;
 				max_len = 0;
 				mb = sbsndmbuf(&so->so_snd, off, &moff);
 
@@ -1253,6 +1263,7 @@ send:
 		ipov->ih_len = save;
 	}
 #endif /* TCPDEBUG */
+	TCP_PROBE3(debug__input, tp, th, mtod(m, const char *));
 
 	/*
 	 * Fill in IP length and desired time to live and
diff --git a/sys/netinet/tcp_reass.c b/sys/netinet/tcp_reass.c
index 3913ef33e237d..467715459567f 100644
--- a/sys/netinet/tcp_reass.c
+++ b/sys/netinet/tcp_reass.c
@@ -327,7 +327,6 @@ present:
 		tp->t_segqlen--;
 		q = nq;
 	} while (q && q->tqe_th->th_seq == tp->rcv_nxt);
-	ND6_HINT(tp);
 	sorwakeup_locked(so);
 	return (flags);
 }
diff --git a/sys/netinet/tcp_subr.c b/sys/netinet/tcp_subr.c
index 207ad0312c345..a20bd8109316d 100644
--- a/sys/netinet/tcp_subr.c
+++ b/sys/netinet/tcp_subr.c
@@ -720,6 +720,7 @@ tcp_respond(struct tcpcb *tp, void *ipgen, struct tcphdr *th, struct mbuf *m,
 	if (tp == NULL || (inp->inp_socket->so_options & SO_DEBUG))
 		tcp_trace(TA_OUTPUT, 0, tp, mtod(m, void *), th, 0);
 #endif
+	TCP_PROBE3(debug__input, tp, th, mtod(m, const char *));
 	if (flags & TH_RST)
 		TCP_PROBE5(accept__refused, NULL, NULL, mtod(m, const char *),
 		    tp, nth);
@@ -1514,74 +1515,74 @@ tcp_ctlinput(int cmd, struct sockaddr *sa, void *vip)
 		ip = NULL;
 	else if ((unsigned)cmd >= PRC_NCMDS || inetctlerrmap[cmd] == 0)
 		return;
-	if (ip != NULL) {
-		icp = (struct icmp *)((caddr_t)ip
-				      - offsetof(struct icmp, icmp_ip));
-		th = (struct tcphdr *)((caddr_t)ip
-				       + (ip->ip_hl << 2));
-		INP_INFO_RLOCK(&V_tcbinfo);
-		inp = in_pcblookup(&V_tcbinfo, faddr, th->th_dport,
-		    ip->ip_src, th->th_sport, INPLOOKUP_WLOCKPCB, NULL);
-		if (inp != NULL)  {
-			if (!(inp->inp_flags & INP_TIMEWAIT) &&
-			    !(inp->inp_flags & INP_DROPPED) &&
-			    !(inp->inp_socket == NULL)) {
-				icmp_tcp_seq = htonl(th->th_seq);
-				tp = intotcpcb(inp);
-				if (SEQ_GEQ(icmp_tcp_seq, tp->snd_una) &&
-				    SEQ_LT(icmp_tcp_seq, tp->snd_max)) {
-					if (cmd == PRC_MSGSIZE) {
-					    /*
-					     * MTU discovery:
-					     * If we got a needfrag set the MTU
-					     * in the route to the suggested new
-					     * value (if given) and then notify.
-					     */
-					    bzero(&inc, sizeof(inc));
-					    inc.inc_faddr = faddr;
-					    inc.inc_fibnum =
-						inp->inp_inc.inc_fibnum;
 
-					    mtu = ntohs(icp->icmp_nextmtu);
-					    /*
-					     * If no alternative MTU was
-					     * proposed, try the next smaller
-					     * one.
-					     */
-					    if (!mtu)
+	if (ip == NULL) {
+		in_pcbnotifyall(&V_tcbinfo, faddr, inetctlerrmap[cmd], notify);
+		return;
+	}
+
+	icp = (struct icmp *)((caddr_t)ip - offsetof(struct icmp, icmp_ip));
+	th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2));
+	INP_INFO_RLOCK(&V_tcbinfo);
+	inp = in_pcblookup(&V_tcbinfo, faddr, th->th_dport, ip->ip_src,
+	    th->th_sport, INPLOOKUP_WLOCKPCB, NULL);
+	if (inp != NULL)  {
+		if (!(inp->inp_flags & INP_TIMEWAIT) &&
+		    !(inp->inp_flags & INP_DROPPED) &&
+		    !(inp->inp_socket == NULL)) {
+			icmp_tcp_seq = ntohl(th->th_seq);
+			tp = intotcpcb(inp);
+			if (SEQ_GEQ(icmp_tcp_seq, tp->snd_una) &&
+			    SEQ_LT(icmp_tcp_seq, tp->snd_max)) {
+				if (cmd == PRC_MSGSIZE) {
+					/*
+					 * MTU discovery:
+					 * If we got a needfrag set the MTU
+					 * in the route to the suggested new
+					 * value (if given) and then notify.
+					 */
+				    	mtu = ntohs(icp->icmp_nextmtu);
+					/*
+					 * If no alternative MTU was
+					 * proposed, try the next smaller
+					 * one.
+					 */
+					if (!mtu)
 						mtu = ip_next_mtu(
-						 ntohs(ip->ip_len), 1);
-					    if (mtu < V_tcp_minmss
-						 + sizeof(struct tcpiphdr))
-						mtu = V_tcp_minmss
-						 + sizeof(struct tcpiphdr);
-					    /*
-					     * Only cache the MTU if it
-					     * is smaller than the interface
-					     * or route MTU.  tcp_mtudisc()
-					     * will do right thing by itself.
-					     */
-					    if (mtu <= tcp_maxmtu(&inc, NULL))
+						    ntohs(ip->ip_len), 1);
+					if (mtu < V_tcp_minmss +
+					    sizeof(struct tcpiphdr))
+						mtu = V_tcp_minmss +
+						    sizeof(struct tcpiphdr);
+					/*
+					 * Only process the offered MTU if it
+					 * is smaller than the current one.
+					 */
+					if (mtu < tp->t_maxopd +
+					    sizeof(struct tcpiphdr)) {
+						bzero(&inc, sizeof(inc));
+						inc.inc_faddr = faddr;
+						inc.inc_fibnum =
+						    inp->inp_inc.inc_fibnum;
 						tcp_hc_updatemtu(&inc, mtu);
-					    tcp_mtudisc(inp, mtu);
-					} else
-						inp = (*notify)(inp,
-						    inetctlerrmap[cmd]);
-				}
+						tcp_mtudisc(inp, mtu);
+					}
+				} else
+					inp = (*notify)(inp,
+					    inetctlerrmap[cmd]);
 			}
-			if (inp != NULL)
-				INP_WUNLOCK(inp);
-		} else {
-			bzero(&inc, sizeof(inc));
-			inc.inc_fport = th->th_dport;
-			inc.inc_lport = th->th_sport;
-			inc.inc_faddr = faddr;
-			inc.inc_laddr = ip->ip_src;
-			syncache_unreach(&inc, th);
 		}
-		INP_INFO_RUNLOCK(&V_tcbinfo);
-	} else
-		in_pcbnotifyall(&V_tcbinfo, faddr, inetctlerrmap[cmd], notify);
+		if (inp != NULL)
+			INP_WUNLOCK(inp);
+	} else {
+		bzero(&inc, sizeof(inc));
+		inc.inc_fport = th->th_dport;
+		inc.inc_lport = th->th_sport;
+		inc.inc_faddr = faddr;
+		inc.inc_laddr = ip->ip_src;
+		syncache_unreach(&inc, th);
+	}
+	INP_INFO_RUNLOCK(&V_tcbinfo);
 }
 #endif /* INET */
 
diff --git a/sys/netinet/tcp_timer.c b/sys/netinet/tcp_timer.c
index 799064685adab..60b882f869f7a 100644
--- a/sys/netinet/tcp_timer.c
+++ b/sys/netinet/tcp_timer.c
@@ -57,6 +57,7 @@ __FBSDID("$FreeBSD$");
 
 #include <netinet/cc.h>
 #include <netinet/in.h>
+#include <netinet/in_kdtrace.h>
 #include <netinet/in_pcb.h>
 #include <netinet/in_rss.h>
 #include <netinet/in_systm.h>
@@ -369,6 +370,8 @@ tcp_timer_2msl(void *xtp)
 		tcp_trace(TA_USER, ostate, tp, (void *)0, (struct tcphdr *)0,
 			  PRU_SLOWTIMO);
 #endif
+	TCP_PROBE2(debug__user, tp, PRU_SLOWTIMO);
+
 	if (tp != NULL)
 		INP_WUNLOCK(inp);
 	INP_INFO_RUNLOCK(&V_tcbinfo);
@@ -454,6 +457,7 @@ tcp_timer_keep(void *xtp)
 		tcp_trace(TA_USER, ostate, tp, (void *)0, (struct tcphdr *)0,
 			  PRU_SLOWTIMO);
 #endif
+	TCP_PROBE2(debug__user, tp, PRU_SLOWTIMO);
 	INP_WUNLOCK(inp);
 	INP_INFO_RUNLOCK(&V_tcbinfo);
 	CURVNET_RESTORE();
@@ -468,6 +472,7 @@ dropit:
 		tcp_trace(TA_USER, ostate, tp, (void *)0, (struct tcphdr *)0,
 			  PRU_SLOWTIMO);
 #endif
+	TCP_PROBE2(debug__user, tp, PRU_SLOWTIMO);
 	if (tp != NULL)
 		INP_WUNLOCK(tp->t_inpcb);
 	INP_INFO_RUNLOCK(&V_tcbinfo);
@@ -546,6 +551,7 @@ out:
 	if (tp != NULL && tp->t_inpcb->inp_socket->so_options & SO_DEBUG)
 		tcp_trace(TA_USER, ostate, tp, NULL, NULL, PRU_SLOWTIMO);
 #endif
+	TCP_PROBE2(debug__user, tp, PRU_SLOWTIMO);
 	if (tp != NULL)
 		INP_WUNLOCK(inp);
 	INP_INFO_RUNLOCK(&V_tcbinfo);
@@ -792,6 +798,7 @@ out:
 		tcp_trace(TA_USER, ostate, tp, (void *)0, (struct tcphdr *)0,
 			  PRU_SLOWTIMO);
 #endif
+	TCP_PROBE2(debug__user, tp, PRU_SLOWTIMO);
 	if (tp != NULL)
 		INP_WUNLOCK(inp);
 	if (headlocked)
diff --git a/sys/netinet/tcp_usrreq.c b/sys/netinet/tcp_usrreq.c
index fe390e0e4513a..7d1ec6a0f2a91 100644
--- a/sys/netinet/tcp_usrreq.c
+++ b/sys/netinet/tcp_usrreq.c
@@ -70,6 +70,7 @@ __FBSDID("$FreeBSD$");
 
 #include <netinet/cc.h>
 #include <netinet/in.h>
+#include <netinet/in_kdtrace.h>
 #include <netinet/in_pcb.h>
 #include <netinet/in_systm.h>
 #include <netinet/in_var.h>
@@ -146,6 +147,7 @@ tcp_usr_attach(struct socket *so, int proto, struct thread *td)
 	tp = intotcpcb(inp);
 out:
 	TCPDEBUG2(PRU_ATTACH);
+	TCP_PROBE2(debug__user, tp, PRU_ATTACH);
 	return error;
 }
 
@@ -295,6 +297,7 @@ tcp_usr_bind(struct socket *so, struct sockaddr *nam, struct thread *td)
 	INP_HASH_WUNLOCK(&V_tcbinfo);
 out:
 	TCPDEBUG2(PRU_BIND);
+	TCP_PROBE2(debug__user, tp, PRU_BIND);
 	INP_WUNLOCK(inp);
 
 	return (error);
@@ -355,6 +358,7 @@ tcp6_usr_bind(struct socket *so, struct sockaddr *nam, struct thread *td)
 	INP_HASH_WUNLOCK(&V_tcbinfo);
 out:
 	TCPDEBUG2(PRU_BIND);
+	TCP_PROBE2(debug__user, tp, PRU_BIND);
 	INP_WUNLOCK(inp);
 	return (error);
 }
@@ -399,6 +403,7 @@ tcp_usr_listen(struct socket *so, int backlog, struct thread *td)
 
 out:
 	TCPDEBUG2(PRU_LISTEN);
+	TCP_PROBE2(debug__user, tp, PRU_LISTEN);
 	INP_WUNLOCK(inp);
 	return (error);
 }
@@ -444,6 +449,7 @@ tcp6_usr_listen(struct socket *so, int backlog, struct thread *td)
 
 out:
 	TCPDEBUG2(PRU_LISTEN);
+	TCP_PROBE2(debug__user, tp, PRU_LISTEN);
 	INP_WUNLOCK(inp);
 	return (error);
 }
@@ -592,6 +598,7 @@ tcp6_usr_connect(struct socket *so, struct sockaddr *nam, struct thread *td)
 
 out:
 	TCPDEBUG2(PRU_CONNECT);
+	TCP_PROBE2(debug__user, tp, PRU_CONNECT);
 	INP_WUNLOCK(inp);
 	return (error);
 }
@@ -631,6 +638,7 @@ tcp_usr_disconnect(struct socket *so)
 	tcp_disconnect(tp);
 out:
 	TCPDEBUG2(PRU_DISCONNECT);
+	TCP_PROBE2(debug__user, tp, PRU_DISCONNECT);
 	INP_WUNLOCK(inp);
 	INP_INFO_RUNLOCK(&V_tcbinfo);
 	return (error);
@@ -674,6 +682,7 @@ tcp_usr_accept(struct socket *so, struct sockaddr **nam)
 
 out:
 	TCPDEBUG2(PRU_ACCEPT);
+	TCP_PROBE2(debug__user, tp, PRU_ACCEPT);
 	INP_WUNLOCK(inp);
 	if (error == 0)
 		*nam = in_sockaddr(port, &addr);
@@ -724,6 +733,7 @@ tcp6_usr_accept(struct socket *so, struct sockaddr **nam)
 
 out:
 	TCPDEBUG2(PRU_ACCEPT);
+	TCP_PROBE2(debug__user, tp, PRU_ACCEPT);
 	INP_WUNLOCK(inp);
 	INP_INFO_RUNLOCK(&V_tcbinfo);
 	if (error == 0) {
@@ -764,6 +774,7 @@ tcp_usr_shutdown(struct socket *so)
 
 out:
 	TCPDEBUG2(PRU_SHUTDOWN);
+	TCP_PROBE2(debug__user, tp, PRU_SHUTDOWN);
 	INP_WUNLOCK(inp);
 	INP_INFO_RUNLOCK(&V_tcbinfo);
 
@@ -799,6 +810,7 @@ tcp_usr_rcvd(struct socket *so, int flags)
 
 out:
 	TCPDEBUG2(PRU_RCVD);
+	TCP_PROBE2(debug__user, tp, PRU_RCVD);
 	INP_WUNLOCK(inp);
 	return (error);
 }
@@ -953,6 +965,8 @@ tcp_usr_send(struct socket *so, int flags, struct mbuf *m,
 out:
 	TCPDEBUG2((flags & PRUS_OOB) ? PRU_SENDOOB :
 		  ((flags & PRUS_EOF) ? PRU_SEND_EOF : PRU_SEND));
+	TCP_PROBE2(debug__user, tp, (flags & PRUS_OOB) ? PRU_SENDOOB :
+		   ((flags & PRUS_EOF) ? PRU_SEND_EOF : PRU_SEND));
 	INP_WUNLOCK(inp);
 	if (flags & PRUS_EOF)
 		INP_INFO_RUNLOCK(&V_tcbinfo);
@@ -1013,6 +1027,7 @@ tcp_usr_abort(struct socket *so)
 		TCPDEBUG1();
 		tcp_drop(tp, ECONNABORTED);
 		TCPDEBUG2(PRU_ABORT);
+		TCP_PROBE2(debug__user, tp, PRU_ABORT);
 	}
 	if (!(inp->inp_flags & INP_DROPPED)) {
 		SOCK_LOCK(so);
@@ -1052,6 +1067,7 @@ tcp_usr_close(struct socket *so)
 		TCPDEBUG1();
 		tcp_disconnect(tp);
 		TCPDEBUG2(PRU_CLOSE);
+		TCP_PROBE2(debug__user, tp, PRU_CLOSE);
 	}
 	if (!(inp->inp_flags & INP_DROPPED)) {
 		SOCK_LOCK(so);
@@ -1101,6 +1117,7 @@ tcp_usr_rcvoob(struct socket *so, struct mbuf *m, int flags)
 
 out:
 	TCPDEBUG2(PRU_RCVOOB);
+	TCP_PROBE2(debug__user, tp, PRU_RCVOOB);
 	INP_WUNLOCK(inp);
 	return (error);
 }
@@ -1748,9 +1765,9 @@ tcp_usrclosed(struct tcpcb *tp)
 #ifdef TCP_OFFLOAD
 		tcp_offload_listen_stop(tp);
 #endif
+		tcp_state_change(tp, TCPS_CLOSED);
 		/* FALLTHROUGH */
 	case TCPS_CLOSED:
-		tcp_state_change(tp, TCPS_CLOSED);
 		tp = tcp_close(tp);
 		/*
 		 * tcp_close() should never return NULL here as the socket is
diff --git a/sys/netinet/tcp_var.h b/sys/netinet/tcp_var.h
index e0a6ed142f0ad..61fc41903fd63 100644
--- a/sys/netinet/tcp_var.h
+++ b/sys/netinet/tcp_var.h
@@ -83,18 +83,6 @@ struct tcptemp {
 
 #define tcp6cb		tcpcb  /* for KAME src sync over BSD*'s */
 
-/* Neighbor Discovery, Neighbor Unreachability Detection Upper layer hint. */
-#ifdef INET6
-#define ND6_HINT(tp)						\
-do {								\
-	if ((tp) && (tp)->t_inpcb &&				\
-	    ((tp)->t_inpcb->inp_vflag & INP_IPV6) != 0)		\
-		nd6_nud_hint(NULL, NULL, 0);			\
-} while (0)
-#else
-#define ND6_HINT(tp)
-#endif
-
 /*
  * Tcp control block, one per tcp; fields:
  * Organized for 16 byte cacheline efficiency.
diff --git a/sys/netinet/toecore.c b/sys/netinet/toecore.c
index 28ef32a3b60d9..87d8856537bf0 100644
--- a/sys/netinet/toecore.c
+++ b/sys/netinet/toecore.c
@@ -448,80 +448,6 @@ toe_route_redirect_event(void *arg __unused, struct rtentry *rt0,
 	return;
 }
 
-#ifdef INET6
-/*
- * XXX: no checks to verify that sa is really a neighbor because we assume it is
- * the result of a route lookup and is on-link on the given ifp.
- */
-static int
-toe_nd6_resolve(struct ifnet *ifp, struct sockaddr *sa, uint8_t *lladdr)
-{
-	struct llentry *lle, *lle_tmp;
-	struct sockaddr_in6 *sin6 = (void *)sa;
-	int rc, flags = 0;
-
-restart:
-	IF_AFDATA_RLOCK(ifp);
-	lle = lla_lookup(LLTABLE6(ifp), flags, sa);
-	IF_AFDATA_RUNLOCK(ifp);
-	if (lle == NULL) {
-		lle = nd6_alloc(&sin6->sin6_addr, 0, ifp);
-		if (lle == NULL)
-			return (ENOMEM); /* Couldn't create entry in cache. */
-		lle->ln_state = ND6_LLINFO_INCOMPLETE;
-		IF_AFDATA_WLOCK(ifp);
-		LLE_WLOCK(lle);
-		lle_tmp = nd6_lookup(&sin6->sin6_addr, ND6_EXCLUSIVE, ifp);
-		/* Prefer any existing lle over newly-created one */
-		if (lle_tmp == NULL)
-			lltable_link_entry(LLTABLE6(ifp), lle); 
-		IF_AFDATA_WUNLOCK(ifp);
-		if (lle_tmp == NULL) {
-			/* Arm timer for newly-created entry and send NS */
-			nd6_llinfo_settimer_locked(lle,
-			    (long)ND_IFINFO(ifp)->retrans * hz / 1000);
-			LLE_WUNLOCK(lle);
-
-			nd6_ns_output(ifp, NULL, NULL, &sin6->sin6_addr, 0);
-
-			return (EWOULDBLOCK);
-		} else {
-			/* Drop newly-created lle and switch to existing one */
-			lltable_free_entry(LLTABLE6(ifp), lle);
-			lle = lle_tmp;
-			lle_tmp = NULL;
-		}
-	}
-
-	if (lle->ln_state == ND6_LLINFO_STALE) {
-		if ((flags & LLE_EXCLUSIVE) == 0) {
-			LLE_RUNLOCK(lle);
-			flags |= LLE_EXCLUSIVE;
-			goto restart;
-		}
-
-		LLE_WLOCK_ASSERT(lle);
-
-		lle->la_asked = 0;
-		lle->ln_state = ND6_LLINFO_DELAY;
-		nd6_llinfo_settimer_locked(lle, (long)V_nd6_delay * hz);
-	}
-
-	if (lle->la_flags & LLE_VALID) {
-		memcpy(lladdr, &lle->ll_addr, ifp->if_addrlen);
-		rc = 0;
-	} else
-		rc = EWOULDBLOCK;
-
-	if (flags & LLE_EXCLUSIVE)
-		LLE_WUNLOCK(lle);
-	else
-		LLE_RUNLOCK(lle);
-
-	return (rc);
-}
-#endif
-
 /*
  * Returns 0 or EWOULDBLOCK on success (any other value is an error).  0 means
  * lladdr and vtag are valid on return, EWOULDBLOCK means the TOE driver's
@@ -541,7 +467,7 @@ toe_l2_resolve(struct toedev *tod, struct ifnet *ifp, struct sockaddr *sa,
 #endif
 #ifdef INET6
 	case AF_INET6:
-		rc = toe_nd6_resolve(ifp, sa, lladdr);
+		rc = nd6_resolve(ifp, 0, NULL, sa, lladdr, NULL);
 		break;
 #endif
 	default:
diff --git a/sys/netinet6/icmp6.c b/sys/netinet6/icmp6.c
index 29dd3edf841eb..c918e69627fb6 100644
--- a/sys/netinet6/icmp6.c
+++ b/sys/netinet6/icmp6.c
@@ -2434,27 +2434,39 @@ icmp6_redirect_input(struct mbuf *m, int off)
 	nd6_cache_lladdr(ifp, &redtgt6, lladdr, lladdrlen, ND_REDIRECT,
 	    is_onlink ? ND_REDIRECT_ONLINK : ND_REDIRECT_ROUTER);
 
-	if (!is_onlink) {	/* better router case.  perform rtredirect. */
-		/* perform rtredirect */
+	/*
+	 * Install a gateway route in the better-router case or an interface
+	 * route in the on-link-destination case.
+	 */
+	{
 		struct sockaddr_in6 sdst;
 		struct sockaddr_in6 sgw;
 		struct sockaddr_in6 ssrc;
+		struct sockaddr *gw;
+		int rt_flags;
 		u_int fibnum;
 
 		bzero(&sdst, sizeof(sdst));
-		bzero(&sgw, sizeof(sgw));
 		bzero(&ssrc, sizeof(ssrc));
-		sdst.sin6_family = sgw.sin6_family = ssrc.sin6_family = AF_INET6;
-		sdst.sin6_len = sgw.sin6_len = ssrc.sin6_len =
-			sizeof(struct sockaddr_in6);
-		bcopy(&redtgt6, &sgw.sin6_addr, sizeof(struct in6_addr));
+		sdst.sin6_family = ssrc.sin6_family = AF_INET6;
+		sdst.sin6_len = ssrc.sin6_len = sizeof(struct sockaddr_in6);
 		bcopy(&reddst6, &sdst.sin6_addr, sizeof(struct in6_addr));
 		bcopy(&src6, &ssrc.sin6_addr, sizeof(struct in6_addr));
+		rt_flags = RTF_HOST;
+		if (is_router) {
+			bzero(&sgw, sizeof(sgw));
+			sgw.sin6_family = AF_INET6;
+			sgw.sin6_len = sizeof(struct sockaddr_in6);
+			bcopy(&redtgt6, &sgw.sin6_addr,
+				sizeof(struct in6_addr));
+			gw = (struct sockaddr *)&sgw;
+			rt_flags |= RTF_GATEWAY;
+		} else
+			gw = ifp->if_addr->ifa_addr;
 		for (fibnum = 0; fibnum < rt_numfibs; fibnum++)
-			in6_rtredirect((struct sockaddr *)&sdst,
-			    (struct sockaddr *)&sgw, (struct sockaddr *)NULL,
-			    RTF_GATEWAY | RTF_HOST, (struct sockaddr *)&ssrc,
-			    fibnum);
+			in6_rtredirect((struct sockaddr *)&sdst, gw,
+			    (struct sockaddr *)NULL, rt_flags,
+			    (struct sockaddr *)&ssrc, fibnum);
 	}
 	/* finally update cached route in each socket via pfctlinput */
     {
diff --git a/sys/netinet6/in6.c b/sys/netinet6/in6.c
index b14af016d0fa6..af62ed1d9f99a 100644
--- a/sys/netinet6/in6.c
+++ b/sys/netinet6/in6.c
@@ -1307,9 +1307,6 @@ in6_purgeaddr(struct ifaddr *ifa)
 	/* stop DAD processing */
 	nd6_dad_stop(ifa);
 
-	/* Remove local address entry from lltable. */
-	nd6_rem_ifa_lle(ia);
-
 	/* Leave multicast groups. */
 	while ((imm = LIST_FIRST(&ia->ia6_memberships)) != NULL) {
 		LIST_REMOVE(imm, i6mm_chain);
@@ -1333,6 +1330,7 @@ static void
 in6_unlink_ifa(struct in6_ifaddr *ia, struct ifnet *ifp)
 {
 	char ip6buf[INET6_ADDRSTRLEN];
+	int remove_lle;
 
 	IF_ADDR_WLOCK(ifp);
 	TAILQ_REMOVE(&ifp->if_addrhead, &ia->ia_ifa, ifa_link);
@@ -1353,15 +1351,21 @@ in6_unlink_ifa(struct in6_ifaddr *ia, struct ifnet *ifp)
 	 * Release the reference to the base prefix.  There should be a
 	 * positive reference.
 	 */
+	remove_lle = 0;
 	if (ia->ia6_ndpr == NULL) {
 		nd6log((LOG_NOTICE,
 		    "in6_unlink_ifa: autoconf'ed address "
 		    "%s has no prefix\n", ip6_sprintf(ip6buf, IA6_IN6(ia))));
 	} else {
 		ia->ia6_ndpr->ndpr_refcnt--;
+		/* Do not delete lles within prefix if refcont != 0 */
+		if (ia->ia6_ndpr->ndpr_refcnt == 0)
+			remove_lle = 1;
 		ia->ia6_ndpr = NULL;
 	}
 
+	nd6_rem_ifa_lle(ia, remove_lle);
+
 	/*
 	 * Also, if the address being removed is autoconf'ed, call
 	 * pfxlist_onlink_check() since the release might affect the status of
@@ -2081,15 +2085,33 @@ in6_lltable_new(const struct in6_addr *addr6, u_int flags)
 }
 
 static int
-in6_lltable_match_prefix(const struct sockaddr *prefix,
-    const struct sockaddr *mask, u_int flags, struct llentry *lle)
+in6_lltable_match_prefix(const struct sockaddr *saddr,
+    const struct sockaddr *smask, u_int flags, struct llentry *lle)
 {
-	const struct sockaddr_in6 *pfx = (const struct sockaddr_in6 *)prefix;
-	const struct sockaddr_in6 *msk = (const struct sockaddr_in6 *)mask;
+	const struct in6_addr *addr, *mask, *lle_addr;
+
+	addr = &((const struct sockaddr_in6 *)saddr)->sin6_addr;
+	mask = &((const struct sockaddr_in6 *)smask)->sin6_addr;
+	lle_addr = &lle->r_l3addr.addr6;
+
+	if (IN6_ARE_MASKED_ADDR_EQUAL(lle_addr, addr, mask) == 0)
+		return (0);
+
+	if (lle->la_flags & LLE_IFADDR) {
+
+		/*
+		 * Delete LLE_IFADDR records IFF address & flag matches.
+		 * Note that addr is the interface address within prefix
+		 * being matched.
+		 */
+		if (IN6_ARE_ADDR_EQUAL(addr, lle_addr) &&
+		    (flags & LLE_STATIC) != 0)
+			return (1);
+		return (0);
+	}
 
-	if (IN6_ARE_MASKED_ADDR_EQUAL(&lle->r_l3addr.addr6,
-	    &pfx->sin6_addr, &msk->sin6_addr) &&
-	    ((flags & LLE_STATIC) || !(lle->la_flags & LLE_STATIC)))
+	/* flags & LLE_STATIC means deleting both dynamic and static entries */
+	if ((flags & LLE_STATIC) || !(lle->la_flags & LLE_STATIC))
 		return (1);
 
 	return (0);
@@ -2200,36 +2222,16 @@ in6_lltable_find_dst(struct lltable *llt, const struct in6_addr *dst)
 	return (lle);
 }
 
-static int
-in6_lltable_delete(struct lltable *llt, u_int flags,
-	const struct sockaddr *l3addr)
+static void
+in6_lltable_delete_entry(struct lltable *llt, struct llentry *lle)
 {
-	const struct sockaddr_in6 *sin6 = (const struct sockaddr_in6 *)l3addr;
-	struct llentry *lle;
-
-	IF_AFDATA_LOCK_ASSERT(llt->llt_ifp);
-	KASSERT(l3addr->sa_family == AF_INET6,
-	    ("sin_family %d", l3addr->sa_family));
-
-	lle = in6_lltable_find_dst(llt, &sin6->sin6_addr);
 
-	if (lle == NULL)
-		return (ENOENT);
-
-	if (!(lle->la_flags & LLE_IFADDR) || (flags & LLE_IFADDR)) {
-		LLE_WLOCK(lle);
-		lle->la_flags |= LLE_DELETED;
-		EVENTHANDLER_INVOKE(lle_event, lle, LLENTRY_DELETED);
+	lle->la_flags |= LLE_DELETED;
+	EVENTHANDLER_INVOKE(lle_event, lle, LLENTRY_DELETED);
 #ifdef DIAGNOSTIC
-		log(LOG_INFO, "ifaddr cache = %p is deleted\n", lle);
+	log(LOG_INFO, "ifaddr cache = %p is deleted\n", lle);
 #endif
-		if ((lle->la_flags & (LLE_STATIC | LLE_IFADDR)) == LLE_STATIC)
-			llentry_free(lle);
-		else
-			LLE_WUNLOCK(lle);
-	}
-
-	return (0);
+	llentry_free(lle);
 }
 
 static struct llentry *
@@ -2352,6 +2354,8 @@ in6_lltable_dump_entry(struct lltable *llt, struct llentry *lle,
 			ndpc.rtm.rtm_flags |= (RTF_HOST | RTF_LLDATA);
 			if (lle->la_flags & LLE_STATIC)
 				ndpc.rtm.rtm_flags |= RTF_STATIC;
+			if (lle->la_flags & LLE_IFADDR)
+				ndpc.rtm.rtm_flags |= RTF_PINNED;
 			ndpc.rtm.rtm_index = ifp->if_index;
 			error = SYSCTL_OUT(wr, &ndpc, sizeof(ndpc));
 
@@ -2369,7 +2373,7 @@ in6_lltattach(struct ifnet *ifp)
 
 	llt->llt_lookup = in6_lltable_lookup;
 	llt->llt_alloc_entry = in6_lltable_alloc;
-	llt->llt_delete = in6_lltable_delete;
+	llt->llt_delete_entry = in6_lltable_delete_entry;
 	llt->llt_dump_entry = in6_lltable_dump_entry;
 	llt->llt_hash = in6_lltable_hash;
 	llt->llt_fill_sa_entry = in6_lltable_fill_sa_entry;
diff --git a/sys/netinet6/ip6_forward.c b/sys/netinet6/ip6_forward.c
index 83e58c14d70d9..9df8e4c9dc047 100644
--- a/sys/netinet6/ip6_forward.c
+++ b/sys/netinet6/ip6_forward.c
@@ -571,7 +571,7 @@ pass:
 		goto bad;
 	}
 
-	error = nd6_output(rt->rt_ifp, origifp, m, dst, rt);
+	error = nd6_output_ifp(rt->rt_ifp, origifp, m, dst);
 	if (error) {
 		in6_ifstat_inc(rt->rt_ifp, ifs6_out_discard);
 		IP6STAT_INC(ip6s_cantforward);
diff --git a/sys/netinet6/ip6_ipsec.c b/sys/netinet6/ip6_ipsec.c
index d7f08e91537c0..a99f1db14e8e6 100644
--- a/sys/netinet6/ip6_ipsec.c
+++ b/sys/netinet6/ip6_ipsec.c
@@ -200,9 +200,7 @@ ip6_ipsec_output(struct mbuf **m, struct inpcb *inp, int *error)
 
 		/* NB: callee frees mbuf */
 		*error = ipsec6_process_packet(*m, sp->req);
-		/* Release SP if an error occured */
-		if (*error != 0)
-			KEY_FREESP(&sp);
+		KEY_FREESP(&sp);
 		if (*error == EJUSTRETURN) {
 			/*
 			 * We had a SP with a level of 'use' and no SA. We
diff --git a/sys/netinet6/ip6_output.c b/sys/netinet6/ip6_output.c
index 09d092528f106..181340f0c7713 100644
--- a/sys/netinet6/ip6_output.c
+++ b/sys/netinet6/ip6_output.c
@@ -935,7 +935,7 @@ passout:
 			    m->m_pkthdr.len);
 			ifa_free(&ia6->ia_ifa);
 		}
-		error = nd6_output(ifp, origifp, m, dst, ro->ro_rt);
+		error = nd6_output_ifp(ifp, origifp, m, dst);
 		goto done;
 	}
 
@@ -1034,7 +1034,7 @@ sendorfree:
 				counter_u64_add(ia->ia_ifa.ifa_obytes,
 				    m->m_pkthdr.len);
 			}
-			error = nd6_output(ifp, origifp, m, dst, ro->ro_rt);
+			error = nd6_output_ifp(ifp, origifp, m, dst);
 		} else
 			m_freem(m);
 	}
diff --git a/sys/netinet6/nd6.c b/sys/netinet6/nd6.c
index 11276a509795c..4098ffb88b71d 100644
--- a/sys/netinet6/nd6.c
+++ b/sys/netinet6/nd6.c
@@ -126,19 +126,20 @@ VNET_DEFINE(int, nd6_recalc_reachtm_interval) = ND6_RECALC_REACHTM_INTERVAL;
 
 int	(*send_sendso_input_hook)(struct mbuf *, struct ifnet *, int, int);
 
-static int nd6_is_new_addr_neighbor(struct sockaddr_in6 *,
+static int nd6_is_new_addr_neighbor(const struct sockaddr_in6 *,
 	struct ifnet *);
 static void nd6_setmtu0(struct ifnet *, struct nd_ifinfo *);
 static void nd6_slowtimo(void *);
 static int regen_tmpaddr(struct in6_ifaddr *);
-static struct llentry *nd6_free(struct llentry *, int);
+static void nd6_free(struct llentry *, int);
+static void nd6_free_redirect(const struct llentry *);
 static void nd6_llinfo_timer(void *);
 static void clear_llinfo_pqueue(struct llentry *);
 static void nd6_rtrequest(int, struct rtentry *, struct rt_addrinfo *);
-static int nd6_output_lle(struct ifnet *, struct ifnet *, struct mbuf *,
-	struct sockaddr_in6 *);
-static int nd6_output_ifp(struct ifnet *, struct ifnet *, struct mbuf *,
-    struct sockaddr_in6 *);
+static int nd6_resolve_slow(struct ifnet *, struct mbuf *,
+    const struct sockaddr_in6 *, u_char *, uint32_t *);
+static int nd6_need_cache(struct ifnet *);
+ 
 
 static VNET_DEFINE(struct callout, nd6_slowtimo_ch);
 #define	V_nd6_slowtimo_ch		VNET(nd6_slowtimo_ch)
@@ -538,6 +539,44 @@ nd6_llinfo_get_holdsrc(struct llentry *ln, struct in6_addr *src)
 	return (src);
 }
 
+/*
+ * Switch @lle state to new state optionally arming timers.
+ */
+void
+nd6_llinfo_setstate(struct llentry *lle, int newstate)
+{
+	struct ifnet *ifp;
+	long delay;
+
+	delay = 0;
+
+	switch (newstate) {
+	case ND6_LLINFO_INCOMPLETE:
+		ifp = lle->lle_tbl->llt_ifp;
+		delay = (long)ND_IFINFO(ifp)->retrans * hz / 1000;
+		break;
+	case ND6_LLINFO_REACHABLE:
+		if (!ND6_LLINFO_PERMANENT(lle)) {
+			ifp = lle->lle_tbl->llt_ifp;
+			delay = (long)ND_IFINFO(ifp)->reachable * hz;
+		}
+		break;
+	case ND6_LLINFO_STALE:
+		delay = (long)V_nd6_gctimer * hz;
+		break;
+	case ND6_LLINFO_DELAY:
+		lle->la_asked = 0;
+		delay = (long)V_nd6_delay * hz;
+		break;
+	}
+
+	if (delay > 0)
+		nd6_llinfo_settimer_locked(lle, delay);
+
+	lle->ln_state = newstate;
+}
+
+
 void
 nd6_llinfo_settimer(struct llentry *ln, long tick)
 {
@@ -602,7 +641,7 @@ nd6_llinfo_timer(void *arg)
 	}
 
 	if (ln->la_flags & LLE_DELETED) {
-		(void)nd6_free(ln, 0);
+		nd6_free(ln, 0);
 		ln = NULL;
 		goto done;
 	}
@@ -629,7 +668,7 @@ nd6_llinfo_timer(void *arg)
 				clear_llinfo_pqueue(ln);
 			}
 			EVENTHANDLER_INVOKE(lle_event, ln, LLENTRY_TIMEDOUT);
-			(void)nd6_free(ln, 0);
+			nd6_free(ln, 0);
 			ln = NULL;
 			if (m != NULL)
 				icmp6_error2(m, ICMP6_DST_UNREACH,
@@ -637,17 +676,15 @@ nd6_llinfo_timer(void *arg)
 		}
 		break;
 	case ND6_LLINFO_REACHABLE:
-		if (!ND6_LLINFO_PERMANENT(ln)) {
-			ln->ln_state = ND6_LLINFO_STALE;
-			nd6_llinfo_settimer_locked(ln, (long)V_nd6_gctimer * hz);
-		}
+		if (!ND6_LLINFO_PERMANENT(ln))
+			nd6_llinfo_setstate(ln, ND6_LLINFO_STALE);
 		break;
 
 	case ND6_LLINFO_STALE:
 		/* Garbage Collection(RFC 2461 5.3) */
 		if (!ND6_LLINFO_PERMANENT(ln)) {
 			EVENTHANDLER_INVOKE(lle_event, ln, LLENTRY_EXPIRED);
-			(void)nd6_free(ln, 1);
+			nd6_free(ln, 1);
 			ln = NULL;
 		}
 		break;
@@ -656,12 +693,10 @@ nd6_llinfo_timer(void *arg)
 		if (ndi && (ndi->flags & ND6_IFF_PERFORMNUD) != 0) {
 			/* We need NUD */
 			ln->la_asked = 1;
-			ln->ln_state = ND6_LLINFO_PROBE;
+			nd6_llinfo_setstate(ln, ND6_LLINFO_PROBE);
 			send_ns = 1;
-		} else {
-			ln->ln_state = ND6_LLINFO_STALE; /* XXX */
-			nd6_llinfo_settimer_locked(ln, (long)V_nd6_gctimer * hz);
-		}
+		} else
+			nd6_llinfo_setstate(ln, ND6_LLINFO_STALE); /* XXX */
 		break;
 	case ND6_LLINFO_PROBE:
 		if (ln->la_asked < V_nd6_umaxtries) {
@@ -669,7 +704,7 @@ nd6_llinfo_timer(void *arg)
 			send_ns = 1;
 		} else {
 			EVENTHANDLER_INVOKE(lle_event, ln, LLENTRY_EXPIRED);
-			(void)nd6_free(ln, 0);
+			nd6_free(ln, 0);
 			ln = NULL;
 		}
 		break;
@@ -952,11 +987,10 @@ nd6_purge(struct ifnet *ifp)
  * Returns the llentry locked
  */
 struct llentry *
-nd6_lookup(struct in6_addr *addr6, int flags, struct ifnet *ifp)
+nd6_lookup(const struct in6_addr *addr6, int flags, struct ifnet *ifp)
 {
 	struct sockaddr_in6 sin6;
 	struct llentry *ln;
-	int llflags;
 	
 	bzero(&sin6, sizeof(sin6));
 	sin6.sin6_len = sizeof(struct sockaddr_in6);
@@ -965,14 +999,13 @@ nd6_lookup(struct in6_addr *addr6, int flags, struct ifnet *ifp)
 
 	IF_AFDATA_LOCK_ASSERT(ifp);
 
-	llflags = (flags & ND6_EXCLUSIVE) ? LLE_EXCLUSIVE : 0;
-	ln = lla_lookup(LLTABLE6(ifp), llflags, (struct sockaddr *)&sin6);
+	ln = lla_lookup(LLTABLE6(ifp), flags, (struct sockaddr *)&sin6);
 
 	return (ln);
 }
 
 struct llentry *
-nd6_alloc(struct in6_addr *addr6, int flags, struct ifnet *ifp)
+nd6_alloc(const struct in6_addr *addr6, int flags, struct ifnet *ifp)
 {
 	struct sockaddr_in6 sin6;
 	struct llentry *ln;
@@ -995,7 +1028,7 @@ nd6_alloc(struct in6_addr *addr6, int flags, struct ifnet *ifp)
  * to not reenter the routing code from within itself.
  */
 static int
-nd6_is_new_addr_neighbor(struct sockaddr_in6 *addr, struct ifnet *ifp)
+nd6_is_new_addr_neighbor(const struct sockaddr_in6 *addr, struct ifnet *ifp)
 {
 	struct nd_prefix *pr;
 	struct ifaddr *dstaddr;
@@ -1068,7 +1101,7 @@ nd6_is_new_addr_neighbor(struct sockaddr_in6 *addr, struct ifnet *ifp)
 	 * If the address is assigned on the node of the other side of
 	 * a p2p interface, the address should be a neighbor.
 	 */
-	dstaddr = ifa_ifwithdstaddr((struct sockaddr *)addr, RT_ALL_FIBS);
+	dstaddr = ifa_ifwithdstaddr((const struct sockaddr *)addr, RT_ALL_FIBS);
 	if (dstaddr != NULL) {
 		if (dstaddr->ifa_ifp == ifp) {
 			ifa_free(dstaddr);
@@ -1096,7 +1129,7 @@ nd6_is_new_addr_neighbor(struct sockaddr_in6 *addr, struct ifnet *ifp)
  * XXX: should take care of the destination of a p2p link?
  */
 int
-nd6_is_addr_neighbor(struct sockaddr_in6 *addr, struct ifnet *ifp)
+nd6_is_addr_neighbor(const struct sockaddr_in6 *addr, struct ifnet *ifp)
 {
 	struct llentry *lle;
 	int rc = 0;
@@ -1124,10 +1157,9 @@ nd6_is_addr_neighbor(struct sockaddr_in6 *addr, struct ifnet *ifp)
  * make it global, unless you have a strong reason for the change, and are sure
  * that the change is safe.
  */
-static struct llentry *
+static void
 nd6_free(struct llentry *ln, int gc)
 {
-        struct llentry *next;
 	struct nd_defrouter *dr;
 	struct ifnet *ifp;
 
@@ -1167,10 +1199,9 @@ nd6_free(struct llentry *ln, int gc)
 				nd6_llinfo_settimer_locked(ln,
 				    (long)V_nd6_gctimer * hz);
 
-			next = LIST_NEXT(ln, lle_next);
 			LLE_REMREF(ln);
 			LLE_WUNLOCK(ln);
-			return (next);
+			return;
 		}
 
 		if (dr) {
@@ -1223,83 +1254,65 @@ nd6_free(struct llentry *ln, int gc)
 			defrouter_select();
 		}
 
+		/*
+		 * If this entry was added by an on-link redirect, remove the
+		 * corresponding host route.
+		 */
+		if (ln->la_flags & LLE_REDIRECT)
+			nd6_free_redirect(ln);
+
 		if (ln->ln_router || dr)
 			LLE_WLOCK(ln);
 	}
 
 	/*
-	 * Before deleting the entry, remember the next entry as the
-	 * return value.  We need this because pfxlist_onlink_check() above
-	 * might have freed other entries (particularly the old next entry) as
-	 * a side effect (XXX).
-	 */
-	next = LIST_NEXT(ln, lle_next);
-
-	/*
 	 * Save to unlock. We still hold an extra reference and will not
 	 * free(9) in llentry_free() if someone else holds one as well.
 	 */
 	LLE_WUNLOCK(ln);
 	IF_AFDATA_LOCK(ifp);
 	LLE_WLOCK(ln);
-
 	/* Guard against race with other llentry_free(). */
 	if (ln->la_flags & LLE_LINKED) {
+		/* Remove callout reference */
 		LLE_REMREF(ln);
-		llentry_free(ln);
-	} else
-		LLE_FREE_LOCKED(ln);
-
+		lltable_unlink_entry(ln->lle_tbl, ln);
+	}
 	IF_AFDATA_UNLOCK(ifp);
 
-	return (next);
+	llentry_free(ln);
 }
 
 /*
- * Upper-layer reachability hint for Neighbor Unreachability Detection.
- *
- * XXX cost-effective methods?
+ * Remove the rtentry for the given llentry,
+ * both of which were installed by a redirect.
  */
-void
-nd6_nud_hint(struct rtentry *rt, struct in6_addr *dst6, int force)
+static void
+nd6_free_redirect(const struct llentry *ln)
 {
-	struct llentry *ln;
-	struct ifnet *ifp;
-
-	if ((dst6 == NULL) || (rt == NULL))
-		return;
-
-	ifp = rt->rt_ifp;
-	IF_AFDATA_RLOCK(ifp);
-	ln = nd6_lookup(dst6, ND6_EXCLUSIVE, NULL);
-	IF_AFDATA_RUNLOCK(ifp);
-	if (ln == NULL)
-		return;
+	int fibnum;
+	struct rtentry *rt;
+	struct radix_node_head *rnh;
+	struct sockaddr_in6 sin6;
 
-	if (ln->ln_state < ND6_LLINFO_REACHABLE)
-		goto done;
+	lltable_fill_sa_entry(ln, (struct sockaddr *)&sin6);
+	for (fibnum = 0; fibnum < rt_numfibs; fibnum++) {
+		rnh = rt_tables_get_rnh(fibnum, AF_INET6);
+		if (rnh == NULL)
+			continue;
 
-	/*
-	 * if we get upper-layer reachability confirmation many times,
-	 * it is possible we have false information.
-	 */
-	if (!force) {
-		ln->ln_byhint++;
-		if (ln->ln_byhint > V_nd6_maxnudhint) {
-			goto done;
+		RADIX_NODE_HEAD_LOCK(rnh);
+		rt = in6_rtalloc1((struct sockaddr *)&sin6, 0,
+		    RTF_RNH_LOCKED, fibnum);
+		if (rt) {
+			if (rt->rt_flags == (RTF_UP | RTF_HOST | RTF_DYNAMIC))
+				rt_expunge(rnh, rt);
+			RTFREE_LOCKED(rt);
 		}
+		RADIX_NODE_HEAD_UNLOCK(rnh);
 	}
-
- 	ln->ln_state = ND6_LLINFO_REACHABLE;
-	if (!ND6_LLINFO_PERMANENT(ln)) {
-		nd6_llinfo_settimer_locked(ln,
-		    (long)ND_IFINFO(rt->rt_ifp)->reachable * hz);
-	}
-done:
-	LLE_WUNLOCK(ln);
 }
 
-
 /*
  * Rejuvenate this function for routing operations related
  * processing.
@@ -1567,17 +1580,93 @@ nd6_ioctl(u_long cmd, caddr_t data, struct ifnet *ifp)
 }
 
 /*
+ * Calculates new isRouter value based on provided parameters and
+ * returns it.
+ */
+static int
+nd6_is_router(int type, int code, int is_new, int old_addr, int new_addr,
+    int ln_router)
+{
+
+	/*
+	 * ICMP6 type dependent behavior.
+	 *
+	 * NS: clear IsRouter if new entry
+	 * RS: clear IsRouter
+	 * RA: set IsRouter if there's lladdr
+	 * redir: clear IsRouter if new entry
+	 *
+	 * RA case, (1):
+	 * The spec says that we must set IsRouter in the following cases:
+	 * - If lladdr exist, set IsRouter.  This means (1-5).
+	 * - If it is old entry (!newentry), set IsRouter.  This means (7).
+	 * So, based on the spec, in (1-5) and (7) cases we must set IsRouter.
+	 * A quetion arises for (1) case.  (1) case has no lladdr in the
+	 * neighbor cache, this is similar to (6).
+	 * This case is rare but we figured that we MUST NOT set IsRouter.
+	 *
+	 *   is_new  old_addr new_addr 	    NS  RS  RA	redir
+	 *							D R
+	 *	0	n	n	(1)	c   ?     s
+	 *	0	y	n	(2)	c   s     s
+	 *	0	n	y	(3)	c   s     s
+	 *	0	y	y	(4)	c   s     s
+	 *	0	y	y	(5)	c   s     s
+	 *	1	--	n	(6) c	c	c s
+	 *	1	--	y	(7) c	c   s	c s
+	 *
+	 *					(c=clear s=set)
+	 */
+	switch (type & 0xff) {
+	case ND_NEIGHBOR_SOLICIT:
+		/*
+		 * New entry must have is_router flag cleared.
+		 */
+		if (is_new)					/* (6-7) */
+			ln_router = 0;
+		break;
+	case ND_REDIRECT:
+		/*
+		 * If the icmp is a redirect to a better router, always set the
+		 * is_router flag.  Otherwise, if the entry is newly created,
+		 * clear the flag.  [RFC 2461, sec 8.3]
+		 */
+		if (code == ND_REDIRECT_ROUTER)
+			ln_router = 1;
+		else {
+			if (is_new)				/* (6-7) */
+				ln_router = 0;
+		}
+		break;
+	case ND_ROUTER_SOLICIT:
+		/*
+		 * is_router flag must always be cleared.
+		 */
+		ln_router = 0;
+		break;
+	case ND_ROUTER_ADVERT:
+		/*
+		 * Mark an entry with lladdr as a router.
+		 */
+		if ((!is_new && (old_addr || new_addr)) ||	/* (2-5) */
+		    (is_new && new_addr)) {			/* (7) */
+			ln_router = 1;
+		}
+		break;
+	}
+
+	return (ln_router);
+}
+
+/*
  * Create neighbor cache entry and cache link-layer address,
  * on reception of inbound ND6 packets.  (RS/RA/NS/redirect)
  *
  * type - ICMP6 type
  * code - type dependent information
  *
- * XXXXX
- *  The caller of this function already acquired the ndp 
- *  cache table lock because the cache entry is returned.
  */
-struct llentry *
+void
 nd6_cache_lladdr(struct ifnet *ifp, struct in6_addr *from, char *lladdr,
     int lladdrlen, int type, int code)
 {
@@ -1587,11 +1676,9 @@ nd6_cache_lladdr(struct ifnet *ifp, struct in6_addr *from, char *lladdr,
 	int olladdr;
 	int llchange;
 	int flags;
-	int newstate = 0;
 	uint16_t router = 0;
 	struct sockaddr_in6 sin6;
 	struct mbuf *chain = NULL;
-	int static_route = 0;
 
 	IF_AFDATA_UNLOCK_ASSERT(ifp);
 
@@ -1600,7 +1687,7 @@ nd6_cache_lladdr(struct ifnet *ifp, struct in6_addr *from, char *lladdr,
 
 	/* nothing must be updated for unspecified address */
 	if (IN6_IS_ADDR_UNSPECIFIED(from))
-		return NULL;
+		return;
 
 	/*
 	 * Validation about ifp->if_addrlen and lladdrlen must be done in
@@ -1611,27 +1698,37 @@ nd6_cache_lladdr(struct ifnet *ifp, struct in6_addr *from, char *lladdr,
 	 * Spec says nothing in sections for RA, RS and NA.  There's small
 	 * description on it in NS section (RFC 2461 7.2.3).
 	 */
-	flags = lladdr ? ND6_EXCLUSIVE : 0;
+	flags = lladdr ? LLE_EXCLUSIVE : 0;
 	IF_AFDATA_RLOCK(ifp);
 	ln = nd6_lookup(from, flags, ifp);
 	IF_AFDATA_RUNLOCK(ifp);
 	is_newentry = 0;
 	if (ln == NULL) {
-		flags |= ND6_EXCLUSIVE;
+		flags |= LLE_EXCLUSIVE;
 		ln = nd6_alloc(from, 0, ifp);
 		if (ln == NULL)
-			return (NULL);
+			return;
+
+		/*
+		 * Since we already know all the data for the new entry,
+		 * fill it before insertion.
+		 */
+		if (lladdr != NULL) {
+			bcopy(lladdr, &ln->ll_addr, ifp->if_addrlen);
+			ln->la_flags |= LLE_VALID;
+		}
 		IF_AFDATA_WLOCK(ifp);
 		LLE_WLOCK(ln);
 		/* Prefer any existing lle over newly-created one */
-		ln_tmp = nd6_lookup(from, ND6_EXCLUSIVE, ifp);
+		ln_tmp = nd6_lookup(from, LLE_EXCLUSIVE, ifp);
 		if (ln_tmp == NULL)
 			lltable_link_entry(LLTABLE6(ifp), ln);
 		IF_AFDATA_WUNLOCK(ifp);
-		if (ln_tmp == NULL)
+		if (ln_tmp == NULL) {
 			/* No existing lle, mark as new entry */
 			is_newentry = 1;
-		else {
+			nd6_llinfo_setstate(ln, ND6_LLINFO_STALE);
+		} else {
 			lltable_free_entry(LLTABLE6(ifp), ln);
 			ln = ln_tmp;
 			ln_tmp = NULL;
@@ -1639,28 +1736,37 @@ nd6_cache_lladdr(struct ifnet *ifp, struct in6_addr *from, char *lladdr,
 	} 
 	/* do nothing if static ndp is set */
 	if ((ln->la_flags & LLE_STATIC)) {
-			static_route = 1;
-			goto done;
+		if (flags & LLE_EXCLUSIVE)
+			LLE_WUNLOCK(ln);
+		else
+			LLE_RUNLOCK(ln);
+		return;
 	}
 
 	olladdr = (ln->la_flags & LLE_VALID) ? 1 : 0;
 	if (olladdr && lladdr) {
 		llchange = bcmp(lladdr, &ln->ll_addr,
 		    ifp->if_addrlen);
-	} else
+	} else if (!olladdr && lladdr)
+		llchange = 1;
+	else
 		llchange = 0;
 
 	/*
 	 * newentry olladdr  lladdr  llchange	(*=record)
 	 *	0	n	n	--	(1)
 	 *	0	y	n	--	(2)
-	 *	0	n	y	--	(3) * STALE
+	 *	0	n	y	y	(3) * STALE
 	 *	0	y	y	n	(4) *
 	 *	0	y	y	y	(5) * STALE
 	 *	1	--	n	--	(6)   NOSTATE(= PASSIVE)
 	 *	1	--	y	--	(7) * STALE
 	 */
 
+	do_update = 0;
+	if (!is_newentry && llchange != 0)
+		do_update = 1;	/* (3,5) */
+
 	if (lladdr) {		/* (3-5) and (7) */
 		/*
 		 * Record source link-layer address
@@ -1668,115 +1774,30 @@ nd6_cache_lladdr(struct ifnet *ifp, struct in6_addr *from, char *lladdr,
 		 */
 		bcopy(lladdr, &ln->ll_addr, ifp->if_addrlen);
 		ln->la_flags |= LLE_VALID;
-		EVENTHANDLER_INVOKE(lle_event, ln, LLENTRY_RESOLVED);
-	}
-
-	if (!is_newentry) {
-		if ((!olladdr && lladdr != NULL) ||	/* (3) */
-		    (olladdr && lladdr != NULL && llchange)) {	/* (5) */
-			do_update = 1;
-			newstate = ND6_LLINFO_STALE;
-		} else					/* (1-2,4) */
-			do_update = 0;
-	} else {
-		do_update = 1;
-		if (lladdr == NULL)			/* (6) */
-			newstate = ND6_LLINFO_NOSTATE;
-		else					/* (7) */
-			newstate = ND6_LLINFO_STALE;
-	}
+		nd6_llinfo_setstate(ln, ND6_LLINFO_STALE);
 
-	if (do_update) {
-		/*
-		 * Update the state of the neighbor cache.
-		 */
-		ln->ln_state = newstate;
+		EVENTHANDLER_INVOKE(lle_event, ln, LLENTRY_RESOLVED);
 
-		if (ln->ln_state == ND6_LLINFO_STALE) {
+		if (do_update) {
 			if (ln->la_hold != NULL)
 				nd6_grab_holdchain(ln, &chain, &sin6);
-		} else if (ln->ln_state == ND6_LLINFO_INCOMPLETE) {
-			/* probe right away */
-			nd6_llinfo_settimer_locked((void *)ln, 0);
 		}
 	}
 
-	/*
-	 * ICMP6 type dependent behavior.
-	 *
-	 * NS: clear IsRouter if new entry
-	 * RS: clear IsRouter
-	 * RA: set IsRouter if there's lladdr
-	 * redir: clear IsRouter if new entry
-	 *
-	 * RA case, (1):
-	 * The spec says that we must set IsRouter in the following cases:
-	 * - If lladdr exist, set IsRouter.  This means (1-5).
-	 * - If it is old entry (!newentry), set IsRouter.  This means (7).
-	 * So, based on the spec, in (1-5) and (7) cases we must set IsRouter.
-	 * A quetion arises for (1) case.  (1) case has no lladdr in the
-	 * neighbor cache, this is similar to (6).
-	 * This case is rare but we figured that we MUST NOT set IsRouter.
-	 *
-	 * newentry olladdr  lladdr  llchange	    NS  RS  RA	redir
-	 *							D R
-	 *	0	n	n	--	(1)	c   ?     s
-	 *	0	y	n	--	(2)	c   s     s
-	 *	0	n	y	--	(3)	c   s     s
-	 *	0	y	y	n	(4)	c   s     s
-	 *	0	y	y	y	(5)	c   s     s
-	 *	1	--	n	--	(6) c	c	c s
-	 *	1	--	y	--	(7) c	c   s	c s
-	 *
-	 *					(c=clear s=set)
-	 */
-	switch (type & 0xff) {
-	case ND_NEIGHBOR_SOLICIT:
-		/*
-		 * New entry must have is_router flag cleared.
-		 */
-		if (is_newentry)	/* (6-7) */
-			ln->ln_router = 0;
-		break;
-	case ND_REDIRECT:
-		/*
-		 * If the icmp is a redirect to a better router, always set the
-		 * is_router flag.  Otherwise, if the entry is newly created,
-		 * clear the flag.  [RFC 2461, sec 8.3]
-		 */
-		if (code == ND_REDIRECT_ROUTER)
-			ln->ln_router = 1;
-		else if (is_newentry) /* (6-7) */
-			ln->ln_router = 0;
-		break;
-	case ND_ROUTER_SOLICIT:
-		/*
-		 * is_router flag must always be cleared.
-		 */
-		ln->ln_router = 0;
-		break;
-	case ND_ROUTER_ADVERT:
-		/*
-		 * Mark an entry with lladdr as a router.
-		 */
-		if ((!is_newentry && (olladdr || lladdr)) ||	/* (2-5) */
-		    (is_newentry && lladdr)) {			/* (7) */
-			ln->ln_router = 1;
-		}
-		break;
-	}
+	/* Calculates new router status */
+	router = nd6_is_router(type, code, is_newentry, olladdr,
+	    lladdr != NULL ? 1 : 0, ln->ln_router);
 
-	if (ln != NULL) {
-		static_route = (ln->la_flags & LLE_STATIC);
-		router = ln->ln_router;
+	ln->ln_router = router;
+	/* Mark non-router redirects with special flag */
+	if ((type & 0xFF) == ND_REDIRECT && code != ND_REDIRECT_ROUTER)
+		ln->la_flags |= LLE_REDIRECT;
+
+	if (flags & LLE_EXCLUSIVE)
+		LLE_WUNLOCK(ln);
+	else
+		LLE_RUNLOCK(ln);
 
-		if (flags & ND6_EXCLUSIVE)
-			LLE_WUNLOCK(ln);
-		else
-			LLE_RUNLOCK(ln);
-		if (static_route)
-			ln = NULL;
-	}
 	if (chain != NULL)
 		nd6_flush_holdchain(ifp, ifp, chain, &sin6);
 	
@@ -1795,25 +1816,13 @@ nd6_cache_lladdr(struct ifnet *ifp, struct in6_addr *from, char *lladdr,
 	 * for those are not autoconfigured hosts, we explicitly avoid such
 	 * cases for safety.
 	 */
-	if (do_update && router &&
+	if ((do_update || is_newentry) && router &&
 	    ND_IFINFO(ifp)->flags & ND6_IFF_ACCEPT_RTADV) {
 		/*
 		 * guaranteed recursion
 		 */
 		defrouter_select();
 	}
-	
-	return (ln);
-done:	
-	if (ln != NULL) {
-		if (flags & ND6_EXCLUSIVE)
-			LLE_WUNLOCK(ln);
-		else
-			LLE_RUNLOCK(ln);
-		if (static_route)
-			ln = NULL;
-	}
-	return (ln);
 }
 
 static void
@@ -1868,13 +1877,11 @@ nd6_grab_holdchain(struct llentry *ln, struct mbuf **chain,
 		 * detection on expiration.
 		 * (RFC 2461 7.3.3)
 		 */
-		ln->la_asked = 0;
-		ln->ln_state = ND6_LLINFO_DELAY;
-		nd6_llinfo_settimer_locked(ln, (long)V_nd6_delay * hz);
+		nd6_llinfo_setstate(ln, ND6_LLINFO_DELAY);
 	}
 }
 
-static int
+int
 nd6_output_ifp(struct ifnet *ifp, struct ifnet *origifp, struct mbuf *m,
     struct sockaddr_in6 *dst)
 {
@@ -1920,16 +1927,29 @@ nd6_output_ifp(struct ifnet *ifp, struct ifnet *origifp, struct mbuf *m,
 }
 
 /*
- * IPv6 packet output - light version.
- * Checks if destination LLE exists and is in proper state
- * (e.g no modification required). If not true, fall back to
- * "heavy" version.
+ * Do L2 address resolution for @sa_dst address. Stores found
+ * address in @desten buffer. Copy of lle ln_flags can be also
+ * saved in @pflags if @pflags is non-NULL.
+ *
+ * If destination LLE does not exists or lle state modification
+ * is required, call "slow" version.
+ *
+ * Return values:
+ * - 0 on success (address copied to buffer).
+ * - EWOULDBLOCK (no local error, but address is still unresolved)
+ * - other errors (alloc failure, etc)
  */
 int
-nd6_output(struct ifnet *ifp, struct ifnet *origifp, struct mbuf *m,
-    struct sockaddr_in6 *dst, struct rtentry *rt0)
+nd6_resolve(struct ifnet *ifp, int is_gw, struct mbuf *m,
+    const struct sockaddr *sa_dst, u_char *desten, uint32_t *pflags)
 {
 	struct llentry *ln = NULL;
+	const struct sockaddr_in6 *dst6;
+
+	if (pflags != NULL)
+		*pflags = 0;
+
+	dst6 = (const struct sockaddr_in6 *)sa_dst;
 
 	/* discard the packet if IPv6 operation is disabled on the interface */
 	if ((ND_IFINFO(ifp)->flags & ND6_IFF_IFDISABLED)) {
@@ -1937,20 +1957,31 @@ nd6_output(struct ifnet *ifp, struct ifnet *origifp, struct mbuf *m,
 		return (ENETDOWN); /* better error? */
 	}
 
-	if (IN6_IS_ADDR_MULTICAST(&dst->sin6_addr))
-		goto sendpkt;
-
-	if (nd6_need_cache(ifp) == 0)
-		goto sendpkt;
+	if (m != NULL && m->m_flags & M_MCAST) {
+		switch (ifp->if_type) {
+		case IFT_ETHER:
+		case IFT_FDDI:
+		case IFT_L2VLAN:
+		case IFT_IEEE80211:
+		case IFT_BRIDGE:
+		case IFT_ISO88025:
+			ETHER_MAP_IPV6_MULTICAST(&dst6->sin6_addr,
+						 desten);
+			return (0);
+		default:
+			m_freem(m);
+			return (EAFNOSUPPORT);
+		}
+	}
 
 	IF_AFDATA_RLOCK(ifp);
-	ln = nd6_lookup(&dst->sin6_addr, 0, ifp);
+	ln = nd6_lookup(&dst6->sin6_addr, 0, ifp);
 	IF_AFDATA_RUNLOCK(ifp);
 
 	/*
 	 * Perform fast path for the following cases:
 	 * 1) lle state is REACHABLE
-	 * 2) lle state is DELAY (NS message sentNS message sent)
+	 * 2) lle state is DELAY (NS message sent)
 	 *
 	 * Every other case involves lle modification, so we handle
 	 * them separately.
@@ -1960,44 +1991,34 @@ nd6_output(struct ifnet *ifp, struct ifnet *origifp, struct mbuf *m,
 		/* Fall back to slow processing path */
 		if (ln != NULL)
 			LLE_RUNLOCK(ln);
-		return (nd6_output_lle(ifp, origifp, m, dst));
+		return (nd6_resolve_slow(ifp, m, dst6, desten, pflags));
 	}
 
-sendpkt:
-	if (ln != NULL)
-		LLE_RUNLOCK(ln);
 
-	return (nd6_output_ifp(ifp, origifp, m, dst));
+	bcopy(&ln->ll_addr, desten, ifp->if_addrlen);
+	if (pflags != NULL)
+		*pflags = ln->la_flags;
+	LLE_RUNLOCK(ln);
+	return (0);
 }
 
 
 /*
- * Output IPv6 packet - heavy version.
- * Function assume that either
- * 1) destination LLE does not exist, is invalid or stale, so
- *   ND6_EXCLUSIVE lock needs to be acquired
- * 2) destination lle is provided (with ND6_EXCLUSIVE lock),
- *   in that case packets are queued in &chain.
+ * Do L2 address resolution for @sa_dst address. Stores found
+ * address in @desten buffer. Copy of lle ln_flags can be also
+ * saved in @pflags if @pflags is non-NULL.
  *
+ * Heavy version.
+ * Function assume that destination LLE does not exist,
+ * is invalid or stale, so LLE_EXCLUSIVE lock needs to be acquired.
  */
 static int
-nd6_output_lle(struct ifnet *ifp, struct ifnet *origifp, struct mbuf *m,
-    struct sockaddr_in6 *dst)
+nd6_resolve_slow(struct ifnet *ifp, struct mbuf *m,
+    const struct sockaddr_in6 *dst, u_char *desten, uint32_t *pflags)
 {
 	struct llentry *lle = NULL, *lle_tmp;
-
-	KASSERT(m != NULL, ("NULL mbuf, nothing to send"));
-	/* discard the packet if IPv6 operation is disabled on the interface */
-	if ((ND_IFINFO(ifp)->flags & ND6_IFF_IFDISABLED)) {
-		m_freem(m);
-		return (ENETDOWN); /* better error? */
-	}
-
-	if (IN6_IS_ADDR_MULTICAST(&dst->sin6_addr))
-		goto sendpkt;
-
-	if (nd6_need_cache(ifp) == 0)
-		goto sendpkt;
+	struct in6_addr *psrc, src;
+	int send_ns;
 
 	/*
 	 * Address resolution or Neighbor Unreachability Detection
@@ -2007,7 +2028,7 @@ nd6_output_lle(struct ifnet *ifp, struct ifnet *origifp, struct mbuf *m,
 	 */
 	if (lle == NULL) {
 		IF_AFDATA_RLOCK(ifp);
-		lle = nd6_lookup(&dst->sin6_addr, ND6_EXCLUSIVE, ifp);
+		lle = nd6_lookup(&dst->sin6_addr, LLE_EXCLUSIVE, ifp);
 		IF_AFDATA_RUNLOCK(ifp);
 		if ((lle == NULL) && nd6_is_addr_neighbor(dst, ifp))  {
 			/*
@@ -2025,12 +2046,11 @@ nd6_output_lle(struct ifnet *ifp, struct ifnet *origifp, struct mbuf *m,
 				m_freem(m);
 				return (ENOBUFS);
 			}
-			lle->ln_state = ND6_LLINFO_NOSTATE;
 
 			IF_AFDATA_WLOCK(ifp);
 			LLE_WLOCK(lle);
 			/* Prefer any existing entry over newly-created one */
-			lle_tmp = nd6_lookup(&dst->sin6_addr, ND6_EXCLUSIVE, ifp);
+			lle_tmp = nd6_lookup(&dst->sin6_addr, LLE_EXCLUSIVE, ifp);
 			if (lle_tmp == NULL)
 				lltable_link_entry(LLTABLE6(ifp), lle);
 			IF_AFDATA_WUNLOCK(ifp);
@@ -2042,23 +2062,18 @@ nd6_output_lle(struct ifnet *ifp, struct ifnet *origifp, struct mbuf *m,
 		}
 	} 
 	if (lle == NULL) {
-		if ((ifp->if_flags & IFF_POINTOPOINT) == 0 &&
-		    !(ND_IFINFO(ifp)->flags & ND6_IFF_PERFORMNUD)) {
+		if (!(ND_IFINFO(ifp)->flags & ND6_IFF_PERFORMNUD)) {
 			m_freem(m);
 			return (ENOBUFS);
 		}
-		goto sendpkt;	/* send anyway */
+
+		if (m != NULL)
+			m_freem(m);
+		return (ENOBUFS);
 	}
 
 	LLE_WLOCK_ASSERT(lle);
 
-	/* We don't have to do link-layer address resolution on a p2p link. */
-	if ((ifp->if_flags & IFF_POINTOPOINT) != 0 &&
-	    lle->ln_state < ND6_LLINFO_REACHABLE) {
-		lle->ln_state = ND6_LLINFO_STALE;
-		nd6_llinfo_settimer_locked(lle, (long)V_nd6_gctimer * hz);
-	}
-
 	/*
 	 * The first time we send a packet to a neighbor whose entry is
 	 * STALE, we have to change the state to DELAY and a sets a timer to
@@ -2066,19 +2081,21 @@ nd6_output_lle(struct ifnet *ifp, struct ifnet *origifp, struct mbuf *m,
 	 * neighbor unreachability detection on expiration.
 	 * (RFC 2461 7.3.3)
 	 */
-	if (lle->ln_state == ND6_LLINFO_STALE) {
-		lle->la_asked = 0;
-		lle->ln_state = ND6_LLINFO_DELAY;
-		nd6_llinfo_settimer_locked(lle, (long)V_nd6_delay * hz);
-	}
+	if (lle->ln_state == ND6_LLINFO_STALE)
+		nd6_llinfo_setstate(lle, ND6_LLINFO_DELAY);
 
 	/*
 	 * If the neighbor cache entry has a state other than INCOMPLETE
 	 * (i.e. its link-layer address is already resolved), just
 	 * send the packet.
 	 */
-	if (lle->ln_state > ND6_LLINFO_INCOMPLETE)
-		goto sendpkt;
+	if (lle->ln_state > ND6_LLINFO_INCOMPLETE) {
+		bcopy(&lle->ll_addr, desten, ifp->if_addrlen);
+		if (pflags != NULL)
+			*pflags = lle->la_flags;
+		LLE_WUNLOCK(lle);
+		return (0);
+	}
 
 	/*
 	 * There is a neighbor cache entry, but no ethernet address
@@ -2087,8 +2104,6 @@ nd6_output_lle(struct ifnet *ifp, struct ifnet *origifp, struct mbuf *m,
 	 * does not exceed nd6_maxqueuelen.  When it exceeds nd6_maxqueuelen,
 	 * the oldest packet in the queue will be removed.
 	 */
-	if (lle->ln_state == ND6_LLINFO_NOSTATE)
-		lle->ln_state = ND6_LLINFO_INCOMPLETE;
 
 	if (lle->la_hold != NULL) {
 		struct mbuf *m_hold;
@@ -2115,28 +2130,24 @@ nd6_output_lle(struct ifnet *ifp, struct ifnet *origifp, struct mbuf *m,
 	/*
 	 * If there has been no NS for the neighbor after entering the
 	 * INCOMPLETE state, send the first solicitation.
+	 * Note that for newly-created lle la_asked will be 0,
+	 * so we will transition from ND6_LLINFO_NOSTATE to
+	 * ND6_LLINFO_INCOMPLETE state here.
 	 */
-	if (!ND6_LLINFO_PERMANENT(lle) && lle->la_asked == 0) {
-		struct in6_addr src, *psrc;
+	psrc = NULL;
+	send_ns = 0;
+	if (lle->la_asked == 0) {
 		lle->la_asked++;
-		
-		nd6_llinfo_settimer_locked(lle,
-		    (long)ND_IFINFO(ifp)->retrans * hz / 1000);
+		send_ns = 1;
 		psrc = nd6_llinfo_get_holdsrc(lle, &src);
-		LLE_WUNLOCK(lle);
-		nd6_ns_output(ifp, psrc, NULL, &dst->sin6_addr, NULL);
-	} else {
-		/* We did the lookup so we need to do the unlock here. */
-		LLE_WUNLOCK(lle);
-	}
 
-	return (0);
-
-  sendpkt:
-	if (lle != NULL)
-		LLE_WUNLOCK(lle);
+		nd6_llinfo_setstate(lle, ND6_LLINFO_INCOMPLETE);
+	}
+	LLE_WUNLOCK(lle);
+	if (send_ns != 0)
+		nd6_ns_output(ifp, psrc, NULL, &dst->sin6_addr, NULL);
 
-	return (nd6_output_ifp(ifp, origifp, m, dst));
+	return (EWOULDBLOCK);
 }
 
 
@@ -2162,15 +2173,12 @@ nd6_flush_holdchain(struct ifnet *ifp, struct ifnet *origifp, struct mbuf *chain
 
 	/*
 	 * XXX
-	 * note that intermediate errors are blindly ignored - but this is 
-	 * the same convention as used with nd6_output when called by
-	 * nd6_cache_lladdr
+	 * note that intermediate errors are blindly ignored
 	 */
 	return (error);
 }	
 
-
-int
+static int
 nd6_need_cache(struct ifnet *ifp)
 {
 	/*
@@ -2245,78 +2253,26 @@ nd6_add_ifa_lle(struct in6_ifaddr *ia)
 }
 
 /*
- * Removes ALL lle records for interface address prefix.
- * XXXME: That's probably not we really want to do, we need
- * to remove address record only and keep other records
- * until we determine if given prefix is really going 
- * to be removed.
+ * Removes either all lle entries for given @ia, or lle
+ * corresponding to @ia address.
  */
 void
-nd6_rem_ifa_lle(struct in6_ifaddr *ia)
+nd6_rem_ifa_lle(struct in6_ifaddr *ia, int all)
 {
 	struct sockaddr_in6 mask, addr;
+	struct sockaddr *saddr, *smask;
 	struct ifnet *ifp;
 
 	ifp = ia->ia_ifa.ifa_ifp;
 	memcpy(&addr, &ia->ia_addr, sizeof(ia->ia_addr));
 	memcpy(&mask, &ia->ia_prefixmask, sizeof(ia->ia_prefixmask));
-	lltable_prefix_free(AF_INET6, (struct sockaddr *)&addr,
-	            (struct sockaddr *)&mask, LLE_STATIC);
-}
-
-/*
- * the callers of this function need to be re-worked to drop
- * the lle lock, drop here for now
- */
-int
-nd6_storelladdr(struct ifnet *ifp, struct mbuf *m,
-    const struct sockaddr *dst, u_char *desten, uint32_t *pflags)
-{
-	struct llentry *ln;
-
-	if (pflags != NULL)
-		*pflags = 0;
-	IF_AFDATA_UNLOCK_ASSERT(ifp);
-	if (m != NULL && m->m_flags & M_MCAST) {
-		switch (ifp->if_type) {
-		case IFT_ETHER:
-		case IFT_FDDI:
-		case IFT_L2VLAN:
-		case IFT_IEEE80211:
-		case IFT_BRIDGE:
-		case IFT_ISO88025:
-			ETHER_MAP_IPV6_MULTICAST(&SIN6(dst)->sin6_addr,
-						 desten);
-			return (0);
-		default:
-			m_freem(m);
-			return (EAFNOSUPPORT);
-		}
-	}
-
-
-	/*
-	 * the entry should have been created in nd6_store_lladdr
-	 */
-	IF_AFDATA_RLOCK(ifp);
-	ln = lla_lookup(LLTABLE6(ifp), 0, dst);
-	IF_AFDATA_RUNLOCK(ifp);
-	if ((ln == NULL) || !(ln->la_flags & LLE_VALID)) {
-		if (ln != NULL)
-			LLE_RUNLOCK(ln);
-		/* this could happen, if we could not allocate memory */
-		m_freem(m);
-		return (1);
-	}
+	saddr = (struct sockaddr *)&addr;
+	smask = (struct sockaddr *)&mask;
 
-	bcopy(&ln->ll_addr, desten, ifp->if_addrlen);
-	if (pflags != NULL)
-		*pflags = ln->la_flags;
-	LLE_RUNLOCK(ln);
-	/*
-	 * A *small* use after free race exists here
-	 */
-	return (0);
+	if (all != 0)
+		lltable_prefix_free(AF_INET6, saddr, smask, LLE_STATIC);
+	else
+		lltable_delete_addr(LLTABLE6(ifp), LLE_IFADDR, saddr);
 }
 
 static void 
diff --git a/sys/netinet6/nd6.h b/sys/netinet6/nd6.h
index f4f7f650cf0bc..c5f0eea281c67 100644
--- a/sys/netinet6/nd6.h
+++ b/sys/netinet6/nd6.h
@@ -89,8 +89,6 @@ struct nd_ifinfo {
 #define ND6_IFF_NO_PREFER_IFACE	0x80 /* XXX: not related to ND. */
 #define ND6_IFF_NO_DAD		0x100
 
-#define	ND6_EXCLUSIVE		LLE_EXCLUSIVE
-
 #ifdef _KERNEL
 #define ND_IFINFO(ifp) \
 	(((struct in6_ifextra *)(ifp)->if_afdata[AF_INET6])->nd_ifinfo)
@@ -402,34 +400,31 @@ void nd6_destroy(void);
 #endif
 struct nd_ifinfo *nd6_ifattach(struct ifnet *);
 void nd6_ifdetach(struct nd_ifinfo *);
-int nd6_is_addr_neighbor(struct sockaddr_in6 *, struct ifnet *);
+int nd6_is_addr_neighbor(const struct sockaddr_in6 *, struct ifnet *);
 void nd6_option_init(void *, int, union nd_opts *);
 struct nd_opt_hdr *nd6_option(union nd_opts *);
 int nd6_options(union nd_opts *);
-struct llentry *nd6_lookup(struct in6_addr *, int, struct ifnet *);
-struct llentry *nd6_alloc(struct in6_addr *, int, struct ifnet *);
+struct llentry *nd6_lookup(const struct in6_addr *, int, struct ifnet *);
+struct llentry *nd6_alloc(const struct in6_addr *, int, struct ifnet *);
 void nd6_setmtu(struct ifnet *);
+void nd6_llinfo_setstate(struct llentry *lle, int newstate);
 void nd6_llinfo_settimer(struct llentry *, long);
 void nd6_llinfo_settimer_locked(struct llentry *, long);
 void nd6_timer(void *);
 void nd6_purge(struct ifnet *);
-void nd6_nud_hint(struct rtentry *, struct in6_addr *, int);
-int nd6_resolve(struct ifnet *, struct rtentry *, struct mbuf *,
-	struct sockaddr *, u_char *);
+int nd6_resolve(struct ifnet *, int, struct mbuf *,
+    const struct sockaddr *, u_char *, uint32_t *);
 int nd6_ioctl(u_long, caddr_t, struct ifnet *);
-struct llentry *nd6_cache_lladdr(struct ifnet *, struct in6_addr *,
+void nd6_cache_lladdr(struct ifnet *, struct in6_addr *,
 	char *, int, int, int);
-int nd6_output(struct ifnet *, struct ifnet *, struct mbuf *,
-	struct sockaddr_in6 *, struct rtentry *);
 void nd6_grab_holdchain(struct llentry *, struct mbuf **,
     struct sockaddr_in6 *);
 int nd6_flush_holdchain(struct ifnet *, struct ifnet *, struct mbuf *,
     struct sockaddr_in6 *);
-int nd6_need_cache(struct ifnet *);
 int nd6_add_ifa_lle(struct in6_ifaddr *);
-void nd6_rem_ifa_lle(struct in6_ifaddr *);
-int nd6_storelladdr(struct ifnet *, struct mbuf *,
-	const struct sockaddr *, u_char *, uint32_t *); 
+void nd6_rem_ifa_lle(struct in6_ifaddr *, int);
+int nd6_output_ifp(struct ifnet *, struct ifnet *, struct mbuf *,
+    struct sockaddr_in6 *);
 
 /* nd6_nbr.c */
 void nd6_na_input(struct mbuf *, int, int);
diff --git a/sys/netinet6/nd6_nbr.c b/sys/netinet6/nd6_nbr.c
index 107fba241cc7e..7c3c87ca30b62 100644
--- a/sys/netinet6/nd6_nbr.c
+++ b/sys/netinet6/nd6_nbr.c
@@ -768,17 +768,10 @@ nd6_na_input(struct mbuf *m, int off, int icmp6len)
 		bcopy(lladdr, &ln->ll_addr, ifp->if_addrlen);
 		ln->la_flags |= LLE_VALID;
 		EVENTHANDLER_INVOKE(lle_event, ln, LLENTRY_RESOLVED);
-		if (is_solicited) {
-			ln->ln_state = ND6_LLINFO_REACHABLE;
-			ln->ln_byhint = 0;
-			if (!ND6_LLINFO_PERMANENT(ln)) {
-				nd6_llinfo_settimer_locked(ln,
-				    (long)ND_IFINFO(ln->lle_tbl->llt_ifp)->reachable * hz);
-			}
-		} else {
-			ln->ln_state = ND6_LLINFO_STALE;
-			nd6_llinfo_settimer_locked(ln, (long)V_nd6_gctimer * hz);
-		}
+		if (is_solicited)
+			nd6_llinfo_setstate(ln, ND6_LLINFO_REACHABLE);
+		else
+			nd6_llinfo_setstate(ln, ND6_LLINFO_STALE);
 		if ((ln->ln_router = is_router) != 0) {
 			/*
 			 * This means a router's state has changed from
@@ -829,10 +822,8 @@ nd6_na_input(struct mbuf *m, int off, int icmp6len)
 			 * If state is REACHABLE, make it STALE.
 			 * no other updates should be done.
 			 */
-			if (ln->ln_state == ND6_LLINFO_REACHABLE) {
-				ln->ln_state = ND6_LLINFO_STALE;
-				nd6_llinfo_settimer_locked(ln, (long)V_nd6_gctimer * hz);
-			}
+			if (ln->ln_state == ND6_LLINFO_REACHABLE)
+				nd6_llinfo_setstate(ln, ND6_LLINFO_STALE);
 			goto freeit;
 		} else if (is_override				   /* (2a) */
 			|| (!is_override && (lladdr != NULL && !llchange)) /* (2b) */
@@ -852,19 +843,11 @@ nd6_na_input(struct mbuf *m, int off, int icmp6len)
 			 * If not solicited and the link-layer address was
 			 * changed, make it STALE.
 			 */
-			if (is_solicited) {
-				ln->ln_state = ND6_LLINFO_REACHABLE;
-				ln->ln_byhint = 0;
-				if (!ND6_LLINFO_PERMANENT(ln)) {
-					nd6_llinfo_settimer_locked(ln,
-					    (long)ND_IFINFO(ifp)->reachable * hz);
-				}
-			} else {
-				if (lladdr != NULL && llchange) {
-					ln->ln_state = ND6_LLINFO_STALE;
-					nd6_llinfo_settimer_locked(ln,
-					    (long)V_nd6_gctimer * hz);
-				}
+			if (is_solicited)
+				nd6_llinfo_setstate(ln, ND6_LLINFO_REACHABLE);
+			else {
+				if (lladdr != NULL && llchange)
+					nd6_llinfo_setstate(ln, ND6_LLINFO_STALE);
 			}
 		}
 
diff --git a/sys/netinet6/nd6_rtr.c b/sys/netinet6/nd6_rtr.c
index aa0ff477375cf..cbcca46624cf9 100644
--- a/sys/netinet6/nd6_rtr.c
+++ b/sys/netinet6/nd6_rtr.c
@@ -2105,7 +2105,7 @@ rt6_deleteroute(struct rtentry *rt, void *arg)
 		return (0);
 
 	return (in6_rtrequest(RTM_DELETE, rt_key(rt), rt->rt_gateway,
-	    rt_mask(rt), rt->rt_flags, NULL, rt->rt_fibnum));
+	    rt_mask(rt), rt->rt_flags | RTF_RNH_LOCKED, NULL, rt->rt_fibnum));
 #undef SIN6
 }
 
diff --git a/sys/netinet6/udp6_usrreq.c b/sys/netinet6/udp6_usrreq.c
index 98790a8bcba71..375c00d0a3e24 100644
--- a/sys/netinet6/udp6_usrreq.c
+++ b/sys/netinet6/udp6_usrreq.c
@@ -233,7 +233,7 @@ udp6_input(struct mbuf **mp, int *offp, int proto)
 	plen = ntohs(ip6->ip6_plen) - off + sizeof(*ip6);
 	ulen = ntohs((u_short)uh->uh_ulen);
 
-	nxt = ip6->ip6_nxt;
+	nxt = proto;
 	cscov_partial = (nxt == IPPROTO_UDPLITE) ? 1 : 0;
 	if (nxt == IPPROTO_UDPLITE) {
 		/* Zero means checksum over the complete packet. */
@@ -668,9 +668,11 @@ udp6_output(struct inpcb *inp, struct mbuf *m, struct sockaddr *addr6,
 			return (error);
 	}
 
+	nxt = (inp->inp_socket->so_proto->pr_protocol == IPPROTO_UDP) ?
+	    IPPROTO_UDP : IPPROTO_UDPLITE;
 	if (control) {
 		if ((error = ip6_setpktopts(control, &opt,
-		    inp->in6p_outputopts, td->td_ucred, IPPROTO_UDP)) != 0)
+		    inp->in6p_outputopts, td->td_ucred, nxt)) != 0)
 			goto release;
 		optp = &opt;
 	} else
@@ -786,7 +788,7 @@ udp6_output(struct inpcb *inp, struct mbuf *m, struct sockaddr *addr6,
 	 * for UDP and IP6 headers.
 	 */
 	M_PREPEND(m, hlen + sizeof(struct udphdr), M_NOWAIT);
-	if (m == 0) {
+	if (m == NULL) {
 		error = ENOBUFS;
 		goto release;
 	}
@@ -794,8 +796,6 @@ udp6_output(struct inpcb *inp, struct mbuf *m, struct sockaddr *addr6,
 	/*
 	 * Stuff checksum and output datagram.
 	 */
-	nxt = (inp->inp_socket->so_proto->pr_protocol == IPPROTO_UDP) ?
-	    IPPROTO_UDP : IPPROTO_UDPLITE;
 	udp6 = (struct udphdr *)(mtod(m, caddr_t) + hlen);
 	udp6->uh_sport = inp->inp_lport; /* lport is always set in the PCB */
 	udp6->uh_dport = fport;
@@ -912,17 +912,21 @@ udp6_abort(struct socket *so)
 	inp = sotoinpcb(so);
 	KASSERT(inp != NULL, ("udp6_abort: inp == NULL"));
 
+	INP_WLOCK(inp);
 #ifdef INET
 	if (inp->inp_vflag & INP_IPV4) {
 		struct pr_usrreqs *pru;
+		uint8_t nxt;
 
-		pru = inetsw[ip_protox[IPPROTO_UDP]].pr_usrreqs;
+		nxt = (inp->inp_socket->so_proto->pr_protocol == IPPROTO_UDP) ?
+		    IPPROTO_UDP : IPPROTO_UDPLITE;
+		INP_WUNLOCK(inp);
+		pru = inetsw[ip_protox[nxt]].pr_usrreqs;
 		(*pru->pru_abort)(so);
 		return;
 	}
 #endif
 
-	INP_WLOCK(inp);
 	if (!IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr)) {
 		INP_HASH_WLOCK(pcbinfo);
 		in6_pcbdisconnect(inp);
@@ -1036,16 +1040,20 @@ udp6_close(struct socket *so)
 	inp = sotoinpcb(so);
 	KASSERT(inp != NULL, ("udp6_close: inp == NULL"));
 
+	INP_WLOCK(inp);
 #ifdef INET
 	if (inp->inp_vflag & INP_IPV4) {
 		struct pr_usrreqs *pru;
+		uint8_t nxt;
 
-		pru = inetsw[ip_protox[IPPROTO_UDP]].pr_usrreqs;
+		nxt = (inp->inp_socket->so_proto->pr_protocol == IPPROTO_UDP) ?
+		    IPPROTO_UDP : IPPROTO_UDPLITE;
+		INP_WUNLOCK(inp);
+		pru = inetsw[ip_protox[nxt]].pr_usrreqs;
 		(*pru->pru_disconnect)(so);
 		return;
 	}
 #endif
-	INP_WLOCK(inp);
 	if (!IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr)) {
 		INP_HASH_WLOCK(pcbinfo);
 		in6_pcbdisconnect(inp);
@@ -1151,18 +1159,21 @@ udp6_disconnect(struct socket *so)
 	inp = sotoinpcb(so);
 	KASSERT(inp != NULL, ("udp6_disconnect: inp == NULL"));
 
+	INP_WLOCK(inp);
 #ifdef INET
 	if (inp->inp_vflag & INP_IPV4) {
 		struct pr_usrreqs *pru;
+		uint8_t nxt;
 
-		pru = inetsw[ip_protox[IPPROTO_UDP]].pr_usrreqs;
+		nxt = (inp->inp_socket->so_proto->pr_protocol == IPPROTO_UDP) ?
+		    IPPROTO_UDP : IPPROTO_UDPLITE;
+		INP_WUNLOCK(inp);
+		pru = inetsw[ip_protox[nxt]].pr_usrreqs;
 		(void)(*pru->pru_disconnect)(so);
 		return (0);
 	}
 #endif
 
-	INP_WLOCK(inp);
-
 	if (IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr)) {
 		error = ENOTCONN;
 		goto out;
@@ -1218,7 +1229,10 @@ udp6_send(struct socket *so, int flags, struct mbuf *m,
 		}
 		if (hasv4addr) {
 			struct pr_usrreqs *pru;
+			uint8_t nxt;
 
+			nxt = (inp->inp_socket->so_proto->pr_protocol ==
+			    IPPROTO_UDP) ? IPPROTO_UDP : IPPROTO_UDPLITE;
 			/*
 			 * XXXRW: We release UDP-layer locks before calling
 			 * udp_send() in order to avoid recursion.  However,
@@ -1230,7 +1244,7 @@ udp6_send(struct socket *so, int flags, struct mbuf *m,
 			INP_WUNLOCK(inp);
 			if (sin6)
 				in6_sin6_2_sin_in_sock(addr);
-			pru = inetsw[ip_protox[IPPROTO_UDP]].pr_usrreqs;
+			pru = inetsw[ip_protox[nxt]].pr_usrreqs;
 			/* addr will just be freed in sendit(). */
 			return ((*pru->pru_send)(so, flags, m, addr, control,
 			    td));
diff --git a/sys/netipsec/ipsec_output.c b/sys/netipsec/ipsec_output.c
index a6611a7e7fcb1..f5cdf5a417b62 100644
--- a/sys/netipsec/ipsec_output.c
+++ b/sys/netipsec/ipsec_output.c
@@ -166,10 +166,6 @@ ipsec_process_done(struct mbuf *m, struct ipsecrequest *isr)
 	 * If this is a problem we'll need to introduce a queue
 	 * to set the packet on so we can unwind the stack before
 	 * doing further processing.
-	 *
-	 * If ipsec[46]_process_packet() will successfully queue
-	 * the request, we need to take additional reference to SP,
-	 * because xform callback will release reference.
 	 */
 	if (isr->next) {
 		/* XXX-BZ currently only support same AF bundles. */
@@ -177,11 +173,7 @@ ipsec_process_done(struct mbuf *m, struct ipsecrequest *isr)
 #ifdef INET
 		case AF_INET:
 			IPSECSTAT_INC(ips_out_bundlesa);
-			key_addref(isr->sp);
-			error = ipsec4_process_packet(m, isr->next);
-			if (error != 0)
-				KEY_FREESP(&isr->sp);
-			return (error);
+			return (ipsec4_process_packet(m, isr->next));
 			/* NOTREACHED */
 #endif
 #ifdef notyet
@@ -189,11 +181,7 @@ ipsec_process_done(struct mbuf *m, struct ipsecrequest *isr)
 		case AF_INET6:
 			/* XXX */
 			IPSEC6STAT_INC(ips_out_bundlesa);
-			key_addref(isr->sp);
-			error = ipsec6_process_packet(m, isr->next);
-			if (error != 0)
-				KEY_FREESP(&isr->sp);
-			return (error);
+			return (ipsec6_process_packet(m, isr->next));
 			/* NOTREACHED */
 #endif /* INET6 */
 #endif
diff --git a/sys/netipsec/xform_ah.c b/sys/netipsec/xform_ah.c
index 350a735a89b3c..0d39eeba49ad9 100644
--- a/sys/netipsec/xform_ah.c
+++ b/sys/netipsec/xform_ah.c
@@ -1068,6 +1068,7 @@ ah_output(struct mbuf *m, struct ipsecrequest *isr, struct mbuf **mp,
 	crp->crp_opaque = (caddr_t) tc;
 
 	/* These are passed as-is to the callback. */
+	key_addref(isr->sp);
 	tc->tc_isr = isr;
 	KEY_ADDREFSA(sav);
 	tc->tc_sav = sav;
diff --git a/sys/netipsec/xform_esp.c b/sys/netipsec/xform_esp.c
index a48c0386b9afb..9f645c0c4f55b 100644
--- a/sys/netipsec/xform_esp.c
+++ b/sys/netipsec/xform_esp.c
@@ -874,6 +874,7 @@ esp_output(struct mbuf *m, struct ipsecrequest *isr, struct mbuf **mp,
 	}
 
 	/* Callback parameters */
+	key_addref(isr->sp);
 	tc->tc_isr = isr;
 	KEY_ADDREFSA(sav);
 	tc->tc_sav = sav;
diff --git a/sys/netipsec/xform_ipcomp.c b/sys/netipsec/xform_ipcomp.c
index ef460cdc90dc1..122fc72613785 100644
--- a/sys/netipsec/xform_ipcomp.c
+++ b/sys/netipsec/xform_ipcomp.c
@@ -449,6 +449,7 @@ ipcomp_output(struct mbuf *m, struct ipsecrequest *isr, struct mbuf **mp,
 		goto bad;
 	}
 
+	key_addref(isr->sp);
 	tc->tc_isr = isr;
 	KEY_ADDREFSA(sav);
 	tc->tc_sav = sav;
diff --git a/sys/netpfil/pf/pf.c b/sys/netpfil/pf/pf.c
index 2afd77fbb7bfa..1f6d5a2416588 100644
--- a/sys/netpfil/pf/pf.c
+++ b/sys/netpfil/pf/pf.c
@@ -5534,7 +5534,7 @@ pf_route6(struct mbuf **m, struct pf_rule *r, int dir, struct ifnet *oifp,
 	if (IN6_IS_SCOPE_EMBED(&dst.sin6_addr))
 		dst.sin6_addr.s6_addr16[1] = htons(ifp->if_index);
 	if ((u_long)m0->m_pkthdr.len <= ifp->if_mtu)
-		nd6_output(ifp, ifp, m0, &dst, NULL);
+		nd6_output_ifp(ifp, ifp, m0, &dst);
 	else {
 		in6_ifstat_inc(ifp, ifs6_in_toobig);
 		if (r->rt != PF_DUPTO)
diff --git a/sys/nfs/nfs_diskless.c b/sys/nfs/nfs_diskless.c
index 3882007dff0f1..864aae0bc8ee6 100644
--- a/sys/nfs/nfs_diskless.c
+++ b/sys/nfs/nfs_diskless.c
@@ -57,6 +57,8 @@ __FBSDID("$FreeBSD$");
 #include <nfsclient/nfs.h>
 #include <nfs/nfsdiskless.h>
 
+#define	NFS_IFACE_TIMEOUT_SECS	10 /* Timeout for interface to appear. */
+
 static int inaddr_to_sockaddr(char *ev, struct sockaddr_in *sa);
 static int hwaddr_to_sockaddr(char *ev, struct sockaddr_dl *sa);
 static int decode_nfshandle(char *ev, u_char *fh, int maxfh);
@@ -170,6 +172,7 @@ nfs_setup_diskless(void)
 	char *cp;
 	int cnt, fhlen, is_nfsv3;
 	uint32_t len;
+	time_t timeout_at;
 
 	if (nfs_diskless_valid != 0)
 		return;
@@ -214,6 +217,8 @@ nfs_setup_diskless(void)
 		return;
 	}
 	ifa = NULL;
+	timeout_at = time_uptime + NFS_IFACE_TIMEOUT_SECS;
+retry:
 	CURVNET_SET(TD_TO_VNET(curthread));
 	IFNET_RLOCK();
 	TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
@@ -234,6 +239,10 @@ nfs_setup_diskless(void)
 	}
 	IFNET_RUNLOCK();
 	CURVNET_RESTORE();
+	if (time_uptime < timeout_at) {
+		pause("nfssdl", hz / 5);
+		goto retry;
+	}
 	printf("nfs_diskless: no interface\n");
 	return;	/* no matching interface */
 match_done:
diff --git a/sys/ofed/drivers/infiniband/core/addr.c b/sys/ofed/drivers/infiniband/core/addr.c
index e85b5546858d3..2a918cb38ade0 100644
--- a/sys/ofed/drivers/infiniband/core/addr.c
+++ b/sys/ofed/drivers/infiniband/core/addr.c
@@ -332,7 +332,7 @@ mcast:
 #endif
 #ifdef INET6
 	case AF_INET6:
-		error = nd6_storelladdr(ifp, NULL, dst_in, (u_char *)edst, NULL);
+		error = nd6_resolve(ifp, is_gw, NULL, dst_in, edst, NULL);
 		break;
 #endif
 	default:
diff --git a/sys/ofed/drivers/infiniband/core/mad.c b/sys/ofed/drivers/infiniband/core/mad.c
index 3eedca1d6b6ae..a78dd3ac10fc3 100644
--- a/sys/ofed/drivers/infiniband/core/mad.c
+++ b/sys/ofed/drivers/infiniband/core/mad.c
@@ -1053,7 +1053,7 @@ static void unregister_mad_agent(struct ib_mad_agent_private *mad_agent_priv)
 	 */
 	cancel_mads(mad_agent_priv);
 	port_priv = mad_agent_priv->qp_info->port_priv;
-	cancel_delayed_work(&mad_agent_priv->timed_work);
+	cancel_delayed_work_sync(&mad_agent_priv->timed_work);
 
 	spin_lock_irqsave(&port_priv->reg_lock, flags);
 	remove_mad_reg_req(mad_agent_priv);
diff --git a/sys/ofed/drivers/infiniband/ulp/ipoib/ipoib_main.c b/sys/ofed/drivers/infiniband/ulp/ipoib/ipoib_main.c
index 35e16417a5f68..53ac66d304d7f 100644
--- a/sys/ofed/drivers/infiniband/ulp/ipoib/ipoib_main.c
+++ b/sys/ofed/drivers/infiniband/ulp/ipoib/ipoib_main.c
@@ -1333,7 +1333,7 @@ ipoib_output(struct ifnet *ifp, struct mbuf *m,
 		else if (m->m_flags & M_MCAST)
 			ipv6_ib_mc_map(&((struct sockaddr_in6 *)dst)->sin6_addr, ifp->if_broadcastaddr, edst);
 		else
-			error = nd6_storelladdr(ifp, m, dst, (u_char *)edst, NULL);
+			error = nd6_resolve(ifp, is_gw, m, dst, edst, NULL);
 		if (error)
 			return error;
 		type = htons(ETHERTYPE_IPV6);
diff --git a/sys/powerpc/ofw/ofw_pci.c b/sys/powerpc/ofw/ofw_pci.c
index 8543035d91521..d9ea5c05d0a03 100644
--- a/sys/powerpc/ofw/ofw_pci.c
+++ b/sys/powerpc/ofw/ofw_pci.c
@@ -240,7 +240,7 @@ ofw_pci_attach(device_t dev)
 			return (error);
 	}
 
-	device_add_child(dev, "pci", device_get_unit(dev));
+	device_add_child(dev, "pci", -1);
 	return (bus_generic_attach(dev));
 }
 
diff --git a/sys/powerpc/powerpc/elf32_machdep.c b/sys/powerpc/powerpc/elf32_machdep.c
index bdcd9babb8c5f..bfe23ac763e6a 100644
--- a/sys/powerpc/powerpc/elf32_machdep.c
+++ b/sys/powerpc/powerpc/elf32_machdep.c
@@ -183,6 +183,7 @@ elf_reloc_internal(linker_file_t lf, Elf_Addr relocbase, const void *data,
 	Elf_Addr addend;
 	Elf_Word rtype, symidx;
 	const Elf_Rela *rela;
+	int error;
 
 	switch (type) {
 	case ELF_RELOC_REL:
@@ -202,19 +203,19 @@ elf_reloc_internal(linker_file_t lf, Elf_Addr relocbase, const void *data,
 
 	switch (rtype) {
 
-       	case R_PPC_NONE:
-	       	break;
+	case R_PPC_NONE:
+		break;
 
 	case R_PPC_ADDR32: /* word32 S + A */
-       		addr = lookup(lf, symidx, 1);
-	       	if (addr == 0)
-	       		return -1;
+		error = lookup(lf, symidx, 1, &addr);
+		if (error != 0)
+			return -1;
 		*where = elf_relocaddr(lf, addr + addend);
-	       	break;
+			break;
 
-       	case R_PPC_ADDR16_LO: /* #lo(S) */
-		addr = lookup(lf, symidx, 1);
-		if (addr == 0)
+	case R_PPC_ADDR16_LO: /* #lo(S) */
+		error = lookup(lf, symidx, 1, &addr);
+		if (error != 0)
 			return -1;
 		/*
 		 * addend values are sometimes relative to sections
@@ -228,8 +229,8 @@ elf_reloc_internal(linker_file_t lf, Elf_Addr relocbase, const void *data,
 		break;
 
 	case R_PPC_ADDR16_HA: /* #ha(S) */
-		addr = lookup(lf, symidx, 1);
-		if (addr == 0)
+		error = lookup(lf, symidx, 1, &addr);
+		if (error != 0)
 			return -1;
 		/*
 		 * addend values are sometimes relative to sections
@@ -239,17 +240,17 @@ elf_reloc_internal(linker_file_t lf, Elf_Addr relocbase, const void *data,
 		if (addr > relocbase && addr <= (relocbase + addend))
 			addr = relocbase;
 		addr = elf_relocaddr(lf, addr + addend);
-	       	*hwhere = ((addr >> 16) + ((addr & 0x8000) ? 1 : 0))
+		*hwhere = ((addr >> 16) + ((addr & 0x8000) ? 1 : 0))
 		    & 0xffff;
 		break;
 
 	case R_PPC_RELATIVE: /* word32 B + A */
-       		*where = elf_relocaddr(lf, relocbase + addend);
-	       	break;
+		*where = elf_relocaddr(lf, relocbase + addend);
+		break;
 
 	default:
-       		printf("kldload: unexpected relocation type %d\n",
-	       	    (int) rtype);
+		printf("kldload: unexpected relocation type %d\n",
+		    (int) rtype);
 		return -1;
 	}
 	return(0);
diff --git a/sys/powerpc/powerpc/elf64_machdep.c b/sys/powerpc/powerpc/elf64_machdep.c
index 47fbbff2febd0..e311d0b600f99 100644
--- a/sys/powerpc/powerpc/elf64_machdep.c
+++ b/sys/powerpc/powerpc/elf64_machdep.c
@@ -154,6 +154,7 @@ elf_reloc_internal(linker_file_t lf, Elf_Addr relocbase, const void *data,
 	Elf_Addr addend;
 	Elf_Word rtype, symidx;
 	const Elf_Rela *rela;
+	int error;
 
 	switch (type) {
 	case ELF_RELOC_REL:
@@ -172,30 +173,30 @@ elf_reloc_internal(linker_file_t lf, Elf_Addr relocbase, const void *data,
 
 	switch (rtype) {
 
-       	case R_PPC_NONE:
-	       	break;
+	case R_PPC_NONE:
+		break;
 
 	case R_PPC64_ADDR64:	/* doubleword64 S + A */
-       		addr = lookup(lf, symidx, 1);
-	       	if (addr == 0)
-	       		return -1;
+		error = lookup(lf, symidx, 1, &addr);
+		if (error != 0)
+			return -1;
 		addr += addend;
-	       	*where = addr;
-	       	break;
+		*where = addr;
+		break;
 
 	case R_PPC_RELATIVE:	/* doubleword64 B + A */
-       		*where = elf_relocaddr(lf, relocbase + addend);
-	       	break;
+		*where = elf_relocaddr(lf, relocbase + addend);
+		break;
 
 	case R_PPC_JMP_SLOT:	/* function descriptor copy */
-		addr = lookup(lf, symidx, 1);
+		lookup(lf, symidx, 1, &addr);
 		memcpy(where, (Elf_Addr *)addr, 3*sizeof(Elf_Addr));
 		__asm __volatile("dcbst 0,%0; sync" :: "r"(where) : "memory");
 		break;
 
 	default:
-       		printf("kldload: unexpected relocation type %d\n",
-	       	    (int) rtype);
+		printf("kldload: unexpected relocation type %d\n",
+		    (int) rtype);
 		return -1;
 	}
 	return(0);
diff --git a/sys/rpc/rpcsec_gss/svc_rpcsec_gss.c b/sys/rpc/rpcsec_gss/svc_rpcsec_gss.c
index 64e691e30e5d2..b3a920aafaf09 100644
--- a/sys/rpc/rpcsec_gss/svc_rpcsec_gss.c
+++ b/sys/rpc/rpcsec_gss/svc_rpcsec_gss.c
@@ -121,9 +121,6 @@ enum svc_rpc_gss_client_state {
 };
 
 #define SVC_RPC_GSS_SEQWINDOW	128
-#ifndef RPCAUTH_UNIXGIDS
-#define RPCAUTH_UNIXGIDS	16
-#endif
 
 struct svc_rpc_gss_clientid {
 	unsigned long		ci_hostid;
@@ -150,7 +147,7 @@ struct svc_rpc_gss_client {
 	int			cl_rpcflavor;	/* RPC pseudo sec flavor */
 	bool_t			cl_done_callback; /* TRUE after call */
 	void			*cl_cookie;	/* user cookie from callback */
-	gid_t			cl_gid_storage[RPCAUTH_UNIXGIDS];
+	gid_t			cl_gid_storage[NGROUPS];
 	gss_OID			cl_mech;	/* mechanism */
 	gss_qop_t		cl_qop;		/* quality of protection */
 	uint32_t		cl_seqlast;	/* sequence window origin */
@@ -776,7 +773,7 @@ svc_rpc_gss_build_ucred(struct svc_rpc_gss_client *client,
 	uc->gid = 65534;
 	uc->gidlist = client->cl_gid_storage;
 
-	numgroups = RPCAUTH_UNIXGIDS;
+	numgroups = NGROUPS;
 	maj_stat = gss_pname_to_unix_cred(&min_stat, name, client->cl_mech,
 	    &uc->uid, &uc->gid, &numgroups, &uc->gidlist[0]);
 	if (GSS_ERROR(maj_stat))
diff --git a/sys/sparc64/ebus/ebus.c b/sys/sparc64/ebus/ebus.c
index 93ad3422a0620..0967c5d7d8af1 100644
--- a/sys/sparc64/ebus/ebus.c
+++ b/sys/sparc64/ebus/ebus.c
@@ -69,7 +69,6 @@ __FBSDID("$FreeBSD$");
 #include <sys/kernel.h>
 #include <sys/malloc.h>
 #include <sys/module.h>
-
 #include <sys/rman.h>
 
 #include <dev/ofw/ofw_bus.h>
@@ -294,7 +293,7 @@ ebus_nexus_attach(device_t dev)
 	sc->sc_nrange = OF_getprop_alloc(node, "ranges",
 	    sizeof(struct ebus_nexus_ranges), &sc->sc_range);
 	if (sc->sc_nrange == -1) {
-		printf("%s: could not get ranges property\n", __func__);
+		device_printf(dev, "could not get ranges property\n");
 		return (ENXIO);
 	}
 	return (ebus_attach(dev, sc, node));
@@ -306,6 +305,7 @@ ebus_pci_attach(device_t dev)
 	struct ebus_softc *sc;
 	struct ebus_rinfo *eri;
 	struct resource *res;
+	struct isa_ranges *range;
 	phandle_t node;
 	int i, rnum, rid;
 
@@ -322,7 +322,7 @@ ebus_pci_attach(device_t dev)
 	sc->sc_nrange = OF_getprop_alloc(node, "ranges",
 	    sizeof(struct isa_ranges), &sc->sc_range);
 	if (sc->sc_nrange == -1) {
-		printf("%s: could not get ranges property\n", __func__);
+		device_printf(dev, "could not get ranges property\n");
 		return (ENXIO);
 	}
 
@@ -332,21 +332,34 @@ ebus_pci_attach(device_t dev)
 	/* For every range, there must be a matching resource. */
 	for (rnum = 0; rnum < sc->sc_nrange; rnum++) {
 		eri = &sc->sc_rinfo[rnum];
-		eri->eri_rtype = ofw_isa_range_restype(
-		    &((struct isa_ranges *)sc->sc_range)[rnum]);
+		range = &((struct isa_ranges *)sc->sc_range)[rnum];
+		eri->eri_rtype = ofw_isa_range_restype(range);
 		rid = PCIR_BAR(rnum);
 		res = bus_alloc_resource_any(dev, eri->eri_rtype, &rid,
 		    RF_ACTIVE);
 		if (res == NULL) {
-			printf("%s: failed to allocate range resource!\n",
-			    __func__);
+			device_printf(dev,
+			    "could not allocate range resource %d\n", rnum);
+			goto fail;
+		}
+		if (rman_get_start(res) != ISA_RANGE_PHYS(range)) {
+			device_printf(dev,
+			    "mismatch in start of range %d (0x%lx/0x%lx)\n",
+			    rnum, rman_get_start(res), ISA_RANGE_PHYS(range));
+			goto fail;
+		}
+		if (rman_get_size(res) != range->size) {
+			device_printf(dev,
+			    "mismatch in size of range %d (0x%lx/0x%x)\n",
+			    rnum, rman_get_size(res), range->size);
 			goto fail;
 		}
 		eri->eri_res = res;
 		eri->eri_rman.rm_type = RMAN_ARRAY;
 		eri->eri_rman.rm_descr = "EBus range";
 		if (rman_init_from_resource(&eri->eri_rman, res) != 0) {
-			printf("%s: failed to initialize rman!", __func__);
+			device_printf(dev,
+			    "could not initialize rman for range %d", rnum);
 			goto fail;
 		}
 	}
@@ -452,7 +465,7 @@ ebus_alloc_resource(device_t bus, device_t child, int type, int *rid,
 			 * Map EBus ranges to PCI ranges.  This may include
 			 * changing the allocation type.
 			 */
-			(void)ofw_isa_range_map(sc->sc_range, sc->sc_nrange,
+			type = ofw_isa_range_map(sc->sc_range, sc->sc_nrange,
 			    &start, &end, &ridx);
 			eri = &sc->sc_rinfo[ridx];
 			res = rman_reserve_resource(&eri->eri_rman, start,
@@ -507,7 +520,7 @@ ebus_activate_resource(device_t bus, device_t child, int type, int rid,
 	int i, rv;
 
 	sc = device_get_softc(bus);
-	if ((sc->sc_flags & EBUS_PCI) != 0 && type == SYS_RES_MEMORY) {
+	if ((sc->sc_flags & EBUS_PCI) != 0 && type != SYS_RES_IRQ) {
 		for (i = 0; i < sc->sc_nrange; i++) {
 			eri = &sc->sc_rinfo[i];
 			if (rman_is_region_manager(res, &eri->eri_rman) != 0) {
@@ -550,7 +563,7 @@ ebus_release_resource(device_t bus, device_t child, int type, int rid,
 	passthrough = (device_get_parent(child) != bus);
 	rl = BUS_GET_RESOURCE_LIST(bus, child);
 	sc = device_get_softc(bus);
-	if ((sc->sc_flags & EBUS_PCI) != 0 && type == SYS_RES_MEMORY) {
+	if ((sc->sc_flags & EBUS_PCI) != 0 && type != SYS_RES_IRQ) {
 		if ((rman_get_flags(res) & RF_ACTIVE) != 0 ){
 			rv = bus_deactivate_resource(child, type, rid, res);
 			if (rv != 0)
diff --git a/sys/sparc64/include/smp.h b/sys/sparc64/include/smp.h
index c46c4f8ad14e9..266187e83ca57 100644
--- a/sys/sparc64/include/smp.h
+++ b/sys/sparc64/include/smp.h
@@ -47,6 +47,7 @@
 #include <sys/sched.h>
 #include <sys/smp.h>
 
+#include <machine/atomic.h>
 #include <machine/intr_machdep.h>
 #include <machine/tte.h>
 
@@ -143,7 +144,7 @@ ipi_all_but_self(u_int ipi)
 {
 	cpuset_t cpus;
 
-	if (__predict_false(smp_started == 0))
+	if (__predict_false(atomic_load_acq_int(&smp_started) == 0))
 		return;
 	cpus = all_cpus;
 	sched_pin();
@@ -158,7 +159,8 @@ static __inline void
 ipi_selected(cpuset_t cpus, u_int ipi)
 {
 
-	if (__predict_false(smp_started == 0 || CPU_EMPTY(&cpus)))
+	if (__predict_false(atomic_load_acq_int(&smp_started) == 0 ||
+	    CPU_EMPTY(&cpus)))
 		return;
 	mtx_lock_spin(&ipi_mtx);
 	cpu_ipi_selected(cpus, 0, (u_long)tl_ipi_level, ipi);
@@ -169,7 +171,7 @@ static __inline void
 ipi_cpu(int cpu, u_int ipi)
 {
 
-	if (__predict_false(smp_started == 0))
+	if (__predict_false(atomic_load_acq_int(&smp_started) == 0))
 		return;
 	mtx_lock_spin(&ipi_mtx);
 	cpu_ipi_single(cpu, 0, (u_long)tl_ipi_level, ipi);
@@ -183,7 +185,7 @@ ipi_dcache_page_inval(void *func, vm_paddr_t pa)
 {
 	struct ipi_cache_args *ica;
 
-	if (__predict_false(smp_started == 0))
+	if (__predict_false(atomic_load_acq_int(&smp_started) == 0))
 		return (NULL);
 	sched_pin();
 	ica = &ipi_cache_args;
@@ -200,7 +202,7 @@ ipi_icache_page_inval(void *func, vm_paddr_t pa)
 {
 	struct ipi_cache_args *ica;
 
-	if (__predict_false(smp_started == 0))
+	if (__predict_false(atomic_load_acq_int(&smp_started) == 0))
 		return (NULL);
 	sched_pin();
 	ica = &ipi_cache_args;
@@ -217,7 +219,7 @@ ipi_rd(u_int cpu, void *func, u_long *val)
 {
 	struct ipi_rd_args *ira;
 
-	if (__predict_false(smp_started == 0))
+	if (__predict_false(atomic_load_acq_int(&smp_started) == 0))
 		return (NULL);
 	sched_pin();
 	ira = &ipi_rd_args;
@@ -234,7 +236,7 @@ ipi_tlb_context_demap(struct pmap *pm)
 	struct ipi_tlb_args *ita;
 	cpuset_t cpus;
 
-	if (__predict_false(smp_started == 0))
+	if (__predict_false(atomic_load_acq_int(&smp_started) == 0))
 		return (NULL);
 	sched_pin();
 	cpus = pm->pm_active;
@@ -259,7 +261,7 @@ ipi_tlb_page_demap(struct pmap *pm, vm_offset_t va)
 	struct ipi_tlb_args *ita;
 	cpuset_t cpus;
 
-	if (__predict_false(smp_started == 0))
+	if (__predict_false(atomic_load_acq_int(&smp_started) == 0))
 		return (NULL);
 	sched_pin();
 	cpus = pm->pm_active;
@@ -284,7 +286,7 @@ ipi_tlb_range_demap(struct pmap *pm, vm_offset_t start, vm_offset_t end)
 	struct ipi_tlb_args *ita;
 	cpuset_t cpus;
 
-	if (__predict_false(smp_started == 0))
+	if (__predict_false(atomic_load_acq_int(&smp_started) == 0))
 		return (NULL);
 	sched_pin();
 	cpus = pm->pm_active;
diff --git a/sys/sparc64/pci/fire.c b/sys/sparc64/pci/fire.c
index 2755260152540..be0c64babea42 100644
--- a/sys/sparc64/pci/fire.c
+++ b/sys/sparc64/pci/fire.c
@@ -59,7 +59,6 @@ __FBSDID("$FreeBSD$");
 #include <sys/timetc.h>
 
 #include <dev/ofw/ofw_bus.h>
-#include <dev/ofw/ofw_pci.h>
 #include <dev/ofw/openfirm.h>
 
 #include <vm/vm.h>
@@ -68,7 +67,6 @@ __FBSDID("$FreeBSD$");
 #include <machine/bus.h>
 #include <machine/bus_common.h>
 #include <machine/bus_private.h>
-#include <machine/fsr.h>
 #include <machine/iommureg.h>
 #include <machine/iommuvar.h>
 #include <machine/pmap.h>
@@ -111,19 +109,14 @@ static driver_filter_t fire_xcb;
 /*
  * Methods
  */
-static bus_activate_resource_t fire_activate_resource;
-static bus_adjust_resource_t fire_adjust_resource;
 static pcib_alloc_msi_t fire_alloc_msi;
 static pcib_alloc_msix_t fire_alloc_msix;
 static bus_alloc_resource_t fire_alloc_resource;
 static device_attach_t fire_attach;
-static bus_get_dma_tag_t fire_get_dma_tag;
-static ofw_bus_get_node_t fire_get_node;
 static pcib_map_msi_t fire_map_msi;
 static pcib_maxslots_t fire_maxslots;
 static device_probe_t fire_probe;
 static pcib_read_config_t fire_read_config;
-static bus_read_ivar_t fire_read_ivar;
 static pcib_release_msi_t fire_release_msi;
 static pcib_release_msix_t fire_release_msix;
 static pcib_route_interrupt_t fire_route_interrupt;
@@ -140,15 +133,15 @@ static device_method_t fire_methods[] = {
 	DEVMETHOD(device_resume,	bus_generic_resume),
 
 	/* Bus interface */
-	DEVMETHOD(bus_read_ivar,	fire_read_ivar),
+	DEVMETHOD(bus_read_ivar,	ofw_pci_read_ivar),
 	DEVMETHOD(bus_setup_intr,	fire_setup_intr),
 	DEVMETHOD(bus_teardown_intr,	fire_teardown_intr),
 	DEVMETHOD(bus_alloc_resource,	fire_alloc_resource),
-	DEVMETHOD(bus_activate_resource, fire_activate_resource),
+	DEVMETHOD(bus_activate_resource, ofw_pci_activate_resource),
 	DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource),
-	DEVMETHOD(bus_adjust_resource,	fire_adjust_resource),
+	DEVMETHOD(bus_adjust_resource,	ofw_pci_adjust_resource),
 	DEVMETHOD(bus_release_resource,	bus_generic_release_resource),
-	DEVMETHOD(bus_get_dma_tag,	fire_get_dma_tag),
+	DEVMETHOD(bus_get_dma_tag,	ofw_pci_get_dma_tag),
 
 	/* pcib interface */
 	DEVMETHOD(pcib_maxslots,	fire_maxslots),
@@ -162,7 +155,7 @@ static device_method_t fire_methods[] = {
 	DEVMETHOD(pcib_map_msi,		fire_map_msi),
 
 	/* ofw_bus interface */
-	DEVMETHOD(ofw_bus_get_node,	fire_get_node),
+	DEVMETHOD(ofw_bus_get_node,	ofw_pci_get_node),
 
 	DEVMETHOD_END
 };
@@ -296,7 +289,7 @@ fire_attach(device_t dev)
 	struct ofw_pci_msi_eq_to_devino msi_eq_to_devino;
 	struct fire_msiqarg *fmqa;
 	struct timecounter *tc;
-	struct ofw_pci_ranges *range;
+	bus_dma_tag_t dmat;
 	uint64_t ino_bitmap, val;
 	phandle_t node;
 	uint32_t prop, prop_array[2];
@@ -310,7 +303,6 @@ fire_attach(device_t dev)
 	mode = desc->fd_mode;
 
 	sc->sc_dev = dev;
-	sc->sc_node = node;
 	sc->sc_mode = mode;
 	sc->sc_flags = 0;
 
@@ -715,79 +707,21 @@ fire_attach(device_t dev)
 	sc->sc_is.is_bushandle = rman_get_bushandle(sc->sc_mem_res[FIRE_PCI]);
 	sc->sc_is.is_iommu = FO_PCI_MMU;
 	val = FIRE_PCI_READ_8(sc, FO_PCI_MMU + IMR_CTL);
-	iommu_init(device_get_nameunit(sc->sc_dev), &sc->sc_is, 7, -1, 0);
+	iommu_init(device_get_nameunit(dev), &sc->sc_is, 7, -1, 0);
 #ifdef FIRE_DEBUG
 	device_printf(dev, "FO_PCI_MMU + IMR_CTL 0x%016llx -> 0x%016llx\n",
 	    (long long unsigned)val, (long long unsigned)sc->sc_is.is_cr);
 #endif
-
-	/* Initialize memory and I/O rmans. */
-	sc->sc_pci_io_rman.rm_type = RMAN_ARRAY;
-	sc->sc_pci_io_rman.rm_descr = "Fire PCI I/O Ports";
-	if (rman_init(&sc->sc_pci_io_rman) != 0 ||
-	    rman_manage_region(&sc->sc_pci_io_rman, 0, FO_IO_SIZE) != 0)
-		panic("%s: failed to set up I/O rman", __func__);
-	sc->sc_pci_mem_rman.rm_type = RMAN_ARRAY;
-	sc->sc_pci_mem_rman.rm_descr = "Fire PCI Memory";
-	if (rman_init(&sc->sc_pci_mem_rman) != 0 ||
-	    rman_manage_region(&sc->sc_pci_mem_rman, 0, FO_MEM_SIZE) != 0)
-		panic("%s: failed to set up memory rman", __func__);
-
-	i = OF_getprop_alloc(node, "ranges", sizeof(*range), (void **)&range);
-	/*
-	 * Make sure that the expected ranges are present.  The
-	 * OFW_PCI_CS_MEM64 one is not currently used though.
-	 */
-	if (i != FIRE_NRANGE)
-		panic("%s: unsupported number of ranges", __func__);
-	/*
-	 * Find the addresses of the various bus spaces.
-	 * There should not be multiple ones of one kind.
-	 * The physical start addresses of the ranges are the configuration,
-	 * memory and I/O handles.
-	 */
-	for (i = 0; i < FIRE_NRANGE; i++) {
-		j = OFW_PCI_RANGE_CS(&range[i]);
-		if (sc->sc_pci_bh[j] != 0)
-			panic("%s: duplicate range for space %d",
-			    __func__, j);
-		sc->sc_pci_bh[j] = OFW_PCI_RANGE_PHYS(&range[i]);
-	}
-	free(range, M_OFWPROP);
-
-	/* Allocate our tags. */
-	sc->sc_pci_iot = sparc64_alloc_bus_tag(NULL, PCI_IO_BUS_SPACE);
-	if (sc->sc_pci_iot == NULL)
-		panic("%s: could not allocate PCI I/O tag", __func__);
-	sc->sc_pci_cfgt = sparc64_alloc_bus_tag(NULL, PCI_CONFIG_BUS_SPACE);
-	if (sc->sc_pci_cfgt == NULL)
-		panic("%s: could not allocate PCI configuration space tag",
-		    __func__);
+	/* Create our DMA tag. */
 	if (bus_dma_tag_create(bus_get_dma_tag(dev), 8, 0x100000000,
 	    sc->sc_is.is_pmaxaddr, ~0, NULL, NULL, sc->sc_is.is_pmaxaddr,
-	    0xff, 0xffffffff, 0, NULL, NULL, &sc->sc_pci_dmat) != 0)
+	    0xff, 0xffffffff, 0, NULL, NULL, &dmat) != 0)
 		panic("%s: could not create PCI DMA tag", __func__);
-	/* Customize the tag. */
-	sc->sc_pci_dmat->dt_cookie = &sc->sc_is;
-	sc->sc_pci_dmat->dt_mt = &sc->sc_dma_methods;
-
-	/*
-	 * Get the bus range from the firmware.
-	 * NB: Neither Fire nor Oberon support PCI bus reenumeration.
-	 */
-	i = OF_getprop(node, "bus-range", (void *)prop_array,
-	    sizeof(prop_array));
-	if (i == -1)
-		panic("%s: could not get bus-range", __func__);
-	if (i != sizeof(prop_array))
-		panic("%s: broken bus-range (%d)", __func__, i);
-	sc->sc_pci_secbus = prop_array[0];
-	sc->sc_pci_subbus = prop_array[1];
-	if (bootverbose != 0)
-		device_printf(dev, "bus range %u to %u; PCI bus %d\n",
-		    sc->sc_pci_secbus, sc->sc_pci_subbus, sc->sc_pci_secbus);
+	dmat->dt_cookie = &sc->sc_is;
+	dmat->dt_mt = &sc->sc_dma_methods;
 
-	ofw_bus_setup_iinfo(node, &sc->sc_pci_iinfo, sizeof(ofw_pci_intr_t));
+	if (ofw_pci_attach_common(dev, dmat, FO_IO_SIZE, FO_MEM_SIZE) != 0)
+		panic("%s: ofw_pci_attach_common() failed", __func__);
 
 #define	FIRE_SYSCTL_ADD_UINT(name, arg, desc)				\
 	SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),			\
@@ -1390,136 +1324,44 @@ static uint32_t
 fire_read_config(device_t dev, u_int bus, u_int slot, u_int func, u_int reg,
     int width)
 {
-	struct fire_softc *sc;
-	bus_space_handle_t bh;
-	u_long offset = 0;
-	uint32_t r, wrd;
-	int i;
-	uint16_t shrt;
-	uint8_t byte;
-
-	sc = device_get_softc(dev);
-	if (bus < sc->sc_pci_secbus || bus > sc->sc_pci_subbus ||
-	    slot > PCI_SLOTMAX || func > PCI_FUNCMAX || reg > PCIE_REGMAX)
-		return (-1);
-
-	offset = FO_CONF_OFF(bus, slot, func, reg);
-	bh = sc->sc_pci_bh[OFW_PCI_CS_CONFIG];
-	switch (width) {
-	case 1:
-		i = bus_space_peek_1(sc->sc_pci_cfgt, bh, offset, &byte);
-		r = byte;
-		break;
-	case 2:
-		i = bus_space_peek_2(sc->sc_pci_cfgt, bh, offset, &shrt);
-		r = shrt;
-		break;
-	case 4:
-		i = bus_space_peek_4(sc->sc_pci_cfgt, bh, offset, &wrd);
-		r = wrd;
-		break;
-	default:
-		panic("%s: bad width", __func__);
-		/* NOTREACHED */
-	}
 
-	if (i) {
-#ifdef FIRE_DEBUG
-		printf("%s: read data error reading: %d.%d.%d: 0x%x\n",
-		    __func__, bus, slot, func, reg);
-#endif
-		r = -1;
-	}
-	return (r);
+	return (ofw_pci_read_config_common(dev, PCIE_REGMAX, FO_CONF_OFF(bus,
+	    slot, func, reg), bus, slot, func, reg, width));
 }
 
 static void
 fire_write_config(device_t dev, u_int bus, u_int slot, u_int func, u_int reg,
     uint32_t val, int width)
 {
-	struct fire_softc *sc;
-	bus_space_handle_t bh;
-	u_long offset = 0;
 
-	sc = device_get_softc(dev);
-	if (bus < sc->sc_pci_secbus || bus > sc->sc_pci_subbus ||
-	    slot > PCI_SLOTMAX || func > PCI_FUNCMAX || reg > PCIE_REGMAX)
-		return;
-
-	offset = FO_CONF_OFF(bus, slot, func, reg);
-	bh = sc->sc_pci_bh[OFW_PCI_CS_CONFIG];
-	switch (width) {
-	case 1:
-		bus_space_write_1(sc->sc_pci_cfgt, bh, offset, val);
-		break;
-	case 2:
-		bus_space_write_2(sc->sc_pci_cfgt, bh, offset, val);
-		break;
-	case 4:
-		bus_space_write_4(sc->sc_pci_cfgt, bh, offset, val);
-		break;
-	default:
-		panic("%s: bad width", __func__);
-		/* NOTREACHED */
-	}
+	ofw_pci_write_config_common(dev, PCIE_REGMAX, FO_CONF_OFF(bus, slot,
+	    func, reg), bus, slot, func, reg, val, width);
 }
 
 static int
 fire_route_interrupt(device_t bridge, device_t dev, int pin)
 {
-	struct fire_softc *sc;
-	struct ofw_pci_register reg;
-	ofw_pci_intr_t pintr, mintr;
-
-	sc = device_get_softc(bridge);
-	pintr = pin;
-	if (ofw_bus_lookup_imap(ofw_bus_get_node(dev), &sc->sc_pci_iinfo,
-	    &reg, sizeof(reg), &pintr, sizeof(pintr), &mintr, sizeof(mintr),
-	    NULL) != 0)
-		return (mintr);
-
-	device_printf(bridge, "could not route pin %d for device %d.%d\n",
-	    pin, pci_get_slot(dev), pci_get_function(dev));
-	return (PCI_INVALID_IRQ);
-}
-
-static int
-fire_read_ivar(device_t dev, device_t child, int which, uintptr_t *result)
-{
-	struct fire_softc *sc;
+	ofw_pci_intr_t mintr;
 
-	sc = device_get_softc(dev);
-	switch (which) {
-	case PCIB_IVAR_DOMAIN:
-		*result = device_get_unit(dev);
-		return (0);
-	case PCIB_IVAR_BUS:
-		*result = sc->sc_pci_secbus;
-		return (0);
-	}
-	return (ENOENT);
+	mintr = ofw_pci_route_interrupt_common(bridge, dev, pin);
+	if (!PCI_INTERRUPT_VALID(mintr))
+		device_printf(bridge,
+		    "could not route pin %d for device %d.%d\n",
+		    pin, pci_get_slot(dev), pci_get_function(dev));
+	return (mintr);
 }
 
 static void
 fire_dmamap_sync(bus_dma_tag_t dt __unused, bus_dmamap_t map,
     bus_dmasync_op_t op)
 {
-	static u_char buf[VIS_BLOCKSIZE] __aligned(VIS_BLOCKSIZE);
-	register_t reg, s;
 
 	if ((map->dm_flags & DMF_LOADED) == 0)
 		return;
 
-	if ((op & BUS_DMASYNC_POSTREAD) != 0) {
-		s = intr_disable();
-		reg = rd(fprs);
-		wr(fprs, reg | FPRS_FEF, 0);
-		__asm __volatile("stda %%f0, [%0] %1"
-		    : : "r" (buf), "n" (ASI_BLK_COMMIT_S));
-		membar(Sync);
-		wr(fprs, reg, 0);
-		intr_restore(s);
-	} else if ((op & BUS_DMASYNC_PREWRITE) != 0)
+	if ((op & BUS_DMASYNC_POSTREAD) != 0)
+		ofw_pci_dmamap_sync_stst_order_common();
+	else if ((op & BUS_DMASYNC_PREWRITE) != 0)
 		membar(Sync);
 }
 
@@ -2013,121 +1855,13 @@ fire_alloc_resource(device_t bus, device_t child, int type, int *rid,
     u_long start, u_long end, u_long count, u_int flags)
 {
 	struct fire_softc *sc;
-	struct resource *rv;
-	struct rman *rm;
-
-	sc = device_get_softc(bus);
-	switch (type) {
-	case SYS_RES_IRQ:
-		/*
-		 * XXX: Don't accept blank ranges for now, only single
-		 * interrupts.  The other case should not happen with
-		 * the MI PCI code...
-		 * XXX: This may return a resource that is out of the
-		 * range that was specified.  Is this correct...?
-		 */
-		if (start != end)
-			panic("%s: XXX: interrupt range", __func__);
-		if (*rid == 0)
-			start = end = INTMAP_VEC(sc->sc_ign, end);
-		return (bus_generic_alloc_resource(bus, child, type, rid,
-		    start, end, count, flags));
-	case SYS_RES_MEMORY:
-		rm = &sc->sc_pci_mem_rman;
-		break;
-	case SYS_RES_IOPORT:
-		rm = &sc->sc_pci_io_rman;
-		break;
-	default:
-		return (NULL);
-	}
-
-	rv = rman_reserve_resource(rm, start, end, count, flags & ~RF_ACTIVE,
-	    child);
-	if (rv == NULL)
-		return (NULL);
-	rman_set_rid(rv, *rid);
-
-	if ((flags & RF_ACTIVE) != 0 && bus_activate_resource(child, type,
-	    *rid, rv) != 0) {
-		rman_release_resource(rv);
-		return (NULL);
-	}
-	return (rv);
-}
-
-static int
-fire_activate_resource(device_t bus, device_t child, int type, int rid,
-    struct resource *r)
-{
-	struct fire_softc *sc;
-	struct bus_space_tag *tag;
 
-	sc = device_get_softc(bus);
-	switch (type) {
-	case SYS_RES_IRQ:
-		return (bus_generic_activate_resource(bus, child, type, rid,
-		    r));
-	case SYS_RES_MEMORY:
-		tag = sparc64_alloc_bus_tag(r, PCI_MEMORY_BUS_SPACE);
-		if (tag == NULL)
-			return (ENOMEM);
-		rman_set_bustag(r, tag);
-		rman_set_bushandle(r, sc->sc_pci_bh[OFW_PCI_CS_MEM32] +
-		    rman_get_start(r));
-		break;
-	case SYS_RES_IOPORT:
-		rman_set_bustag(r, sc->sc_pci_iot);
-		rman_set_bushandle(r, sc->sc_pci_bh[OFW_PCI_CS_IO] +
-		    rman_get_start(r));
-		break;
+	if (type == SYS_RES_IRQ && *rid == 0) {
+		sc = device_get_softc(bus);
+		start = end = INTMAP_VEC(sc->sc_ign, end);
 	}
-	return (rman_activate_resource(r));
-}
-
-static int
-fire_adjust_resource(device_t bus, device_t child, int type,
-    struct resource *r, u_long start, u_long end)
-{
-	struct fire_softc *sc;
-	struct rman *rm;
-
-	sc = device_get_softc(bus);
-	switch (type) {
-	case SYS_RES_IRQ:
-		return (bus_generic_adjust_resource(bus, child, type, r,
-		    start, end));
-	case SYS_RES_MEMORY:
-		rm = &sc->sc_pci_mem_rman;
-		break;
-	case SYS_RES_IOPORT:
-		rm = &sc->sc_pci_io_rman;
-		break;
-	default:
-		return (EINVAL);
-	}
-	if (rman_is_region_manager(r, rm) == 0)
-		return (EINVAL);
-	return (rman_adjust_resource(r, start, end));
-}
-
-static bus_dma_tag_t
-fire_get_dma_tag(device_t bus, device_t child __unused)
-{
-	struct fire_softc *sc;
-
-	sc = device_get_softc(bus);
-	return (sc->sc_pci_dmat);
-}
-
-static phandle_t
-fire_get_node(device_t bus, device_t child __unused)
-{
-	struct fire_softc *sc;
-
-	sc = device_get_softc(bus);
-	/* We only have one child, the PCI bus, which needs our own node. */
-	return (sc->sc_node);
+	return (ofw_pci_alloc_resource(bus, child, type, rid, start, end,
+	    count, flags));
 }
 
 static u_int
diff --git a/sys/sparc64/pci/firereg.h b/sys/sparc64/pci/firereg.h
index 247a3f7d33a8d..715d3b7b3a484 100644
--- a/sys/sparc64/pci/firereg.h
+++ b/sys/sparc64/pci/firereg.h
@@ -30,7 +30,6 @@
 #define	_SPARC64_PCI_FIREREG_H_
 
 #define	FIRE_NINTR				3	/* 2 OFW + 1 MSIq */
-#define	FIRE_NRANGE				4
 #define	FIRE_NREG				2
 
 #define	FIRE_PCI				0
diff --git a/sys/sparc64/pci/firevar.h b/sys/sparc64/pci/firevar.h
index d414c1adc5038..36fd8ea8470cf 100644
--- a/sys/sparc64/pci/firevar.h
+++ b/sys/sparc64/pci/firevar.h
@@ -32,6 +32,12 @@
 #define	_SPARC64_PCI_FIREVAR_H_
 
 struct fire_softc {
+	/*
+	 * This is here so that we can hook up the common bus interface
+	 * methods in ofw_pci.c directly.
+	 */
+	struct ofw_pci_softc		sc_ops;
+
 	struct iommu_state		sc_is;
 	struct bus_dma_methods		sc_dma_methods;
 
@@ -42,13 +48,6 @@ struct fire_softc {
 	struct resource			*sc_irq_res[FIRE_NINTR];
 	void				*sc_ihand[FIRE_NINTR];
 
-	struct rman			sc_pci_mem_rman;
-	struct rman			sc_pci_io_rman;
-	bus_space_handle_t		sc_pci_bh[FIRE_NRANGE];
-	bus_space_tag_t			sc_pci_cfgt;
-	bus_space_tag_t			sc_pci_iot;
-	bus_dma_tag_t			sc_pci_dmat;
-
 	device_t			sc_dev;
 
 	uint64_t			*sc_msiq;
@@ -66,8 +65,6 @@ struct fire_softc {
 	uint32_t			sc_msiq_first;
 	uint32_t			sc_msiq_ino_first;
 
-	phandle_t			sc_node;
-
 	u_int				sc_mode;
 #define	FIRE_MODE_FIRE			0
 #define	FIRE_MODE_OBERON		1
@@ -87,11 +84,6 @@ struct fire_softc {
 	uint32_t			sc_stats_tlu_oe_rx_err;
 	uint32_t			sc_stats_tlu_oe_tx_err;
 	uint32_t			sc_stats_ubc_dmardue;
-
-	uint8_t				sc_pci_secbus;
-	uint8_t				sc_pci_subbus;
-
-	struct ofw_bus_iinfo		sc_pci_iinfo;
 };
 
 #endif /* !_SPARC64_PCI_FIREVAR_H_ */
diff --git a/sys/sparc64/pci/ofw_pci.c b/sys/sparc64/pci/ofw_pci.c
new file mode 100644
index 0000000000000..a18052f7647c2
--- /dev/null
+++ b/sys/sparc64/pci/ofw_pci.c
@@ -0,0 +1,406 @@
+/*-
+ * Copyright (c) 1999, 2000 Matthew R. Green
+ * Copyright (c) 2001 - 2003 by Thomas Moestl <tmm@FreeBSD.org>
+ * Copyright (c) 2005 - 2015 by Marius Strobl <marius@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The name of the author may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	from: NetBSD: psycho.c,v 1.35 2001/09/10 16:17:06 eeh Exp
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include "opt_ofw_pci.h"
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/rman.h>
+
+#include <dev/ofw/ofw_bus.h>
+#include <dev/ofw/ofw_pci.h>
+#include <dev/ofw/openfirm.h>
+
+#include <dev/pci/pcireg.h>
+#include <dev/pci/pcivar.h>
+
+#include <machine/asi.h>
+#include <machine/bus.h>
+#include <machine/bus_private.h>
+#include <machine/cpufunc.h>
+#include <machine/fsr.h>
+#include <machine/resource.h>
+
+#include <sparc64/pci/ofw_pci.h>
+
+int
+ofw_pci_attach_common(device_t dev, bus_dma_tag_t dmat, u_long iosize,
+    u_long memsize)
+{
+	struct ofw_pci_softc *sc;
+	struct ofw_pci_ranges *range;
+	phandle_t node;
+	uint32_t prop_array[2];
+	u_int i, j, nrange;
+
+	sc = device_get_softc(dev);
+	node = ofw_bus_get_node(dev);
+	sc->sc_node = node;
+	sc->sc_pci_dmat = dmat;
+
+	/* Initialize memory and I/O rmans. */
+	sc->sc_pci_io_rman.rm_type = RMAN_ARRAY;
+	sc->sc_pci_io_rman.rm_descr = "PCI I/O Ports";
+	if (rman_init(&sc->sc_pci_io_rman) != 0 ||
+	    rman_manage_region(&sc->sc_pci_io_rman, 0, iosize) != 0) {
+		device_printf(dev, "failed to set up I/O rman\n");
+		return (ENXIO);
+	}
+	sc->sc_pci_mem_rman.rm_type = RMAN_ARRAY;
+	sc->sc_pci_mem_rman.rm_descr = "PCI Memory";
+	if (rman_init(&sc->sc_pci_mem_rman) != 0 ||
+	    rman_manage_region(&sc->sc_pci_mem_rman, 0, memsize) != 0) {
+		device_printf(dev, "failed to set up memory rman\n");
+		return (ENXIO);
+	}
+
+	/*
+	 * Find the addresses of the various bus spaces.  The physical
+	 * start addresses of the ranges are the configuration, I/O and
+	 * memory handles.  There should not be multiple ones of one kind.
+	 */
+	nrange = OF_getprop_alloc(node, "ranges", sizeof(*range),
+	    (void **)&range);
+	for (i = 0; i < nrange; i++) {
+		j = OFW_PCI_RANGE_CS(&range[i]);
+		if (sc->sc_pci_bh[j] != 0) {
+			device_printf(dev, "duplicate range for space %d\n",
+			    j);
+			free(range, M_OFWPROP);
+			return (EINVAL);
+		}
+		sc->sc_pci_bh[j] = OFW_PCI_RANGE_PHYS(&range[i]);
+	}
+	free(range, M_OFWPROP);
+
+	/*
+	 * Make sure that the expected ranges are actually present.
+	 * The OFW_PCI_CS_MEM64 one is not currently used.
+	 */
+	if (sc->sc_pci_bh[OFW_PCI_CS_CONFIG] == 0) {
+		device_printf(dev, "missing CONFIG range\n");
+		return (ENXIO);
+	}
+	if (sc->sc_pci_bh[OFW_PCI_CS_IO] == 0) {
+		device_printf(dev, "missing IO range\n");
+		return (ENXIO);
+	}
+	if (sc->sc_pci_bh[OFW_PCI_CS_MEM32] == 0) {
+		device_printf(dev, "missing MEM32 range\n");
+		return (ENXIO);
+	}
+
+	/* Allocate our tags. */
+	sc->sc_pci_iot = sparc64_alloc_bus_tag(NULL, PCI_IO_BUS_SPACE);
+	if (sc->sc_pci_iot == NULL) {
+		device_printf(dev, "could not allocate PCI I/O tag\n");
+		return (ENXIO);
+	}
+	sc->sc_pci_cfgt = sparc64_alloc_bus_tag(NULL, PCI_CONFIG_BUS_SPACE);
+	if (sc->sc_pci_cfgt == NULL) {
+		device_printf(dev,
+		    "could not allocate PCI configuration space tag\n");
+		return (ENXIO);
+	}
+
+	/*
+	 * Get the bus range from the firmware.
+	 */
+	i = OF_getprop(node, "bus-range", (void *)prop_array,
+	    sizeof(prop_array));
+	if (i == -1) {
+		device_printf(dev, "could not get bus-range\n");
+		return (ENXIO);
+	}
+	if (i != sizeof(prop_array)) {
+		device_printf(dev, "broken bus-range (%d)", i);
+		return (EINVAL);
+	}
+	sc->sc_pci_secbus = prop_array[0];
+	sc->sc_pci_subbus = prop_array[1];
+	if (bootverbose != 0)
+		device_printf(dev, "bus range %u to %u; PCI bus %d\n",
+		    sc->sc_pci_secbus, sc->sc_pci_subbus, sc->sc_pci_secbus);
+
+	ofw_bus_setup_iinfo(node, &sc->sc_pci_iinfo, sizeof(ofw_pci_intr_t));
+
+	return (0);
+}
+
+uint32_t
+ofw_pci_read_config_common(device_t dev, u_int regmax, u_long offset,
+    u_int bus, u_int slot, u_int func, u_int reg, int width)
+{
+	struct ofw_pci_softc *sc;
+	bus_space_handle_t bh;
+	uint32_t r, wrd;
+	int i;
+	uint16_t shrt;
+	uint8_t byte;
+
+	sc = device_get_softc(dev);
+	if (bus < sc->sc_pci_secbus || bus > sc->sc_pci_subbus ||
+	    slot > PCI_SLOTMAX || func > PCI_FUNCMAX || reg > regmax)
+		return (-1);
+
+	bh = sc->sc_pci_bh[OFW_PCI_CS_CONFIG];
+	switch (width) {
+	case 1:
+		i = bus_space_peek_1(sc->sc_pci_cfgt, bh, offset, &byte);
+		r = byte;
+		break;
+	case 2:
+		i = bus_space_peek_2(sc->sc_pci_cfgt, bh, offset, &shrt);
+		r = shrt;
+		break;
+	case 4:
+		i = bus_space_peek_4(sc->sc_pci_cfgt, bh, offset, &wrd);
+		r = wrd;
+		break;
+	default:
+		panic("%s: bad width %d", __func__, width);
+		/* NOTREACHED */
+	}
+
+	if (i) {
+#ifdef OFW_PCI_DEBUG
+		printf("%s: read data error reading: %d.%d.%d: 0x%x\n",
+		    __func__, bus, slot, func, reg);
+#endif
+		r = -1;
+	}
+	return (r);
+}
+
+void
+ofw_pci_write_config_common(device_t dev, u_int regmax, u_long offset,
+    u_int bus, u_int slot, u_int func, u_int reg, uint32_t val, int width)
+{
+	struct ofw_pci_softc *sc;
+	bus_space_handle_t bh;
+
+	sc = device_get_softc(dev);
+	if (bus < sc->sc_pci_secbus || bus > sc->sc_pci_subbus ||
+	    slot > PCI_SLOTMAX || func > PCI_FUNCMAX || reg > regmax)
+		return;
+
+	bh = sc->sc_pci_bh[OFW_PCI_CS_CONFIG];
+	switch (width) {
+	case 1:
+		bus_space_write_1(sc->sc_pci_cfgt, bh, offset, val);
+		break;
+	case 2:
+		bus_space_write_2(sc->sc_pci_cfgt, bh, offset, val);
+		break;
+	case 4:
+		bus_space_write_4(sc->sc_pci_cfgt, bh, offset, val);
+		break;
+	default:
+		panic("%s: bad width %d", __func__, width);
+		/* NOTREACHED */
+	}
+}
+
+ofw_pci_intr_t
+ofw_pci_route_interrupt_common(device_t bridge, device_t dev, int pin)
+{
+	struct ofw_pci_softc *sc;
+	struct ofw_pci_register reg;
+	ofw_pci_intr_t pintr, mintr;
+
+	sc = device_get_softc(bridge);
+	pintr = pin;
+	if (ofw_bus_lookup_imap(ofw_bus_get_node(dev), &sc->sc_pci_iinfo,
+	    &reg, sizeof(reg), &pintr, sizeof(pintr), &mintr, sizeof(mintr),
+	    NULL) != 0)
+		return (mintr);
+	return (PCI_INVALID_IRQ);
+}
+
+void
+ofw_pci_dmamap_sync_stst_order_common(void)
+{
+	static u_char buf[VIS_BLOCKSIZE] __aligned(VIS_BLOCKSIZE);
+	register_t reg, s;
+
+	s = intr_disable();
+	reg = rd(fprs);
+	wr(fprs, reg | FPRS_FEF, 0);
+	__asm __volatile("stda %%f0, [%0] %1"
+	    : : "r" (buf), "n" (ASI_BLK_COMMIT_S));
+	membar(Sync);
+	wr(fprs, reg, 0);
+	intr_restore(s);
+}
+
+int
+ofw_pci_read_ivar(device_t dev, device_t child __unused, int which,
+    uintptr_t *result)
+{
+	struct ofw_pci_softc *sc;
+
+	switch (which) {
+	case PCIB_IVAR_DOMAIN:
+		*result = device_get_unit(dev);
+		return (0);
+	case PCIB_IVAR_BUS:
+		sc = device_get_softc(dev);
+		*result = sc->sc_pci_secbus;
+		return (0);
+	}
+	return (ENOENT);
+}
+
+struct resource *
+ofw_pci_alloc_resource(device_t bus, device_t child, int type, int *rid,
+    u_long start, u_long end, u_long count, u_int flags)
+{
+	struct ofw_pci_softc *sc;
+	struct resource *rv;
+	struct rman *rm;
+
+	sc = device_get_softc(bus);
+	switch (type) {
+	case SYS_RES_IRQ:
+		/*
+		 * XXX: Don't accept blank ranges for now, only single
+		 * interrupts.  The other case should not happen with
+		 * the MI PCI code ...
+		 * XXX: This may return a resource that is out of the
+		 * range that was specified.  Is this correct ...?
+		 */
+		if (start != end)
+			panic("%s: XXX: interrupt range", __func__);
+		return (bus_generic_alloc_resource(bus, child, type, rid,
+		    start, end, count, flags));
+	case SYS_RES_MEMORY:
+		rm = &sc->sc_pci_mem_rman;
+		break;
+	case SYS_RES_IOPORT:
+		rm = &sc->sc_pci_io_rman;
+		break;
+	default:
+		return (NULL);
+	}
+
+	rv = rman_reserve_resource(rm, start, end, count, flags & ~RF_ACTIVE,
+	    child);
+	if (rv == NULL)
+		return (NULL);
+	rman_set_rid(rv, *rid);
+
+	if ((flags & RF_ACTIVE) != 0 && bus_activate_resource(child, type,
+	    *rid, rv) != 0) {
+		rman_release_resource(rv);
+		return (NULL);
+	}
+	return (rv);
+}
+
+int
+ofw_pci_activate_resource(device_t bus, device_t child, int type, int rid,
+    struct resource *r)
+{
+	struct ofw_pci_softc *sc;
+	struct bus_space_tag *tag;
+
+	sc = device_get_softc(bus);
+	switch (type) {
+	case SYS_RES_IRQ:
+		return (bus_generic_activate_resource(bus, child, type, rid,
+		    r));
+	case SYS_RES_MEMORY:
+		tag = sparc64_alloc_bus_tag(r, PCI_MEMORY_BUS_SPACE);
+		if (tag == NULL)
+			return (ENOMEM);
+		rman_set_bustag(r, tag);
+		rman_set_bushandle(r, sc->sc_pci_bh[OFW_PCI_CS_MEM32] +
+		    rman_get_start(r));
+		break;
+	case SYS_RES_IOPORT:
+		rman_set_bustag(r, sc->sc_pci_iot);
+		rman_set_bushandle(r, sc->sc_pci_bh[OFW_PCI_CS_IO] +
+		    rman_get_start(r));
+		break;
+	}
+	return (rman_activate_resource(r));
+}
+
+int
+ofw_pci_adjust_resource(device_t bus, device_t child, int type,
+    struct resource *r, u_long start, u_long end)
+{
+	struct ofw_pci_softc *sc;
+	struct rman *rm;
+
+	sc = device_get_softc(bus);
+	switch (type) {
+	case SYS_RES_IRQ:
+		return (bus_generic_adjust_resource(bus, child, type, r,
+		    start, end));
+	case SYS_RES_MEMORY:
+		rm = &sc->sc_pci_mem_rman;
+		break;
+	case SYS_RES_IOPORT:
+		rm = &sc->sc_pci_io_rman;
+		break;
+	default:
+		return (EINVAL);
+	}
+	if (rman_is_region_manager(r, rm) == 0)
+		return (EINVAL);
+	return (rman_adjust_resource(r, start, end));
+}
+
+bus_dma_tag_t
+ofw_pci_get_dma_tag(device_t bus, device_t child __unused)
+{
+	struct ofw_pci_softc *sc;
+
+	sc = device_get_softc(bus);
+	return (sc->sc_pci_dmat);
+}
+
+phandle_t
+ofw_pci_get_node(device_t bus, device_t child __unused)
+{
+	struct ofw_pci_softc *sc;
+
+	sc = device_get_softc(bus);
+	/* We only have one child, the PCI bus, which needs our own node. */
+	return (sc->sc_node);
+}
diff --git a/sys/sparc64/pci/ofw_pci.h b/sys/sparc64/pci/ofw_pci.h
index 3915fa7f3fcbd..6d1e5d947f78e 100644
--- a/sys/sparc64/pci/ofw_pci.h
+++ b/sys/sparc64/pci/ofw_pci.h
@@ -62,6 +62,8 @@
 #ifndef _SPARC64_PCI_OFW_PCI_H_
 #define	_SPARC64_PCI_OFW_PCI_H_
 
+#include <sys/rman.h>
+
 #include <dev/ofw/ofw_bus_subr.h>
 
 #include "ofw_pci_if.h"
@@ -73,6 +75,7 @@ typedef uint32_t ofw_pci_intr_t;
 #define	OFW_PCI_CS_IO		0x01
 #define	OFW_PCI_CS_MEM32	0x02
 #define	OFW_PCI_CS_MEM64	0x03
+#define	OFW_PCI_NUM_CS		4
 
 /* OFW device types */
 #define	OFW_TYPE_PCI		"pci"
@@ -124,4 +127,44 @@ struct ofw_pci_ranges {
 /* default values */
 #define	OFW_PCI_LATENCY	64
 
+/*
+ * Common and generic parts of host-PCI-bridge support
+ */
+
+struct ofw_pci_softc {
+	struct rman		sc_pci_mem_rman;
+	struct rman		sc_pci_io_rman;
+
+	bus_space_handle_t	sc_pci_bh[OFW_PCI_NUM_CS];
+	bus_space_tag_t		sc_pci_cfgt;
+	bus_space_tag_t		sc_pci_iot;
+	bus_dma_tag_t		sc_pci_dmat;
+
+	struct ofw_bus_iinfo	sc_pci_iinfo;
+
+	phandle_t		sc_node;
+
+	uint8_t			sc_pci_secbus;
+	uint8_t			sc_pci_subbus;
+};
+
+int ofw_pci_attach_common(device_t dev, bus_dma_tag_t dmat, u_long iosize,
+    u_long memsize);
+uint32_t ofw_pci_read_config_common(device_t dev, u_int regmax, u_long offset,
+    u_int bus, u_int slot, u_int func, u_int reg, int width);
+void ofw_pci_write_config_common(device_t dev, u_int regmax, u_long offset,
+    u_int bus, u_int slot, u_int func, u_int reg, uint32_t val, int width);
+ofw_pci_intr_t ofw_pci_route_interrupt_common(device_t bridge, device_t dev,
+    int pin);
+
+void ofw_pci_dmamap_sync_stst_order_common(void);
+
+bus_activate_resource_t ofw_pci_activate_resource;
+bus_adjust_resource_t ofw_pci_adjust_resource;
+bus_alloc_resource_t ofw_pci_alloc_resource;
+bus_get_dma_tag_t ofw_pci_get_dma_tag;
+bus_read_ivar_t ofw_pci_read_ivar;
+
+ofw_bus_get_node_t ofw_pci_get_node;
+
 #endif /* ! _SPARC64_PCI_OFW_PCI_H_ */
diff --git a/sys/sparc64/pci/psycho.c b/sys/sparc64/pci/psycho.c
index 57e0f4817b5ae..38f4656abc634 100644
--- a/sys/sparc64/pci/psycho.c
+++ b/sys/sparc64/pci/psycho.c
@@ -57,7 +57,6 @@ __FBSDID("$FreeBSD$");
 #include <sys/sysctl.h>
 
 #include <dev/ofw/ofw_bus.h>
-#include <dev/ofw/ofw_pci.h>
 #include <dev/ofw/openfirm.h>
 
 #include <machine/bus.h>
@@ -110,17 +109,12 @@ static void psycho_iommu_init(struct psycho_softc *, int, uint32_t);
  */
 static device_probe_t psycho_probe;
 static device_attach_t psycho_attach;
-static bus_read_ivar_t psycho_read_ivar;
 static bus_setup_intr_t psycho_setup_intr;
 static bus_alloc_resource_t psycho_alloc_resource;
-static bus_activate_resource_t psycho_activate_resource;
-static bus_adjust_resource_t psycho_adjust_resource;
-static bus_get_dma_tag_t psycho_get_dma_tag;
 static pcib_maxslots_t psycho_maxslots;
 static pcib_read_config_t psycho_read_config;
 static pcib_write_config_t psycho_write_config;
 static pcib_route_interrupt_t psycho_route_interrupt;
-static ofw_bus_get_node_t psycho_get_node;
 static ofw_pci_setup_device_t psycho_setup_device;
 
 static device_method_t psycho_methods[] = {
@@ -132,15 +126,15 @@ static device_method_t psycho_methods[] = {
 	DEVMETHOD(device_resume,	bus_generic_resume),
 
 	/* Bus interface */
-	DEVMETHOD(bus_read_ivar,	psycho_read_ivar),
+	DEVMETHOD(bus_read_ivar,	ofw_pci_read_ivar),
 	DEVMETHOD(bus_setup_intr,	psycho_setup_intr),
 	DEVMETHOD(bus_teardown_intr,	bus_generic_teardown_intr),
 	DEVMETHOD(bus_alloc_resource,	psycho_alloc_resource),
-	DEVMETHOD(bus_activate_resource, psycho_activate_resource),
+	DEVMETHOD(bus_activate_resource, ofw_pci_activate_resource),
 	DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource),
-	DEVMETHOD(bus_adjust_resource,	psycho_adjust_resource),
+	DEVMETHOD(bus_adjust_resource,	ofw_pci_adjust_resource),
 	DEVMETHOD(bus_release_resource,	bus_generic_release_resource),
-	DEVMETHOD(bus_get_dma_tag,	psycho_get_dma_tag),
+	DEVMETHOD(bus_get_dma_tag,	ofw_pci_get_dma_tag),
 
 	/* pcib interface */
 	DEVMETHOD(pcib_maxslots,	psycho_maxslots),
@@ -149,7 +143,7 @@ static device_method_t psycho_methods[] = {
 	DEVMETHOD(pcib_route_interrupt,	psycho_route_interrupt),
 
 	/* ofw_bus interface */
-	DEVMETHOD(ofw_bus_get_node,	psycho_get_node),
+	DEVMETHOD(ofw_bus_get_node,	ofw_pci_get_node),
 
 	/* ofw_pci interface */
 	DEVMETHOD(ofw_pci_setup_device,	psycho_setup_device),
@@ -287,12 +281,12 @@ psycho_attach(device_t dev)
 {
 	struct psycho_icarg *pica;
 	struct psycho_softc *asc, *sc, *osc;
-	struct ofw_pci_ranges *range;
 	const struct psycho_desc *desc;
 	bus_addr_t intrclr, intrmap;
+	bus_dma_tag_t dmat;
 	uint64_t csr, dr;
 	phandle_t node;
-	uint32_t dvmabase, prop, prop_array[2];
+	uint32_t dvmabase, prop;
 	u_int rerun, ver;
 	int i, j;
 
@@ -300,7 +294,6 @@ psycho_attach(device_t dev)
 	sc = device_get_softc(dev);
 	desc = psycho_get_desc(dev);
 
-	sc->sc_node = node;
 	sc->sc_dev = dev;
 	sc->sc_mode = desc->pd_mode;
 
@@ -366,6 +359,7 @@ psycho_attach(device_t dev)
 			panic("%s: mutex not initialized", __func__);
 		sc->sc_mtx = osc->sc_mtx;
 	}
+	SLIST_INSERT_HEAD(&psycho_softcs, sc, sc_link);
 
 	csr = PSYCHO_READ8(sc, PSR_CS);
 	ver = PSYCHO_GCSR_VERS(csr);
@@ -434,43 +428,6 @@ psycho_attach(device_t dev)
 	} else
 		dvmabase = -1;
 
-	/* Initialize memory and I/O rmans. */
-	sc->sc_pci_io_rman.rm_type = RMAN_ARRAY;
-	sc->sc_pci_io_rman.rm_descr = "Psycho PCI I/O Ports";
-	if (rman_init(&sc->sc_pci_io_rman) != 0 ||
-	    rman_manage_region(&sc->sc_pci_io_rman, 0, PSYCHO_IO_SIZE) != 0)
-		panic("%s: failed to set up I/O rman", __func__);
-	sc->sc_pci_mem_rman.rm_type = RMAN_ARRAY;
-	sc->sc_pci_mem_rman.rm_descr = "Psycho PCI Memory";
-	if (rman_init(&sc->sc_pci_mem_rman) != 0 ||
-	    rman_manage_region(&sc->sc_pci_mem_rman, 0, PSYCHO_MEM_SIZE) != 0)
-		panic("%s: failed to set up memory rman", __func__);
-
-	i = OF_getprop_alloc(node, "ranges", sizeof(*range), (void **)&range);
-	/*
-	 * Make sure that the expected ranges are present.  The
-	 * OFW_PCI_CS_MEM64 one is not currently used though.
-	 */
-	if (i != PSYCHO_NRANGE)
-		panic("%s: unsupported number of ranges", __func__);
-	/*
-	 * Find the addresses of the various bus spaces.
-	 * There should not be multiple ones of one kind.
-	 * The physical start addresses of the ranges are the configuration,
-	 * memory and I/O handles.
-	 */
-	for (i = 0; i < PSYCHO_NRANGE; i++) {
-		j = OFW_PCI_RANGE_CS(&range[i]);
-		if (sc->sc_pci_bh[j] != 0)
-			panic("%s: duplicate range for space %d",
-			    __func__, j);
-		sc->sc_pci_bh[j] = OFW_PCI_RANGE_PHYS(&range[i]);
-	}
-	free(range, M_OFWPROP);
-
-	/* Register the softc, this is needed for paired Psychos. */
-	SLIST_INSERT_HEAD(&psycho_softcs, sc, sc_link);
-
 	/*
 	 * If we're a Hummingbird/Sabre or the first of a pair of Psychos
 	 * to arrive here, do the interrupt setup and start up the IOMMU.
@@ -571,37 +528,21 @@ psycho_attach(device_t dev)
 		iommu_reset(sc->sc_is);
 	}
 
-	/* Allocate our tags. */
-	sc->sc_pci_iot = sparc64_alloc_bus_tag(NULL, PCI_IO_BUS_SPACE);
-	if (sc->sc_pci_iot == NULL)
-		panic("%s: could not allocate PCI I/O tag", __func__);
-	sc->sc_pci_cfgt = sparc64_alloc_bus_tag(NULL, PCI_CONFIG_BUS_SPACE);
-	if (sc->sc_pci_cfgt == NULL)
-		panic("%s: could not allocate PCI configuration space tag",
-		    __func__);
+	/* Create our DMA tag. */
 	if (bus_dma_tag_create(bus_get_dma_tag(dev), 8, 0,
 	    sc->sc_is->is_pmaxaddr, ~0, NULL, NULL, sc->sc_is->is_pmaxaddr,
-	    0xff, 0xffffffff, 0, NULL, NULL, &sc->sc_pci_dmat) != 0)
+	    0xff, 0xffffffff, 0, NULL, NULL, &dmat) != 0)
 		panic("%s: could not create PCI DMA tag", __func__);
-	/* Customize the tag. */
-	sc->sc_pci_dmat->dt_cookie = sc->sc_is;
-	sc->sc_pci_dmat->dt_mt = sc->sc_dma_methods;
+	dmat->dt_cookie = sc->sc_is;
+	dmat->dt_mt = sc->sc_dma_methods;
 
-	i = OF_getprop(node, "bus-range", (void *)prop_array,
-	    sizeof(prop_array));
-	if (i == -1)
-		panic("%s: could not get bus-range", __func__);
-	if (i != sizeof(prop_array))
-		panic("%s: broken bus-range (%d)", __func__, i);
-	sc->sc_pci_secbus = prop_array[0];
-	sc->sc_pci_subbus = prop_array[1];
-	if (bootverbose)
-		device_printf(dev, "bus range %u to %u; PCI bus %d\n",
-		    sc->sc_pci_secbus, sc->sc_pci_subbus, sc->sc_pci_secbus);
+	if (ofw_pci_attach_common(dev, dmat, PSYCHO_IO_SIZE,
+	    PSYCHO_MEM_SIZE) != 0)
+		panic("%s: ofw_pci_attach_common() failed", __func__);
 
 	/* Clear any pending PCI error bits. */
-	PCIB_WRITE_CONFIG(dev, sc->sc_pci_secbus, PCS_DEVICE, PCS_FUNC,
-	    PCIR_STATUS, PCIB_READ_CONFIG(dev, sc->sc_pci_secbus,
+	PCIB_WRITE_CONFIG(dev, sc->sc_ops.sc_pci_secbus, PCS_DEVICE, PCS_FUNC,
+	    PCIR_STATUS, PCIB_READ_CONFIG(dev, sc->sc_ops.sc_pci_secbus,
 	    PCS_DEVICE, PCS_FUNC, PCIR_STATUS, 2), 2);
 	PCICTL_WRITE8(sc, PCR_CS, PCICTL_READ8(sc, PCR_CS));
 	PCICTL_WRITE8(sc, PCR_AFS, PCICTL_READ8(sc, PCR_AFS));
@@ -664,20 +605,20 @@ psycho_attach(device_t dev)
 	 * Set the latency timer register as this isn't always done by the
 	 * firmware.
 	 */
-	PCIB_WRITE_CONFIG(dev, sc->sc_pci_secbus, PCS_DEVICE, PCS_FUNC,
+	PCIB_WRITE_CONFIG(dev, sc->sc_ops.sc_pci_secbus, PCS_DEVICE, PCS_FUNC,
 	    PCIR_LATTIMER, OFW_PCI_LATENCY, 1);
 
 	for (i = PCIR_VENDOR; i < PCIR_STATUS; i += sizeof(uint16_t))
-		le16enc(&sc->sc_pci_hpbcfg[i], bus_space_read_2(
-		    sc->sc_pci_cfgt, sc->sc_pci_bh[OFW_PCI_CS_CONFIG],
-		    PSYCHO_CONF_OFF(sc->sc_pci_secbus, PCS_DEVICE,
+		le16enc(&sc->sc_pci_hpbcfg[i],
+		    bus_space_read_2(sc->sc_ops.sc_pci_cfgt,
+		    sc->sc_ops.sc_pci_bh[OFW_PCI_CS_CONFIG],
+		    PSYCHO_CONF_OFF(sc->sc_ops.sc_pci_secbus, PCS_DEVICE,
 		    PCS_FUNC, i)));
 	for (i = PCIR_REVID; i <= PCIR_BIST; i += sizeof(uint8_t))
-		sc->sc_pci_hpbcfg[i] = bus_space_read_1(sc->sc_pci_cfgt,
-		    sc->sc_pci_bh[OFW_PCI_CS_CONFIG], PSYCHO_CONF_OFF(
-		    sc->sc_pci_secbus, PCS_DEVICE, PCS_FUNC, i));
+		sc->sc_pci_hpbcfg[i] = bus_space_read_1(sc->sc_ops.sc_pci_cfgt,
+		    sc->sc_ops.sc_pci_bh[OFW_PCI_CS_CONFIG], PSYCHO_CONF_OFF(
+		    sc->sc_ops.sc_pci_secbus, PCS_DEVICE, PCS_FUNC, i));
 
-	ofw_bus_setup_iinfo(node, &sc->sc_pci_iinfo, sizeof(ofw_pci_intr_t));
 	/*
 	 * On E250 the interrupt map entry for the EBus bridge is wrong,
 	 * causing incorrect interrupts to be assigned to some devices on
@@ -688,9 +629,9 @@ psycho_attach(device_t dev)
 	 * EBus devices will be used directly instead.
 	 */
 	if (strcmp(sparc64_model, "SUNW,Ultra-250") == 0 &&
-	    sc->sc_pci_iinfo.opi_imapmsk != NULL)
-		*(ofw_pci_intr_t *)(&sc->sc_pci_iinfo.opi_imapmsk[
-		    sc->sc_pci_iinfo.opi_addrc]) = INTMAP_INO_MASK;
+	    sc->sc_ops.sc_pci_iinfo.opi_imapmsk != NULL)
+		*(ofw_pci_intr_t *)(&sc->sc_ops.sc_pci_iinfo.opi_imapmsk[
+		    sc->sc_ops.sc_pci_iinfo.opi_addrc]) = INTMAP_INO_MASK;
 
 	device_add_child(dev, "pci", -1);
 	return (bus_generic_attach(dev));
@@ -924,20 +865,8 @@ psycho_read_config(device_t dev, u_int bus, u_int slot, u_int func, u_int reg,
     int width)
 {
 	struct psycho_softc *sc;
-	bus_space_handle_t bh;
-	u_long offset = 0;
-	uint8_t byte;
-	uint16_t shrt;
-	uint32_t r, wrd;
-	int i;
 
 	sc = device_get_softc(dev);
-	if (bus < sc->sc_pci_secbus || bus > sc->sc_pci_subbus ||
-	    slot > PCI_SLOTMAX || func > PCI_FUNCMAX || reg > PCI_REGMAX)
-		return (-1);
-
-	bh = sc->sc_pci_bh[OFW_PCI_CS_CONFIG];
-
 	/*
 	 * The Hummingbird and Sabre bridges are picky in that they
 	 * only allow their config space to be accessed using the
@@ -953,9 +882,9 @@ psycho_read_config(device_t dev, u_int bus, u_int slot, u_int func, u_int reg,
 	 * The Psycho bridges contain a dupe of their header at 0x80
 	 * which we nullify that way also.
 	 */
-	if (bus == sc->sc_pci_secbus && slot == PCS_DEVICE &&
+	if (bus == sc->sc_ops.sc_pci_secbus && slot == PCS_DEVICE &&
 	    func == PCS_FUNC) {
-		if (offset % width != 0)
+		if (reg % width != 0)
 			return (-1);
 
 		if (reg >= sizeof(sc->sc_pci_hpbcfg))
@@ -964,8 +893,9 @@ psycho_read_config(device_t dev, u_int bus, u_int slot, u_int func, u_int reg,
 		if ((reg < PCIR_STATUS && reg + width > PCIR_STATUS) ||
 		    reg == PCIR_STATUS || reg == PCIR_STATUS + 1)
 			le16enc(&sc->sc_pci_hpbcfg[PCIR_STATUS],
-			    bus_space_read_2(sc->sc_pci_cfgt, bh,
-			    PSYCHO_CONF_OFF(sc->sc_pci_secbus,
+			    bus_space_read_2(sc->sc_ops.sc_pci_cfgt,
+			    sc->sc_ops.sc_pci_bh[OFW_PCI_CS_CONFIG],
+			    PSYCHO_CONF_OFF(sc->sc_ops.sc_pci_secbus,
 			    PCS_DEVICE, PCS_FUNC, PCIR_STATUS)));
 
 		switch (width) {
@@ -978,79 +908,29 @@ psycho_read_config(device_t dev, u_int bus, u_int slot, u_int func, u_int reg,
 		}
 	}
 
-	offset = PSYCHO_CONF_OFF(bus, slot, func, reg);
-	switch (width) {
-	case 1:
-		i = bus_space_peek_1(sc->sc_pci_cfgt, bh, offset, &byte);
-		r = byte;
-		break;
-	case 2:
-		i = bus_space_peek_2(sc->sc_pci_cfgt, bh, offset, &shrt);
-		r = shrt;
-		break;
-	case 4:
-		i = bus_space_peek_4(sc->sc_pci_cfgt, bh, offset, &wrd);
-		r = wrd;
-		break;
-	default:
-		panic("%s: bad width", __func__);
-		/* NOTREACHED */
-	}
-
-	if (i) {
-#ifdef PSYCHO_DEBUG
-		printf("%s: read data error reading: %d.%d.%d: 0x%x\n",
-		    __func__, bus, slot, func, reg);
-#endif
-		r = -1;
-	}
-	return (r);
+	return (ofw_pci_read_config_common(dev, PCI_REGMAX,
+	    PSYCHO_CONF_OFF(bus, slot, func, reg), bus, slot, func, reg,
+	    width));
 }
 
 static void
 psycho_write_config(device_t dev, u_int bus, u_int slot, u_int func,
     u_int reg, uint32_t val, int width)
 {
-	struct psycho_softc *sc;
-	bus_space_handle_t bh;
-	u_long offset = 0;
 
-	sc = device_get_softc(dev);
-	if (bus < sc->sc_pci_secbus || bus > sc->sc_pci_subbus ||
-	    slot > PCI_SLOTMAX || func > PCI_FUNCMAX || reg > PCI_REGMAX)
-		return;
-
-	offset = PSYCHO_CONF_OFF(bus, slot, func, reg);
-	bh = sc->sc_pci_bh[OFW_PCI_CS_CONFIG];
-	switch (width) {
-	case 1:
-		bus_space_write_1(sc->sc_pci_cfgt, bh, offset, val);
-		break;
-	case 2:
-		bus_space_write_2(sc->sc_pci_cfgt, bh, offset, val);
-		break;
-	case 4:
-		bus_space_write_4(sc->sc_pci_cfgt, bh, offset, val);
-		break;
-	default:
-		panic("%s: bad width", __func__);
-		/* NOTREACHED */
-	}
+	ofw_pci_write_config_common(dev, PCI_REGMAX, PSYCHO_CONF_OFF(bus,
+	    slot, func, reg), bus, slot, func, reg, val, width);
 }
 
 static int
 psycho_route_interrupt(device_t bridge, device_t dev, int pin)
 {
 	struct psycho_softc *sc;
-	struct ofw_pci_register reg;
 	bus_addr_t intrmap;
-	ofw_pci_intr_t pintr, mintr;
+	ofw_pci_intr_t mintr;
 
-	sc = device_get_softc(bridge);
-	pintr = pin;
-	if (ofw_bus_lookup_imap(ofw_bus_get_node(dev), &sc->sc_pci_iinfo,
-	    &reg, sizeof(reg), &pintr, sizeof(pintr), &mintr, sizeof(mintr),
-	    NULL))
+	mintr = ofw_pci_route_interrupt_common(bridge, dev, pin);
+	if (PCI_INTERRUPT_VALID(mintr))
 		return (mintr);
 	/*
 	 * If this is outside of the range for an intpin, it's likely a full
@@ -1069,6 +949,7 @@ psycho_route_interrupt(device_t bridge, device_t dev, int pin)
 	 * for bus A are one-based, while those for bus B seemingly have an
 	 * offset of 2 (hence the factor of 3 below).
 	 */
+	sc = device_get_softc(dev);
 	intrmap = PSR_PCIA0_INT_MAP +
 	    8 * (pci_get_slot(dev) - 1 + 3 * sc->sc_half);
 	mintr = INTINO(PSYCHO_READ8(sc, intrmap)) + pin - 1;
@@ -1078,23 +959,6 @@ psycho_route_interrupt(device_t bridge, device_t dev, int pin)
 	return (mintr);
 }
 
-static int
-psycho_read_ivar(device_t dev, device_t child, int which, uintptr_t *result)
-{
-	struct psycho_softc *sc;
-
-	sc = device_get_softc(dev);
-	switch (which) {
-	case PCIB_IVAR_DOMAIN:
-		*result = device_get_unit(dev);
-		return (0);
-	case PCIB_IVAR_BUS:
-		*result = sc->sc_pci_secbus;
-		return (0);
-	}
-	return (ENOENT);
-}
-
 static void
 sabre_dmamap_sync(bus_dma_tag_t dt, bus_dmamap_t map, bus_dmasync_op_t op)
 {
@@ -1177,120 +1041,13 @@ psycho_alloc_resource(device_t bus, device_t child, int type, int *rid,
     u_long start, u_long end, u_long count, u_int flags)
 {
 	struct psycho_softc *sc;
-	struct resource *rv;
-	struct rman *rm;
 
-	sc = device_get_softc(bus);
-	switch (type) {
-	case SYS_RES_IRQ:
-		/*
-		 * XXX: Don't accept blank ranges for now, only single
-		 * interrupts.  The other case should not happen with
-		 * the MI PCI code...
-		 * XXX: This may return a resource that is out of the
-		 * range that was specified.  Is this correct...?
-		 */
-		if (start != end)
-			panic("%s: XXX: interrupt range", __func__);
+	if (type == SYS_RES_IRQ) {
+		sc = device_get_softc(bus);
 		start = end = INTMAP_VEC(sc->sc_ign, end);
-		return (bus_generic_alloc_resource(bus, child, type, rid,
-		     start, end, count, flags));
-	case SYS_RES_MEMORY:
-		rm = &sc->sc_pci_mem_rman;
-		break;
-	case SYS_RES_IOPORT:
-		rm = &sc->sc_pci_io_rman;
-		break;
-	default:
-		return (NULL);
-	}
-
-	rv = rman_reserve_resource(rm, start, end, count, flags & ~RF_ACTIVE,
-	    child);
-	if (rv == NULL)
-		return (NULL);
-	rman_set_rid(rv, *rid);
-
-	if ((flags & RF_ACTIVE) != 0 && bus_activate_resource(child, type,
-	    *rid, rv) != 0) {
-		rman_release_resource(rv);
-		return (NULL);
-	}
-	return (rv);
-}
-
-static int
-psycho_activate_resource(device_t bus, device_t child, int type, int rid,
-    struct resource *r)
-{
-	struct psycho_softc *sc;
-	struct bus_space_tag *tag;
-
-	sc = device_get_softc(bus);
-	switch (type) {
-	case SYS_RES_IRQ:
-		return (bus_generic_activate_resource(bus, child, type, rid,
-		    r));
-	case SYS_RES_MEMORY:
-		tag = sparc64_alloc_bus_tag(r, PCI_MEMORY_BUS_SPACE);
-		if (tag == NULL)
-			return (ENOMEM);
-		rman_set_bustag(r, tag);
-		rman_set_bushandle(r, sc->sc_pci_bh[OFW_PCI_CS_MEM32] +
-		    rman_get_start(r));
-		break;
-	case SYS_RES_IOPORT:
-		rman_set_bustag(r, sc->sc_pci_iot);
-		rman_set_bushandle(r, sc->sc_pci_bh[OFW_PCI_CS_IO] +
-		    rman_get_start(r));
-		break;
-	}
-	return (rman_activate_resource(r));
-}
-
-static int
-psycho_adjust_resource(device_t bus, device_t child, int type,
-    struct resource *r, u_long start, u_long end)
-{
-	struct psycho_softc *sc;
-	struct rman *rm;
-
-	sc = device_get_softc(bus);
-	switch (type) {
-	case SYS_RES_IRQ:
-		return (bus_generic_adjust_resource(bus, child, type, r,
-		    start, end));
-	case SYS_RES_MEMORY:
-		rm = &sc->sc_pci_mem_rman;
-		break;
-	case SYS_RES_IOPORT:
-		rm = &sc->sc_pci_io_rman;
-		break;
-	default:
-		return (EINVAL);
 	}
-	if (rman_is_region_manager(r, rm) == 0)
-		return (EINVAL);
-	return (rman_adjust_resource(r, start, end));
-}
-
-static bus_dma_tag_t
-psycho_get_dma_tag(device_t bus, device_t child __unused)
-{
-	struct psycho_softc *sc;
-
-	sc = device_get_softc(bus);
-	return (sc->sc_pci_dmat);
-}
-
-static phandle_t
-psycho_get_node(device_t bus, device_t child __unused)
-{
-	struct psycho_softc *sc;
-
-	sc = device_get_softc(bus);
-	/* We only have one child, the PCI bus, which needs our own node. */
-	return (sc->sc_node);
+	return (ofw_pci_alloc_resource(bus, child, type, rid, start, end,
+	    count, flags));
 }
 
 static void
diff --git a/sys/sparc64/pci/psychoreg.h b/sys/sparc64/pci/psychoreg.h
index ede593abf79f3..6e4ea39f2259a 100644
--- a/sys/sparc64/pci/psychoreg.h
+++ b/sys/sparc64/pci/psychoreg.h
@@ -73,7 +73,6 @@
  */
 
 #define	PSYCHO_NINTR		6
-#define	PSYCHO_NRANGE		4
 
 /*
  * Psycho register offsets
@@ -121,7 +120,7 @@
 #define	PSR_PWRMGT_INT_MAP	0x1090	/* power mgmt wake interrupt map reg */
 #define	PSR_FFB0_INT_MAP	0x1098	/* FFB0 graphics interrupt map reg */
 #define	PSR_FFB1_INT_MAP	0x10a0	/* FFB1 graphics interrupt map reg */
-/* Note: clear interrupt 0 registers are not really used */
+/* Note: Clear interrupt 0 registers are not really used. */
 #define	PSR_PCIA0_INT_CLR	0x1400	/* PCI a slot 0 clear int regs 0..3 */
 #define	PSR_PCIA1_INT_CLR	0x1420	/* PCI a slot 1 clear int regs 0..3 */
 #define	PSR_PCIA2_INT_CLR	0x1440	/* PCI a slot 2 clear int regs 0..3 */
@@ -165,6 +164,7 @@
 #define	PSR_PCI_INT_DIAG	0xa800	/* PCI int state diag reg */
 #define	PSR_OBIO_INT_DIAG	0xa808	/* OBIO and misc int state diag reg */
 #define	PSR_STRBUF_DIAG		0xb000	/* Streaming buffer diag regs */
+
 /*
  * Here is the rest of the map, which we're not specifying:
  *
@@ -176,7 +176,7 @@
  * 1ff.0000.0000 - 1ff.7fff.ffff	PCI A memory space
  * 1ff.8000.0000 - 1ff.ffff.ffff	PCI B memory space
  *
- * NB: config and I/O space can use 1-4 byte accesses, not 8 byte
+ * NB: Config and I/O space can use 1-4 byte accesses, not 8 byte
  * accesses.  Memory space can use any sized accesses.
  *
  * Note that the SUNW,sabre/SUNW,simba combinations found on the
diff --git a/sys/sparc64/pci/psychovar.h b/sys/sparc64/pci/psychovar.h
index 5532d16bbe6f8..0f828ee586590 100644
--- a/sys/sparc64/pci/psychovar.h
+++ b/sys/sparc64/pci/psychovar.h
@@ -36,49 +36,38 @@
  * per pair of psychos.
  */
 struct psycho_softc {
-	struct bus_dma_methods		*sc_dma_methods;
+	/*
+	 * This is here so that we can hook up the common bus interface
+	 * methods in ofw_pci.c directly.
+	 */
+	struct ofw_pci_softc		sc_ops;
 
-	device_t			sc_dev;
+	struct iommu_state		*sc_is;
+	struct bus_dma_methods		*sc_dma_methods;
 
 	struct mtx			*sc_mtx;
 
-	/* Interrupt Group Number for this device */
-	uint32_t			sc_ign;
-
-	bus_addr_t			sc_pcictl;
-
-	phandle_t			sc_node;	/* Firmware node */
-	u_int				sc_mode;
-#define	PSYCHO_MODE_SABRE		0
-#define	PSYCHO_MODE_PSYCHO		1
-
-	/* Bus A or B of a psycho pair? */
-	u_int				sc_half;
-
-	struct iommu_state		*sc_is;
-
 	struct resource			*sc_mem_res;
 	struct resource			*sc_irq_res[PSYCHO_NINTR];
 	void				*sc_ihand[PSYCHO_NINTR];
 
-	struct ofw_bus_iinfo		sc_pci_iinfo;
+	uint8_t				sc_pci_hpbcfg[16];
 
-	/* Tags for PCI access */
-	bus_space_tag_t			sc_pci_cfgt;
-	bus_space_tag_t			sc_pci_iot;
-	bus_dma_tag_t			sc_pci_dmat;
+	SLIST_ENTRY(psycho_softc)	sc_link;
 
-	bus_space_handle_t		sc_pci_bh[PSYCHO_NRANGE];
+	device_t			sc_dev;
 
-	struct rman			sc_pci_mem_rman;
-	struct rman			sc_pci_io_rman;
+	bus_addr_t			sc_pcictl;
 
-	uint8_t				sc_pci_secbus;
-	uint8_t				sc_pci_subbus;
+	u_int				sc_mode;
+#define	PSYCHO_MODE_SABRE		0
+#define	PSYCHO_MODE_PSYCHO		1
 
-	uint8_t				sc_pci_hpbcfg[16];
+	/* Bus A or B of a psycho pair? */
+	u_int				sc_half;
 
-	SLIST_ENTRY(psycho_softc)	sc_link;
+	/* Interrupt Group Number for this device */
+	uint32_t			sc_ign;
 };
 
 #endif /* !_SPARC64_PCI_PSYCHOVAR_H_ */
diff --git a/sys/sparc64/pci/schizo.c b/sys/sparc64/pci/schizo.c
index e7ebaf1898780..ed1cccdaac19b 100644
--- a/sys/sparc64/pci/schizo.c
+++ b/sys/sparc64/pci/schizo.c
@@ -57,13 +57,11 @@ __FBSDID("$FreeBSD$");
 #include <sys/timetc.h>
 
 #include <dev/ofw/ofw_bus.h>
-#include <dev/ofw/ofw_pci.h>
 #include <dev/ofw/openfirm.h>
 
 #include <machine/bus.h>
 #include <machine/bus_common.h>
 #include <machine/bus_private.h>
-#include <machine/fsr.h>
 #include <machine/iommureg.h>
 #include <machine/iommuvar.h>
 #include <machine/resource.h>
@@ -108,17 +106,12 @@ static void schizo_iommu_init(struct schizo_softc *, int, uint32_t);
  */
 static device_probe_t schizo_probe;
 static device_attach_t schizo_attach;
-static bus_read_ivar_t schizo_read_ivar;
 static bus_setup_intr_t schizo_setup_intr;
 static bus_alloc_resource_t schizo_alloc_resource;
-static bus_activate_resource_t schizo_activate_resource;
-static bus_adjust_resource_t schizo_adjust_resource;
-static bus_get_dma_tag_t schizo_get_dma_tag;
 static pcib_maxslots_t schizo_maxslots;
 static pcib_read_config_t schizo_read_config;
 static pcib_write_config_t schizo_write_config;
 static pcib_route_interrupt_t schizo_route_interrupt;
-static ofw_bus_get_node_t schizo_get_node;
 static ofw_pci_setup_device_t schizo_setup_device;
 
 static device_method_t schizo_methods[] = {
@@ -130,15 +123,15 @@ static device_method_t schizo_methods[] = {
 	DEVMETHOD(device_resume,	bus_generic_resume),
 
 	/* Bus interface */
-	DEVMETHOD(bus_read_ivar,	schizo_read_ivar),
+	DEVMETHOD(bus_read_ivar,	ofw_pci_read_ivar),
 	DEVMETHOD(bus_setup_intr,	schizo_setup_intr),
 	DEVMETHOD(bus_teardown_intr,	bus_generic_teardown_intr),
 	DEVMETHOD(bus_alloc_resource,	schizo_alloc_resource),
-	DEVMETHOD(bus_activate_resource, schizo_activate_resource),
+	DEVMETHOD(bus_activate_resource, ofw_pci_activate_resource),
 	DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource),
-	DEVMETHOD(bus_adjust_resource,	schizo_adjust_resource),
+	DEVMETHOD(bus_adjust_resource,	ofw_pci_adjust_resource),
 	DEVMETHOD(bus_release_resource,	bus_generic_release_resource),
-	DEVMETHOD(bus_get_dma_tag,	schizo_get_dma_tag),
+	DEVMETHOD(bus_get_dma_tag,	ofw_pci_get_dma_tag),
 
 	/* pcib interface */
 	DEVMETHOD(pcib_maxslots,	schizo_maxslots),
@@ -147,7 +140,7 @@ static device_method_t schizo_methods[] = {
 	DEVMETHOD(pcib_route_interrupt,	schizo_route_interrupt),
 
 	/* ofw_bus interface */
-	DEVMETHOD(ofw_bus_get_node,	schizo_get_node),
+	DEVMETHOD(ofw_bus_get_node,	ofw_pci_get_node),
 
 	/* ofw_pci interface */
 	DEVMETHOD(ofw_pci_setup_device,	schizo_setup_device),
@@ -270,10 +263,10 @@ schizo_probe(device_t dev)
 static int
 schizo_attach(device_t dev)
 {
-	struct ofw_pci_ranges *range;
 	const struct schizo_desc *desc;
 	struct schizo_softc *asc, *sc, *osc;
 	struct timecounter *tc;
+	bus_dma_tag_t dmat;
 	uint64_t ino_bitmap, reg;
 	phandle_t node;
 	uint32_t prop, prop_array[2];
@@ -285,7 +278,6 @@ schizo_attach(device_t dev)
 	mode = desc->sd_mode;
 
 	sc->sc_dev = dev;
-	sc->sc_node = node;
 	sc->sc_mode = mode;
 	sc->sc_flags = 0;
 
@@ -347,6 +339,7 @@ schizo_attach(device_t dev)
 			panic("%s: mutex not initialized", __func__);
 		sc->sc_mtx = osc->sc_mtx;
 	}
+	SLIST_INSERT_HEAD(&schizo_softcs, sc, sc_link);
 
 	if (OF_getprop(node, "portid", &sc->sc_ign, sizeof(sc->sc_ign)) == -1)
 		panic("%s: could not determine IGN", __func__);
@@ -542,80 +535,23 @@ schizo_attach(device_t dev)
 
 #undef TSBCASE
 
-	/* Initialize memory and I/O rmans. */
-	sc->sc_pci_io_rman.rm_type = RMAN_ARRAY;
-	sc->sc_pci_io_rman.rm_descr = "Schizo PCI I/O Ports";
-	if (rman_init(&sc->sc_pci_io_rman) != 0 ||
-	    rman_manage_region(&sc->sc_pci_io_rman, 0, STX_IO_SIZE) != 0)
-		panic("%s: failed to set up I/O rman", __func__);
-	sc->sc_pci_mem_rman.rm_type = RMAN_ARRAY;
-	sc->sc_pci_mem_rman.rm_descr = "Schizo PCI Memory";
-	if (rman_init(&sc->sc_pci_mem_rman) != 0 ||
-	    rman_manage_region(&sc->sc_pci_mem_rman, 0, STX_MEM_SIZE) != 0)
-		panic("%s: failed to set up memory rman", __func__);
-
-	i = OF_getprop_alloc(node, "ranges", sizeof(*range), (void **)&range);
-	/*
-	 * Make sure that the expected ranges are present.  The
-	 * OFW_PCI_CS_MEM64 one is not currently used though.
-	 */
-	if (i != STX_NRANGE)
-		panic("%s: unsupported number of ranges", __func__);
-	/*
-	 * Find the addresses of the various bus spaces.
-	 * There should not be multiple ones of one kind.
-	 * The physical start addresses of the ranges are the configuration,
-	 * memory and I/O handles.
-	 */
-	for (i = 0; i < STX_NRANGE; i++) {
-		j = OFW_PCI_RANGE_CS(&range[i]);
-		if (sc->sc_pci_bh[j] != 0)
-			panic("%s: duplicate range for space %d",
-			    __func__, j);
-		sc->sc_pci_bh[j] = OFW_PCI_RANGE_PHYS(&range[i]);
-	}
-	free(range, M_OFWPROP);
-
-	/* Register the softc, this is needed for paired Schizos. */
-	SLIST_INSERT_HEAD(&schizo_softcs, sc, sc_link);
-
-	/* Allocate our tags. */
-	sc->sc_pci_iot = sparc64_alloc_bus_tag(NULL, PCI_IO_BUS_SPACE);
-	if (sc->sc_pci_iot == NULL)
-		panic("%s: could not allocate PCI I/O tag", __func__);
-	sc->sc_pci_cfgt = sparc64_alloc_bus_tag(NULL, PCI_CONFIG_BUS_SPACE);
-	if (sc->sc_pci_cfgt == NULL)
-		panic("%s: could not allocate PCI configuration space tag",
-		    __func__);
+	/* Create our DMA tag. */
 	if (bus_dma_tag_create(bus_get_dma_tag(dev), 8, 0,
 	    sc->sc_is.sis_is.is_pmaxaddr, ~0, NULL, NULL,
 	    sc->sc_is.sis_is.is_pmaxaddr, 0xff, 0xffffffff, 0, NULL, NULL,
-	    &sc->sc_pci_dmat) != 0)
+	    &dmat) != 0)
 		panic("%s: could not create PCI DMA tag", __func__);
-	/* Customize the tag. */
-	sc->sc_pci_dmat->dt_cookie = &sc->sc_is;
-	sc->sc_pci_dmat->dt_mt = &sc->sc_dma_methods;
+	dmat->dt_cookie = &sc->sc_is;
+	dmat->dt_mt = &sc->sc_dma_methods;
 
-	/*
-	 * Get the bus range from the firmware.
-	 * NB: Tomatillos don't support PCI bus reenumeration.
-	 */
-	i = OF_getprop(node, "bus-range", (void *)prop_array,
-	    sizeof(prop_array));
-	if (i == -1)
-		panic("%s: could not get bus-range", __func__);
-	if (i != sizeof(prop_array))
-		panic("%s: broken bus-range (%d)", __func__, i);
-	sc->sc_pci_secbus = prop_array[0];
-	sc->sc_pci_subbus = prop_array[1];
-	if (bootverbose)
-		device_printf(dev, "bus range %u to %u; PCI bus %d\n",
-		    sc->sc_pci_secbus, sc->sc_pci_subbus, sc->sc_pci_secbus);
+	if (ofw_pci_attach_common(dev, dmat, STX_IO_SIZE, STX_MEM_SIZE) != 0)
+		panic("%s: ofw_pci_attach_common() failed", __func__);
 
 	/* Clear any pending PCI error bits. */
-	PCIB_WRITE_CONFIG(dev, sc->sc_pci_secbus, STX_CS_DEVICE, STX_CS_FUNC,
-	    PCIR_STATUS, PCIB_READ_CONFIG(dev, sc->sc_pci_secbus,
-	    STX_CS_DEVICE, STX_CS_FUNC, PCIR_STATUS, 2), 2);
+	PCIB_WRITE_CONFIG(dev, sc->sc_ops.sc_pci_secbus, STX_CS_DEVICE,
+	    STX_CS_FUNC, PCIR_STATUS, PCIB_READ_CONFIG(dev,
+	    sc->sc_ops.sc_pci_secbus, STX_CS_DEVICE, STX_CS_FUNC, PCIR_STATUS,
+	    2), 2);
 	SCHIZO_PCI_SET(sc, STX_PCI_CTRL, SCHIZO_PCI_READ_8(sc, STX_PCI_CTRL));
 	SCHIZO_PCI_SET(sc, STX_PCI_AFSR, SCHIZO_PCI_READ_8(sc, STX_PCI_AFSR));
 
@@ -743,10 +679,8 @@ schizo_attach(device_t dev)
 	 * Set the latency timer register as this isn't always done by the
 	 * firmware.
 	 */
-	PCIB_WRITE_CONFIG(dev, sc->sc_pci_secbus, STX_CS_DEVICE, STX_CS_FUNC,
-	    PCIR_LATTIMER, OFW_PCI_LATENCY, 1);
-
-	ofw_bus_setup_iinfo(node, &sc->sc_pci_iinfo, sizeof(ofw_pci_intr_t));
+	PCIB_WRITE_CONFIG(dev, sc->sc_ops.sc_pci_secbus, STX_CS_DEVICE,
+	    STX_CS_FUNC, PCIR_LATTIMER, OFW_PCI_LATENCY, 1);
 
 #define	SCHIZO_SYSCTL_ADD_UINT(name, arg, desc)				\
 	SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),			\
@@ -870,7 +804,7 @@ schizo_pci_bus(void *arg)
 		xstat = SCHIZO_PCI_READ_8(sc, XMS_PCI_X_ERR_STAT);
 	else
 		xstat = 0;
-	status = PCIB_READ_CONFIG(sc->sc_dev, sc->sc_pci_secbus,
+	status = PCIB_READ_CONFIG(sc->sc_dev, sc->sc_ops.sc_pci_secbus,
 	    STX_CS_DEVICE, STX_CS_FUNC, PCIR_STATUS, 2);
 
 	/*
@@ -911,7 +845,7 @@ schizo_pci_bus(void *arg)
 	    (unsigned long long)iommu, (unsigned long long)xstat, status);
 
 	/* Clear the error bits that we caught. */
-	PCIB_WRITE_CONFIG(sc->sc_dev, sc->sc_pci_secbus, STX_CS_DEVICE,
+	PCIB_WRITE_CONFIG(sc->sc_dev, sc->sc_ops.sc_pci_secbus, STX_CS_DEVICE,
 	    STX_CS_FUNC, PCIR_STATUS, status, 2);
 	SCHIZO_PCI_WRITE_8(sc, STX_PCI_CTRL, csr);
 	SCHIZO_PCI_WRITE_8(sc, STX_PCI_AFSR, afsr);
@@ -1032,121 +966,40 @@ schizo_read_config(device_t dev, u_int bus, u_int slot, u_int func, u_int reg,
     int width)
 {
 	struct schizo_softc *sc;
-	bus_space_handle_t bh;
-	u_long offset = 0;
-	uint32_t r, wrd;
-	int i;
-	uint16_t shrt;
-	uint8_t byte;
 
 	sc = device_get_softc(dev);
-	if (bus < sc->sc_pci_secbus || bus > sc->sc_pci_subbus ||
-	    slot > PCI_SLOTMAX || func > PCI_FUNCMAX || reg > PCI_REGMAX)
-		return (-1);
-
 	/*
 	 * The Schizo bridges contain a dupe of their header at 0x80.
 	 */
-	if (sc->sc_mode == SCHIZO_MODE_SCZ && bus == sc->sc_pci_secbus &&
-	    slot == STX_CS_DEVICE && func == STX_CS_FUNC &&
-	    reg + width > 0x80)
+	if (sc->sc_mode == SCHIZO_MODE_SCZ &&
+	    bus == sc->sc_ops.sc_pci_secbus && slot == STX_CS_DEVICE &&
+	    func == STX_CS_FUNC && reg + width > 0x80)
 		return (0);
 
-	offset = STX_CONF_OFF(bus, slot, func, reg);
-	bh = sc->sc_pci_bh[OFW_PCI_CS_CONFIG];
-	switch (width) {
-	case 1:
-		i = bus_space_peek_1(sc->sc_pci_cfgt, bh, offset, &byte);
-		r = byte;
-		break;
-	case 2:
-		i = bus_space_peek_2(sc->sc_pci_cfgt, bh, offset, &shrt);
-		r = shrt;
-		break;
-	case 4:
-		i = bus_space_peek_4(sc->sc_pci_cfgt, bh, offset, &wrd);
-		r = wrd;
-		break;
-	default:
-		panic("%s: bad width", __func__);
-		/* NOTREACHED */
-	}
-
-	if (i) {
-#ifdef SCHIZO_DEBUG
-		printf("%s: read data error reading: %d.%d.%d: 0x%x\n",
-		    __func__, bus, slot, func, reg);
-#endif
-		r = -1;
-	}
-	return (r);
+	return (ofw_pci_read_config_common(dev, PCI_REGMAX, STX_CONF_OFF(bus,
+	    slot, func, reg), bus, slot, func, reg, width));
 }
 
 static void
 schizo_write_config(device_t dev, u_int bus, u_int slot, u_int func,
     u_int reg, uint32_t val, int width)
 {
-	struct schizo_softc *sc;
-	bus_space_handle_t bh;
-	u_long offset = 0;
-
-	sc = device_get_softc(dev);
-	if (bus < sc->sc_pci_secbus || bus > sc->sc_pci_subbus ||
-	    slot > PCI_SLOTMAX || func > PCI_FUNCMAX || reg > PCI_REGMAX)
-		return;
 
-	offset = STX_CONF_OFF(bus, slot, func, reg);
-	bh = sc->sc_pci_bh[OFW_PCI_CS_CONFIG];
-	switch (width) {
-	case 1:
-		bus_space_write_1(sc->sc_pci_cfgt, bh, offset, val);
-		break;
-	case 2:
-		bus_space_write_2(sc->sc_pci_cfgt, bh, offset, val);
-		break;
-	case 4:
-		bus_space_write_4(sc->sc_pci_cfgt, bh, offset, val);
-		break;
-	default:
-		panic("%s: bad width", __func__);
-		/* NOTREACHED */
-	}
+	ofw_pci_write_config_common(dev, PCI_REGMAX, STX_CONF_OFF(bus, slot,
+	    func, reg), bus, slot, func, reg, val, width);
 }
 
 static int
 schizo_route_interrupt(device_t bridge, device_t dev, int pin)
 {
-	struct schizo_softc *sc;
-	struct ofw_pci_register reg;
-	ofw_pci_intr_t pintr, mintr;
-
-	sc = device_get_softc(bridge);
-	pintr = pin;
-	if (ofw_bus_lookup_imap(ofw_bus_get_node(dev), &sc->sc_pci_iinfo,
-	    &reg, sizeof(reg), &pintr, sizeof(pintr), &mintr, sizeof(mintr),
-	    NULL))
-		return (mintr);
+	ofw_pci_intr_t mintr;
 
-	device_printf(bridge, "could not route pin %d for device %d.%d\n",
-	    pin, pci_get_slot(dev), pci_get_function(dev));
-	return (PCI_INVALID_IRQ);
-}
-
-static int
-schizo_read_ivar(device_t dev, device_t child, int which, uintptr_t *result)
-{
-	struct schizo_softc *sc;
-
-	sc = device_get_softc(dev);
-	switch (which) {
-	case PCIB_IVAR_DOMAIN:
-		*result = device_get_unit(dev);
-		return (0);
-	case PCIB_IVAR_BUS:
-		*result = sc->sc_pci_secbus;
-		return (0);
-	}
-	return (ENOENT);
+	mintr = ofw_pci_route_interrupt_common(bridge, dev, pin);
+	if (!PCI_INTERRUPT_VALID(mintr))
+		device_printf(bridge,
+		    "could not route pin %d for device %d.%d\n",
+		    pin, pci_get_slot(dev), pci_get_function(dev));
+	return (mintr);
 }
 
 static void
@@ -1214,11 +1067,10 @@ schizo_dmamap_sync(bus_dma_tag_t dt, bus_dmamap_t map, bus_dmasync_op_t op)
 static void
 ichip_dmamap_sync(bus_dma_tag_t dt, bus_dmamap_t map, bus_dmasync_op_t op)
 {
-	static u_char buf[VIS_BLOCKSIZE] __aligned(VIS_BLOCKSIZE);
 	struct timeval cur, end;
 	struct schizo_iommu_state *sis = dt->dt_cookie;
 	struct schizo_softc *sc = sis->sis_sc;
-	register_t reg, s;
+	uint64_t reg;
 
 	if ((map->dm_flags & DMF_STREAMED) != 0) {
 		iommu_dma_methods.dm_dmamap_sync(dt, map, op);
@@ -1246,14 +1098,7 @@ ichip_dmamap_sync(bus_dma_tag_t dt, bus_dmamap_t map, bus_dmasync_op_t op)
 		if (sc->sc_mode == SCHIZO_MODE_XMS)
 			mtx_unlock_spin(&sc->sc_sync_mtx);
 		else if ((sc->sc_flags & SCHIZO_FLAGS_BSWAR) != 0) {
-			s = intr_disable();
-			reg = rd(fprs);
-			wr(fprs, reg | FPRS_FEF, 0);
-			__asm __volatile("stda %%f0, [%0] %1"
-			    : : "r" (buf), "n" (ASI_BLK_COMMIT_S));
-			membar(Sync);
-			wr(fprs, reg, 0);
-			intr_restore(s);
+			ofw_pci_dmamap_sync_stst_order_common();
 			return;
 		}
 	}
@@ -1354,120 +1199,13 @@ schizo_alloc_resource(device_t bus, device_t child, int type, int *rid,
     u_long start, u_long end, u_long count, u_int flags)
 {
 	struct schizo_softc *sc;
-	struct resource *rv;
-	struct rman *rm;
 
-	sc = device_get_softc(bus);
-	switch (type) {
-	case SYS_RES_IRQ:
-		/*
-		 * XXX: Don't accept blank ranges for now, only single
-		 * interrupts.  The other case should not happen with
-		 * the MI PCI code...
-		 * XXX: This may return a resource that is out of the
-		 * range that was specified.  Is this correct...?
-		 */
-		if (start != end)
-			panic("%s: XXX: interrupt range", __func__);
+	if (type == SYS_RES_IRQ) {
+		sc = device_get_softc(bus);
 		start = end = INTMAP_VEC(sc->sc_ign, end);
-		return (bus_generic_alloc_resource(bus, child, type, rid,
-		    start, end, count, flags));
-	case SYS_RES_MEMORY:
-		rm = &sc->sc_pci_mem_rman;
-		break;
-	case SYS_RES_IOPORT:
-		rm = &sc->sc_pci_io_rman;
-		break;
-	default:
-		return (NULL);
-	}
-
-	rv = rman_reserve_resource(rm, start, end, count, flags & ~RF_ACTIVE,
-	    child);
-	if (rv == NULL)
-		return (NULL);
-	rman_set_rid(rv, *rid);
-
-	if ((flags & RF_ACTIVE) != 0 && bus_activate_resource(child, type,
-	    *rid, rv) != 0) {
-		rman_release_resource(rv);
-		return (NULL);
-	}
-	return (rv);
-}
-
-static int
-schizo_activate_resource(device_t bus, device_t child, int type, int rid,
-    struct resource *r)
-{
-	struct schizo_softc *sc;
-	struct bus_space_tag *tag;
-
-	sc = device_get_softc(bus);
-	switch (type) {
-	case SYS_RES_IRQ:
-		return (bus_generic_activate_resource(bus, child, type, rid,
-		    r));
-	case SYS_RES_MEMORY:
-		tag = sparc64_alloc_bus_tag(r, PCI_MEMORY_BUS_SPACE);
-		if (tag == NULL)
-			return (ENOMEM);
-		rman_set_bustag(r, tag);
-		rman_set_bushandle(r, sc->sc_pci_bh[OFW_PCI_CS_MEM32] +
-		    rman_get_start(r));
-		break;
-	case SYS_RES_IOPORT:
-		rman_set_bustag(r, sc->sc_pci_iot);
-		rman_set_bushandle(r, sc->sc_pci_bh[OFW_PCI_CS_IO] +
-		    rman_get_start(r));
-		break;
 	}
-	return (rman_activate_resource(r));
-}
-
-static int
-schizo_adjust_resource(device_t bus, device_t child, int type,
-    struct resource *r, u_long start, u_long end)
-{
-	struct schizo_softc *sc;
-	struct rman *rm;
-
-	sc = device_get_softc(bus);
-	switch (type) {
-	case SYS_RES_IRQ:
-		return (bus_generic_adjust_resource(bus, child, type, r,
-		    start, end));
-	case SYS_RES_MEMORY:
-		rm = &sc->sc_pci_mem_rman;
-		break;
-	case SYS_RES_IOPORT:
-		rm = &sc->sc_pci_io_rman;
-		break;
-	default:
-		return (EINVAL);
-	}
-	if (rman_is_region_manager(r, rm) == 0)
-		return (EINVAL);
-	return (rman_adjust_resource(r, start, end));
-}
-
-static bus_dma_tag_t
-schizo_get_dma_tag(device_t bus, device_t child __unused)
-{
-	struct schizo_softc *sc;
-
-	sc = device_get_softc(bus);
-	return (sc->sc_pci_dmat);
-}
-
-static phandle_t
-schizo_get_node(device_t bus, device_t child __unused)
-{
-	struct schizo_softc *sc;
-
-	sc = device_get_softc(bus);
-	/* We only have one child, the PCI bus, which needs our own node. */
-	return (sc->sc_node);
+	return (ofw_pci_alloc_resource(bus, child, type, rid, start, end,
+	    count, flags));
 }
 
 static void
diff --git a/sys/sparc64/pci/schizoreg.h b/sys/sparc64/pci/schizoreg.h
index cd816b57672bc..59c00b6c632eb 100644
--- a/sys/sparc64/pci/schizoreg.h
+++ b/sys/sparc64/pci/schizoreg.h
@@ -32,7 +32,6 @@
 #define	_SPARC64_PCI_SCHIZOREG_H_
 
 #define	STX_NINTR			5	/* 4 via OFW + 1 CDMA */
-#define	STX_NRANGE			4
 #define	SCZ_NREG			3
 #define	TOM_NREG			4
 
@@ -279,7 +278,7 @@
 
 /*
  * Safari/JBus performance control register
- * NB: for Tomatillo only events 0x00 through 0x08 are documented as
+ * NB: For Tomatillo only events 0x00 through 0x08 are documented as
  * implemented.
  */
 #define	SCZ_CTRL_PERF_ZDATA_OUT		0x0000000000000016ULL
@@ -345,7 +344,7 @@
 
 /* Non-Standard registers in the configration space */
 /*
- * NB: for Tomatillo the secondary and subordinate bus number registers
+ * NB: For Tomatillo the secondary and subordinate bus number registers
  * apparently are read-only although documented otherwise; writing to
  * them just triggers a PCI bus error interrupt or has no effect at best.
  */
diff --git a/sys/sparc64/pci/schizovar.h b/sys/sparc64/pci/schizovar.h
index ab339c83894a1..1b58cf41bbb3d 100644
--- a/sys/sparc64/pci/schizovar.h
+++ b/sys/sparc64/pci/schizovar.h
@@ -39,16 +39,27 @@ struct schizo_iommu_state {
 };
 
 struct schizo_softc {
-	struct bus_dma_methods          sc_dma_methods;
+	/*
+	 * This is here so that we can hook up the common bus interface
+	 * methods in ofw_pci.c directly.
+	 */
+	struct ofw_pci_softc		sc_ops;
 
-	device_t			sc_dev;
+	struct schizo_iommu_state	sc_is;
+	struct bus_dma_methods          sc_dma_methods;
 
 	struct mtx			sc_sync_mtx;
 	uint64_t			sc_sync_val;
 
 	struct mtx			*sc_mtx;
 
-	phandle_t			sc_node;
+	struct resource			*sc_mem_res[TOM_NREG];
+	struct resource			*sc_irq_res[STX_NINTR];
+	void				*sc_ihand[STX_NINTR];
+
+	SLIST_ENTRY(schizo_softc)	sc_link;
+
+	device_t			sc_dev;
 
 	u_int				sc_mode;
 #define	SCHIZO_MODE_SCZ			0
@@ -72,28 +83,8 @@ struct schizo_softc {
 	uint32_t			sc_ver;
 	uint32_t			sc_mrev;
 
-	struct resource			*sc_mem_res[TOM_NREG];
-	struct resource			*sc_irq_res[STX_NINTR];
-	void				*sc_ihand[STX_NINTR];
-
-	struct schizo_iommu_state	sc_is;
-
-	struct rman			sc_pci_mem_rman;
-	struct rman			sc_pci_io_rman;
-	bus_space_handle_t		sc_pci_bh[STX_NRANGE];
-	bus_space_tag_t			sc_pci_cfgt;
-	bus_space_tag_t			sc_pci_iot;
-	bus_dma_tag_t			sc_pci_dmat;
-
 	uint32_t			sc_stats_dma_ce;
 	uint32_t			sc_stats_pci_non_fatal;
-
-	uint8_t				sc_pci_secbus;
-	uint8_t				sc_pci_subbus;
-
-	struct ofw_bus_iinfo		sc_pci_iinfo;
-
-	SLIST_ENTRY(schizo_softc)	sc_link;
 };
 
 #endif /* !_SPARC64_PCI_SCHIZOVAR_H_ */
diff --git a/sys/sparc64/sparc64/elf_machdep.c b/sys/sparc64/sparc64/elf_machdep.c
index 93045757e436f..0dab76d59e1ff 100644
--- a/sys/sparc64/sparc64/elf_machdep.c
+++ b/sys/sparc64/sparc64/elf_machdep.c
@@ -344,6 +344,7 @@ elf_reloc(linker_file_t lf, Elf_Addr relocbase, const void *data, int type,
 	Elf_Addr value;
 	Elf_Addr mask;
 	Elf_Addr addr;
+	int error;
 
 	if (type != ELF_RELOC_RELA)
 		return (-1);
@@ -372,8 +373,8 @@ elf_reloc(linker_file_t lf, Elf_Addr relocbase, const void *data, int type,
 	value = rela->r_addend;
 
 	if (RELOC_RESOLVE_SYMBOL(rtype)) {
-		addr = lookup(lf, symidx, 1);
-		if (addr == 0)
+		error = lookup(lf, symidx, 1, &addr);
+		if (error != 0)
 			return (-1);
 		value += addr;
 		if (RELOC_BARE_SYMBOL(rtype))
diff --git a/sys/sys/buf.h b/sys/sys/buf.h
index d5ce0e51d87ae..bdc457eef35ff 100644
--- a/sys/sys/buf.h
+++ b/sys/sys/buf.h
@@ -204,7 +204,7 @@ struct buf {
 #define	B_PERSISTENT	0x00000100	/* Perm. ref'ed while EXT2FS mounted. */
 #define	B_DONE		0x00000200	/* I/O completed. */
 #define	B_EINTR		0x00000400	/* I/O was interrupted */
-#define	B_00000800	0x00000800	/* Available flag. */
+#define	B_NOREUSE	0x00000800	/* Contents not reused once released. */
 #define	B_00001000	0x00001000	/* Available flag. */
 #define	B_INVAL		0x00002000	/* Does not contain valid info. */
 #define	B_BARRIER	0x00004000	/* Write this and all preceeding first. */
@@ -229,7 +229,7 @@ struct buf {
 #define PRINT_BUF_FLAGS "\20\40remfree\37cluster\36vmio\35ram\34managed" \
 	"\33paging\32infreecnt\31nocopy\30b23\27relbuf\26dirty\25b20" \
 	"\24b19\23b18\22clusterok\21malloc\20nocache\17b14\16inval" \
-	"\15b12\14b11\13eintr\12done\11persist\10delwri" \
+	"\15b12\14noreuse\13eintr\12done\11persist\10delwri" \
 	"\7validsuspwrt\6cache\5deferred\4direct\3async\2needcommit\1age"
 
 /*
diff --git a/sys/sys/elf_common.h b/sys/sys/elf_common.h
index 2e784d67a769e..67f37d8610aaa 100644
--- a/sys/sys/elf_common.h
+++ b/sys/sys/elf_common.h
@@ -452,6 +452,7 @@ typedef struct {
 #define	SHT_MIPS_EH_REGION	0x70000027
 #define	SHT_MIPS_XLATE_OLD	0x70000028
 #define	SHT_MIPS_PDR_EXCEPTION	0x70000029
+#define	SHT_MIPS_ABIFLAGS	0x7000002a
 
 #define	SHT_SPARC_GOTDATA	0x70000000
 
diff --git a/sys/sys/file.h b/sys/sys/file.h
index cb51c27f76b18..68d33e0dfd257 100644
--- a/sys/sys/file.h
+++ b/sys/sys/file.h
@@ -160,8 +160,6 @@ struct fadvise_info {
 	int		fa_advice;	/* (f) FADV_* type. */
 	off_t		fa_start;	/* (f) Region start. */
 	off_t		fa_end;		/* (f) Region end. */
-	off_t		fa_prevstart;	/* (f) Previous NOREUSE start. */
-	off_t		fa_prevend;	/* (f) Previous NOREUSE end. */
 };
 
 struct file {
diff --git a/sys/sys/linker.h b/sys/sys/linker.h
index 5d24f003944d5..7e562a65662c5 100644
--- a/sys/sys/linker.h
+++ b/sys/sys/linker.h
@@ -264,7 +264,7 @@ extern int kld_debug;
 
 #endif
 
-typedef Elf_Addr elf_lookup_fn(linker_file_t, Elf_Size, int);
+typedef int elf_lookup_fn(linker_file_t, Elf_Size, int, Elf_Addr *);
 
 /* Support functions */
 int	elf_reloc(linker_file_t _lf, Elf_Addr base, const void *_rel, int _type, elf_lookup_fn _lu);
diff --git a/sys/sys/reboot.h b/sys/sys/reboot.h
index 6b8e25e61cdd1..ebe688e869f26 100644
--- a/sys/sys/reboot.h
+++ b/sys/sys/reboot.h
@@ -59,6 +59,7 @@
 #define	RB_RESERVED1	0x40000	/* reserved for internal use of boot blocks */
 #define	RB_RESERVED2	0x80000	/* reserved for internal use of boot blocks */
 #define	RB_PAUSE	0x100000 /* pause after each output line during probe */
+#define	RB_REROOT	0x200000 /* unmount the rootfs and mount it again */
 #define	RB_MULTIPLE	0x20000000	/* use multiple consoles */
 
 #define	RB_BOOTINFO	0x80000000	/* have `struct bootinfo *' arg */
diff --git a/sys/sys/sdt.h b/sys/sys/sdt.h
index ca820f68bce1b..5dd0b67af6097 100644
--- a/sys/sys/sdt.h
+++ b/sys/sys/sdt.h
@@ -398,7 +398,7 @@ struct sdt_probe {
 	struct sdt_provider *prov;	/* Ptr to the provider structure. */
 	TAILQ_ENTRY(sdt_probe)
 			probe_entry;	/* SDT probe list entry. */
-	TAILQ_HEAD(argtype_list_head, sdt_argtype) argtype_list;
+	TAILQ_HEAD(, sdt_argtype) argtype_list;
 	const char	*mod;
 	const char	*func;
 	const char	*name;
diff --git a/sys/sys/sysctl.h b/sys/sys/sysctl.h
index 988dbae94087e..4c66431e55f4f 100644
--- a/sys/sys/sysctl.h
+++ b/sys/sys/sysctl.h
@@ -807,8 +807,8 @@ int	userland_sysctl(struct thread *td, int *name, u_int namelen, void *old,
 	    size_t *retval, int flags);
 int	sysctl_find_oid(int *name, u_int namelen, struct sysctl_oid **noid,
 	    int *nindx, struct sysctl_req *req);
-void	sysctl_xlock(void);
-void	sysctl_xunlock(void);
+void	sysctl_wlock(void);
+void	sysctl_wunlock(void);
 int	sysctl_wire_old_buffer(struct sysctl_req *req, size_t len);
 
 struct sbuf;
diff --git a/sys/sys/systm.h b/sys/sys/systm.h
index cb05a00008e7e..592112a0c860c 100644
--- a/sys/sys/systm.h
+++ b/sys/sys/systm.h
@@ -82,12 +82,12 @@ void	kassert_panic(const char *fmt, ...)  __printflike(1, 2);
 #ifdef	INVARIANTS		/* The option is always available */
 #define	KASSERT(exp,msg) do {						\
 	if (__predict_false(!(exp)))					\
-		kassert_panic msg;						\
+		kassert_panic msg;					\
 } while (0)
 #define	VNASSERT(exp, vp, msg) do {					\
 	if (__predict_false(!(exp))) {					\
 		vn_printf(vp, "VNASSERT failed\n");			\
-		kassert_panic msg;						\
+		kassert_panic msg;					\
 	}								\
 } while (0)
 #else
diff --git a/sys/sys/types.h b/sys/sys/types.h
index 4a66a4eb20b62..4dfd8ef19b4d9 100644
--- a/sys/sys/types.h
+++ b/sys/sys/types.h
@@ -290,9 +290,6 @@ typedef	_Bool	bool;
  * The following are all things that really shouldn't exist in this header,
  * since its purpose is to provide typedefs, not miscellaneous doodads.
  */
-#if __BSD_VISIBLE
-
-#include <sys/select.h>
 
 #ifdef __POPCNT__
 #define	__bitcount64(x)	__builtin_popcountll((__uint64_t)(x))
@@ -356,6 +353,10 @@ __bitcount64(__uint64_t _x)
 #define	__bitcount(x)	__bitcount32((unsigned int)(x))
 #endif
 
+#if __BSD_VISIBLE
+
+#include <sys/select.h>
+
 /*
  * minor() gives a cookie instead of an index since we don't want to
  * change the meanings of bits 0-15 or waste time and space shifting
diff --git a/sys/sys/vnode.h b/sys/sys/vnode.h
index 800d340edbf53..cce39ef83a1bc 100644
--- a/sys/sys/vnode.h
+++ b/sys/sys/vnode.h
@@ -781,6 +781,7 @@ void	vop_lookup_post(void *a, int rc);
 void	vop_lookup_pre(void *a);
 void	vop_mkdir_post(void *a, int rc);
 void	vop_mknod_post(void *a, int rc);
+void	vop_reclaim_post(void *a, int rc);
 void	vop_remove_post(void *a, int rc);
 void	vop_rename_post(void *a, int rc);
 void	vop_rename_pre(void *a);
@@ -796,7 +797,8 @@ void	vop_rename_fail(struct vop_rename_args *ap);
 
 #define	VOP_WRITE_PRE(ap)						\
 	struct vattr va;						\
-	int error, osize, ooffset, noffset;				\
+	int error;							\
+	off_t osize, ooffset, noffset;					\
 									\
 	osize = ooffset = noffset = 0;					\
 	if (!VN_KNLIST_EMPTY((ap)->a_vp)) {				\
@@ -804,7 +806,7 @@ void	vop_rename_fail(struct vop_rename_args *ap);
 		if (error)						\
 			return (error);					\
 		ooffset = (ap)->a_uio->uio_offset;			\
-		osize = va.va_size;					\
+		osize = (off_t)va.va_size;				\
 	}
 
 #define VOP_WRITE_POST(ap, ret)						\
diff --git a/sys/tools/vnode_if.awk b/sys/tools/vnode_if.awk
index 7d106e7f9c67c..5bb3971c4387d 100644
--- a/sys/tools/vnode_if.awk
+++ b/sys/tools/vnode_if.awk
@@ -359,7 +359,7 @@ while ((getline < srcfile) > 0) {
 		printc("\t    vop->"name" == NULL && vop->vop_bypass == NULL)")
 		printc("\t\tvop = vop->vop_default;")
 		printc("\tVNASSERT(vop != NULL, a->a_" args[0]", (\"No "name"(%p, %p)\", a->a_" args[0]", a));")
-		printc("\tSDT_PROBE(vfs, vop, " name ", entry, a->a_" args[0] ", a, 0, 0, 0);\n");
+		printc("\tSDT_PROBE2(vfs, vop, " name ", entry, a->a_" args[0] ", a);\n");
 		for (i = 0; i < numargs; ++i)
 			add_debug_code(name, args[i], "Entry", "\t");
 		printc("\tKTR_START" ctrstr);
@@ -370,7 +370,7 @@ while ((getline < srcfile) > 0) {
 		printc("\telse")
 		printc("\t\trc = vop->vop_bypass(&a->a_gen);")
 		printc("\tVFS_EPILOGUE(a->a_" args[0]"->v_mount);")
-		printc("\tSDT_PROBE(vfs, vop, " name ", return, a->a_" args[0] ", a, rc, 0, 0);\n");
+		printc("\tSDT_PROBE3(vfs, vop, " name ", return, a->a_" args[0] ", a, rc);\n");
 		printc("\tif (rc == 0) {");
 		for (i = 0; i < numargs; ++i)
 			add_debug_code(name, args[i], "OK", "\t\t");
diff --git a/sys/vm/uma_core.c b/sys/vm/uma_core.c
index 2a4a2cde4c537..8169256994dc8 100644
--- a/sys/vm/uma_core.c
+++ b/sys/vm/uma_core.c
@@ -1153,7 +1153,7 @@ noobj_alloc(uma_zone_t zone, vm_size_t bytes, uint8_t *flags, int wait)
 		 * exit.
 		 */
 		TAILQ_FOREACH_SAFE(p, &alloctail, listq, p_next) {
-			vm_page_unwire(p, PQ_INACTIVE);
+			vm_page_unwire(p, PQ_NONE);
 			vm_page_free(p); 
 		}
 		return (NULL);
diff --git a/sys/vm/vm_kern.c b/sys/vm/vm_kern.c
index fd16bf217b592..f0aacf11a1c74 100644
--- a/sys/vm/vm_kern.c
+++ b/sys/vm/vm_kern.c
@@ -159,11 +159,10 @@ kmem_alloc_attr(vmem_t *vmem, vm_size_t size, int flags, vm_paddr_t low,
     vm_paddr_t high, vm_memattr_t memattr)
 {
 	vm_object_t object = vmem == kmem_arena ? kmem_object : kernel_object;
-	vm_offset_t addr;
+	vm_offset_t addr, i;
 	vm_ooffset_t offset;
 	vm_page_t m;
 	int pflags, tries;
-	int i;
 
 	size = round_page(size);
 	if (vmem_alloc(vmem, size, M_BESTFIT | flags, &addr))
@@ -184,18 +183,7 @@ retry:
 				tries++;
 				goto retry;
 			}
-			/* 
-			 * Unmap and free the pages.
-			 */
-			if (i != 0)
-				pmap_remove(kernel_pmap, addr, addr + i);
-			while (i != 0) {
-				i -= PAGE_SIZE;
-				m = vm_page_lookup(object,
-				    OFF_TO_IDX(offset + i));
-				vm_page_unwire(m, PQ_INACTIVE);
-				vm_page_free(m);
-			}
+			kmem_unback(object, addr, i);
 			vmem_free(vmem, addr, size);
 			return (0);
 		}
@@ -353,25 +341,13 @@ retry:
 		 * aren't on any queues.
 		 */
 		if (m == NULL) {
+			VM_OBJECT_WUNLOCK(object);
 			if ((flags & M_NOWAIT) == 0) {
-				VM_OBJECT_WUNLOCK(object);
 				VM_WAIT;
 				VM_OBJECT_WLOCK(object);
 				goto retry;
 			}
-			/* 
-			 * Unmap and free the pages.
-			 */
-			if (i != 0)
-				pmap_remove(kernel_pmap, addr, addr + i);
-			while (i != 0) {
-				i -= PAGE_SIZE;
-				m = vm_page_lookup(object,
-						   OFF_TO_IDX(offset + i));
-				vm_page_unwire(m, PQ_INACTIVE);
-				vm_page_free(m);
-			}
-			VM_OBJECT_WUNLOCK(object);
+			kmem_unback(object, addr, i);
 			return (KERN_NO_SPACE);
 		}
 		if (flags & M_ZERO && (m->flags & PG_ZERO) == 0)
@@ -387,6 +363,15 @@ retry:
 	return (KERN_SUCCESS);
 }
 
+/*
+ *	kmem_unback:
+ *
+ *	Unmap and free the physical pages underlying the specified virtual
+ *	address range.
+ *
+ *	A physical page must exist within the specified object at each index
+ *	that is being unmapped.
+ */
 void
 kmem_unback(vm_object_t object, vm_offset_t addr, vm_size_t size)
 {
diff --git a/sys/vm/vm_object.c b/sys/vm/vm_object.c
index a4aac95c7adb2..0a3c2efdf8cfe 100644
--- a/sys/vm/vm_object.c
+++ b/sys/vm/vm_object.c
@@ -1963,15 +1963,15 @@ skipmemq:
 }
 
 /*
- *	vm_object_page_cache:
+ *	vm_object_page_noreuse:
  *
- *	For the given object, attempt to move the specified clean
- *	pages to the cache queue.  If a page is wired for any reason,
- *	then it will not be changed.  Pages are specified by the given
- *	range ["start", "end").  As a special case, if "end" is zero,
- *	then the range extends from "start" to the end of the object.
- *	Any mappings to the specified pages are removed before the
- *	pages are moved to the cache queue.
+ *	For the given object, attempt to move the specified pages to
+ *	the head of the inactive queue.  This bypasses regular LRU
+ *	operation and allows the pages to be reused quickly under memory
+ *	pressure.  If a page is wired for any reason, then it will not
+ *	be queued.  Pages are specified by the range ["start", "end").
+ *	As a special case, if "end" is zero, then the range extends from
+ *	"start" to the end of the object.
  *
  *	This operation should only be performed on objects that
  *	contain non-fictitious, managed pages.
@@ -1979,14 +1979,14 @@ skipmemq:
  *	The object must be locked.
  */
 void
-vm_object_page_cache(vm_object_t object, vm_pindex_t start, vm_pindex_t end)
+vm_object_page_noreuse(vm_object_t object, vm_pindex_t start, vm_pindex_t end)
 {
 	struct mtx *mtx, *new_mtx;
 	vm_page_t p, next;
 
 	VM_OBJECT_ASSERT_WLOCKED(object);
 	KASSERT((object->flags & (OBJ_FICTITIOUS | OBJ_UNMANAGED)) == 0,
-	    ("vm_object_page_cache: illegal object %p", object));
+	    ("vm_object_page_noreuse: illegal object %p", object));
 	if (object->resident_page_count == 0)
 		return;
 	p = vm_page_find_least(object, start);
@@ -2009,7 +2009,7 @@ vm_object_page_cache(vm_object_t object, vm_pindex_t start, vm_pindex_t end)
 			mtx = new_mtx;
 			mtx_lock(mtx);
 		}
-		vm_page_try_to_cache(p);
+		vm_page_deactivate_noreuse(p);
 	}
 	if (mtx != NULL)
 		mtx_unlock(mtx);
diff --git a/sys/vm/vm_object.h b/sys/vm/vm_object.h
index 7e433aee02288..894a8d5616bab 100644
--- a/sys/vm/vm_object.h
+++ b/sys/vm/vm_object.h
@@ -304,10 +304,10 @@ void vm_object_terminate (vm_object_t);
 void vm_object_set_writeable_dirty (vm_object_t);
 void vm_object_init (void);
 void vm_object_madvise(vm_object_t, vm_pindex_t, vm_pindex_t, int);
-void vm_object_page_cache(vm_object_t object, vm_pindex_t start,
-    vm_pindex_t end);
 boolean_t vm_object_page_clean(vm_object_t object, vm_ooffset_t start,
     vm_ooffset_t end, int flags);
+void vm_object_page_noreuse(vm_object_t object, vm_pindex_t start,
+    vm_pindex_t end);
 void vm_object_page_remove(vm_object_t object, vm_pindex_t start,
     vm_pindex_t end, int options);
 boolean_t vm_object_populate(vm_object_t, vm_pindex_t, vm_pindex_t);
diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c
index 9c4890f2c303f..a3a9a10417a49 100644
--- a/sys/vm/vm_page.c
+++ b/sys/vm/vm_page.c
@@ -2476,42 +2476,46 @@ vm_page_wire(vm_page_t m)
 /*
  * vm_page_unwire:
  *
- * Release one wiring of the specified page, potentially enabling it to be
- * paged again.  If paging is enabled, then the value of the parameter
- * "queue" determines the queue to which the page is added.
+ * Release one wiring of the specified page, potentially allowing it to be
+ * paged out.  Returns TRUE if the number of wirings transitions to zero and
+ * FALSE otherwise.
  *
- * However, unless the page belongs to an object, it is not enqueued because
- * it cannot be paged out.
+ * Only managed pages belonging to an object can be paged out.  If the number
+ * of wirings transitions to zero and the page is eligible for page out, then
+ * the page is added to the specified paging queue (unless PQ_NONE is
+ * specified).
  *
  * If a page is fictitious, then its wire count must always be one.
  *
  * A managed page must be locked.
  */
-void
+boolean_t
 vm_page_unwire(vm_page_t m, uint8_t queue)
 {
 
-	KASSERT(queue < PQ_COUNT,
+	KASSERT(queue < PQ_COUNT || queue == PQ_NONE,
 	    ("vm_page_unwire: invalid queue %u request for page %p",
 	    queue, m));
 	if ((m->oflags & VPO_UNMANAGED) == 0)
-		vm_page_lock_assert(m, MA_OWNED);
+		vm_page_assert_locked(m);
 	if ((m->flags & PG_FICTITIOUS) != 0) {
 		KASSERT(m->wire_count == 1,
 	    ("vm_page_unwire: fictitious page %p's wire count isn't one", m));
-		return;
+		return (FALSE);
 	}
 	if (m->wire_count > 0) {
 		m->wire_count--;
 		if (m->wire_count == 0) {
 			atomic_subtract_int(&vm_cnt.v_wire_count, 1);
-			if ((m->oflags & VPO_UNMANAGED) != 0 ||
-			    m->object == NULL)
-				return;
-			if (queue == PQ_INACTIVE)
-				m->flags &= ~PG_WINATCFLS;
-			vm_page_enqueue(queue, m);
-		}
+			if ((m->oflags & VPO_UNMANAGED) == 0 &&
+			    m->object != NULL && queue != PQ_NONE) {
+				if (queue == PQ_INACTIVE)
+					m->flags &= ~PG_WINATCFLS;
+				vm_page_enqueue(queue, m);
+			}
+			return (TRUE);
+		} else
+			return (FALSE);
 	} else
 		panic("vm_page_unwire: page %p's wire count is zero", m);
 }
@@ -2585,6 +2589,19 @@ vm_page_deactivate(vm_page_t m)
 }
 
 /*
+ * Move the specified page to the inactive queue with the expectation
+ * that it is unlikely to be reused.
+ *
+ * The page must be locked.
+ */
+void
+vm_page_deactivate_noreuse(vm_page_t m)
+{
+
+	_vm_page_deactivate(m, 1);
+}
+
+/*
  * vm_page_try_to_cache:
  *
  * Returns 0 on failure, 1 on success
@@ -2736,8 +2753,7 @@ vm_page_cache(vm_page_t m)
 /*
  * vm_page_advise
  *
- * 	Deactivate or do nothing, as appropriate.  This routine is used
- * 	by madvise() and vop_stdadvise().
+ * 	Deactivate or do nothing, as appropriate.
  *
  *	The object and page must be locked.
  */
@@ -3096,7 +3112,8 @@ vm_page_set_invalid(vm_page_t m, int base, int size)
 		bits = VM_PAGE_BITS_ALL;
 	else
 		bits = vm_page_bits(base, size);
-	if (m->valid == VM_PAGE_BITS_ALL && bits != 0)
+	if (object->ref_count != 0 && m->valid == VM_PAGE_BITS_ALL &&
+	    bits != 0)
 		pmap_remove_all(m);
 	KASSERT((bits == 0 && m->valid == VM_PAGE_BITS_ALL) ||
 	    !pmap_page_is_mapped(m),
diff --git a/sys/vm/vm_page.h b/sys/vm/vm_page.h
index dcbd54c4a79d2..dedd6ac9eb0c2 100644
--- a/sys/vm/vm_page.h
+++ b/sys/vm/vm_page.h
@@ -451,6 +451,7 @@ void vm_page_cache_transfer(vm_object_t, vm_pindex_t, vm_object_t);
 int vm_page_try_to_cache (vm_page_t);
 int vm_page_try_to_free (vm_page_t);
 void vm_page_deactivate (vm_page_t);
+void vm_page_deactivate_noreuse(vm_page_t);
 void vm_page_dequeue(vm_page_t m);
 void vm_page_dequeue_locked(vm_page_t m);
 vm_page_t vm_page_find_least(vm_object_t, vm_pindex_t);
@@ -480,7 +481,7 @@ vm_offset_t vm_page_startup(vm_offset_t vaddr);
 void vm_page_sunbusy(vm_page_t m);
 int vm_page_trysbusy(vm_page_t m);
 void vm_page_unhold_pages(vm_page_t *ma, int count);
-void vm_page_unwire (vm_page_t m, uint8_t queue);
+boolean_t vm_page_unwire(vm_page_t m, uint8_t queue);
 void vm_page_updatefake(vm_page_t m, vm_paddr_t paddr, vm_memattr_t memattr);
 void vm_page_wire (vm_page_t);
 void vm_page_xunbusy_hard(vm_page_t m);
diff --git a/sys/vm/vm_pageout.c b/sys/vm/vm_pageout.c
index 664119391cd98..0a18e6db68931 100644
--- a/sys/vm/vm_pageout.c
+++ b/sys/vm/vm_pageout.c
@@ -1638,9 +1638,15 @@ vm_pageout_worker(void *arg)
 		}
 		if (vm_pages_needed) {
 			/*
-			 * Still not done, take a second pass without waiting
-			 * (unlimited dirty cleaning), otherwise sleep a bit
-			 * and try again.
+			 * We're still not done.  Either vm_pages_needed was
+			 * set by another thread during the previous scan
+			 * (typically, this happens during a level 0 scan) or
+			 * vm_pages_needed was already set and the scan failed
+			 * to free enough pages.  If we haven't yet performed
+			 * a level >= 2 scan (unlimited dirty cleaning), then
+			 * upgrade the level and scan again now.  Otherwise,
+			 * sleep a bit and try again later.  While sleeping,
+			 * vm_pages_needed can be cleared.
 			 */
 			if (domain->vmd_pass > 1)
 				msleep(&vm_pages_needed,
@@ -1651,15 +1657,14 @@ vm_pageout_worker(void *arg)
 			 * Good enough, sleep until required to refresh
 			 * stats.
 			 */
-			domain->vmd_pass = 0;
 			msleep(&vm_pages_needed, &vm_page_queue_free_mtx,
 			    PVM, "psleep", hz);
-
 		}
 		if (vm_pages_needed) {
 			vm_cnt.v_pdwakeups++;
 			domain->vmd_pass++;
-		}
+		} else
+			domain->vmd_pass = 0;
 		mtx_unlock(&vm_page_queue_free_mtx);
 		vm_pageout_scan(domain, domain->vmd_pass);
 	}
diff --git a/sys/x86/acpica/madt.c b/sys/x86/acpica/madt.c
index 9b99651c3c603..62554bdad4a7b 100644
--- a/sys/x86/acpica/madt.c
+++ b/sys/x86/acpica/madt.c
@@ -182,7 +182,19 @@ madt_setup_local(void)
 			    CPUID_TO_FAMILY(cpu_id) == 0x6 &&
 			    CPUID_TO_MODEL(cpu_id) == 0x2a) {
 				x2apic_mode = 0;
-		reason = "for a suspected Lenovo SandyBridge BIOS bug";
+				reason =
+				    "for a suspected Lenovo SandyBridge BIOS bug";
+			}
+			/*
+			 * Same reason, ASUS SandyBridge.
+			 */
+			if (hw_vendor != NULL &&
+			    !strcmp(hw_vendor, "ASUSTeK Computer Inc.") &&
+			    CPUID_TO_FAMILY(cpu_id) == 0x6 &&
+			    CPUID_TO_MODEL(cpu_id) == 0x2a) {
+				x2apic_mode = 0;
+				reason =
+				    "for a suspected ASUS SandyBridge BIOS bug";
 			}
 			freeenv(hw_vendor);
 		}
diff --git a/sys/x86/iommu/intel_ctx.c b/sys/x86/iommu/intel_ctx.c
index 402612b5a6b67..71de1672054d1 100644
--- a/sys/x86/iommu/intel_ctx.c
+++ b/sys/x86/iommu/intel_ctx.c
@@ -74,7 +74,6 @@ static void dmar_domain_unload_task(void *arg, int pending);
 static void dmar_unref_domain_locked(struct dmar_unit *dmar,
     struct dmar_domain *domain);
 static void dmar_domain_destroy(struct dmar_domain *domain);
-static void dmar_ctx_dtr(struct dmar_ctx *ctx);
 
 static void
 dmar_ensure_ctx_page(struct dmar_unit *dmar, int bus)
diff --git a/sys/x86/pci/pci_bus.c b/sys/x86/pci/pci_bus.c
index f64659eec2ada..821265dd12e66 100644
--- a/sys/x86/pci/pci_bus.c
+++ b/sys/x86/pci/pci_bus.c
@@ -524,7 +524,7 @@ legacy_pcib_attach(device_t dev)
 			device_probe_and_attach(pir);
 	}
 #endif
-	device_add_child(dev, "pci", bus);
+	device_add_child(dev, "pci", -1);
 	return bus_generic_attach(dev);
 }
 
diff --git a/sys/x86/pci/qpi.c b/sys/x86/pci/qpi.c
index ae29daa1d3496..2650986ea5a76 100644
--- a/sys/x86/pci/qpi.c
+++ b/sys/x86/pci/qpi.c
@@ -218,7 +218,7 @@ static int
 qpi_pcib_attach(device_t dev)
 {
 
-	device_add_child(dev, "pci", pcib_get_bus(dev));      
+	device_add_child(dev, "pci", -1);
         return (bus_generic_attach(dev));
 }
 
diff --git a/sys/x86/x86/mptable_pci.c b/sys/x86/x86/mptable_pci.c
index 3e2c1cbe145ff..31165ae9d2dd1 100644
--- a/sys/x86/x86/mptable_pci.c
+++ b/sys/x86/x86/mptable_pci.c
@@ -69,7 +69,7 @@ mptable_hostb_attach(device_t dev)
 #ifdef NEW_PCIB
 	mptable_pci_host_res_init(dev);
 #endif
-	device_add_child(dev, "pci", pcib_get_bus(dev));
+	device_add_child(dev, "pci", -1);
 	return (bus_generic_attach(dev));
 }