53 files changed, 328 insertions, 157 deletions

diff --git a/usr.sbin/Makefile b/usr.sbin/Makefile
index b06dba709b35..fa0c9951440d 100644
--- a/usr.sbin/Makefile
+++ b/usr.sbin/Makefile
@@ -1,7 +1,7 @@
 #	From: @(#)Makefile	5.20 (Berkeley) 6/12/93
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 SUBDIR=	adduser \
 	arp \
diff --git a/usr.sbin/amd/Makefile.inc b/usr.sbin/amd/Makefile.inc
index cc616f1592f8..e4425f52a2c5 100644
--- a/usr.sbin/amd/Makefile.inc
+++ b/usr.sbin/amd/Makefile.inc
@@ -9,7 +9,7 @@
 #	Portions derived from amd/libamu/Makefile
 #	$NetBSD: Makefile,v 1.8 1998/08/08 22:33:37 christos Exp $
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 CFLAGS+= -I. -I${.CURDIR} 
 CFLAGS+= -I${.CURDIR}/../include
diff --git a/usr.sbin/amd/amd/Makefile b/usr.sbin/amd/amd/Makefile
index c7200037f628..1fc41bfe20d0 100644
--- a/usr.sbin/amd/amd/Makefile
+++ b/usr.sbin/amd/amd/Makefile
@@ -6,7 +6,7 @@
 # $FreeBSD$
 #
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 .PATH: ${.CURDIR}/../../../contrib/amd/amd
 
diff --git a/usr.sbin/amd/include/Makefile b/usr.sbin/amd/include/Makefile
index df770d1caf81..e594d71a4fb3 100644
--- a/usr.sbin/amd/include/Makefile
+++ b/usr.sbin/amd/include/Makefile
@@ -7,7 +7,7 @@
 #	Portions derived from amd/libamu/Makefile
 #	$NetBSD: Makefile,v 1.8 1998/08/08 22:33:37 christos Exp $
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 SRCS=	config_local.h
 .if ${MK_NIS} == "no"
diff --git a/usr.sbin/bhyve/bhyve.8 b/usr.sbin/bhyve/bhyve.8
index 88e72142b2ca..aa97a670997d 100644
--- a/usr.sbin/bhyve/bhyve.8
+++ b/usr.sbin/bhyve/bhyve.8
@@ -35,7 +35,7 @@
 .Op Fl aehwxAHPW
 .Op Fl c Ar numcpus
 .Op Fl g Ar gdbport
-.Op Fl p Ar pinnedcpu
+.Op Fl p Ar vcpu:hostcpu
 .Op Fl s Ar slot,emulation Ns Op , Ns Ar conf
 .Op Fl l Ar lpcdev Ns Op , Ns Ar conf
 .Ar vmname
@@ -80,12 +80,11 @@ For
 allow a remote kernel kgdb to be relayed to the guest kernel gdb stub
 via a local IPv4 address and this port.
 This option will be deprecated in a future version.
-.It Fl p Ar pinnedcpu
-Force guest virtual CPUs to be pinned to host CPUs.
-Virtual CPU
-.Em n
-is pinned to host CPU
-.Em pinnedcpu+n .
+.It Fl p Ar vcpu:hostcpu
+Pin guest's virtual CPU
+.Em vcpu
+to
+.Em hostcpu .
 .It Fl P
 Force the guest virtual CPU to exit when a PAUSE instruction is detected.
 .It Fl W
@@ -236,6 +235,8 @@ This is intended for debug purposes.
 Ignore accesses to unimplemented Model Specific Registers (MSRs). This is intended for debug purposes.
 .It Fl x
 The guest's local APIC is configured in x2APIC mode.
+.It Fl Y
+Disable MPtable generation.
 .It Fl h
 Print help message and exit.
 .It Ar vmname
diff --git a/usr.sbin/bhyve/bhyverun.c b/usr.sbin/bhyve/bhyverun.c
index e8010e6fbfa8..e662ca39d9c4 100644
--- a/usr.sbin/bhyve/bhyverun.c
+++ b/usr.sbin/bhyve/bhyverun.c
@@ -85,7 +85,6 @@ char *vmname;
 int guest_ncpus;
 char *guest_uuid_str;
 
-static int pincpu = -1;
 static int guest_vmexit_on_hlt, guest_vmexit_on_pause;
 static int virtio_msix = 1;
 static int x2apic_mode = 0;	/* default is xAPIC */
@@ -114,6 +113,7 @@ struct bhyvestats {
         uint64_t        cpu_switch_rotate;
         uint64_t        cpu_switch_direct;
         int             io_reset;
+	int		io_poweroff;
 } stats;
 
 struct mt_vmm_info {
@@ -122,18 +122,20 @@ struct mt_vmm_info {
 	int		mt_vcpu;	
 } mt_vmm_info[VM_MAXCPU];
 
+static cpuset_t *vcpumap[VM_MAXCPU] = { NULL };
+
 static void
 usage(int code)
 {
 
         fprintf(stderr,
-                "Usage: %s [-aehwAHIPW] [-g <gdb port>] [-s <pci>]\n"
-		"       %*s [-c vcpus] [-p pincpu] [-m mem] [-l <lpc>] <vm>\n"
+                "Usage: %s [-aehwAHIPW] [-g <gdb port>] [-s <pci>] [-c vcpus]\n"
+		"       %*s [-p vcpu:hostcpu] [-m mem] [-l <lpc>] <vm>\n"
 		"       -a: local apic is in xAPIC mode (deprecated)\n"
 		"       -A: create an ACPI table\n"
 		"       -g: gdb port\n"
 		"       -c: # cpus (default 1)\n"
-		"       -p: pin vcpu 'n' to host cpu 'pincpu + n'\n"
+		"       -p: pin 'vcpu' to 'hostcpu'\n"
 		"       -H: vmexit from the guest on hlt\n"
 		"       -P: vmexit from the guest on pause\n"
 		"       -W: force virtio to use single-vector MSI\n"
@@ -144,12 +146,46 @@ usage(int code)
 		"       -m: memory size in MB\n"
 		"       -w: ignore unimplemented MSRs\n"
 		"       -x: local apic is in x2APIC mode\n"
+		"       -Y: disable MPtable generation\n"
 		"       -U: uuid\n",
 		progname, (int)strlen(progname), "");
 
 	exit(code);
 }
 
+static int
+pincpu_parse(const char *opt)
+{
+	int vcpu, pcpu;
+
+	if (sscanf(opt, "%d:%d", &vcpu, &pcpu) != 2) {
+		fprintf(stderr, "invalid format: %s\n", opt);
+		return (-1);
+	}
+
+	if (vcpu < 0 || vcpu >= VM_MAXCPU) {
+		fprintf(stderr, "vcpu '%d' outside valid range from 0 to %d\n",
+		    vcpu, VM_MAXCPU - 1);
+		return (-1);
+	}
+
+	if (pcpu < 0 || pcpu >= CPU_SETSIZE) {
+		fprintf(stderr, "hostcpu '%d' outside valid range from "
+		    "0 to %d\n", pcpu, CPU_SETSIZE - 1);
+		return (-1);
+	}
+
+	if (vcpumap[vcpu] == NULL) {
+		if ((vcpumap[vcpu] = malloc(sizeof(cpuset_t))) == NULL) {
+			perror("malloc");
+			return (-1);
+		}
+		CPU_ZERO(vcpumap[vcpu]);
+	}
+	CPU_SET(pcpu, vcpumap[vcpu]);
+	return (0);
+}
+
 void *
 paddr_guest2host(struct vmctx *ctx, uintptr_t gaddr, size_t len)
 {
@@ -227,8 +263,7 @@ fbsdrun_deletecpu(struct vmctx *ctx, int vcpu)
 {
 
 	if (!CPU_ISSET(vcpu, &cpumask)) {
-		fprintf(stderr, "addcpu: attempting to delete unknown cpu %d\n",
-		    vcpu);
+		fprintf(stderr, "Attempting to delete unknown cpu %d\n", vcpu);
 		exit(1);
 	}
 
@@ -237,13 +272,6 @@ fbsdrun_deletecpu(struct vmctx *ctx, int vcpu)
 }
 
 static int
-vmexit_catch_reset(void)
-{
-        stats.io_reset++;
-        return (VMEXIT_RESET);
-}
-
-static int
 vmexit_catch_inout(void)
 {
 	return (VMEXIT_ABORT);
@@ -293,8 +321,10 @@ vmexit_inout(struct vmctx *ctx, struct vm_exit *vme, int *pvcpu)
 	case INOUT_OK:
 		return (VMEXIT_CONTINUE);
 	case INOUT_RESET:
+		stats.io_reset++;
 		return (VMEXIT_RESET);
 	case INOUT_POWEROFF:
+		stats.io_poweroff++;
 		return (VMEXIT_POWEROFF);
 	default:
 		fprintf(stderr, "Unhandled %s%c 0x%04x\n",
@@ -365,17 +395,6 @@ vmexit_spinup_ap(struct vmctx *ctx, struct vm_exit *vme, int *pvcpu)
 }
 
 static int
-vmexit_spindown_cpu(struct vmctx *ctx, struct vm_exit *vme, int *pvcpu)
-{
-	int lastcpu;
-
-	lastcpu = fbsdrun_deletecpu(ctx, *pvcpu);
-	if (!lastcpu)
-		pthread_exit(NULL);
-	return (vmexit_catch_reset());
-}
-
-static int
 vmexit_vmx(struct vmctx *ctx, struct vm_exit *vmexit, int *pvcpu)
 {
 
@@ -461,17 +480,17 @@ vmexit_inst_emul(struct vmctx *ctx, struct vm_exit *vmexit, int *pvcpu)
 
 static pthread_mutex_t resetcpu_mtx = PTHREAD_MUTEX_INITIALIZER;
 static pthread_cond_t resetcpu_cond = PTHREAD_COND_INITIALIZER;
-static int resetcpu = -1;
 
 static int
 vmexit_suspend(struct vmctx *ctx, struct vm_exit *vmexit, int *pvcpu)
 {
-	
-	assert(resetcpu != -1);
+	enum vm_suspend_how how;
+
+	how = vmexit->u.suspended.how;
 
 	fbsdrun_deletecpu(ctx, *pvcpu);
 
-	if (*pvcpu != resetcpu) {
+	if (*pvcpu != BSP) {
 		pthread_mutex_lock(&resetcpu_mtx);
 		pthread_cond_signal(&resetcpu_cond);
 		pthread_mutex_unlock(&resetcpu_mtx);
@@ -483,7 +502,19 @@ vmexit_suspend(struct vmctx *ctx, struct vm_exit *vmexit, int *pvcpu)
 		pthread_cond_wait(&resetcpu_cond, &resetcpu_mtx);
 	}
 	pthread_mutex_unlock(&resetcpu_mtx);
-	exit(0);
+
+	switch (how) {
+	case VM_SUSPEND_RESET:
+		exit(0);
+	case VM_SUSPEND_POWEROFF:
+		exit(1);
+	case VM_SUSPEND_HALT:
+		exit(2);
+	default:
+		fprintf(stderr, "vmexit_suspend: invalid reason %d\n", how);
+		exit(100);
+	}
+	return (0);	/* NOTREACHED */
 }
 
 static vmexit_handler_t handler[VM_EXITCODE_MAX] = {
@@ -495,22 +526,19 @@ static vmexit_handler_t handler[VM_EXITCODE_MAX] = {
 	[VM_EXITCODE_MTRAP]  = vmexit_mtrap,
 	[VM_EXITCODE_INST_EMUL] = vmexit_inst_emul,
 	[VM_EXITCODE_SPINUP_AP] = vmexit_spinup_ap,
-	[VM_EXITCODE_SPINDOWN_CPU] = vmexit_spindown_cpu,
 	[VM_EXITCODE_SUSPENDED] = vmexit_suspend
 };
 
 static void
 vm_loop(struct vmctx *ctx, int vcpu, uint64_t rip)
 {
-	cpuset_t mask;
 	int error, rc, prevcpu;
 	enum vm_exitcode exitcode;
+	enum vm_suspend_how how;
 
-	if (pincpu >= 0) {
-		CPU_ZERO(&mask);
-		CPU_SET(pincpu + vcpu, &mask);
+	if (vcpumap[vcpu] != NULL) {
 		error = pthread_setaffinity_np(pthread_self(),
-					       sizeof(mask), &mask);
+		    sizeof(cpuset_t), vcpumap[vcpu]);
 		assert(error == 0);
 	}
 
@@ -538,10 +566,13 @@ vm_loop(struct vmctx *ctx, int vcpu, uint64_t rip)
                         rip = vmexit[vcpu].rip;
 			break;
 		case VMEXIT_RESET:
-			if (vm_suspend(ctx) == 0) {
-				assert(resetcpu == -1);
-				resetcpu = vcpu;
-			}
+		case VMEXIT_POWEROFF:
+			if (rc == VMEXIT_RESET)
+				how = VM_SUSPEND_RESET;
+			else
+				how = VM_SUSPEND_POWEROFF;
+			error = vm_suspend(ctx, how);
+			assert(error == 0 || errno == EALREADY);
                         rip = vmexit[vcpu].rip + vmexit[vcpu].inst_length;
 			break;
 		default:
@@ -616,7 +647,7 @@ int
 main(int argc, char *argv[])
 {
 	int c, error, gdb_port, err, bvmcons;
-	int max_vcpus;
+	int max_vcpus, mptgen;
 	struct vmctx *ctx;
 	uint64_t rip;
 	size_t memsize;
@@ -626,8 +657,9 @@ main(int argc, char *argv[])
 	gdb_port = 0;
 	guest_ncpus = 1;
 	memsize = 256 * MB;
+	mptgen = 1;
 
-	while ((c = getopt(argc, argv, "abehwxAHIPWp:g:c:s:m:l:U:")) != -1) {
+	while ((c = getopt(argc, argv, "abehwxAHIPWYp:g:c:s:m:l:U:")) != -1) {
 		switch (c) {
 		case 'a':
 			x2apic_mode = 0;
@@ -639,7 +671,10 @@ main(int argc, char *argv[])
 			bvmcons = 1;
 			break;
 		case 'p':
-			pincpu = atoi(optarg);
+                        if (pincpu_parse(optarg) != 0) {
+                            errx(EX_USAGE, "invalid vcpu pinning "
+                                 "configuration '%s'", optarg);
+                        }
 			break;
                 case 'c':
 			guest_ncpus = atoi(optarg);
@@ -693,6 +728,9 @@ main(int argc, char *argv[])
 		case 'x':
 			x2apic_mode = 1;
 			break;
+		case 'Y':
+			mptgen = 0;
+			break;
 		case 'h':
 			usage(0);			
 		default:
@@ -752,7 +790,11 @@ main(int argc, char *argv[])
 	/*
 	 * build the guest tables, MP etc.
 	 */
-	mptable_build(ctx, guest_ncpus);
+	if (mptgen) {
+		error = mptable_build(ctx, guest_ncpus);
+		if (error)
+			exit(1);
+	}
 
 	error = smbios_build(ctx);
 	assert(error == 0);
diff --git a/usr.sbin/bhyve/mevent.c b/usr.sbin/bhyve/mevent.c
index b83ec6904d63..07d3baf72721 100644
--- a/usr.sbin/bhyve/mevent.c
+++ b/usr.sbin/bhyve/mevent.c
@@ -52,9 +52,10 @@ __FBSDID("$FreeBSD$");
 
 #define	MEVENT_MAX	64
 
-#define MEV_ENABLE	1
-#define MEV_DISABLE	2
-#define MEV_DEL_PENDING	3
+#define	MEV_ADD		1
+#define	MEV_ENABLE	2
+#define	MEV_DISABLE	3
+#define	MEV_DEL_PENDING	4
 
 extern char *vmname;
 
@@ -147,10 +148,11 @@ mevent_kq_flags(struct mevent *mevp)
 	int ret;
 
 	switch (mevp->me_state) {
+	case MEV_ADD:
+		ret = EV_ADD;		/* implicitly enabled */
+		break;
 	case MEV_ENABLE:
-		ret = EV_ADD;
-		if (mevp->me_type == EVF_TIMER)
-			ret |= EV_ENABLE;
+		ret = EV_ENABLE;
 		break;
 	case MEV_DISABLE:
 		ret = EV_DISABLE;
@@ -158,6 +160,9 @@ mevent_kq_flags(struct mevent *mevp)
 	case MEV_DEL_PENDING:
 		ret = EV_DELETE;
 		break;
+	default:
+		assert(0);
+		break;
 	}
 
 	return (ret);
@@ -284,7 +289,7 @@ mevent_add(int tfd, enum ev_type type,
 
 	LIST_INSERT_HEAD(&change_head, mevp, me_list);
 	mevp->me_cq = 1;
-	mevp->me_state = MEV_ENABLE;
+	mevp->me_state = MEV_ADD;
 	mevent_notify();
 
 exit:
diff --git a/usr.sbin/bhyve/mptbl.c b/usr.sbin/bhyve/mptbl.c
index e8958359adb6..4c2167e98c85 100644
--- a/usr.sbin/bhyve/mptbl.c
+++ b/usr.sbin/bhyve/mptbl.c
@@ -303,16 +303,31 @@ mptable_build(struct vmctx *ctx, int ncpu)
 	proc_entry_ptr		mpep;
 	mpfps_t			mpfp;
 	int_entry_ptr		mpie;
-	int			ioints;
+	int			ioints, bus;
 	char 			*curraddr;
 	char 			*startaddr;
 
 	startaddr = paddr_guest2host(ctx, MPTABLE_BASE, MPTABLE_MAX_LENGTH);
 	if (startaddr == NULL) {
-		printf("mptable requires mapped mem\n");
+		fprintf(stderr, "mptable requires mapped mem\n");
 		return (ENOMEM);
 	}
 
+	/*
+	 * There is no way to advertise multiple PCI hierarchies via MPtable
+	 * so require that there is no PCI hierarchy with a non-zero bus
+	 * number.
+	 */
+	for (bus = 1; bus <= PCI_BUSMAX; bus++) {
+		if (pci_bus_configured(bus)) {
+			fprintf(stderr, "MPtable is incompatible with "
+			    "multiple PCI hierarchies.\r\n");
+			fprintf(stderr, "MPtable generation can be disabled "
+			    "by passing the -Y option to bhyve(8).\r\n");
+			return (EINVAL);
+		}
+	}
+
 	curraddr = startaddr;
 	mpfp = (mpfps_t)curraddr;
 	mpt_build_mpfp(mpfp, MPTABLE_BASE);
diff --git a/usr.sbin/bhyve/pci_ahci.c b/usr.sbin/bhyve/pci_ahci.c
index 3f9fc7f1c33c..9f6110730b09 100644
--- a/usr.sbin/bhyve/pci_ahci.c
+++ b/usr.sbin/bhyve/pci_ahci.c
@@ -192,6 +192,7 @@ struct pci_ahci_softc {
 	uint32_t em_ctl;
 	uint32_t cap2;
 	uint32_t bohc;
+	uint32_t lintr;
 	struct ahci_port port[MAX_PORTS];
 };
 #define	ahci_ctx(sc)	((sc)->asc_pi->pi_vmctx)
@@ -211,8 +212,11 @@ static inline void lba_to_msf(uint8_t *buf, int lba)
 static void
 ahci_generate_intr(struct pci_ahci_softc *sc)
 {
+	struct pci_devinst *pi;
 	int i;
 
+	pi = sc->asc_pi;
+
 	for (i = 0; i < sc->ports; i++) {
 		struct ahci_port *pr;
 		pr = &sc->port[i];
@@ -222,8 +226,28 @@ ahci_generate_intr(struct pci_ahci_softc *sc)
 
 	DPRINTF("%s %x\n", __func__, sc->is);
 
-	if (sc->is && (sc->ghc & AHCI_GHC_IE))
-		pci_generate_msi(sc->asc_pi, 0);
+	if (sc->is && (sc->ghc & AHCI_GHC_IE)) {		
+		if (pci_msi_enabled(pi)) {
+			/*
+			 * Generate an MSI interrupt on every edge
+			 */
+			pci_generate_msi(pi, 0);
+		} else if (!sc->lintr) {
+			/*
+			 * Only generate a pin-based interrupt if one wasn't
+			 * in progress
+			 */
+			sc->lintr = 1;
+			pci_lintr_assert(pi);
+		}
+	} else if (sc->lintr) {
+		/*
+		 * No interrupts: deassert pin-based signal if it had
+		 * been asserted
+		 */
+		pci_lintr_deassert(pi);
+		sc->lintr = 0;
+	}
 }
 
 static void
@@ -367,6 +391,12 @@ ahci_reset(struct pci_ahci_softc *sc)
 
 	sc->ghc = AHCI_GHC_AE;
 	sc->is = 0;
+
+	if (sc->lintr) {
+		pci_lintr_deassert(sc->asc_pi);
+		sc->lintr = 0;
+	}
+
 	for (i = 0; i < sc->ports; i++) {
 		sc->port[i].ie = 0;
 		sc->port[i].is = 0;
@@ -1814,6 +1844,8 @@ pci_ahci_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts, int atapi)
 	pci_emul_alloc_bar(pi, 5, PCIBAR_MEM32,
 	    AHCI_OFFSET + sc->ports * AHCI_STEP);
 
+	pci_lintr_request(pi);
+
 open_fail:
 	if (ret) {
 		blockif_close(sc->port[0].bctx);
diff --git a/usr.sbin/bhyve/pci_emul.c b/usr.sbin/bhyve/pci_emul.c
index 15f2fad53485..5b87da7f0604 100644
--- a/usr.sbin/bhyve/pci_emul.c
+++ b/usr.sbin/bhyve/pci_emul.c
@@ -1261,6 +1261,13 @@ pci_write_dsdt(void)
 }
 
 int
+pci_bus_configured(int bus)
+{
+	assert(bus >= 0 && bus < MAXBUSES);
+	return (pci_businfo[bus] != NULL);
+}
+
+int
 pci_msi_enabled(struct pci_devinst *pi)
 {
 	return (pi->pi_msi.enabled);
diff --git a/usr.sbin/bhyve/pci_emul.h b/usr.sbin/bhyve/pci_emul.h
index 7ad0a5d1f6b2..e1040a8b715a 100644
--- a/usr.sbin/bhyve/pci_emul.h
+++ b/usr.sbin/bhyve/pci_emul.h
@@ -233,6 +233,7 @@ uint64_t pci_emul_msix_tread(struct pci_devinst *pi, uint64_t offset, int size);
 int	pci_count_lintr(int bus);
 void	pci_walk_lintr(int bus, pci_lintr_cb cb, void *arg);
 void	pci_write_dsdt(void);
+int	pci_bus_configured(int bus);
 
 static __inline void 
 pci_set_cfgdata8(struct pci_devinst *pi, int offset, uint8_t val)
diff --git a/usr.sbin/bhyve/uart_emul.c b/usr.sbin/bhyve/uart_emul.c
index 38e14a95ab03..4242e5ce9548 100644
--- a/usr.sbin/bhyve/uart_emul.c
+++ b/usr.sbin/bhyve/uart_emul.c
@@ -110,6 +110,7 @@ struct uart_softc {
 	uint8_t dlh;		/* Baudrate divisor latch MSB */
 
 	struct fifo rxfifo;
+	struct mevent *mev;
 
 	struct ttyfd tty;
 	bool	thre_int_pending;	/* THRE interrupt pending */
@@ -145,34 +146,15 @@ ttyopen(struct ttyfd *tf)
 	}
 }
 
-static bool
-tty_char_available(struct ttyfd *tf)
-{
-	fd_set rfds;
-	struct timeval tv;
-
-	FD_ZERO(&rfds);
-	FD_SET(tf->fd, &rfds);
-	tv.tv_sec = 0;
-	tv.tv_usec = 0;
-	if (select(tf->fd + 1, &rfds, NULL, NULL, &tv) > 0 ) {
-		return (true);
-	} else {
-		return (false);
-	}
-}
-
 static int
 ttyread(struct ttyfd *tf)
 {
-	char rb;
+	unsigned char rb;
 
-	if (tty_char_available(tf)) {
-		read(tf->fd, &rb, 1);
-		return (rb & 0xff);
-	} else {
+	if (read(tf->fd, &rb, 1) == 1)
+		return (rb);
+	else
 		return (-1);
-	}
 }
 
 static void
@@ -183,62 +165,111 @@ ttywrite(struct ttyfd *tf, unsigned char wb)
 }
 
 static void
-fifo_reset(struct fifo *fifo, int size)
+rxfifo_reset(struct uart_softc *sc, int size)
 {
+	char flushbuf[32];
+	struct fifo *fifo;
+	ssize_t nread;
+	int error;
 
+	fifo = &sc->rxfifo;
 	bzero(fifo, sizeof(struct fifo));
 	fifo->size = size;
+
+	if (sc->tty.opened) {
+		/*
+		 * Flush any unread input from the tty buffer.
+		 */
+		while (1) {
+			nread = read(sc->tty.fd, flushbuf, sizeof(flushbuf));
+			if (nread != sizeof(flushbuf))
+				break;
+		}
+
+		/*
+		 * Enable mevent to trigger when new characters are available
+		 * on the tty fd.
+		 */
+		error = mevent_enable(sc->mev);
+		assert(error == 0);
+	}
 }
 
 static int
-fifo_putchar(struct fifo *fifo, uint8_t ch)
+rxfifo_available(struct uart_softc *sc)
 {
+	struct fifo *fifo;
+
+	fifo = &sc->rxfifo;
+	return (fifo->num < fifo->size);
+}
+
+static int
+rxfifo_putchar(struct uart_softc *sc, uint8_t ch)
+{
+	struct fifo *fifo;
+	int error;
+
+	fifo = &sc->rxfifo;
 
 	if (fifo->num < fifo->size) {
 		fifo->buf[fifo->windex] = ch;
 		fifo->windex = (fifo->windex + 1) % fifo->size;
 		fifo->num++;
+		if (!rxfifo_available(sc)) {
+			if (sc->tty.opened) {
+				/*
+				 * Disable mevent callback if the FIFO is full.
+				 */
+				error = mevent_disable(sc->mev);
+				assert(error == 0);
+			}
+		}
 		return (0);
 	} else
 		return (-1);
 }
 
 static int
-fifo_getchar(struct fifo *fifo)
+rxfifo_getchar(struct uart_softc *sc)
 {
-	int c;
+	struct fifo *fifo;
+	int c, error, wasfull;
 
+	wasfull = 0;
+	fifo = &sc->rxfifo;
 	if (fifo->num > 0) {
+		if (!rxfifo_available(sc))
+			wasfull = 1;
 		c = fifo->buf[fifo->rindex];
 		fifo->rindex = (fifo->rindex + 1) % fifo->size;
 		fifo->num--;
+		if (wasfull) {
+			if (sc->tty.opened) {
+				error = mevent_enable(sc->mev);
+				assert(error == 0);
+			}
+		}
 		return (c);
 	} else
 		return (-1);
 }
 
 static int
-fifo_numchars(struct fifo *fifo)
+rxfifo_numchars(struct uart_softc *sc)
 {
+	struct fifo *fifo = &sc->rxfifo;
 
 	return (fifo->num);
 }
 
-static int
-fifo_available(struct fifo *fifo)
-{
-
-	return (fifo->num < fifo->size);
-}
-
 static void
 uart_opentty(struct uart_softc *sc)
 {
-	struct mevent *mev;
 
 	ttyopen(&sc->tty);
-	mev = mevent_add(sc->tty.fd, EVF_READ, uart_drain, sc);
-	assert(mev);
+	sc->mev = mevent_add(sc->tty.fd, EVF_READ, uart_drain, sc);
+	assert(sc->mev != NULL);
 }
 
 /*
@@ -255,7 +286,7 @@ uart_intr_reason(struct uart_softc *sc)
 
 	if ((sc->lsr & LSR_OE) != 0 && (sc->ier & IER_ERLS) != 0)
 		return (IIR_RLS);
-	else if (fifo_numchars(&sc->rxfifo) > 0 && (sc->ier & IER_ERXRDY) != 0)
+	else if (rxfifo_numchars(sc) > 0 && (sc->ier & IER_ERXRDY) != 0)
 		return (IIR_RXTOUT);
 	else if (sc->thre_int_pending && (sc->ier & IER_ETXRDY) != 0)
 		return (IIR_TXRDY);
@@ -274,7 +305,7 @@ uart_reset(struct uart_softc *sc)
 	sc->dll = divisor;
 	sc->dlh = divisor >> 16;
 
-	fifo_reset(&sc->rxfifo, 1);	/* no fifo until enabled by software */
+	rxfifo_reset(sc, 1);	/* no fifo until enabled by software */
 }
 
 /*
@@ -315,9 +346,9 @@ uart_drain(int fd, enum ev_type ev, void *arg)
 	if ((sc->mcr & MCR_LOOPBACK) != 0) {
 		(void) ttyread(&sc->tty);
 	} else {
-		while (fifo_available(&sc->rxfifo) &&
+		while (rxfifo_available(sc) &&
 		       ((ch = ttyread(&sc->tty)) != -1)) {
-			fifo_putchar(&sc->rxfifo, ch);
+			rxfifo_putchar(sc, ch);
 		}
 		uart_toggle_intr(sc);
 	}
@@ -351,7 +382,7 @@ uart_write(struct uart_softc *sc, int offset, uint8_t value)
         switch (offset) {
 	case REG_DATA:
 		if (sc->mcr & MCR_LOOPBACK) {
-			if (fifo_putchar(&sc->rxfifo, value) != 0)
+			if (rxfifo_putchar(sc, value) != 0)
 				sc->lsr |= LSR_OE;
 		} else if (sc->tty.opened) {
 			ttywrite(&sc->tty, value);
@@ -372,7 +403,7 @@ uart_write(struct uart_softc *sc, int offset, uint8_t value)
 			 */
 			if ((sc->fcr & FCR_ENABLE) ^ (value & FCR_ENABLE)) {
 				fifosz = (value & FCR_ENABLE) ? FIFOSZ : 1;
-				fifo_reset(&sc->rxfifo, fifosz);
+				rxfifo_reset(sc, fifosz);
 			}
 
 			/*
@@ -383,7 +414,7 @@ uart_write(struct uart_softc *sc, int offset, uint8_t value)
 				sc->fcr = 0;
 			} else {
 				if ((value & FCR_RCV_RST) != 0)
-					fifo_reset(&sc->rxfifo, FIFOSZ);
+					rxfifo_reset(sc, FIFOSZ);
 
 				sc->fcr = value &
 					 (FCR_ENABLE | FCR_DMA | FCR_RX_MASK);
@@ -480,7 +511,7 @@ uart_read(struct uart_softc *sc, int offset)
 
 	switch (offset) {
 	case REG_DATA:
-		reg = fifo_getchar(&sc->rxfifo);
+		reg = rxfifo_getchar(sc);
 		break;
 	case REG_IER:
 		reg = sc->ier;
@@ -511,7 +542,7 @@ uart_read(struct uart_softc *sc, int offset)
 		sc->lsr |= LSR_TEMT | LSR_THRE;
 
 		/* Check for new receive data */
-		if (fifo_numchars(&sc->rxfifo) > 0)
+		if (rxfifo_numchars(sc) > 0)
 			sc->lsr |= LSR_RXRDY;
 		else
 			sc->lsr &= ~LSR_RXRDY;
@@ -615,6 +646,10 @@ uart_set_backend(struct uart_softc *sc, const char *opts)
 		retval = 0;
 	}
 
+	/* Make the backend file descriptor non-blocking */
+	if (retval == 0)
+		retval = fcntl(sc->tty.fd, F_SETFL, O_NONBLOCK);
+
 	if (retval == 0)
 		uart_opentty(sc);
 
diff --git a/usr.sbin/bhyve/xmsr.c b/usr.sbin/bhyve/xmsr.c
index ba94125a0900..63522bf023be 100644
--- a/usr.sbin/bhyve/xmsr.c
+++ b/usr.sbin/bhyve/xmsr.c
@@ -47,6 +47,8 @@ emulate_wrmsr(struct vmctx *ctx, int vcpu, uint32_t code, uint64_t val)
 	case 0xd04:			/* Sandy Bridge uncore PMC MSRs */
 	case 0xc24:
 		return (0);
+	case 0x79:
+		return (0);		/* IA32_BIOS_UPDT_TRIG MSR */
 	default:
 		break;
 	}
diff --git a/usr.sbin/bhyvectl/bhyvectl.c b/usr.sbin/bhyvectl/bhyvectl.c
index 2e732b512595..ceee33a2aeeb 100644
--- a/usr.sbin/bhyvectl/bhyvectl.c
+++ b/usr.sbin/bhyvectl/bhyvectl.c
@@ -191,13 +191,16 @@ usage(void)
 	"       [--get-highmem]\n"
 	"       [--get-gpa-pmap]\n"
 	"       [--assert-lapic-lvt=<pin>]\n"
-	"       [--inject-nmi]\n",
+	"       [--inject-nmi]\n"
+	"       [--force-reset]\n"
+	"       [--force-poweroff]\n",
 	progname);
 	exit(1);
 }
 
 static int get_stats, getcap, setcap, capval, get_gpa_pmap;
 static int inject_nmi, assert_lapic_lvt;
+static int force_reset, force_poweroff;
 static const char *capname;
 static int create, destroy, get_lowmem, get_highmem;
 static uint64_t memsize;
@@ -565,6 +568,8 @@ main(int argc, char *argv[])
 		{ "create",	NO_ARG,		&create,	1 },
 		{ "destroy",	NO_ARG,		&destroy,	1 },
 		{ "inject-nmi",	NO_ARG,		&inject_nmi,	1 },
+		{ "force-reset",	NO_ARG,	&force_reset,	1 },
+		{ "force-poweroff", NO_ARG,	&force_poweroff, 1 },
 		{ NULL,		0,		NULL,		0 }
 	};
 
@@ -1535,6 +1540,12 @@ main(int argc, char *argv[])
 			printf("vm_run error %d\n", error);
 	}
 
+	if (!error && force_reset)
+		error = vm_suspend(ctx, VM_SUSPEND_RESET);
+
+	if (!error && force_poweroff)
+		error = vm_suspend(ctx, VM_SUSPEND_POWEROFF);
+
 	if (error)
 		printf("errno = %d\n", errno);
 
diff --git a/usr.sbin/bootparamd/bootparamd/Makefile b/usr.sbin/bootparamd/bootparamd/Makefile
index a5163a3873ab..0596993ec67c 100644
--- a/usr.sbin/bootparamd/bootparamd/Makefile
+++ b/usr.sbin/bootparamd/bootparamd/Makefile
@@ -1,7 +1,7 @@
 #	from: @(#)Makefile	5.8 (Berkeley) 7/28/90
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 PROG=	bootparamd
 MAN=	bootparams.5 bootparamd.8
diff --git a/usr.sbin/bsdinstall/distextract/Makefile b/usr.sbin/bsdinstall/distextract/Makefile
index 1910b23d956c..e359bba590e5 100644
--- a/usr.sbin/bsdinstall/distextract/Makefile
+++ b/usr.sbin/bsdinstall/distextract/Makefile
@@ -8,7 +8,7 @@ LDADD=	-larchive -ldialog -lm
 WARNS?=	6
 MAN=
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 .if ${MK_NCURSESW} == "no"
 DPADD+=	${LIBNCURSES}
diff --git a/usr.sbin/bsdinstall/distfetch/Makefile b/usr.sbin/bsdinstall/distfetch/Makefile
index 08eb91c07352..2ac9eb611b73 100644
--- a/usr.sbin/bsdinstall/distfetch/Makefile
+++ b/usr.sbin/bsdinstall/distfetch/Makefile
@@ -8,7 +8,7 @@ LDADD=	-lfetch -ldialog -lm
 WARNS?=	6
 MAN=
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 .if ${MK_NCURSESW} == "no"
 DPADD+=	${LIBNCURSES}
diff --git a/usr.sbin/bsdinstall/partedit/Makefile b/usr.sbin/bsdinstall/partedit/Makefile
index 3eb0d3f8946d..328259c1ec11 100644
--- a/usr.sbin/bsdinstall/partedit/Makefile
+++ b/usr.sbin/bsdinstall/partedit/Makefile
@@ -22,7 +22,7 @@ SRCS=	diskeditor.c partedit.c gpart_ops.c partedit_${PARTEDIT_ARCH}.c \
 WARNS?=	3
 MAN= sade.8
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 .if ${MK_NCURSESW} == "no"
 DPADD+=	${LIBNCURSES}
diff --git a/usr.sbin/bsnmpd/bsnmpd/Makefile b/usr.sbin/bsnmpd/bsnmpd/Makefile
index 3e6df47fadac..b20ba31d604c 100644
--- a/usr.sbin/bsnmpd/bsnmpd/Makefile
+++ b/usr.sbin/bsnmpd/bsnmpd/Makefile
@@ -2,7 +2,7 @@
 #
 # Author: Harti Brandt <harti@freebsd.org>
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 CONTRIB=${.CURDIR}/../../../contrib/bsnmp
 .PATH: ${CONTRIB}/snmpd
diff --git a/usr.sbin/bsnmpd/modules/Makefile b/usr.sbin/bsnmpd/modules/Makefile
index 45558b7abcc0..24f9f0298c84 100644
--- a/usr.sbin/bsnmpd/modules/Makefile
+++ b/usr.sbin/bsnmpd/modules/Makefile
@@ -1,6 +1,6 @@
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 .PATH: ${.CURDIR}/../../../contrib/bsnmp/snmpd
 
diff --git a/usr.sbin/bsnmpd/modules/snmp_hast/Makefile b/usr.sbin/bsnmpd/modules/snmp_hast/Makefile
index 636991836137..e05ce31020ed 100644
--- a/usr.sbin/bsnmpd/modules/snmp_hast/Makefile
+++ b/usr.sbin/bsnmpd/modules/snmp_hast/Makefile
@@ -1,6 +1,6 @@
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 .PATH:	${.CURDIR}/../../../../sbin/hastd
 
diff --git a/usr.sbin/bsnmpd/tools/bsnmptools/Makefile b/usr.sbin/bsnmpd/tools/bsnmptools/Makefile
index 94a1cea17242..3310420a4f38 100644
--- a/usr.sbin/bsnmpd/tools/bsnmptools/Makefile
+++ b/usr.sbin/bsnmpd/tools/bsnmptools/Makefile
@@ -1,7 +1,7 @@
 # $FreeBSD$
 # Author: Shteryana Shopova <syrinx@FreeBSD.org>
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 .PATH: ${.CURDIR}
 
diff --git a/usr.sbin/etcupdate/Makefile b/usr.sbin/etcupdate/Makefile
index 765533a331ad..155151de02f8 100644
--- a/usr.sbin/etcupdate/Makefile
+++ b/usr.sbin/etcupdate/Makefile
@@ -1,6 +1,6 @@
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 SCRIPTS=etcupdate.sh
 MAN=	etcupdate.8
diff --git a/usr.sbin/flowctl/Makefile b/usr.sbin/flowctl/Makefile
index a2dd37788c38..acee16ba8eff 100644
--- a/usr.sbin/flowctl/Makefile
+++ b/usr.sbin/flowctl/Makefile
@@ -2,7 +2,7 @@
 # $FreeBSD$
 #
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 	
 PROG=	flowctl
 MAN=	flowctl.8
diff --git a/usr.sbin/gssd/Makefile b/usr.sbin/gssd/Makefile
index 3663bf9e8040..9573702b1b62 100644
--- a/usr.sbin/gssd/Makefile
+++ b/usr.sbin/gssd/Makefile
@@ -1,6 +1,6 @@
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 PROG=	gssd
 MAN=	gssd.8
diff --git a/usr.sbin/ifmcstat/Makefile b/usr.sbin/ifmcstat/Makefile
index fab7ea69e2e6..5b967c7b9eaf 100644
--- a/usr.sbin/ifmcstat/Makefile
+++ b/usr.sbin/ifmcstat/Makefile
@@ -1,7 +1,7 @@
 #	@(#)Makefile	8.1 (Berkeley) 6/5/93
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 PROG=	ifmcstat
 SRCS=	ifmcstat.c printb.c
diff --git a/usr.sbin/inetd/Makefile b/usr.sbin/inetd/Makefile
index 29aacd1813d6..f31027af8765 100644
--- a/usr.sbin/inetd/Makefile
+++ b/usr.sbin/inetd/Makefile
@@ -1,7 +1,7 @@
 #	@(#)Makefile	8.1 (Berkeley) 6/6/93
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 PROG=	inetd
 MAN=	inetd.8
diff --git a/usr.sbin/jail/Makefile b/usr.sbin/jail/Makefile
index fb36defa8bf6..58324ac3c088 100644
--- a/usr.sbin/jail/Makefile
+++ b/usr.sbin/jail/Makefile
@@ -1,6 +1,6 @@
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 PROG=	jail
 MAN=	jail.8 jail.conf.5
diff --git a/usr.sbin/jls/Makefile b/usr.sbin/jls/Makefile
index b297cc46d653..431fd0c80c1d 100644
--- a/usr.sbin/jls/Makefile
+++ b/usr.sbin/jls/Makefile
@@ -1,6 +1,6 @@
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 PROG=	jls
 MAN=	jls.8
diff --git a/usr.sbin/kldxref/ef.c b/usr.sbin/kldxref/ef.c
index b68389801345..88fbc34a43b6 100644
--- a/usr.sbin/kldxref/ef.c
+++ b/usr.sbin/kldxref/ef.c
@@ -47,6 +47,7 @@
 
 #include "ef.h"
 
+#define	MAXSEGS 2
 struct ef_file {
 	char*		ef_name;
 	struct elf_file *ef_efile;
@@ -68,7 +69,7 @@ struct ef_file {
 	Elf_Off		ef_symoff;
 	Elf_Sym*	ef_symtab;
 	int		ef_nsegs;
-	Elf_Phdr *	ef_segs[2];
+	Elf_Phdr *	ef_segs[MAXSEGS];
 	int		ef_verbose;
 	Elf_Rel *	ef_rel;			/* relocation table */
 	int		ef_relsz;		/* number of entries */
@@ -580,12 +581,9 @@ ef_open(const char *filename, struct elf_file *efile, int verbose)
 				ef_print_phdr(phdr);
 			switch (phdr->p_type) {
 			case PT_LOAD:
-				if (nsegs == 2) {
-					warnx("%s: too many sections",
-					    filename);
-					break;
-				}
-				ef->ef_segs[nsegs++] = phdr;
+				if (nsegs < MAXSEGS)
+					ef->ef_segs[nsegs] = phdr;
+				nsegs++;
 				break;
 			case PT_PHDR:
 				break;
@@ -597,12 +595,15 @@ ef_open(const char *filename, struct elf_file *efile, int verbose)
 		}
 		if (verbose > 1)
 			printf("\n");
-		ef->ef_nsegs = nsegs;
 		if (phdyn == NULL) {
 			warnx("Skipping %s: not dynamically-linked",
 			    filename);
 			break;
+		} else if (nsegs > MAXSEGS) {
+			warnx("%s: too many sections", filename);
+			break;
 		}
+		ef->ef_nsegs = nsegs;
 		if (ef_read_entry(ef, phdyn->p_offset,
 			phdyn->p_filesz, (void**)&ef->ef_dyn) != 0) {
 			printf("ef_read_entry failed\n");
diff --git a/usr.sbin/lpr/Makefile.inc b/usr.sbin/lpr/Makefile.inc
index ab667dd119a1..65535e6abac0 100644
--- a/usr.sbin/lpr/Makefile.inc
+++ b/usr.sbin/lpr/Makefile.inc
@@ -1,6 +1,6 @@
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 .if ${MK_INET6_SUPPORT} != "no"
 CFLAGS+= -DINET6
diff --git a/usr.sbin/mailwrapper/Makefile b/usr.sbin/mailwrapper/Makefile
index 8811f6f17587..8e49db7edd55 100644
--- a/usr.sbin/mailwrapper/Makefile
+++ b/usr.sbin/mailwrapper/Makefile
@@ -1,6 +1,6 @@
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 .if ${MK_MAILWRAPPER} != "no"
 PROG=	mailwrapper
diff --git a/usr.sbin/mtest/Makefile b/usr.sbin/mtest/Makefile
index 9104c682002b..22c417c8141f 100644
--- a/usr.sbin/mtest/Makefile
+++ b/usr.sbin/mtest/Makefile
@@ -1,6 +1,6 @@
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 PROG=	mtest
 MAN=	mtest.8
diff --git a/usr.sbin/newsyslog/Makefile b/usr.sbin/newsyslog/Makefile
index 4e3ae557034e..1c63e73e0a87 100644
--- a/usr.sbin/newsyslog/Makefile
+++ b/usr.sbin/newsyslog/Makefile
@@ -1,6 +1,6 @@
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 PROG=	newsyslog
 MAN=	newsyslog.8 newsyslog.conf.5
diff --git a/usr.sbin/ngctl/Makefile b/usr.sbin/ngctl/Makefile
index 071f1c38ab95..a1d1e6b375c8 100644
--- a/usr.sbin/ngctl/Makefile
+++ b/usr.sbin/ngctl/Makefile
@@ -1,7 +1,7 @@
 # $FreeBSD$
 # $Whistle: Makefile,v 1.3 1999/01/16 00:10:11 archie Exp $
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 PROG=	ngctl
 MAN=	ngctl.8
diff --git a/usr.sbin/ntp/Makefile.inc b/usr.sbin/ntp/Makefile.inc
index 5095e89bbfbc..cadd4c7a941c 100644
--- a/usr.sbin/ntp/Makefile.inc
+++ b/usr.sbin/ntp/Makefile.inc
@@ -1,6 +1,6 @@
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 DEFS_LOCAL= -DPARSE -DHAVE_CONFIG_H
 NTPDEFS=   -DSYS_FREEBSD
diff --git a/usr.sbin/ntp/doc/Makefile b/usr.sbin/ntp/doc/Makefile
index 64e357155bea..9c42b0970174 100644
--- a/usr.sbin/ntp/doc/Makefile
+++ b/usr.sbin/ntp/doc/Makefile
@@ -1,6 +1,6 @@
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 FILESDIR=	${SHAREDIR}/doc/ntp
 
diff --git a/usr.sbin/ntp/ntp-keygen/Makefile b/usr.sbin/ntp/ntp-keygen/Makefile
index 26716c2ff001..fea148cb076c 100644
--- a/usr.sbin/ntp/ntp-keygen/Makefile
+++ b/usr.sbin/ntp/ntp-keygen/Makefile
@@ -2,7 +2,7 @@
 
 MAN=
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 .PATH: ${.CURDIR}/../../../contrib/ntp/util \
        ${.CURDIR}/../../../contrib/ntp/ntpd
diff --git a/usr.sbin/ntp/ntpd/Makefile b/usr.sbin/ntp/ntpd/Makefile
index 1ac4c7249f71..53f537f71343 100644
--- a/usr.sbin/ntp/ntpd/Makefile
+++ b/usr.sbin/ntp/ntpd/Makefile
@@ -2,7 +2,7 @@
 
 MAN=
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 .PATH: ${.CURDIR}/../../../contrib/ntp/ntpd
 
diff --git a/usr.sbin/pmcstat/pmcstat.8 b/usr.sbin/pmcstat/pmcstat.8
index 5555759939ad..97a2f626ca6d 100644
--- a/usr.sbin/pmcstat/pmcstat.8
+++ b/usr.sbin/pmcstat/pmcstat.8
@@ -25,7 +25,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd September 19, 2008
+.Dd April 29, 2014
 .Dt PMCSTAT 8
 .Os
 .Sh NAME
@@ -46,6 +46,7 @@
 .Op Fl S Ar event-spec
 .Op Fl T
 .Op Fl W
+.Op Fl a Ar pathname
 .Op Fl c Ar cpu-spec
 .Op Fl d
 .Op Fl f Ar pluginopt
@@ -221,6 +222,21 @@ This is an experimental feature intended to help analyse the
 dynamic behaviour of processes in the system.
 It may incur substantial overhead if enabled.
 The default is for this feature to be disabled.
+.It Fl a Ar pathname
+Perform a symbol and file:line lookup for each address in each
+callgraph and save the output to
+.Ar pathname .
+Unlike
+.Fl m
+that only resolves the first symbol in the graph, this resolves
+every node in the callgraph, or prints out addresses if no
+lookup information is available.
+This option requires the
+.Fl R
+option to read in samples that were previously collected and
+saved with the 
+.Fl o
+option.
 .It Fl c Ar cpu-spec
 Set the cpus for subsequent system mode PMCs specified on the
 command line to
diff --git a/usr.sbin/pmcstat/pmcstat.c b/usr.sbin/pmcstat/pmcstat.c
index 3edba3f20db4..ae93b9387da2 100644
--- a/usr.sbin/pmcstat/pmcstat.c
+++ b/usr.sbin/pmcstat/pmcstat.c
@@ -503,7 +503,7 @@ pmcstat_show_usage(void)
 	    "\t -S spec\t allocate a system-wide sampling PMC\n"
 	    "\t -T\t\t start in top mode\n"
 	    "\t -W\t\t (toggle) show counts per context switch\n"
-	    "\t -a <file>\t print sampled PCs and callgraph to \"file\"\n"
+	    "\t -a file\t print sampled PCs and callgraph to \"file\"\n"
 	    "\t -c cpu-list\t set cpus for subsequent system-wide PMCs\n"
 	    "\t -d\t\t (toggle) track descendants\n"
 	    "\t -f spec\t pass \"spec\" to as plugin option\n"
diff --git a/usr.sbin/ppp/Makefile b/usr.sbin/ppp/Makefile
index 99436659344b..a61ebf57501e 100644
--- a/usr.sbin/ppp/Makefile
+++ b/usr.sbin/ppp/Makefile
@@ -1,6 +1,6 @@
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 PROG=	ppp
 MAN=	ppp.8
diff --git a/usr.sbin/rpcbind/Makefile b/usr.sbin/rpcbind/Makefile
index dd042f217a98..278000160fbb 100644
--- a/usr.sbin/rpcbind/Makefile
+++ b/usr.sbin/rpcbind/Makefile
@@ -1,7 +1,7 @@
 #	$NetBSD: Makefile,v 1.3 2000/06/20 13:56:43 fvdl Exp $
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 PROG=	rpcbind
 MAN=	rpcbind.8
diff --git a/usr.sbin/sa/Makefile b/usr.sbin/sa/Makefile
index 23f9bf274a85..c3c79b0b7350 100644
--- a/usr.sbin/sa/Makefile
+++ b/usr.sbin/sa/Makefile
@@ -1,6 +1,6 @@
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 .PATH: ${.CURDIR}/../../usr.bin/lastcomm
 
diff --git a/usr.sbin/sendmail/Makefile b/usr.sbin/sendmail/Makefile
index a9f7fbbafd69..f8a910bc1844 100644
--- a/usr.sbin/sendmail/Makefile
+++ b/usr.sbin/sendmail/Makefile
@@ -1,7 +1,7 @@
 #	@(#)Makefile	8.8 (Berkeley) 3/28/97
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 SENDMAIL_DIR=${.CURDIR}/../../contrib/sendmail
 SMDIR=	${SENDMAIL_DIR}/src
diff --git a/usr.sbin/syslogd/Makefile b/usr.sbin/syslogd/Makefile
index 47640130bb14..069e093ccd21 100644
--- a/usr.sbin/syslogd/Makefile
+++ b/usr.sbin/syslogd/Makefile
@@ -1,7 +1,7 @@
 #	@(#)Makefile	8.1 (Berkeley) 6/6/93
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 .PATH: ${.CURDIR}/../../usr.bin/wall
 
diff --git a/usr.sbin/tcpdchk/Makefile b/usr.sbin/tcpdchk/Makefile
index a49ffc60b748..e9799165c61f 100644
--- a/usr.sbin/tcpdchk/Makefile
+++ b/usr.sbin/tcpdchk/Makefile
@@ -1,6 +1,6 @@
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 .PATH: ${.CURDIR}/../../contrib/tcp_wrappers
 
diff --git a/usr.sbin/tcpdmatch/Makefile b/usr.sbin/tcpdmatch/Makefile
index caae746805d8..f8ccf8adc7da 100644
--- a/usr.sbin/tcpdmatch/Makefile
+++ b/usr.sbin/tcpdmatch/Makefile
@@ -1,6 +1,6 @@
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 .PATH: ${.CURDIR}/../../contrib/tcp_wrappers
 
diff --git a/usr.sbin/tcpdump/tcpdump/Makefile b/usr.sbin/tcpdump/tcpdump/Makefile
index ac7636acb164..498287320737 100644
--- a/usr.sbin/tcpdump/tcpdump/Makefile
+++ b/usr.sbin/tcpdump/tcpdump/Makefile
@@ -1,6 +1,6 @@
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 TCPDUMP_DISTDIR?= ${.CURDIR}/../../../contrib/tcpdump
 .PATH: ${TCPDUMP_DISTDIR}
diff --git a/usr.sbin/trpt/Makefile b/usr.sbin/trpt/Makefile
index 5c62ce049a6a..e49fff1e9d33 100644
--- a/usr.sbin/trpt/Makefile
+++ b/usr.sbin/trpt/Makefile
@@ -1,7 +1,7 @@
 #	@(#)Makefile	8.1 (Berkeley) 6/6/93
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 PROG=	trpt
 MAN=	trpt.8
diff --git a/usr.sbin/tzsetup/Makefile b/usr.sbin/tzsetup/Makefile
index 5c80a480058b..5646b2acec04 100644
--- a/usr.sbin/tzsetup/Makefile
+++ b/usr.sbin/tzsetup/Makefile
@@ -10,7 +10,7 @@ WARNS?=	3
 DPADD=	${LIBDIALOG} ${LIBM}
 LDADD=	-ldialog -lm
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 .if ${MK_NCURSESW} == "no"
 DPADD+=	${LIBNCURSES}
diff --git a/usr.sbin/wpa/hostapd/Makefile b/usr.sbin/wpa/hostapd/Makefile
index 530cf020d2c5..7f0304c6af15 100644
--- a/usr.sbin/wpa/hostapd/Makefile
+++ b/usr.sbin/wpa/hostapd/Makefile
@@ -1,5 +1,6 @@
 # $FreeBSD$
 
+.include <src.opts.mk>
 .include "${.CURDIR}/../Makefile.inc"
 
 .PATH.c:${HOSTAPD_DISTDIR} \
diff --git a/usr.sbin/wpa/wpa_supplicant/Makefile b/usr.sbin/wpa/wpa_supplicant/Makefile
index 3424413f827b..11cccf3ee492 100644
--- a/usr.sbin/wpa/wpa_supplicant/Makefile
+++ b/usr.sbin/wpa/wpa_supplicant/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <src.opts.mk>
+
 .include "${.CURDIR}/../Makefile.inc"
 
 .PATH.c:${WPA_SUPPLICANT_DISTDIR} \