6 files changed, 231 insertions, 29 deletions

diff --git a/share/man/man4/mfi.4 b/share/man/man4/mfi.4
index 36f1ec1e8dcc..6174caa392d5 100644
--- a/share/man/man4/mfi.4
+++ b/share/man/man4/mfi.4
@@ -63,6 +63,15 @@ a new driver.
 Older SCSI and SATA MegaRAID cards are supported by
 .Xr amr 4
 and will not work with this driver.
+.Pp
+Two sysctls are provided to tune the
+.Nm
+driver's behavior when a request is made to remove a mounted volume.
+By default the driver will disallow any requests to remove a mounted volume.
+If the sysctl
+.Va dev.mfi.%d.delete_busy_volumes
+is set to 1,
+then the driver will allow mounted volumes to be removed.
 .Sh HARDWARE
 The
 .Nm
@@ -83,6 +92,11 @@ array/logical disk interface
 .It Pa /dev/mfi?
 management interface
 .El
+.Sh DIAGNOSTICS
+.Bl -diag
+.It "mfid%d: Unable to delete busy device"
+An attempt was made to remove a mounted volume.
+.El
 .Sh SEE ALSO
 .Xr amr 4 ,
 .Xr pci 4
diff --git a/sys/dev/mfi/mfi.c b/sys/dev/mfi/mfi.c
index 583ecadce3ce..b5b2eb9d5a83 100644
--- a/sys/dev/mfi/mfi.c
+++ b/sys/dev/mfi/mfi.c
@@ -185,6 +185,7 @@ mfi_attach(struct mfi_softc *sc)
 	int frames, unit, max_fw_sge;
 
 	mtx_init(&sc->mfi_io_lock, "MFI I/O lock", NULL, MTX_DEF);
+	sx_init(&sc->mfi_config_lock, "MFI config");
 	TAILQ_INIT(&sc->mfi_ld_tqh);
 	TAILQ_INIT(&sc->mfi_aen_pids);
 	TAILQ_INIT(&sc->mfi_cam_ccbq);
@@ -393,6 +394,15 @@ mfi_attach(struct mfi_softc *sc)
 		make_dev_alias(sc->mfi_cdev, "megaraid_sas_ioctl_node");
 	if (sc->mfi_cdev != NULL)
 		sc->mfi_cdev->si_drv1 = sc;
+	SYSCTL_ADD_INT(device_get_sysctl_ctx(sc->mfi_dev),
+	    SYSCTL_CHILDREN(device_get_sysctl_tree(sc->mfi_dev)),
+	    OID_AUTO, "delete_busy_volumes", CTLFLAG_RW,
+	    &sc->mfi_delete_busy_volumes, 0, "Allow removal of busy volumes");
+	SYSCTL_ADD_INT(device_get_sysctl_ctx(sc->mfi_dev),
+	    SYSCTL_CHILDREN(device_get_sysctl_tree(sc->mfi_dev)),
+	    OID_AUTO, "keep_deleted_volumes", CTLFLAG_RW,
+	    &sc->mfi_keep_deleted_volumes, 0,
+	    "Don't detach the mfid device for a busy volume that is deleted");
 
 	device_add_child(sc->mfi_dev, "mfip", -1);
 	bus_generic_attach(sc->mfi_dev);
@@ -750,8 +760,10 @@ mfi_free(struct mfi_softc *sc)
 	if (sc->mfi_parent_dmat != NULL)
 		bus_dma_tag_destroy(sc->mfi_parent_dmat);
 
-	if (mtx_initialized(&sc->mfi_io_lock))
+	if (mtx_initialized(&sc->mfi_io_lock)) {
 		mtx_destroy(&sc->mfi_io_lock);
+		sx_destroy(&sc->mfi_config_lock);
+	}
 
 	return;
 }
@@ -766,9 +778,11 @@ mfi_startup(void *arg)
 	config_intrhook_disestablish(&sc->mfi_ich);
 
 	mfi_enable_intr(sc);
+	sx_xlock(&sc->mfi_config_lock);
 	mtx_lock(&sc->mfi_io_lock);
 	mfi_ldprobe(sc);
 	mtx_unlock(&sc->mfi_io_lock);
+	sx_xunlock(&sc->mfi_config_lock);
 }
 
 static void
@@ -857,8 +871,10 @@ mfi_ldprobe(struct mfi_softc *sc)
 	struct mfi_frame_header *hdr;
 	struct mfi_command *cm = NULL;
 	struct mfi_ld_list *list = NULL;
+	struct mfi_disk *ld;
 	int error, i;
 
+	sx_assert(&sc->mfi_config_lock, SA_XLOCKED);
 	mtx_assert(&sc->mfi_io_lock, MA_OWNED);
 
 	error = mfi_dcmd_command(sc, &cm, MFI_DCMD_LD_GET_LIST,
@@ -879,8 +895,14 @@ mfi_ldprobe(struct mfi_softc *sc)
 		goto out;
 	}
 
-	for (i = 0; i < list->ld_count; i++)
+	for (i = 0; i < list->ld_count; i++) {
+		TAILQ_FOREACH(ld, &sc->mfi_ld_tqh, ld_link) {
+			if (ld->ld_id == list->ld_list[i].ld.v.target_id)
+				goto skip_add;
+		}
 		mfi_add_ld(sc, list->ld_list[i].ld.v.target_id);
+	skip_add:;
+	}
 out:
 	if (list)
 		free(list, M_MFIBUF);
@@ -1790,6 +1812,111 @@ mfi_close(struct cdev *dev, int flags, int fmt, d_thread_t *td)
 }
 
 static int
+mfi_config_lock(struct mfi_softc *sc, uint32_t opcode)
+{
+
+	switch (opcode) {
+	case MFI_DCMD_LD_DELETE:
+	case MFI_DCMD_CFG_ADD:
+	case MFI_DCMD_CFG_CLEAR:
+		sx_xlock(&sc->mfi_config_lock);
+		return (1);
+	default:
+		return (0);
+	}
+}
+
+static void
+mfi_config_unlock(struct mfi_softc *sc, int locked)
+{
+
+	if (locked)
+		sx_xunlock(&sc->mfi_config_lock);
+}
+
+/* Perform pre-issue checks on commands from userland and possibly veto them. */
+static int
+mfi_check_command_pre(struct mfi_softc *sc, struct mfi_command *cm)
+{
+	struct mfi_disk *ld, *ld2;
+	int error;
+
+	mtx_assert(&sc->mfi_io_lock, MA_OWNED);
+	error = 0;
+	switch (cm->cm_frame->dcmd.opcode) {
+	case MFI_DCMD_LD_DELETE:
+		TAILQ_FOREACH(ld, &sc->mfi_ld_tqh, ld_link) {
+			if (ld->ld_id == cm->cm_frame->dcmd.mbox[0])
+				break;
+		}
+		if (ld == NULL)
+			error = ENOENT;
+		else
+			error = mfi_disk_disable(ld);
+		break;
+	case MFI_DCMD_CFG_CLEAR:
+		TAILQ_FOREACH(ld, &sc->mfi_ld_tqh, ld_link) {
+			error = mfi_disk_disable(ld);
+			if (error)
+				break;
+		}
+		if (error) {
+			TAILQ_FOREACH(ld2, &sc->mfi_ld_tqh, ld_link) {
+				if (ld2 == ld)
+					break;
+				mfi_disk_enable(ld2);
+			}
+		}
+		break;
+	default:
+		break;
+	}
+	return (error);
+}
+
+/* Perform post-issue checks on commands from userland. */
+static void
+mfi_check_command_post(struct mfi_softc *sc, struct mfi_command *cm)
+{
+	struct mfi_disk *ld, *ldn;
+
+	switch (cm->cm_frame->dcmd.opcode) {
+	case MFI_DCMD_LD_DELETE:
+		TAILQ_FOREACH(ld, &sc->mfi_ld_tqh, ld_link) {
+			if (ld->ld_id == cm->cm_frame->dcmd.mbox[0])
+				break;
+		}
+		KASSERT(ld != NULL, ("volume dissappeared"));
+		if (cm->cm_frame->header.cmd_status == MFI_STAT_OK) {
+			mtx_unlock(&sc->mfi_io_lock);
+			mtx_lock(&Giant);
+			device_delete_child(sc->mfi_dev, ld->ld_dev);
+			mtx_unlock(&Giant);
+			mtx_lock(&sc->mfi_io_lock);
+		} else
+			mfi_disk_enable(ld);
+		break;
+	case MFI_DCMD_CFG_CLEAR:
+		if (cm->cm_frame->header.cmd_status == MFI_STAT_OK) {
+			mtx_unlock(&sc->mfi_io_lock);
+			mtx_lock(&Giant);
+			TAILQ_FOREACH_SAFE(ld, &sc->mfi_ld_tqh, ld_link, ldn) {
+				device_delete_child(sc->mfi_dev, ld->ld_dev);
+			}
+			mtx_unlock(&Giant);
+			mtx_lock(&sc->mfi_io_lock);
+		} else {
+			TAILQ_FOREACH(ld, &sc->mfi_ld_tqh, ld_link)
+				mfi_disk_enable(ld);
+		}
+		break;
+	case MFI_DCMD_CFG_ADD:
+		mfi_ldprobe(sc);
+		break;
+	}
+}
+
+static int
 mfi_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int flag, d_thread_t *td)
 {
 	struct mfi_softc *sc;
@@ -1801,7 +1928,7 @@ mfi_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int flag, d_thread_t *td)
 	uint8_t *sense_ptr;
 	uint8_t *data = NULL, *temp;
 	int i;
-	int error;
+	int error, locked;
 
 	sc = dev->si_drv1;
 	error = 0;
@@ -1855,6 +1982,7 @@ mfi_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int flag, d_thread_t *td)
 			return (EBUSY);
 		}
 		mtx_unlock(&sc->mfi_io_lock);
+		locked = 0;
 
 		/*
 		 * save off original context since copying from user
@@ -1892,7 +2020,16 @@ mfi_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int flag, d_thread_t *td)
 			temp = &temp[ioc->mfi_sgl[i].iov_len];
 		}
 
+		if (cm->cm_frame->header.cmd == MFI_CMD_DCMD)
+			locked = mfi_config_lock(sc, cm->cm_frame->dcmd.opcode);
+
 		mtx_lock(&sc->mfi_io_lock);
+		error = mfi_check_command_pre(sc, cm);
+		if (error) {
+			mtx_unlock(&sc->mfi_io_lock);
+			goto out;
+		}
+
 		if ((error = mfi_wait_command(sc, cm)) != 0) {
 			device_printf(sc->mfi_dev,
 			    "Controller polled failed\n");
@@ -1900,6 +2037,7 @@ mfi_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int flag, d_thread_t *td)
 			goto out;
 		}
 
+		mfi_check_command_post(sc, cm);
 		mtx_unlock(&sc->mfi_io_lock);
 
 		temp = data;
@@ -1929,17 +2067,8 @@ mfi_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int flag, d_thread_t *td)
 		}
 
 		ioc->mfi_frame.hdr.cmd_status = cm->cm_frame->header.cmd_status;
-		if (cm->cm_frame->header.cmd_status == MFI_STAT_OK) {
-			switch (cm->cm_frame->dcmd.opcode) {
-			case MFI_DCMD_CFG_CLEAR:
-			case MFI_DCMD_CFG_ADD:
-/*
-				mfi_ldrescan(sc);
-*/
-				break;
-			}
-		}
 out:
+		mfi_config_unlock(sc, locked);
 		if (data)
 			free(data, M_MFIBUF);
 		if (cm) {
@@ -2019,7 +2148,7 @@ mfi_linux_ioctl_int(struct cdev *dev, u_long cmd, caddr_t arg, int flag, d_threa
 	uint8_t *data = NULL, *temp;
 	void *temp_convert;
 	int i;
-	int error;
+	int error, locked;
 
 	sc = dev->si_drv1;
 	error = 0;
@@ -2039,6 +2168,7 @@ mfi_linux_ioctl_int(struct cdev *dev, u_long cmd, caddr_t arg, int flag, d_threa
 			return (EBUSY);
 		}
 		mtx_unlock(&sc->mfi_io_lock);
+		locked = 0;
 
 		/*
 		 * save off original context since copying from user
@@ -2074,7 +2204,16 @@ mfi_linux_ioctl_int(struct cdev *dev, u_long cmd, caddr_t arg, int flag, d_threa
 			temp = &temp[l_ioc.lioc_sgl[i].iov_len];
 		}
 
+		if (cm->cm_frame->header.cmd == MFI_CMD_DCMD)
+			locked = mfi_config_lock(sc, cm->cm_frame->dcmd.opcode);
+
 		mtx_lock(&sc->mfi_io_lock);
+		error = mfi_check_command_pre(sc, cm);
+		if (error) {
+			mtx_unlock(&sc->mfi_io_lock);
+			goto out;
+		}
+
 		if ((error = mfi_wait_command(sc, cm)) != 0) {
 			device_printf(sc->mfi_dev,
 			    "Controller polled failed\n");
@@ -2082,6 +2221,7 @@ mfi_linux_ioctl_int(struct cdev *dev, u_long cmd, caddr_t arg, int flag, d_threa
 			goto out;
 		}
 
+		mfi_check_command_post(sc, cm);
 		mtx_unlock(&sc->mfi_io_lock);
 
 		temp = data;
@@ -2122,15 +2262,8 @@ mfi_linux_ioctl_int(struct cdev *dev, u_long cmd, caddr_t arg, int flag, d_threa
 			goto out;
 		}
 
-		if (cm->cm_frame->header.cmd_status == MFI_STAT_OK) {
-			switch (cm->cm_frame->dcmd.opcode) {
-			case MFI_DCMD_CFG_CLEAR:
-			case MFI_DCMD_CFG_ADD:
-				/* mfi_ldrescan(sc); */
-				break;
-			}
-		}
 out:
+		mfi_config_unlock(sc, locked);
 		if (data)
 			free(data, M_MFIBUF);
 		if (cm) {
diff --git a/sys/dev/mfi/mfi_disk.c b/sys/dev/mfi/mfi_disk.c
index 290bf475e4c0..0a553fa36ce0 100644
--- a/sys/dev/mfi/mfi_disk.c
+++ b/sys/dev/mfi/mfi_disk.c
@@ -108,7 +108,9 @@ mfi_disk_attach(device_t dev)
 
 	sectors = ld_info->size;
 	secsize = MFI_SECTOR_LEN;
+	mtx_lock(&sc->ld_controller->mfi_io_lock);
 	TAILQ_INSERT_TAIL(&sc->ld_controller->mfi_ld_tqh, sc, ld_link);
+	mtx_unlock(&sc->ld_controller->mfi_io_lock);
 
 	switch (ld_info->ld_config.params.state) {
 	case MFI_LD_STATE_OFFLINE:
@@ -162,12 +164,21 @@ mfi_disk_detach(device_t dev)
 
 	sc = device_get_softc(dev);
 
-	if ((sc->ld_disk->d_flags & DISKFLAG_OPEN) ||
-	    (sc->ld_flags & MFI_DISK_FLAGS_OPEN))
+	mtx_lock(&sc->ld_controller->mfi_io_lock);
+	if (((sc->ld_disk->d_flags & DISKFLAG_OPEN) ||
+	    (sc->ld_flags & MFI_DISK_FLAGS_OPEN)) &&
+	    (sc->ld_controller->mfi_keep_deleted_volumes ||
+	    sc->ld_controller->mfi_detaching)) {
+		mtx_unlock(&sc->ld_controller->mfi_io_lock);
 		return (EBUSY);
+	}
+	mtx_unlock(&sc->ld_controller->mfi_io_lock);
 
-	free(sc->ld_info, M_MFIBUF);
 	disk_destroy(sc->ld_disk);
+	mtx_lock(&sc->ld_controller->mfi_io_lock);
+	TAILQ_REMOVE(&sc->ld_controller->mfi_ld_tqh, sc, ld_link);
+	mtx_unlock(&sc->ld_controller->mfi_io_lock);
+	free(sc->ld_info, M_MFIBUF);
 	return (0);
 }
 
@@ -175,13 +186,19 @@ static int
 mfi_disk_open(struct disk *dp)
 {
 	struct mfi_disk *sc;
+	int error;
 
 	sc = dp->d_drv1;
 	mtx_lock(&sc->ld_controller->mfi_io_lock);
-	sc->ld_flags |= MFI_DISK_FLAGS_OPEN;
+	if (sc->ld_flags & MFI_DISK_FLAGS_DISABLED)
+		error = ENXIO;
+	else {
+		sc->ld_flags |= MFI_DISK_FLAGS_OPEN;
+		error = 0;
+	}
 	mtx_unlock(&sc->ld_controller->mfi_io_lock);
 
-	return (0);
+	return (error);
 }
 
 static int
@@ -197,6 +214,29 @@ mfi_disk_close(struct disk *dp)
 	return (0);
 }
 
+int
+mfi_disk_disable(struct mfi_disk *sc)
+{
+
+	mtx_assert(&sc->ld_controller->mfi_io_lock, MA_OWNED);
+	if (sc->ld_flags & MFI_DISK_FLAGS_OPEN) {
+		if (sc->ld_controller->mfi_delete_busy_volumes)
+			return (0);
+		device_printf(sc->ld_dev, "Unable to delete busy device\n");
+		return (EBUSY);
+	}
+	sc->ld_flags |= MFI_DISK_FLAGS_DISABLED;
+	return (0);
+}
+
+void
+mfi_disk_enable(struct mfi_disk *sc)
+{
+
+	mtx_assert(&sc->ld_controller->mfi_io_lock, MA_OWNED);
+	sc->ld_flags &= ~MFI_DISK_FLAGS_DISABLED;
+}
+
 static void
 mfi_disk_strategy(struct bio *bio)
 {
diff --git a/sys/dev/mfi/mfi_pci.c b/sys/dev/mfi/mfi_pci.c
index ccec1d0c3a17..a335bd21eef0 100644
--- a/sys/dev/mfi/mfi_pci.c
+++ b/sys/dev/mfi/mfi_pci.c
@@ -200,19 +200,23 @@ mfi_pci_detach(device_t dev)
 
 	sc = device_get_softc(dev);
 
+	sx_xlock(&sc->mfi_config_lock);
 	mtx_lock(&sc->mfi_io_lock);
 	if ((sc->mfi_flags & MFI_FLAGS_OPEN) != 0) {
 		mtx_unlock(&sc->mfi_io_lock);
 		return (EBUSY);
 	}
+	sc->mfi_detaching = 1;
+	mtx_unlock(&sc->mfi_io_lock);
 
 	while ((ld = TAILQ_FIRST(&sc->mfi_ld_tqh)) != NULL) {
 		if ((error = device_delete_child(dev, ld->ld_dev)) != 0) {
-			mtx_unlock(&sc->mfi_io_lock);
+			sc->mfi_detaching = 0;
+			sx_xunlock(&sc->mfi_config_lock);
 			return (error);
 		}
-		TAILQ_REMOVE(&sc->mfi_ld_tqh, ld, ld_link);
 	}
+	sx_xunlock(&sc->mfi_config_lock);
 
 	EVENTHANDLER_DEREGISTER(shutdown_final, sc->mfi_eh);
 
diff --git a/sys/dev/mfi/mfireg.h b/sys/dev/mfi/mfireg.h
index da4df4e25011..841756c4b7c9 100644
--- a/sys/dev/mfi/mfireg.h
+++ b/sys/dev/mfi/mfireg.h
@@ -112,6 +112,7 @@ typedef enum {
 	MFI_DCMD_LD_GET_INFO =		0x03020000,
 	MFI_DCMD_LD_GET_PROP =		0x03030000,
 	MFI_DCMD_LD_SET_PROP =		0x03040000,
+	MFI_DCMD_LD_DELETE =		0x03090000,
 	MFI_DCMD_CFG_READ =		0x04010000,
 	MFI_DCMD_CFG_ADD =		0x04020000,
 	MFI_DCMD_CFG_CLEAR =		0x04030000,
diff --git a/sys/dev/mfi/mfivar.h b/sys/dev/mfi/mfivar.h
index 874ae1fdaeba..80714a0fab3a 100644
--- a/sys/dev/mfi/mfivar.h
+++ b/sys/dev/mfi/mfivar.h
@@ -30,6 +30,9 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include <sys/lock.h>
+#include <sys/sx.h>
+
 /*
  * SCSI structures and definitions are used from here, but no linking
  * requirements are made to CAM.
@@ -87,6 +90,7 @@ struct mfi_disk {
 	struct disk	*ld_disk;
 	int		ld_flags;
 #define MFI_DISK_FLAGS_OPEN	0x01
+#define	MFI_DISK_FLAGS_DISABLED	0x02
 };
 
 struct mfi_aen {
@@ -131,6 +135,9 @@ struct mfi_softc {
 	uint32_t			mfi_aen_triggered;
 	uint32_t			mfi_poll_waiting;
 	struct selinfo			mfi_select;
+	int				mfi_delete_busy_volumes;
+	int				mfi_keep_deleted_volumes;
+	int				mfi_detaching;
 
 	bus_dma_tag_t			mfi_sense_dmat;
 	bus_dmamap_t			mfi_sense_dmamap;
@@ -185,6 +192,7 @@ struct mfi_softc {
 	struct mfi_command *		(* mfi_cam_start)(void *);
 	struct callout			mfi_watchdog_callout;
 	struct mtx			mfi_io_lock;
+	struct sx			mfi_config_lock;
 };
 
 extern int mfi_attach(struct mfi_softc *);
@@ -192,6 +200,8 @@ extern void mfi_free(struct mfi_softc *);
 extern int mfi_shutdown(struct mfi_softc *);
 extern void mfi_startio(struct mfi_softc *);
 extern void mfi_disk_complete(struct bio *);
+extern int mfi_disk_disable(struct mfi_disk *);
+extern void mfi_disk_enable(struct mfi_disk *);
 extern int mfi_dump_blocks(struct mfi_softc *, int id, uint64_t, void *, int);
 
 #define MFIQ_ADD(sc, qname)					\