9 files changed, 271 insertions, 11 deletions

diff --git a/cddl/usr.sbin/zfsd/case_file.cc b/cddl/usr.sbin/zfsd/case_file.cc
index 34234a5736a5..5064d3c266dd 100644
--- a/cddl/usr.sbin/zfsd/case_file.cc
+++ b/cddl/usr.sbin/zfsd/case_file.cc
@@ -116,6 +116,26 @@ CaseFile::Find(Guid poolGUID, Guid vdevGUID)
 	return (NULL);
 }
 
+void
+CaseFile::Find(Guid poolGUID, Guid vdevGUID, CaseFileList &cases)
+{
+	for (CaseFileList::iterator curCase = s_activeCases.begin();
+	    curCase != s_activeCases.end(); curCase++) {
+		if (((*curCase)->PoolGUID() != poolGUID &&
+		    Guid::InvalidGuid() != poolGUID) ||
+		    (*curCase)->VdevGUID() != vdevGUID)
+			continue;
+
+		/*
+		 * We can have multiple cases for spare vdevs
+		 */
+		cases.push_back(*curCase);
+		if (!(*curCase)->IsSpare()) {
+			return;
+		}
+	}
+}
+
 CaseFile *
 CaseFile::Find(const string &physPath)
 {
@@ -217,6 +237,12 @@ CaseFile::PurgeAll()
 
 }
 
+int
+CaseFile::IsSpare()
+{
+	return (m_is_spare);
+}
+
 //- CaseFile Public Methods ----------------------------------------------------
 bool
 CaseFile::RefreshVdevState()
@@ -240,6 +266,7 @@ CaseFile::ReEvaluate(const string &devPath, const string &physPath, Vdev *vdev)
 {
 	ZpoolList zpl(ZpoolList::ZpoolByGUID, &m_poolGUID);
 	zpool_handle_t *pool(zpl.empty() ? NULL : zpl.front());
+	int flags = ZFS_ONLINE_CHECKREMOVE | ZFS_ONLINE_UNSPARE;
 
 	if (pool == NULL || !RefreshVdevState()) {
 		/*
@@ -280,9 +307,10 @@ CaseFile::ReEvaluate(const string &devPath, const string &physPath, Vdev *vdev)
 	   || vdev->PoolGUID() == Guid::InvalidGuid())
 	 && vdev->GUID() == m_vdevGUID) {
 
+		if (IsSpare())
+			flags |= ZFS_ONLINE_SPARE;
 		if (zpool_vdev_online(pool, vdev->GUIDString().c_str(),
-		    ZFS_ONLINE_CHECKREMOVE | ZFS_ONLINE_UNSPARE,
-		    &m_vdevState) != 0) {
+		    flags, &m_vdevState) != 0) {
 			syslog(LOG_ERR,
 			    "Failed to online vdev(%s/%s:%s): %s: %s\n",
 			    zpool_get_name(pool), vdev->GUIDString().c_str(),
@@ -790,7 +818,8 @@ CaseFile::CaseFile(const Vdev &vdev)
  : m_poolGUID(vdev.PoolGUID()),
    m_vdevGUID(vdev.GUID()),
    m_vdevState(vdev.State()),
-   m_vdevPhysPath(vdev.PhysicalPath())
+   m_vdevPhysPath(vdev.PhysicalPath()),
+   m_is_spare(vdev.IsSpare())
 {
 	stringstream guidString;
 
diff --git a/cddl/usr.sbin/zfsd/case_file.h b/cddl/usr.sbin/zfsd/case_file.h
index b4dc2dee5d96..d7475d331a76 100644
--- a/cddl/usr.sbin/zfsd/case_file.h
+++ b/cddl/usr.sbin/zfsd/case_file.h
@@ -99,6 +99,19 @@ public:
 	static CaseFile *Find(DevdCtl::Guid poolGUID, DevdCtl::Guid vdevGUID);
 
 	/**
+	 * \brief Find multiple CaseFile objects by a vdev's pool/vdev
+	 *        GUID tuple (special case for spare vdevs)
+	 *
+	 * \param poolGUID  Pool GUID for the vdev of the CaseFile to find.
+	 * 		    If InvalidGuid, then only match the vdev GUID
+	 * 		    instead of both pool and vdev GUIDs.
+	 * \param vdevGUID  Vdev GUID for the vdev of the CaseFile to find.
+	 * \param caseList  List of cases associated with the vdev.
+	 */
+	static void  Find(DevdCtl::Guid poolGUID, DevdCtl::Guid vdevGUID,
+				     CaseFileList &caseList);
+
+	/**
 	 * \brief Find a CaseFile object by a vdev's current/last known
 	 *        physical path.
 	 *
@@ -219,6 +232,11 @@ public:
 	 */
 	bool ShouldFault() const;
 
+	/**
+	 * \brief If this vdev is spare
+	 */
+	int IsSpare();
+
 protected:
 	enum {
 		/**
@@ -384,6 +402,7 @@ protected:
 	string		   m_poolGUIDString;
 	string		   m_vdevGUIDString;
 	string		   m_vdevPhysPath;
+	int		   m_is_spare;
 
 	/**
 	 * \brief Callout activated when a grace period
diff --git a/cddl/usr.sbin/zfsd/vdev_iterator.cc b/cddl/usr.sbin/zfsd/vdev_iterator.cc
index b5a4f22c1c60..c23399ed68a8 100644
--- a/cddl/usr.sbin/zfsd/vdev_iterator.cc
+++ b/cddl/usr.sbin/zfsd/vdev_iterator.cc
@@ -78,8 +78,10 @@ VdevIterator::Reset()
 {
 	nvlist_t  *rootVdev;
 	nvlist	  **cache_child;
+	nvlist	  **spare_child;
 	int	   result;
 	uint_t   cache_children;
+	uint_t	 spare_children;
 
 	result = nvlist_lookup_nvlist(m_poolConfig,
 				      ZPOOL_CONFIG_VDEV_TREE,
@@ -95,6 +97,13 @@ VdevIterator::Reset()
 	if (result == 0)
 		for (uint_t c = 0; c < cache_children; c++)
 			m_vdevQueue.push_back(cache_child[c]);
+	result = nvlist_lookup_nvlist_array(rootVdev,
+					    ZPOOL_CONFIG_SPARES,
+					    &spare_child,
+					    &spare_children);
+	if (result == 0)
+		for (uint_t c = 0; c < spare_children; c++)
+			m_vdevQueue.push_back(spare_child[c]);
 }
 
 nvlist_t *
diff --git a/cddl/usr.sbin/zfsd/zfsd_event.cc b/cddl/usr.sbin/zfsd/zfsd_event.cc
index 688e7c0354a2..45fc7adbb31b 100644
--- a/cddl/usr.sbin/zfsd/zfsd_event.cc
+++ b/cddl/usr.sbin/zfsd/zfsd_event.cc
@@ -229,7 +229,9 @@ bool
 GeomEvent::OnlineByLabel(const string &devPath, const string& physPath,
 			      nvlist_t *devConfig)
 {
+	bool ret = false;
 	try {
+		CaseFileList case_list;
 		/*
 		 * A device with ZFS label information has been
 		 * inserted.  If it matches a device for which we
@@ -238,10 +240,12 @@ GeomEvent::OnlineByLabel(const string &devPath, const string& physPath,
 		syslog(LOG_INFO, "Interrogating VDEV label for %s\n",
 		       devPath.c_str());
 		Vdev vdev(devConfig);
-		CaseFile *caseFile(CaseFile::Find(vdev.PoolGUID(),
-						  vdev.GUID()));
-		if (caseFile != NULL)
-			return (caseFile->ReEvaluate(devPath, physPath, &vdev));
+		CaseFile::Find(vdev.PoolGUID(),vdev.GUID(), case_list);
+		for (CaseFileList::iterator curr = case_list.begin();
+		    curr != case_list.end(); curr++) {
+			ret |= (*curr)->ReEvaluate(devPath, physPath, &vdev);
+		}
+		return (ret);
 
 	} catch (ZfsdException &exp) {
 		string context("GeomEvent::OnlineByLabel: " + devPath + ": ");
@@ -249,7 +253,7 @@ GeomEvent::OnlineByLabel(const string &devPath, const string& physPath,
 		exp.GetString().insert(0, context);
 		exp.Log();
 	}
-	return (false);
+	return (ret);
 }
 
 
diff --git a/tests/sys/cddl/zfs/tests/zfsd/Makefile b/tests/sys/cddl/zfs/tests/zfsd/Makefile
index 6bdbae7886a6..fa662e0c41b6 100644
--- a/tests/sys/cddl/zfs/tests/zfsd/Makefile
+++ b/tests/sys/cddl/zfs/tests/zfsd/Makefile
@@ -34,5 +34,7 @@ ${PACKAGE}FILES+=	zfsd_import_001_pos.ksh
 ${PACKAGE}FILES+=	zfsd_replace_001_pos.ksh
 ${PACKAGE}FILES+=	zfsd_replace_002_pos.ksh
 ${PACKAGE}FILES+=	zfsd_replace_003_pos.ksh
+${PACKAGE}FILES+=	zfsd_replace_004_pos.ksh
+${PACKAGE}FILES+=	zfsd_replace_005_pos.ksh
 
 .include <bsd.test.mk>
diff --git a/tests/sys/cddl/zfs/tests/zfsd/zfsd.kshlib b/tests/sys/cddl/zfs/tests/zfsd/zfsd.kshlib
index 4a7f9b09e182..a14136677ba8 100644
--- a/tests/sys/cddl/zfs/tests/zfsd/zfsd.kshlib
+++ b/tests/sys/cddl/zfs/tests/zfsd/zfsd.kshlib
@@ -35,15 +35,20 @@ function wait_for_pool_dev_state_change
 	typeset -i timeout=$1
 	typeset disk=$2
 	typeset state=$3
+	typeset pool=$4
+
+	if [ -z "$pool" ]; then
+		pool=$TESTPOOL
+	fi
 
 	log_note "Waiting up to $timeout seconds for $disk to become $state ..."
 	for ((; $timeout > 0; timeout=$timeout-1)); do
-		check_state $TESTPOOL "$disk" "$state"
+		check_state $pool "$disk" "$state"
 		[ $? -eq 0 ] && return
 		$SLEEP 1
 	done
-	log_must $ZPOOL status $TESTPOOL
-	log_fail "ERROR: Disk $disk not marked as $state in $TESTPOOL"
+	log_must $ZPOOL status $pool
+	log_fail "ERROR: Disk $disk not marked as $state in $pool"
 }
 
 function wait_for_pool_removal
diff --git a/tests/sys/cddl/zfs/tests/zfsd/zfsd_replace_004_pos.ksh b/tests/sys/cddl/zfs/tests/zfsd/zfsd_replace_004_pos.ksh
new file mode 100644
index 000000000000..343c30bf1da6
--- /dev/null
+++ b/tests/sys/cddl/zfs/tests/zfsd/zfsd_replace_004_pos.ksh
@@ -0,0 +1,64 @@
+#!/usr/local/bin/ksh93 -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright 2023 Axcient.  All rights reserved.
+# Use is subject to license terms.
+#
+# $FreeBSD$
+
+. $STF_SUITE/tests/hotspare/hotspare.kshlib
+. $STF_SUITE/tests/zfsd/zfsd.kshlib
+. $STF_SUITE/include/libtest.kshlib
+. $STF_SUITE/include/libgnop.kshlib
+
+log_assert "ZFSD will automatically replace a spare that disappears and reappears in the same location, with the same devname"
+
+ensure_zfsd_running
+
+set_disks
+
+typeset DISK0_NOP=${DISK0}.nop
+typeset DISK1_NOP=${DISK1}.nop
+
+log_must create_gnops $DISK0 $DISK1
+
+# Create a pool on the supplied disks
+create_pool $TESTPOOL $DISK0_NOP spare $DISK1_NOP
+
+# Disable the first disk.
+log_must destroy_gnop $DISK1
+
+# Check to make sure ZFS sees the disk as removed
+wait_for_pool_dev_state_change 20 $DISK1_NOP REMOVED
+
+# Re-enable the disk
+log_must create_gnop $DISK1
+
+# Disk should auto-join the zpool
+wait_for_pool_dev_state_change 20 $DISK1_NOP AVAIL
+
+$ZPOOL status $TESTPOOL
+destroy_pool $TESTPOOL
+log_must $RM -rf /$TESTPOOL
+
+log_pass
diff --git a/tests/sys/cddl/zfs/tests/zfsd/zfsd_replace_005_pos.ksh b/tests/sys/cddl/zfs/tests/zfsd/zfsd_replace_005_pos.ksh
new file mode 100644
index 000000000000..d21454168048
--- /dev/null
+++ b/tests/sys/cddl/zfs/tests/zfsd/zfsd_replace_005_pos.ksh
@@ -0,0 +1,70 @@
+#!/usr/local/bin/ksh93 -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright 2023 Axcient.  All rights reserved.
+# Use is subject to license terms.
+#
+# $FreeBSD$
+
+. $STF_SUITE/tests/hotspare/hotspare.kshlib
+. $STF_SUITE/tests/zfsd/zfsd.kshlib
+. $STF_SUITE/include/libtest.kshlib
+. $STF_SUITE/include/libgnop.kshlib
+
+log_assert "ZFSD will automatically replace a multi-pool spare that disappears and reappears"
+
+ensure_zfsd_running
+
+set_disks
+
+typeset DISK0_NOP=${DISK0}.nop
+typeset DISK1_NOP=${DISK1}.nop
+typeset DISK2_NOP=${DISK2}.nop
+
+log_must create_gnops $DISK0 $DISK1 $DISK2
+
+# Create pools on the supplied disks
+create_pool $TESTPOOL $DISK0_NOP spare $DISK2_NOP
+create_pool $TESTPOOL1 $DISK1_NOP spare $DISK2_NOP
+
+# Disable the spare disk.
+log_must destroy_gnop $DISK2
+
+# Check to make sure ZFS sees the disk as removed
+wait_for_pool_dev_state_change 20 $DISK2_NOP REMOVED
+wait_for_pool_dev_state_change 20 $DISK2_NOP REMOVED $TESTPOOL1
+
+# Re-enable the disk
+log_must create_gnop $DISK2
+
+# Disk should auto-join the zpools
+wait_for_pool_dev_state_change 20 $DISK2_NOP AVAIL
+wait_for_pool_dev_state_change 20 $DISK2_NOP AVAIL $TESTPOOL1
+
+$ZPOOL status $TESTPOOL
+$ZPOOL status $TESTPOOL1
+destroy_pool $TESTPOOL
+destroy_pool $TESTPOOL1
+log_must $RM -rf /$TESTPOOL
+
+log_pass
diff --git a/tests/sys/cddl/zfs/tests/zfsd/zfsd_test.sh b/tests/sys/cddl/zfs/tests/zfsd/zfsd_test.sh
index e6bc16a56442..b15208973bfe 100755
--- a/tests/sys/cddl/zfs/tests/zfsd/zfsd_test.sh
+++ b/tests/sys/cddl/zfs/tests/zfsd/zfsd_test.sh
@@ -538,6 +538,62 @@ zfsd_replace_003_pos_cleanup()
 	ksh93 $(atf_get_srcdir)/cleanup.ksh || atf_fail "Cleanup failed"
 }
 
+atf_test_case zfsd_replace_004_pos cleanup
+zfsd_replace_004_pos_head()
+{
+	atf_set "descr" "ZFSD will automatically replace a spare that disappears and reappears in the same location, with the same devname"
+	atf_set "require.progs" "ksh93 zpool zfs gnop"
+}
+zfsd_replace_004_pos_body()
+{
+	. $(atf_get_srcdir)/../../include/default.cfg
+	. $(atf_get_srcdir)/zfsd.cfg
+
+	verify_disk_count "$DISKS" 2
+	verify_zfsd_running
+	ksh93 $(atf_get_srcdir)/setup.ksh || atf_fail "Setup failed"
+	ksh93 $(atf_get_srcdir)/zfsd_replace_004_pos.ksh
+	if [[ $? != 0 ]]; then
+		save_artifacts
+		atf_fail "Testcase failed"
+	fi
+}
+zfsd_replace_004_pos_cleanup()
+{
+	. $(atf_get_srcdir)/../../include/default.cfg
+	. $(atf_get_srcdir)/zfsd.cfg
+
+	ksh93 $(atf_get_srcdir)/cleanup.ksh || atf_fail "Cleanup failed"
+}
+
+atf_test_case zfsd_replace_005_pos cleanup
+zfsd_replace_005_pos_head()
+{
+	atf_set "descr" "ZFSD will automatically replace a multi-pool spare that disappears and reappears"
+	atf_set "require.progs" "ksh93 zpool zfs gnop"
+}
+zfsd_replace_005_pos_body()
+{
+	. $(atf_get_srcdir)/../../include/default.cfg
+	. $(atf_get_srcdir)/zfsd.cfg
+
+	verify_disk_count "$DISKS" 3
+	verify_zfsd_running
+	ksh93 $(atf_get_srcdir)/setup.ksh || atf_fail "Setup failed"
+	ksh93 $(atf_get_srcdir)/zfsd_replace_005_pos.ksh
+	if [[ $? != 0 ]]; then
+		save_artifacts
+		atf_fail "Testcase failed"
+	fi
+}
+zfsd_replace_005_pos_cleanup()
+{
+	. $(atf_get_srcdir)/../../include/default.cfg
+	. $(atf_get_srcdir)/zfsd.cfg
+
+	ksh93 $(atf_get_srcdir)/cleanup.ksh || atf_fail "Cleanup failed"
+}
+
 atf_test_case zfsd_import_001_pos cleanup
 zfsd_import_001_pos_head()
 {
@@ -591,6 +647,8 @@ atf_init_test_cases()
 	atf_add_test_case zfsd_replace_001_pos
 	atf_add_test_case zfsd_replace_002_pos
 	atf_add_test_case zfsd_replace_003_pos
+	atf_add_test_case zfsd_replace_004_pos
+	atf_add_test_case zfsd_replace_005_pos
 	atf_add_test_case zfsd_import_001_pos
 }