MFC r368461: kern: cpuset: resolve race between cpuset_lookup/cpuset_rel

The race plays out like so between threads A and B:

1. A ref's cpuset 10
2. B does a lookup of cpuset 10, grabs the cpuset lock and searches
   cpuset_ids
3. A rel's cpuset 10 and observes the last ref, waits on the cpuset lock
   while B is still searching and not yet ref'd
4. B ref's cpuset 10 and drops the cpuset lock
5. A proceeds to free the cpuset out from underneath B

Resolve the race by only releasing the last reference under the cpuset lock.
Thread A now picks up the spinlock and observes that the cpuset has been
revived, returning immediately for B to deal with later.

1 files changed, 10 insertions, 2 deletions

diff --git a/sys/kern/kern_cpuset.c b/sys/kern/kern_cpuset.c
index 7e1e3c4da2d4..07b40c179edc 100644
--- a/sys/kern/kern_cpuset.c
+++ b/sys/kern/kern_cpuset.c
@@ -207,9 +207,13 @@ cpuset_rel(struct cpuset *set)
 {
 	cpusetid_t id;
 
-	if (refcount_release(&set->cs_ref) == 0)
+	if (refcount_release_if_not_last(&set->cs_ref))
 		return;
 	mtx_lock_spin(&cpuset_lock);
+	if (!refcount_release(&set->cs_ref)) {
+		mtx_unlock_spin(&cpuset_lock);
+		return;
+	}
 	LIST_REMOVE(set, cs_siblings);
 	id = set->cs_id;
 	if (id != CPUSET_INVALID)
@@ -229,9 +233,13 @@ static void
 cpuset_rel_defer(struct setlist *head, struct cpuset *set)
 {
 
-	if (refcount_release(&set->cs_ref) == 0)
+	if (refcount_release_if_not_last(&set->cs_ref))
 		return;
 	mtx_lock_spin(&cpuset_lock);
+	if (!refcount_release(&set->cs_ref)) {
+		mtx_unlock_spin(&cpuset_lock);
+		return;
+	}
 	LIST_REMOVE(set, cs_siblings);
 	if (set->cs_id != CPUSET_INVALID)
 		LIST_REMOVE(set, cs_link);