cpuset(8): move to /bin/

Summary:
This change moves /usr/bin/cpuset to /bin/cpuset so it is more readily
available on hosts where the /usr partition might not be mounted at the
time / has been mounted.

Remove some complexity from rc.subr(8) since /bin is assumed to always
be present if/when / is mounted.

MFC after:	2 weeks
MFC with:	0661f93892a2, 271d552379af

Test Plan:
- Test out rc.subr change.
- Confirm that the installation logic does what's needed.

Relnotes:	yes (moving cpuset may impact strict file permissions/mode checking)
Reviewers:	kevans
Subscribers:	imp
Differential Revision: https://reviews.freebsd.org/D40057

5 files changed, 731 insertions, 0 deletions

diff --git a/bin/Makefile b/bin/Makefile
index 83bf79dcb9f9..075c91394f04 100644
--- a/bin/Makefile
+++ b/bin/Makefile
@@ -8,6 +8,7 @@ SUBDIR= cat \
 	chio \
 	chmod \
 	cp \
+	cpuset \
 	date \
 	dd \
 	df \
diff --git a/bin/cpuset/Makefile b/bin/cpuset/Makefile
new file mode 100644
index 000000000000..8626303bb254
--- /dev/null
+++ b/bin/cpuset/Makefile
@@ -0,0 +1,9 @@
+# $FreeBSD$
+
+PROG=   cpuset
+
+LIBADD=	jail
+
+SYMLINKS+=	${BINDIR}/cpuset	/usr/bin/cpuset
+
+.include <bsd.prog.mk>
diff --git a/bin/cpuset/Makefile.depend b/bin/cpuset/Makefile.depend
new file mode 100644
index 000000000000..1c40cf7442bb
--- /dev/null
+++ b/bin/cpuset/Makefile.depend
@@ -0,0 +1,17 @@
+# $FreeBSD$
+# Autogenerated - do NOT edit!
+
+DIRDEPS = \
+	include \
+	include/xlocale \
+	lib/${CSU_DIR} \
+	lib/libc \
+	lib/libcompiler_rt \
+	lib/libjail \
+
+
+.include <dirdeps.mk>
+
+.if ${DEP_RELDIR} == ${_DEP_RELDIR}
+# local dependencies - needed for -jN in clean tree
+.endif
diff --git a/bin/cpuset/cpuset.1 b/bin/cpuset/cpuset.1
new file mode 100644
index 000000000000..1d0180c98991
--- /dev/null
+++ b/bin/cpuset/cpuset.1
@@ -0,0 +1,229 @@
+.\" Copyright (c) 2008 Christian Brueffer
+.\" Copyright (c) 2008 Jeffrey Roberson
+.\" All rights reserved.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\" $FreeBSD$
+.\"
+.Dd July 3, 2018
+.Dt CPUSET 1
+.Os
+.Sh NAME
+.Nm cpuset
+.Nd "configure processor sets"
+.Sh SYNOPSIS
+.Nm
+.Op Fl l Ar cpu-list
+.Op Fl n Ar policy:domain-list
+.Op Fl s Ar setid
+.Ar cmd ...
+.Nm
+.Op Fl l Ar cpu-list
+.Op Fl n Ar policy:domain-list
+.Op Fl s Ar setid
+.Fl p Ar pid
+.Nm
+.Op Fl c
+.Op Fl l Ar cpu-list
+.Op Fl n Ar policy:domain-list
+.Fl C
+.Fl p Ar pid
+.Nm
+.Op Fl c
+.Op Fl l Ar cpu-list
+.Op Fl n Ar policy:domain-list
+.Op Fl j Ar jail | Fl p Ar pid | Fl t Ar tid | Fl s Ar setid | Fl x Ar irq
+.Nm
+.Fl g
+.Op Fl cir
+.Op Fl d Ar domain | Fl j Ar jail | Fl p Ar pid | Fl t Ar tid | Fl s Ar setid | Fl x Ar irq
+.Sh DESCRIPTION
+The
+.Nm
+command can be used to assign processor sets to processes, run commands
+constrained to a given set or list of processors and memory domains, and query
+information about processor binding, memory binding and policy, sets, and
+available processors and memory domains in the system.
+.Pp
+.Nm
+requires a target to modify or query.
+The target may be specified as a command, process id, thread id, a
+cpuset id, an irq, a jail, or a NUMA domain.
+Using
+.Fl g
+the target's set id or mask may be queried.
+Using
+.Fl l
+or
+.Fl s
+the target's CPU mask or set id may be set.
+If no target is specified,
+.Nm
+operates on itself.
+Not all combinations of operations and targets are supported.
+For example,
+you may not set the id of an existing set or query and launch a command
+at the same time.
+.Pp
+There are two sets applicable to each process and one private mask per thread.
+Every process in the system belongs to a cpuset.
+By default processes are started in set 1.
+The mask or id may be queried using
+.Fl c .
+Each thread also has a private mask of CPUs it is allowed to run
+on that must be a subset of the assigned set.
+And finally, there is a root set, numbered 0, that is immutable.
+This last set is the list of all possible CPUs in the system and is
+queried using
+.Fl r .
+.Pp
+Most sets include NUMA memory domain and policy information.
+This can be inspected with
+.Fl g
+and set with
+.Fl n .
+This will specify which NUMA domains are visible to the process and
+affect where anonymous memory and file pages will be stored on first access.
+Files accessed first by other processes may specify conflicting policy.
+.Pp
+When running a command it may join a set specified with
+.Fl s
+otherwise a new set is created.
+In addition, a mask for the command may be specified using
+.Fl l .
+When used in conjunction with
+.Fl c
+the mask modifies the supplied or created set rather than the private mask
+for the thread.
+.Pp
+The options are as follows:
+.Bl -tag -width ".Fl l Ar cpu-list"
+.It Fl C
+Create a new cpuset and assign the target process to that set.
+.It Fl c
+The requested operation should reference the cpuset available via the
+target specifier.
+.It Fl d Ar domain
+Specifies a NUMA domain id as the target of the operation.
+This can only be used to query the cpus visible in each numberd domain.
+.It Fl g
+Causes
+.Nm
+to print either a list of valid CPUs or, using
+.Fl i ,
+the id of the target.
+.It Fl i
+When used with the
+.Fl g
+option print the id rather than the valid mask of the target.
+.It Fl j Ar jail
+Specifies a jail id or name as the target of the operation.
+.It Fl l Ar cpu-list
+Specifies a list of CPUs to apply to a target.
+Specification may include
+numbers separated by '-' for ranges and commas separating individual numbers.
+A special list of
+.Dq all
+may be specified in which case the list includes all CPUs from the root set.
+.It Fl n Ar policy:domain-list
+Specifies a list of domains and allocation policy to apply to a target.
+Ranges may be specified as in
+.Fl l .
+Valid policies include first-touch (ft), round-robin (rr), prefer and
+interleave (il).
+First-touch allocates on the local domain when memory is available.
+Round-robin alternates between every possible domain page at a time.
+The prefer policy accepts only a single domain in the set.
+The parent of the set is consulted if the preferred domain is unavailable.
+Interleave operates like round-robin with an implementation defined stripe
+width.
+See
+.Xr domainset 9
+for more details on policies.
+.It Fl p Ar pid
+Specifies a pid as the target of the operation.
+.It Fl s Ar setid
+Specifies a set id as the target of the operation.
+.It Fl r
+The requested operation should reference the root set available via the
+target specifier.
+.It Fl t Ar tid
+Specifies a thread id as the target of the operation.
+.It Fl x Ar irq
+Specifies an irq as the target of the operation.
+.El
+.Sh EXIT STATUS
+.Ex -std
+.Sh EXAMPLES
+Create a new group with CPUs 0-4 inclusive and run
+.Pa /bin/sh
+on it:
+.Dl cpuset -c -l 0-4 /bin/sh
+.Pp
+Query the mask of CPUs the
+.Aq sh pid
+is allowed to run on:
+.Dl cpuset -g -p <sh pid>
+.Pp
+Restrict
+.Pa /bin/sh
+to run on CPUs 0 and 2 while its group is still allowed to run on
+CPUs 0-4:
+.Dl cpuset -l 0,2 -p <sh pid>
+.Pp
+Modify the cpuset
+.Pa /bin/sh
+belongs to restricting it to CPUs 0 and 2:
+.Dl cpuset -l 0,2 -c -p <sh pid>
+.Pp
+Modify the cpuset all threads are in by default to contain only
+the first 4 CPUs, leaving the rest idle:
+.Dl cpuset -l 0-3 -s 1
+.Pp
+Print the id of the cpuset
+.Pa /bin/sh
+is in:
+.Dl cpuset -g -i -p <sh pid>
+.Pp
+Move the
+.Ar pid
+into the specified cpuset
+.Ar setid
+so it may be managed with other pids in that set:
+.Dl cpuset -s <setid> -p <pid>
+.Pp
+Create a new cpuset that is restricted to CPUs 0 and 2 and move
+.Ar pid
+into the new set:
+.Dl cpuset -C -c -l 0,2 -p <pid>
+.Sh SEE ALSO
+.Xr nproc 1 ,
+.Xr cpuset 2 ,
+.Xr rctl 8
+.Sh HISTORY
+The
+.Nm
+command first appeared in
+.Fx 7.1 .
+.Sh AUTHORS
+.An Jeffrey Roberson Aq Mt jeff@FreeBSD.org
diff --git a/bin/cpuset/cpuset.c b/bin/cpuset/cpuset.c
new file mode 100644
index 000000000000..43aadb33f1e0
--- /dev/null
+++ b/bin/cpuset/cpuset.c
@@ -0,0 +1,475 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2007, 2008 	Jeffrey Roberson <jeff@freebsd.org>
+ * All rights reserved.
+ *
+ * Copyright (c) 2008 Nokia Corporation
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#define _WANT_FREEBSD_BITSET
+
+#include <sys/param.h>
+#include <sys/types.h>
+#include <sys/time.h>
+#include <sys/resource.h>
+#include <sys/cpuset.h>
+#include <sys/domainset.h>
+
+#include <ctype.h>
+#include <err.h>
+#include <errno.h>
+#include <jail.h>
+#include <limits.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <unistd.h>
+#include <string.h>
+
+static int Cflag;
+static int cflag;
+static int dflag;
+static int gflag;
+static int iflag;
+static int jflag;
+static int lflag;
+static int nflag;
+static int pflag;
+static int rflag;
+static int sflag;
+static int tflag;
+static int xflag;
+static id_t id;
+static cpulevel_t level;
+static cpuwhich_t which;
+
+static void usage(void) __dead2;
+
+struct numa_policy {
+	const char 	*name;
+	int		policy;
+};
+
+static struct numa_policy policies[] = {
+	{ "round-robin", DOMAINSET_POLICY_ROUNDROBIN },
+	{ "rr", DOMAINSET_POLICY_ROUNDROBIN },
+	{ "first-touch", DOMAINSET_POLICY_FIRSTTOUCH },
+	{ "ft", DOMAINSET_POLICY_FIRSTTOUCH },
+	{ "prefer", DOMAINSET_POLICY_PREFER },
+	{ "interleave", DOMAINSET_POLICY_INTERLEAVE},
+	{ "il", DOMAINSET_POLICY_INTERLEAVE},
+	{ NULL, DOMAINSET_POLICY_INVALID }
+};
+
+static void printset(struct bitset *mask, int size);
+
+static void
+parselist(char *list, struct bitset *mask, int size)
+{
+	enum { NONE, NUM, DASH } state;
+	int lastnum;
+	int curnum;
+	char *l;
+
+	state = NONE;
+	curnum = lastnum = 0;
+	for (l = list; *l != '\0';) {
+		if (isdigit(*l)) {
+			curnum = atoi(l);
+			if (curnum >= size)
+				errx(EXIT_FAILURE,
+				    "List entry %d exceeds maximum of %d",
+				    curnum, size - 1);
+			while (isdigit(*l))
+				l++;
+			switch (state) {
+			case NONE:
+				lastnum = curnum;
+				state = NUM;
+				break;
+			case DASH:
+				for (; lastnum <= curnum; lastnum++)
+					BIT_SET(size, lastnum, mask);
+				state = NONE;
+				break;
+			case NUM:
+			default:
+				goto parserr;
+			}
+			continue;
+		}
+		switch (*l) {
+		case ',':
+			switch (state) {
+			case NONE:
+				break;
+			case NUM:
+				BIT_SET(size, curnum, mask);
+				state = NONE;
+				break;
+			case DASH:
+				goto parserr;
+				break;
+			}
+			break;
+		case '-':
+			if (state != NUM)
+				goto parserr;
+			state = DASH;
+			break;
+		default:
+			goto parserr;
+		}
+		l++;
+	}
+	switch (state) {
+		case NONE:
+			break;
+		case NUM:
+			BIT_SET(size, curnum, mask);
+			break;
+		case DASH:
+			goto parserr;
+	}
+	return;
+parserr:
+	errx(EXIT_FAILURE, "Malformed list %s", list);
+}
+
+static void
+parsecpulist(char *list, cpuset_t *mask)
+{
+
+	if (strcasecmp(list, "all") == 0) {
+		if (cpuset_getaffinity(CPU_LEVEL_ROOT, CPU_WHICH_PID, -1,
+		    sizeof(*mask), mask) != 0)
+			err(EXIT_FAILURE, "getaffinity");
+		return;
+	}
+	parselist(list, (struct bitset *)mask, CPU_SETSIZE);
+}
+
+/*
+ * permissively parse policy:domain list
+ * allow:
+ *	round-robin:0-4		explicit
+ *	round-robin:all		explicit root domains
+ *	0-4			implicit root policy
+ *	round-robin		implicit root domains
+ *	all			explicit root domains and implicit policy
+ */
+static void
+parsedomainlist(char *list, domainset_t *mask, int *policyp)
+{
+	domainset_t rootmask;
+	struct numa_policy *policy;
+	char *l;
+	int p;
+
+	/*
+	 * Use the rootset's policy as the default for unspecified policies.
+	 */
+	if (cpuset_getdomain(CPU_LEVEL_ROOT, CPU_WHICH_PID, -1,
+	    sizeof(rootmask), &rootmask, &p) != 0)
+		err(EXIT_FAILURE, "getdomain");
+
+	l = list;
+	for (policy = &policies[0]; policy->name != NULL; policy++) {
+		if (strncasecmp(l, policy->name, strlen(policy->name)) == 0) {
+			p = policy->policy;
+			l += strlen(policy->name);
+			if (*l != ':' && *l != '\0')
+				errx(EXIT_FAILURE, "Malformed list %s", list);
+			if (*l == ':')
+				l++;
+			break;
+		}
+	}
+	*policyp = p;
+	if (strcasecmp(l, "all") == 0 || *l == '\0') {
+		DOMAINSET_COPY(&rootmask, mask);
+		return;
+	}
+	parselist(l, (struct bitset *)mask, DOMAINSET_SETSIZE);
+}
+
+static void
+printset(struct bitset *mask, int size)
+{
+	int once;
+	int bit;
+
+	for (once = 0, bit = 0; bit < size; bit++) {
+		if (BIT_ISSET(size, bit, mask)) {
+			if (once == 0) {
+				printf("%d", bit);
+				once = 1;
+			} else
+				printf(", %d", bit);
+		}
+	}
+	printf("\n");
+}
+
+static const char *whichnames[] = { NULL, "tid", "pid", "cpuset", "irq", "jail",
+				    "domain" };
+static const char *levelnames[] = { NULL, " root", " cpuset", "" };
+static const char *policynames[] = { "invalid", "round-robin", "first-touch",
+				    "prefer", "interleave" };
+
+static void
+printaffinity(void)
+{
+	domainset_t domain;
+	cpuset_t mask;
+	int policy;
+
+	if (cpuset_getaffinity(level, which, id, sizeof(mask), &mask) != 0)
+		err(EXIT_FAILURE, "getaffinity");
+	printf("%s %jd%s mask: ", whichnames[which], (intmax_t)id,
+	    levelnames[level]);
+	printset((struct bitset *)&mask, CPU_SETSIZE);
+	if (dflag || xflag)
+		goto out;
+	if (cpuset_getdomain(level, which, id, sizeof(domain), &domain,
+	    &policy) != 0)
+		err(EXIT_FAILURE, "getdomain");
+	printf("%s %jd%s domain policy: %s mask: ", whichnames[which],
+	    (intmax_t)id, levelnames[level], policynames[policy]);
+	printset((struct bitset *)&domain, DOMAINSET_SETSIZE);
+out:
+	exit(EXIT_SUCCESS);
+}
+
+static void
+printsetid(void)
+{
+	cpusetid_t setid;
+
+	/*
+	 * Only LEVEL_WHICH && WHICH_CPUSET has a numbered id.
+	 */
+	if (level == CPU_LEVEL_WHICH && !sflag)
+		level = CPU_LEVEL_CPUSET;
+	if (cpuset_getid(level, which, id, &setid))
+		err(errno, "getid");
+	printf("%s %jd%s id: %d\n", whichnames[which], (intmax_t)id,
+	    levelnames[level], setid);
+}
+
+int
+main(int argc, char *argv[])
+{
+	domainset_t domains;
+	cpusetid_t setid;
+	cpuset_t mask;
+	int policy;
+	lwpid_t tid;
+	pid_t pid;
+	int ch;
+
+	CPU_ZERO(&mask);
+	DOMAINSET_ZERO(&domains);
+	policy = DOMAINSET_POLICY_INVALID;
+	level = CPU_LEVEL_WHICH;
+	which = CPU_WHICH_PID;
+	id = pid = tid = setid = -1;
+	while ((ch = getopt(argc, argv, "Ccd:gij:l:n:p:rs:t:x:")) != -1) {
+		switch (ch) {
+		case 'C':
+			Cflag = 1;
+			break;
+		case 'c':
+			cflag = 1;
+			level = CPU_LEVEL_CPUSET;
+			break;
+		case 'd':
+			dflag = 1;
+			which = CPU_WHICH_DOMAIN;
+			id = atoi(optarg);
+			break;
+		case 'g':
+			gflag = 1;
+			break;
+		case 'i':
+			iflag = 1;
+			break;
+		case 'j':
+			jflag = 1;
+			which = CPU_WHICH_JAIL;
+			id = jail_getid(optarg);
+			if (id < 0)
+				errx(EXIT_FAILURE, "%s", jail_errmsg);
+			break;
+		case 'l':
+			lflag = 1;
+			parsecpulist(optarg, &mask);
+			break;
+		case 'n':
+			nflag = 1;
+			parsedomainlist(optarg, &domains, &policy);
+			break;
+		case 'p':
+			pflag = 1;
+			which = CPU_WHICH_PID;
+			id = pid = atoi(optarg);
+			break;
+		case 'r':
+			level = CPU_LEVEL_ROOT;
+			rflag = 1;
+			break;
+		case 's':
+			sflag = 1;
+			which = CPU_WHICH_CPUSET;
+			id = setid = atoi(optarg);
+			break;
+		case 't':
+			tflag = 1;
+			which = CPU_WHICH_TID;
+			id = tid = atoi(optarg);
+			break;
+		case 'x':
+			xflag = 1;
+			which = CPU_WHICH_IRQ;
+			id = atoi(optarg);
+			break;
+		default:
+			usage();
+		}
+	}
+	argc -= optind;
+	argv += optind;
+	if (gflag) {
+		if (argc || Cflag || lflag || nflag)
+			usage();
+		/* Only one identity specifier. */
+		if (dflag + jflag + xflag + sflag + pflag + tflag > 1)
+			usage();
+		if (iflag)
+			printsetid();
+		else
+			printaffinity();
+		exit(EXIT_SUCCESS);
+	}
+
+	if (dflag || iflag || rflag)
+		usage();
+	/*
+	 * The user wants to run a command with a set and possibly cpumask.
+	 */
+	if (argc) {
+		if (Cflag || pflag || tflag || xflag || jflag)
+			usage();
+		if (sflag) {
+			if (cpuset_setid(CPU_WHICH_PID, -1, setid))
+				err(argc, "setid");
+		} else {
+			if (cpuset(&setid))
+				err(argc, "newid");
+		}
+		if (lflag) {
+			if (cpuset_setaffinity(level, CPU_WHICH_PID,
+			    -1, sizeof(mask), &mask) != 0)
+				err(EXIT_FAILURE, "setaffinity");
+		}
+		if (nflag) {
+			if (cpuset_setdomain(level, CPU_WHICH_PID,
+			    -1, sizeof(domains), &domains, policy) != 0)
+				err(EXIT_FAILURE, "setdomain");
+		}
+		errno = 0;
+		execvp(*argv, argv);
+		err(errno == ENOENT ? 127 : 126, "%s", *argv);
+	}
+	/*
+	 * We're modifying something that presently exists.
+	 */
+	if (Cflag && (jflag || !pflag || sflag || tflag || xflag))
+		usage();
+	if ((!lflag && !nflag) && cflag)
+		usage();
+	if ((!lflag && !nflag) && !(Cflag || sflag))
+		usage();
+	/* You can only set a mask on a thread. */
+	if (tflag && (sflag | pflag | xflag | jflag))
+		usage();
+	/* You can only set a mask on an irq. */
+	if (xflag && (jflag | pflag | sflag | tflag))
+		usage();
+	if (Cflag) {
+		/*
+		 * Create a new cpuset and move the specified process
+		 * into the set.
+		 */
+		if (cpuset(&setid) < 0)
+			err(EXIT_FAILURE, "newid");
+		sflag = 1;
+	}
+	if (pflag && sflag) {
+		if (cpuset_setid(CPU_WHICH_PID, pid, setid))
+			err(EXIT_FAILURE, "setid");
+		/*
+		 * If the user specifies a set and a list we want the mask
+		 * to effect the pid and not the set.
+		 */
+		which = CPU_WHICH_PID;
+		id = pid;
+	}
+	if (lflag) {
+		if (cpuset_setaffinity(level, which, id, sizeof(mask),
+		    &mask) != 0)
+			err(EXIT_FAILURE, "setaffinity");
+	}
+	if (nflag) {
+		if (cpuset_setdomain(level, which, id, sizeof(domains),
+		    &domains, policy) != 0)
+			err(EXIT_FAILURE, "setdomain");
+	}
+
+	exit(EXIT_SUCCESS);
+}
+
+static void
+usage(void)
+{
+
+	fprintf(stderr,
+    "usage: cpuset [-l cpu-list] [-n policy:domain-list] [-s setid] cmd ...\n");
+	fprintf(stderr,
+    "       cpuset [-l cpu-list] [-n policy:domain-list] [-s setid] -p pid\n");
+	fprintf(stderr,
+    "       cpuset [-c] [-l cpu-list] [-n policy:domain-list] -C -p pid\n");
+	fprintf(stderr,
+    "       cpuset [-c] [-l cpu-list] [-n policy:domain-list]\n"
+    "              [-j jailid | -p pid | -t tid | -s setid | -x irq]\n");
+	fprintf(stderr,
+    "       cpuset -g [-cir]\n"
+    "              [-d domain | -j jailid | -p pid | -t tid | -s setid | -x irq]\n");
+	exit(1);
+}