dtrace: do not overload libproc flags

dtrace stored its PR_RLC and PR_KLC flags in the proc_handle's flags,
where they collided with PATTACH_FORCE and PATTACH_RDONLY, respectively.
Thus, Psetflags(PR_KLC) effectively also set the PATTACH_RDONLY flag.

Since the flags are private to dtrace (at least on FreeBSD), store them in
dtrace's own dt_proc structure instead.

On FreeBSD, either PR_RLC or PR_KLC was always set, so remove code that
would handle the case where neither was set.

Reviewed by:	markj
MFC after:	1 week
Sponsored by:	Dell EMC Isilon
Differential Revision:	https://reviews.freebsd.org/D41121

4 files changed, 20 insertions, 27 deletions

diff --git a/cddl/compat/opensolaris/include/libproc.h b/cddl/compat/opensolaris/include/libproc.h
index 428fa6cf53fe..92c6d6ab0323 100644
--- a/cddl/compat/opensolaris/include/libproc.h
+++ b/cddl/compat/opensolaris/include/libproc.h
@@ -35,9 +35,6 @@
 #define ps_prochandle	proc_handle
 #define Lmid_t		int
 
-#define PR_RLC		0x0001
-#define PR_KLC		0x0002
-
 #include_next <libproc.h>
 
 #endif
diff --git a/cddl/contrib/opensolaris/lib/libdtrace/common/dt_proc.c b/cddl/contrib/opensolaris/lib/libdtrace/common/dt_proc.c
index 40b1f4108d1a..958d6ca34e85 100644
--- a/cddl/contrib/opensolaris/lib/libdtrace/common/dt_proc.c
+++ b/cddl/contrib/opensolaris/lib/libdtrace/common/dt_proc.c
@@ -490,10 +490,11 @@ dt_proc_control(void *arg)
 	dt_proc_attach(dpr, B_FALSE);		/* enable rtld breakpoints */
 
 	/*
-	 * If PR_KLC is set, we created the process; otherwise we grabbed it.
-	 * Check for an appropriate stop request and wait for dt_proc_continue.
+	 * If DT_CLOSE_KILL is set, we created the process; otherwise we
+	 * grabbed it.  Check for an appropriate stop request and wait for
+	 * dt_proc_continue.
 	 */
-	if (proc_getflags(P) & PR_KLC)
+	if (dpr->dpr_close == DT_CLOSE_KILL)
 		dt_proc_stop(dpr, DT_PROC_STOP_CREATE);
 	else
 		dt_proc_stop(dpr, DT_PROC_STOP_GRAB);
@@ -586,7 +587,7 @@ dt_proc_control(void *arg)
 		}
 
 		if (Pstate(P) != PS_UNDEAD) {
-			if (dpr->dpr_quit && (proc_getflags(P) & PR_KLC)) {
+			if (dpr->dpr_quit && dpr->dpr_close == DT_CLOSE_KILL) {
 				/*
 				 * We're about to kill the child, so don't
 				 * bother resuming it.  In some cases, such as
@@ -678,20 +679,15 @@ dt_proc_destroy(dtrace_hdl_t *dtp, struct ps_prochandle *P)
 
 	assert(dpr != NULL);
 
-	/*
-	 * If neither PR_KLC nor PR_RLC is set, then the process is stopped by
-	 * an external debugger and we were waiting in dt_proc_waitrun().
-	 * Leave the process in this condition using PRELEASE_HANG.
-	 */
-	if (!(proc_getflags(dpr->dpr_proc) & (PR_KLC | PR_RLC))) {
-		dt_dprintf("abandoning pid %d\n", (int)dpr->dpr_pid);
-		rflag = PRELEASE_HANG;
-	} else if (proc_getflags(dpr->dpr_proc) & PR_KLC) {
+	switch (dpr->dpr_close) {
+	case DT_CLOSE_KILL:
 		dt_dprintf("killing pid %d\n", (int)dpr->dpr_pid);
-		rflag = PRELEASE_KILL; /* apply kill-on-last-close */
-	} else {
+		rflag = PRELEASE_KILL;
+		break;
+	case DT_CLOSE_RUN:
 		dt_dprintf("releasing pid %d\n", (int)dpr->dpr_pid);
-		rflag = 0; /* apply run-on-last-close */
+		rflag = 0;
+		break;
 	}
 
 	if (dpr->dpr_tid) {
@@ -860,9 +856,7 @@ dt_proc_create(dtrace_hdl_t *dtp, const char *file, char *const *argv,
 
 	dpr->dpr_hdl = dtp;
 	dpr->dpr_pid = proc_getpid(dpr->dpr_proc);
-
-	(void) Punsetflags(dpr->dpr_proc, PR_RLC);
-	(void) Psetflags(dpr->dpr_proc, PR_KLC);
+	dpr->dpr_close = DT_CLOSE_KILL;
 
 	if (dt_proc_create_thread(dtp, dpr, dtp->dt_prcmode) != 0)
 		return (NULL); /* dt_proc_error() has been called for us */
@@ -927,9 +921,7 @@ dt_proc_grab(dtrace_hdl_t *dtp, pid_t pid, int flags, int nomonitor)
 
 	dpr->dpr_hdl = dtp;
 	dpr->dpr_pid = pid;
-
-	(void) Punsetflags(dpr->dpr_proc, PR_KLC);
-	(void) Psetflags(dpr->dpr_proc, PR_RLC);
+	dpr->dpr_close = DT_CLOSE_RUN;
 
 	/*
 	 * If we are attempting to grab the process without a monitor
diff --git a/cddl/contrib/opensolaris/lib/libdtrace/common/dt_proc.h b/cddl/contrib/opensolaris/lib/libdtrace/common/dt_proc.h
index beae6f6d5cda..458e9902881f 100644
--- a/cddl/contrib/opensolaris/lib/libdtrace/common/dt_proc.h
+++ b/cddl/contrib/opensolaris/lib/libdtrace/common/dt_proc.h
@@ -40,6 +40,11 @@
 extern "C" {
 #endif
 
+enum dt_close_action {
+	DT_CLOSE_RUN,
+	DT_CLOSE_KILL,
+};
+
 typedef struct dt_proc {
 	dt_list_t dpr_list;		/* prev/next pointers for lru chain */
 	struct dt_proc *dpr_hash;	/* next pointer for pid hash chain */
@@ -60,6 +65,7 @@ typedef struct dt_proc {
 	uint8_t dpr_rdonly;		/* proc flag: opened read-only */
 	pthread_t dpr_tid;		/* control thread (or zero if none) */
 	dt_list_t dpr_bps;		/* list of dt_bkpt_t structures */
+	enum dt_close_action dpr_close;	/* do this to child when exiting */
 } dt_proc_t;
 
 typedef struct dt_proc_notify {
diff --git a/cddl/lib/libdtrace/libproc_compat.h b/cddl/lib/libdtrace/libproc_compat.h
index 14f32506bb8f..d2a04d143e7e 100644
--- a/cddl/lib/libdtrace/libproc_compat.h
+++ b/cddl/lib/libdtrace/libproc_compat.h
@@ -58,10 +58,8 @@
 #define	Prd_agent proc_rdagent
 #define	Prelease proc_detach
 #define	Psetbkpt proc_bkptset
-#define	Psetflags proc_setflags
 #define	Pstate proc_state
 #define	Psymbol_iter_by_addr proc_iter_symbyaddr
-#define	Punsetflags proc_clearflags
 #define	Pupdate_maps proc_rdagent
 #define	Pupdate_syms proc_updatesyms
 #define	Pxecbkpt proc_bkptexec