rtld: add some dlopen tests

dlopen_basic just tests that libthr.so can be dlopen()ed, which will
just serve as a sanity check that "libthr.so" is a thing that can be
dlopened in case we get a weird failure in dlopen_recursing.

dlopen_recursing tests a regression reported after the libsys split,
where some dlopen() may cause infinite recursion and a resulting crash.
This case is inspired by bdrewery's description of what seemed to be
causing his issue.

The corresponding fix landed in commit
968a18975ad ("rtld: ignore load_filtees() calls if we already [...]")

Reviewed by:	kib
Differential Revision:	https://reviews.freebsd.org/D43859

2 files changed, 54 insertions, 0 deletions

diff --git a/libexec/rtld-elf/tests/Makefile b/libexec/rtld-elf/tests/Makefile
index 06e143a441a1..e380e9850fc1 100644
--- a/libexec/rtld-elf/tests/Makefile
+++ b/libexec/rtld-elf/tests/Makefile
@@ -13,6 +13,8 @@ ATF_TESTS_C+=	ld_preload_fds
 SRCS.$t=	$t.c common.c
 .endfor
 
+ATF_TESTS_C+=	dlopen_test
+
 WARNS?=		3
 
 .include <bsd.test.mk>
diff --git a/libexec/rtld-elf/tests/dlopen_test.c b/libexec/rtld-elf/tests/dlopen_test.c
new file mode 100644
index 000000000000..ab1e8da1cb41
--- /dev/null
+++ b/libexec/rtld-elf/tests/dlopen_test.c
@@ -0,0 +1,52 @@
+/*-
+ *
+ * Copyright (C) 2024 Kyle Evans <kevans@FreeBSD.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ */
+
+#include <dlfcn.h>
+
+#include <atf-c.h>
+
+ATF_TC_WITHOUT_HEAD(dlopen_basic);
+ATF_TC_BODY(dlopen_basic, tc)
+{
+	void *hdl, *sym;
+
+	hdl = dlopen("libthr.so", RTLD_NOW);
+	ATF_REQUIRE(hdl != NULL);
+
+	sym = dlsym(hdl, "pthread_create");
+	ATF_REQUIRE(sym != NULL);
+
+	dlclose(hdl);
+
+	sym = dlsym(hdl, "pthread_create");
+	ATF_REQUIRE(sym == NULL);
+}
+
+ATF_TC_WITHOUT_HEAD(dlopen_recursing);
+ATF_TC_BODY(dlopen_recursing, tc)
+{
+	void *hdl;
+
+	/*
+	 * If this doesn't crash, we're OK; a regression at one point caused
+	 * some infinite recursion here.
+	 */
+	hdl = dlopen("libthr.so", RTLD_NOW | RTLD_GLOBAL);
+	ATF_REQUIRE(hdl != NULL);
+
+	dlclose(hdl);
+}
+
+ATF_TP_ADD_TCS(tp)
+{
+
+	ATF_TP_ADD_TC(tp, dlopen_basic);
+	ATF_TP_ADD_TC(tp, dlopen_recursing);
+
+	return atf_no_error();
+}