2 files changed, 71 insertions, 26 deletions

diff --git a/libexec/rtld-elf/rtld.c b/libexec/rtld-elf/rtld.c
index 2a05afe66902..938e4b0b8a6b 100644
--- a/libexec/rtld-elf/rtld.c
+++ b/libexec/rtld-elf/rtld.c
@@ -6878,31 +6878,6 @@ getenv(const char *name)
 	    strlen(name))));
 }
 
-/* malloc */
-void *
-malloc(size_t nbytes)
-{
-	return (__crt_malloc(nbytes));
-}
-
-void *
-calloc(size_t num, size_t size)
-{
-	return (__crt_calloc(num, size));
-}
-
-void
-free(void *cp)
-{
-	__crt_free(cp);
-}
-
-void *
-realloc(void *cp, size_t nbytes)
-{
-	return (__crt_realloc(cp, nbytes));
-}
-
 extern int _rtld_version__FreeBSD_version __exported;
 int _rtld_version__FreeBSD_version = __FreeBSD_version;
 
diff --git a/libexec/rtld-elf/xmalloc.c b/libexec/rtld-elf/xmalloc.c
index 5f7c1b5ba10a..4c67939fc27f 100644
--- a/libexec/rtld-elf/xmalloc.c
+++ b/libexec/rtld-elf/xmalloc.c
@@ -26,6 +26,8 @@
  */
 
 #include <sys/param.h>
+#include <machine/cpu.h>
+#include <machine/cpufunc.h>
 #include <stddef.h>
 #include <stdlib.h>
 #include <string.h>
@@ -35,12 +37,36 @@
 #include "rtld_malloc.h"
 #include "rtld_libc.h"
 
+static int rtld_malloc_spinlock;
+
+/*
+ * Almost always the rtld malloc is called under the write-locked rtld
+ * bind lock.  Due to this, xlock() would need a single CAS to take
+ * the spinlock.  But some places only own read-locked rtld bind lock,
+ * and then the spinlock protects the malloc structures from the
+ * parallel updates.
+ */
+static void
+xlock(void)
+{
+	while (atomic_cmpset_acq_int(&rtld_malloc_spinlock, 0, 1) == 0)
+		cpu_spinwait();
+}
+
+static void
+xunlock(void)
+{
+	atomic_store_rel_int(&rtld_malloc_spinlock, 0);
+}
+
 void *
 xcalloc(size_t number, size_t size)
 {
 	void *p;
 
+	xlock();
 	p = __crt_calloc(number, size);
+	xunlock();
 	if (p == NULL) {
 		rtld_fdputstr(STDERR_FILENO, "Out of memory\n");
 		_exit(1);
@@ -51,10 +77,11 @@ xcalloc(size_t number, size_t size)
 void *
 xmalloc(size_t size)
 {
-
 	void *p;
 
+	xlock();
 	p = __crt_malloc(size);
+	xunlock();
 	if (p == NULL) {
 		rtld_fdputstr(STDERR_FILENO, "Out of memory\n");
 		_exit(1);
@@ -83,10 +110,53 @@ xmalloc_aligned(size_t size, size_t align, size_t offset)
 	if (align < sizeof(void *))
 		align = sizeof(void *);
 
+	xlock();
 	res = __crt_aligned_alloc_offset(align, size, offset);
+	xunlock();
 	if (res == NULL) {
 		rtld_fdputstr(STDERR_FILENO, "Out of memory\n");
 		_exit(1);
 	}
 	return (res);
 }
+
+void *
+malloc(size_t size)
+{
+	void *p;
+
+	xlock();
+	p = __crt_malloc(size);
+	xunlock();
+	return (p);
+}
+
+void *
+calloc(size_t num, size_t size)
+{
+	void *p;
+
+	xlock();
+	p = __crt_calloc(num, size);
+	xunlock();
+	return (p);
+}
+
+void
+free(void *cp)
+{
+	xlock();
+	__crt_free(cp);
+	xunlock();
+}
+
+void *
+realloc(void *cp, size_t nbytes)
+{
+	void *p;
+
+	xlock();
+	p = __crt_realloc(cp, nbytes);
+	xunlock();
+	return (p);
+}