9 files changed, 387 insertions, 174 deletions

diff --git a/lib/libpthread/thread/Makefile.inc b/lib/libpthread/thread/Makefile.inc
index 8c1e5677eb72..0eb1d59ec161 100644
--- a/lib/libpthread/thread/Makefile.inc
+++ b/lib/libpthread/thread/Makefile.inc
@@ -13,6 +13,7 @@ SRCS+= \
 	uthread_attr_destroy.c \
 	uthread_attr_init.c \
 	uthread_attr_getdetachstate.c \
+	uthread_attr_getguardsize.c \
 	uthread_attr_getinheritsched.c \
 	uthread_attr_getschedparam.c \
 	uthread_attr_getschedpolicy.c \
@@ -21,6 +22,7 @@ SRCS+= \
 	uthread_attr_getstacksize.c \
 	uthread_attr_setcreatesuspend_np.c \
 	uthread_attr_setdetachstate.c \
+	uthread_attr_setguardsize.c \
 	uthread_attr_setinheritsched.c \
 	uthread_attr_setschedparam.c \
 	uthread_attr_setschedpolicy.c \
@@ -124,6 +126,7 @@ SRCS+= \
 	uthread_socketpair.c \
 	uthread_spec.c \
 	uthread_spinlock.c \
+	uthread_stack.c \
 	uthread_suspend_np.c \
 	uthread_switch_np.c \
 	uthread_system.c \
diff --git a/lib/libpthread/thread/thr_attr_getguardsize.c b/lib/libpthread/thread/thr_attr_getguardsize.c
new file mode 100644
index 000000000000..849bf27a5025
--- /dev/null
+++ b/lib/libpthread/thread/thr_attr_getguardsize.c
@@ -0,0 +1,52 @@
+/*
+ * Copyright (C) 2001 Jason Evans <jasone@freebsd.org>.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice(s), this list of conditions and the following disclaimer
+ *    unmodified other than the allowable addition of one or more
+ *    copyright notices.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice(s), this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+ * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <errno.h>
+#include <pthread.h>
+#include "pthread_private.h"
+
+__weak_reference(_pthread_attr_getguardsize, pthread_attr_getguardsize);
+
+int
+_pthread_attr_getguardsize(const pthread_attr_t *attr, size_t *guardsize)
+{
+	int	ret;
+
+	/* Check for invalid arguments: */
+	if (attr == NULL || *attr == NULL || guardsize == NULL)
+		ret = EINVAL;
+	else {
+		/* Return the guard size: */
+		*guardsize = (*attr)->guardsize_attr;
+		ret = 0;
+	}
+	return(ret);
+}
diff --git a/lib/libpthread/thread/thr_attr_setguardsize.c b/lib/libpthread/thread/thr_attr_setguardsize.c
new file mode 100644
index 000000000000..0c3de145d0b1
--- /dev/null
+++ b/lib/libpthread/thread/thr_attr_setguardsize.c
@@ -0,0 +1,57 @@
+/*
+ * Copyright (C) 2001 Jason Evans <jasone@freebsd.org>.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice(s), this list of conditions and the following disclaimer
+ *    unmodified other than the allowable addition of one or more
+ *    copyright notices.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice(s), this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+ * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/param.h>
+#include <errno.h>
+#include <pthread.h>
+#include "pthread_private.h"
+
+__weak_reference(_pthread_attr_setguardsize, pthread_attr_setguardsize);
+
+int
+_pthread_attr_setguardsize(pthread_attr_t *attr, size_t guardsize)
+{
+	int	ret;
+
+	/* Check for invalid arguments. */
+	if (attr == NULL || *attr == NULL)
+		ret = EINVAL;
+	else {
+		/* Round guardsize up to the nearest multiple of PAGE_SIZE. */
+		if (guardsize % PAGE_SIZE != 0)
+			guardsize = ((guardsize / PAGE_SIZE) + 1) * PAGE_SIZE;
+
+		/* Save the stack size. */
+		(*attr)->guardsize_attr = guardsize;
+		ret = 0;
+	}
+	return(ret);
+}
diff --git a/lib/libpthread/thread/thr_create.c b/lib/libpthread/thread/thr_create.c
index 40f5364e2d54..7c52b4b41f91 100644
--- a/lib/libpthread/thread/thr_create.c
+++ b/lib/libpthread/thread/thr_create.c
@@ -38,8 +38,6 @@
 #include <unistd.h>
 #include <stddef.h>
 #include <sys/time.h>
-#include <sys/param.h>
-#include <sys/mman.h>
 #include <machine/reg.h>
 #include <pthread.h>
 #include "pthread_private.h"
@@ -99,68 +97,15 @@ _pthread_create(pthread_t * thread, const pthread_attr_t * attr,
 		/* Check if a stack was specified in the thread attributes: */
 		if ((stack = pattr->stackaddr_attr) != NULL) {
 		}
-		/* Allocate memory for a default-size stack: */
-		else if (pattr->stacksize_attr == PTHREAD_STACK_DEFAULT) {
-			struct stack	*spare_stack;
-			
-			/* Allocate or re-use a default-size stack. */
-			
-			/*
-			 * Use the garbage collector mutex for synchronization
-			 * of the spare stack list.
-			 */
-			if (pthread_mutex_lock(&_gc_mutex) != 0)
-				PANIC("Cannot lock gc mutex");
-			
-			if ((spare_stack = SLIST_FIRST(&_stackq)) != NULL) {
-				/* Use the spare stack. */
-				SLIST_REMOVE_HEAD(&_stackq, qe);
-				
-				/* Unlock the garbage collector mutex. */
-				if (pthread_mutex_unlock(&_gc_mutex) != 0)
-					PANIC("Cannot unlock gc mutex");
-				
-				stack = sizeof(struct stack)
-				    + (void *) spare_stack
-				    - PTHREAD_STACK_DEFAULT;
-			} else {
-				/* Allocate a new stack. */
-				stack = _next_stack + PTHREAD_STACK_GUARD;
-
-				/*
-				 * Even if stack allocation fails, we don't want
-				 * to try to use this location again, so
-				 * unconditionally decrement _next_stack.  Under
-				 * normal operating conditions, the most likely
-				 * reason for an mmap() error is a stack
-				 * overflow of the adjacent thread stack.
-				 */
-				_next_stack -= (PTHREAD_STACK_DEFAULT
-				    + PTHREAD_STACK_GUARD);
-
-				/* Unlock the garbage collector mutex. */
-				if (pthread_mutex_unlock(&_gc_mutex) != 0)
-					PANIC("Cannot unlock gc mutex");
-
-				/* Stack: */
-				if (mmap(stack, PTHREAD_STACK_DEFAULT,
-				    PROT_READ | PROT_WRITE, MAP_STACK,
-				    -1, 0) == MAP_FAILED) {
-					ret = EAGAIN;
-					free(new_thread);
-				}
+		/* Allocate a stack: */
+		else {
+			stack = _thread_stack_alloc(pattr->stacksize_attr,
+			    pattr->guardsize_attr);
+			if (stack == NULL) {
+				ret = EAGAIN;
+				free(new_thread);
 			}
 		}
-		/*
-		 * The user wants a stack of a particular size.  Lets hope they
-		 * really know what they want, and simply malloc the stack.
-		 */
-		else if ((stack = (void *) malloc(pattr->stacksize_attr))
-		    == NULL) {
-			/* Insufficient memory to create a thread: */
-			ret = EAGAIN;
-			free(new_thread);
-		}
 
 		/* Check for errors: */
 		if (ret != 0) {
diff --git a/lib/libpthread/thread/thr_fork.c b/lib/libpthread/thread/thr_fork.c
index 4437d886a80e..9d9a647f0659 100644
--- a/lib/libpthread/thread/thr_fork.c
+++ b/lib/libpthread/thread/thr_fork.c
@@ -31,6 +31,7 @@
  *
  * $FreeBSD$
  */
+#include <sys/param.h>
 #include <errno.h>
 #include <string.h>
 #include <stdlib.h>
@@ -220,28 +221,16 @@ _fork(void)
 static void
 free_thread_resources(struct pthread *thread)
 {
-	struct stack	*spare_stack;
 
 	/* Check to see if the threads library allocated the stack. */
 	if ((thread->attr.stackaddr_attr == NULL) && (thread->stack != NULL)) {
-		if (thread->attr.stacksize_attr != PTHREAD_STACK_DEFAULT) {
-			/*
-			 * The threads library malloc()'d the stack;
-			 * just free() it.
-			 */
-			free(thread->stack);
-		} else {
-			/*
-			 * This stack was allocated from the main threads
-			 * stack; cache it for future use.  Since this is
-			 * being called from fork, we are currently single
-			 * threaded so there is no need to protect the
-			 * queue insertion.
-			 */
-			spare_stack = (thread->stack + PTHREAD_STACK_DEFAULT -
-			    sizeof(struct stack));
-			SLIST_INSERT_HEAD(&_stackq, spare_stack, qe);
-		}
+		/*
+		 * Since this is being called from fork, we are currently single
+		 * threaded so there is no need to protect the call to
+		 * _thread_stack_free() with _gc_mutex.
+		 */
+		_thread_stack_free(thread->stack, thread->attr.stacksize_attr,
+		    thread->attr.guardsize_attr);
 	}
 
 	if (thread->specific_data != NULL)
diff --git a/lib/libpthread/thread/thr_gc.c b/lib/libpthread/thread/thr_gc.c
index 83ff38fc966f..9c930288f9e3 100644
--- a/lib/libpthread/thread/thr_gc.c
+++ b/lib/libpthread/thread/thr_gc.c
@@ -34,13 +34,12 @@
  * Garbage collector thread. Frees memory allocated for dead threads.
  *
  */
+#include <sys/param.h>
 #include <errno.h>
 #include <time.h>
 #include <stdlib.h>
 #include <unistd.h>
 #include <sys/types.h>
-#include <sys/types.h>
-#include <sys/mman.h>
 #include <pthread.h>
 #include "pthread_private.h"
 
@@ -123,39 +122,20 @@ _thread_gc(pthread_addr_t arg)
 			 * Check if this thread has detached:
 			 */
 			else if ((pthread->attr.flags &
-				  PTHREAD_DETACHED) != 0) {
+			    PTHREAD_DETACHED) != 0) {
 				/* Remove this thread from the dead list: */
 				TAILQ_REMOVE(&_dead_list, pthread, dle);
 
 				/*
 				 * Check if the stack was not specified by
-				 * the caller to pthread_create and has not
+				 * the caller to pthread_create() and has not
 				 * been destroyed yet: 
 				 */
 				if (pthread->attr.stackaddr_attr == NULL &&
 				    pthread->stack != NULL) {
-					if (pthread->attr.stacksize_attr
-					    == PTHREAD_STACK_DEFAULT) {
-						/*
-						 * Default-size stack.  Cache
-						 * it:
-						 */
-						struct stack	*spare_stack;
-
-						spare_stack
-						    = (pthread->stack
-						       + PTHREAD_STACK_DEFAULT
-						       - sizeof(struct stack));
-						SLIST_INSERT_HEAD(&_stackq,
-								  spare_stack,
-								  qe);
-					} else {
-						/*
-						 * Non-standard stack size.
-                                                 * free() it outside the locks.
-						 */
-						p_stack = pthread->stack;
-					}
+					_thread_stack_free(pthread->stack,
+					    pthread->attr.stacksize_attr,
+					    pthread->attr.guardsize_attr);
 				}
 
 				/*
@@ -170,37 +150,18 @@ _thread_gc(pthread_addr_t arg)
 				 * not destroy it.
 				 *
 				 * Check if the stack was not specified by
-				 * the caller to pthread_create and has not
+				 * the caller to pthread_create() and has not
 				 * been destroyed yet: 
 				 */
 				if (pthread->attr.stackaddr_attr == NULL &&
 				    pthread->stack != NULL) {
-					if (pthread->attr.stacksize_attr
-					    == PTHREAD_STACK_DEFAULT) {
-						/*
-						 * Default-size stack.  Cache
-						 * it:
-						 */
-						struct stack	*spare_stack;
+					_thread_stack_free(pthread->stack,
+					    pthread->attr.stacksize_attr,
+					    pthread->attr.guardsize_attr);
 
-						spare_stack
-						    = (pthread->stack
-						       + PTHREAD_STACK_DEFAULT
-						       - sizeof(struct stack));
-						SLIST_INSERT_HEAD(&_stackq,
-								  spare_stack,
-								  qe);
-					} else {
-						/*
-						 * Non-standard stack size.
-						 * free() it outside the locks:
-						 */
-						p_stack = pthread->stack;
-					}
-					
 					/*
-					 * NULL the stack pointer now
-					 * that the memory has been freed: 
+					 * NULL the stack pointer now that the
+					 * memory has been freed:
 					 */
 					pthread->stack = NULL;
 				}
diff --git a/lib/libpthread/thread/thr_init.c b/lib/libpthread/thread/thr_init.c
index b63a11157737..9f10f740fd1d 100644
--- a/lib/libpthread/thread/thr_init.c
+++ b/lib/libpthread/thread/thr_init.c
@@ -268,9 +268,6 @@ _thread_init(void)
 		memcpy((void *) &_thread_initial->attr, &pthread_attr_default,
 		    sizeof(struct pthread_attr));
 
-		/* Initialize the thread stack cache: */
-		SLIST_INIT(&_stackq);
-
 		/*
 		 * Create a red zone below the main stack.  All other stacks are
 		 * constrained to a maximum size by the paramters passed to
@@ -279,7 +276,7 @@ _thread_init(void)
 		 * thread stack that is just beyond.
 		 */
 		if (mmap((void *) USRSTACK - PTHREAD_STACK_INITIAL -
-		    PTHREAD_STACK_GUARD, PTHREAD_STACK_GUARD, 0, MAP_ANON,
+		    PTHREAD_GUARD_DEFAULT, PTHREAD_GUARD_DEFAULT, 0, MAP_ANON,
 		    -1, 0) == MAP_FAILED)
 			PANIC("Cannot allocate red zone for initial thread");
 
diff --git a/lib/libpthread/thread/thr_private.h b/lib/libpthread/thread/thr_private.h
index 6528c13689bf..2bfac782d887 100644
--- a/lib/libpthread/thread/thr_private.h
+++ b/lib/libpthread/thread/thr_private.h
@@ -389,6 +389,7 @@ struct pthread_attr {
 	void	(*cleanup_attr) ();
 	void	*stackaddr_attr;
 	size_t	stacksize_attr;
+	size_t	guardsize_attr;
 };
 
 /*
@@ -414,13 +415,13 @@ enum pthread_susp {
  */
 #define PTHREAD_STACK_DEFAULT			65536
 /*
- * Size of red zone at the end of each stack.  In actuality, this "red zone" is
- * merely an unmapped region, except in the case of the initial stack.  Since
- * mmap() makes it possible to specify the maximum growth of a MAP_STACK region,
- * an unmapped gap between thread stacks achieves the same effect as explicitly
- * mapped red zones.
+ * Size of default red zone at the end of each stack.  In actuality, this "red
+ * zone" is merely an unmapped region, except in the case of the initial stack.
+ * Since mmap() makes it possible to specify the maximum growth of a MAP_STACK
+ * region, an unmapped gap between thread stacks achieves the same effect as
+ * explicitly mapped red zones.
  */
-#define PTHREAD_STACK_GUARD			PAGE_SIZE
+#define	PTHREAD_GUARD_DEFAULT			PAGE_SIZE
 
 /*
  * Maximum size of initial thread's stack.  This perhaps deserves to be larger
@@ -875,11 +876,6 @@ struct pthread {
 	int			lineno;	/* Source line number.      */
 };
 
-/* Spare thread stack. */
-struct stack {
-	SLIST_ENTRY(stack)	qe; /* Queue entry for this stack. */
-};
-
 /*
  * Global variables for the uthread kernel.
  */
@@ -992,8 +988,9 @@ SCLASS struct pthread *_thread_initial
 /* Default thread attributes: */
 SCLASS struct pthread_attr pthread_attr_default
 #ifdef GLOBAL_PTHREAD_PRIVATE
-= { SCHED_RR, 0, TIMESLICE_USEC, PTHREAD_DEFAULT_PRIORITY, PTHREAD_CREATE_RUNNING,
-	PTHREAD_CREATE_JOINABLE, NULL, NULL, NULL, PTHREAD_STACK_DEFAULT };
+= { SCHED_RR, 0, TIMESLICE_USEC, PTHREAD_DEFAULT_PRIORITY,
+	PTHREAD_CREATE_RUNNING, PTHREAD_CREATE_JOINABLE, NULL, NULL, NULL,
+	PTHREAD_STACK_DEFAULT, PTHREAD_GUARD_DEFAULT };
 #else
 ;
 #endif
@@ -1142,31 +1139,6 @@ SCLASS pthread_switch_routine_t _sched_switch_hook
 ;
 
 /*
- * Spare stack queue.  Stacks of default size are cached in order to reduce
- * thread creation time.  Spare stacks are used in LIFO order to increase cache
- * locality.
- */
-SCLASS SLIST_HEAD(, stack)	_stackq;
-
-/*
- * Base address of next unallocated default-size {stack, red zone}.  Stacks are
- * allocated contiguously, starting below the bottom of the main stack.  When a
- * new stack is created, a red zone is created (actually, the red zone is simply
- * left unmapped) below the bottom of the stack, such that the stack will not be
- * able to grow all the way to the top of the next stack.  This isn't
- * fool-proof.  It is possible for a stack to grow by a large amount, such that
- * it grows into the next stack, and as long as the memory within the red zone
- * is never accessed, nothing will prevent one thread stack from trouncing all
- * over the next.
- */
-SCLASS void *	_next_stack
-#ifdef GLOBAL_PTHREAD_PRIVATE
-/* main stack top   - main stack size       - stack size            - (red zone + main stack red zone) */
-= (void *) USRSTACK - PTHREAD_STACK_INITIAL - PTHREAD_STACK_DEFAULT - (2 * PTHREAD_STACK_GUARD)
-#endif
-;
-
-/*
  * Declare the kernel scheduler jump buffer and stack:
  */
 SCLASS jmp_buf	_thread_kern_sched_jb;
@@ -1210,6 +1182,8 @@ void	_fd_lock_backout(pthread_t);
 int     _find_thread(pthread_t);
 struct pthread *_get_curthread(void);
 void	_set_curthread(struct pthread *);
+void	*_thread_stack_alloc(size_t, size_t);
+void	_thread_stack_free(void *, size_t, size_t);
 int     _thread_create(pthread_t *,const pthread_attr_t *,void *(*start_routine)(void *),void *,pthread_t);
 int     _thread_fd_lock(int, int, struct timespec *);
 int     _thread_fd_lock_debug(int, int, struct timespec *,char *fname,int lineno);
diff --git a/lib/libpthread/thread/thr_stack.c b/lib/libpthread/thread/thr_stack.c
new file mode 100644
index 000000000000..055af076c94f
--- /dev/null
+++ b/lib/libpthread/thread/thr_stack.c
@@ -0,0 +1,235 @@
+/*
+ * Copyright (c) 2001 Daniel Eischen <deischen@freebsd.org>
+ * Copyright (c) 2000-2001 Jason Evans <jasone@freebsd.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+#include <sys/types.h>
+#include <sys/mman.h>
+#include <sys/param.h>
+#include <sys/queue.h>
+#include <sys/user.h>
+#include <stdlib.h>
+#include <pthread.h>
+#include "pthread_private.h"
+
+/* Spare thread stack. */
+struct stack {
+	LIST_ENTRY(stack)	qe;		/* Stack queue linkage. */
+	size_t			stacksize;	/* Stack size (rounded up). */
+	size_t			guardsize;	/* Guard size. */
+	void			*stackaddr;	/* Stack address. */
+};
+
+/*
+ * Default sized (stack and guard) spare stack queue.  Stacks are cached to
+ * avoid additional complexity managing mmap()ed stack regions.  Spare stacks
+ * are used in LIFO order to increase cache locality.
+ */
+static LIST_HEAD(, stack)	_dstackq = LIST_HEAD_INITIALIZER(_dstackq);
+
+/*
+ * Miscellaneous sized (non-default stack and/or guard) spare stack queue.
+ * Stacks are cached to avoid additional complexity managing mmap()ed stack
+ * regions.  This list is unordered, since ordering on both stack size and guard
+ * size would be more trouble than it's worth.  Stacks are allocated from this
+ * cache on a first size match basis.
+ */
+static LIST_HEAD(, stack)	_mstackq = LIST_HEAD_INITIALIZER(_mstackq);
+
+/**
+ * Base address of the last stack allocated (including its red zone, if there is
+ * one).  Stacks are allocated contiguously, starting beyond the top of the main
+ * stack.  When a new stack is created, a red zone is typically created
+ * (actually, the red zone is simply left unmapped) above the top of the stack,
+ * such that the stack will not be able to grow all the way to the bottom of the
+ * next stack.  This isn't fool-proof.  It is possible for a stack to grow by a
+ * large amount, such that it grows into the next stack, and as long as the
+ * memory within the red zone is never accessed, nothing will prevent one thread
+ * stack from trouncing all over the next.
+ *
+ * low memory
+ *     . . . . . . . . . . . . . . . . . . 
+ *    |                                   |
+ *    |             stack 3               | start of 3rd thread stack
+ *    +-----------------------------------+
+ *    |                                   |
+ *    |       Red Zone (guard page)       | red zone for 2nd thread
+ *    |                                   |
+ *    +-----------------------------------+
+ *    |  stack 2 - PTHREAD_STACK_DEFAULT  | top of 2nd thread stack
+ *    |                                   |
+ *    |                                   |
+ *    |                                   |
+ *    |                                   |
+ *    |             stack 2               |
+ *    +-----------------------------------+ <-- start of 2nd thread stack
+ *    |                                   |
+ *    |       Red Zone                    | red zone for 1st thread
+ *    |                                   |
+ *    +-----------------------------------+
+ *    |  stack 1 - PTHREAD_STACK_DEFAULT  | top of 1st thread stack
+ *    |                                   |
+ *    |                                   |
+ *    |                                   |
+ *    |                                   |
+ *    |             stack 1               |
+ *    +-----------------------------------+ <-- start of 1st thread stack
+ *    |                                   |   (initial value of last_stack)
+ *    |       Red Zone                    |
+ *    |                                   | red zone for main thread
+ *    +-----------------------------------+
+ *    | USRSTACK - PTHREAD_STACK_INITIAL  | top of main thread stack
+ *    |                                   | ^
+ *    |                                   | |
+ *    |                                   | |
+ *    |                                   | | stack growth
+ *    |                                   |
+ *    +-----------------------------------+ <-- start of main thread stack
+ *                                              (USRSTACK)
+ * high memory
+ *
+ */
+static void *	last_stack = (void *) USRSTACK - PTHREAD_STACK_INITIAL
+		    - PTHREAD_GUARD_DEFAULT;
+
+void *
+_thread_stack_alloc(size_t stacksize, size_t guardsize)
+{
+	void		*stack = NULL;
+	struct stack	*spare_stack;
+	size_t		stack_size;
+
+	/*
+	 * Round up stack size to nearest multiple of PAGE_SIZE, so that mmap()
+	 * will work.  If the stack size is not an even multiple, we end up
+	 * initializing things such that there is unused space above the
+	 * beginning of the stack, so the stack sits snugly against its guard.
+	 */
+	if (stacksize % PAGE_SIZE != 0)
+		stack_size = ((stacksize / PAGE_SIZE) + 1) * PAGE_SIZE;
+	else
+		stack_size = stacksize;
+
+	/*
+	 * If the stack and guard sizes are default, try to allocate a stack
+	 * from the default-size stack cache:
+	 */
+	if (stack_size == PTHREAD_STACK_DEFAULT &&
+	    guardsize == PTHREAD_GUARD_DEFAULT) {
+		/*
+		 * Use the garbage collector mutex for synchronization of the
+		 * spare stack list.
+		 */
+		if (pthread_mutex_lock(&_gc_mutex) != 0)
+			PANIC("Cannot lock gc mutex");
+
+		if ((spare_stack = LIST_FIRST(&_dstackq)) != NULL) {
+				/* Use the spare stack. */
+			LIST_REMOVE(spare_stack, qe);
+			stack = spare_stack->stackaddr;
+		}
+
+		/* Unlock the garbage collector mutex. */
+		if (pthread_mutex_unlock(&_gc_mutex) != 0)
+			PANIC("Cannot unlock gc mutex");
+	}
+	/*
+	 * The user specified a non-default stack and/or guard size, so try to
+	 * allocate a stack from the non-default size stack cache, using the
+	 * rounded up stack size (stack_size) in the search:
+	 */
+	else {
+		/*
+		 * Use the garbage collector mutex for synchronization of the
+		 * spare stack list.
+		 */
+		if (pthread_mutex_lock(&_gc_mutex) != 0)
+			PANIC("Cannot lock gc mutex");
+
+		LIST_FOREACH(spare_stack, &_mstackq, qe) {
+			if (spare_stack->stacksize == stack_size &&
+			    spare_stack->guardsize == guardsize) {
+				LIST_REMOVE(spare_stack, qe);
+				stack = spare_stack->stackaddr;
+				break;
+			}
+		}
+
+		/* Unlock the garbage collector mutex. */
+		if (pthread_mutex_unlock(&_gc_mutex) != 0)
+			PANIC("Cannot unlock gc mutex");
+	}
+
+	/* Check if a stack was not allocated from a stack cache: */
+	if (stack == NULL) {
+
+		/* Allocate a new stack. */
+
+		stack = last_stack - stack_size;
+
+		/*
+		 * Even if stack allocation fails, we don't want to try to use
+		 * this location again, so unconditionally decrement
+		 * last_stack.  Under normal operating conditions, the most
+		 * likely reason for an mmap() error is a stack overflow of the
+		 * adjacent thread stack.
+		 */
+		last_stack -= (stack_size + guardsize);
+
+		/* Stack: */
+		if (mmap(stack, stack_size, PROT_READ | PROT_WRITE, MAP_STACK,
+		    -1, 0) == MAP_FAILED)
+			stack = NULL;
+	}
+
+	return (stack);
+}
+
+/* This function must be called with _gc_mutex held. */
+void
+_thread_stack_free(void *stack, size_t stacksize, size_t guardsize)
+{
+	struct stack	*spare_stack;
+
+	spare_stack = (stack + stacksize - sizeof(struct stack));
+	/* Round stacksize up to nearest multiple of PAGE_SIZE. */
+	if (stacksize % PAGE_SIZE != 0) {
+		spare_stack->stacksize = ((stacksize / PAGE_SIZE) + 1) *
+		    PAGE_SIZE;
+	} else
+		spare_stack->stacksize = stacksize;
+	spare_stack->guardsize = guardsize;
+	spare_stack->stackaddr = stack;
+
+	if (spare_stack->stacksize == PTHREAD_STACK_DEFAULT &&
+	    spare_stack->guardsize == PTHREAD_GUARD_DEFAULT) {
+		/* Default stack/guard size. */
+		LIST_INSERT_HEAD(&_dstackq, spare_stack, qe);
+	} else {
+		/* Non-default stack/guard size. */
+		LIST_INSERT_HEAD(&_mstackq, spare_stack, qe);
+	}
+}