Implement thread destruction

This change makes libpthread release almost every resource allocated for
a thread, including the kernel thread, its send right, its reply port
and its stack. This improves resource usage after peaks of activity
during which servers can create hundreds or even thousands of threads.

To achieve this, the library relies on the recently added
thread_terminate_release one-way GNU Mach RPC, which allows threads to
release their last resources along with terminating in a single
operation. The pthread_exit function unconditionally releases all the
resources it can, including other kernel objects (namely the port used
for blocking and waking up) and signal states. When releasing the
pthread structure, a reference counter is used so that joinable threads
remain available. Once the reference counter drops to 0, the pthread
structure can be recycled. Thread local storage (TLS) is also recycled
since it needs to remain allocated while terminating the thread, as it
is there that the reply port is stored. TLS could be released too, after
grabbing the reply port name, but it is difficult to make sure no RPC
involving a reply port is used afterwards, so the simpler solution of
recycling TLS was chosen.

* Makefile (libpthread-routines): Replace pt-thread-halt with
pt-thread-terminate.
* pthread/pt-alloc.c (initialize_pthread): Set reference counter and
unconditionally initialize new threads completely.
(__pthread_alloc): Remove call to __pthread_thread_halt, update calls
to initialize_pthread.
* pthread/pt-create.c (__pthread_create_internal): Don't attempt to
reuse stacks, handle reference counter, update failure handling.
* pthread/pt-dealloc.c: Include bits/pt-atomic.h.
(__pthread_dealloc): Make pthread structure available for reuse when
reference counter reaches 0.
* pthread/pt-detach.c (pthread_detach): Assume the target thread takes
care of its own resources and, as a result, simply unreference its
pthread struct.
* pthread/pt-exit.c (__pthread_exit): Release resources and terminate.
* pthread/pt-internal.h (struct __pthread): New `nr_refs' member.
(__pthread_alloc): Update description.
(__pthread_dealloc): Likewise.
(__pthread_thread_dealloc): Likewise.
(__pthread_thread_terminate): New declaration.
* pthread/pt-join.c (pthread_join): Assume the target thread takes care
of its own resources and, as a result, simply unreference its pthread
struct.
* sysdeps/mach/hurd/pt-sigstate-destroy.c
(__pthread_sigstate_destroy): Call _hurd_sigstate_delete.
* sysdeps/mach/hurd/pt-sigstate-init.c (__pthread_sigstate_init): Call
_hurd_thread_sigstate and _hurd_sigstate_set_global_rcv when
appropriate.
* sysdeps/mach/hurd/pt-sysdep.c (__pthread_create_internal): Prevent
the library from releasing the stack of the main thread.
* sysdeps/mach/hurd/pt-sysdep.h (PTHREAD_SYSDEP_MEMBERS): Remove
`have_kernel_resources' from the list of sysdep members.
* sysdeps/mach/pt-thread-alloc.c (__pthread_thread_alloc): Update
thread allocation.
* sysdeps/mach/pt-thread-dealloc.c (__pthread_thread_dealloc): Update
description.
* sysdeps/mach/pt-thread-halt.c: Remove file.
* sysdeps/mach/pt-thread-start.c (__pthread_thread_start): Fix the
conditions under which a thread should actually be started.
* sysdeps/mach/pt-thread-terminate.c: New file.

9 files changed, 108 insertions, 67 deletions

diff --git a/sysdeps/mach/hurd/pt-sigstate-destroy.c b/sysdeps/mach/hurd/pt-sigstate-destroy.c
index 8e56c5c..d5e28d2 100644
--- a/sysdeps/mach/hurd/pt-sigstate-destroy.c
+++ b/sysdeps/mach/hurd/pt-sigstate-destroy.c
@@ -24,5 +24,5 @@
 void
 __pthread_sigstate_destroy (struct __pthread *thread)
 {
-  /* Nothing to do.  */
+  _hurd_sigstate_delete (thread->kernel_thread);
 }
diff --git a/sysdeps/mach/hurd/pt-sigstate-init.c b/sysdeps/mach/hurd/pt-sigstate-init.c
index dd56d90..500b4d4 100644
--- a/sysdeps/mach/hurd/pt-sigstate-init.c
+++ b/sysdeps/mach/hurd/pt-sigstate-init.c
@@ -23,6 +23,22 @@
 error_t
 __pthread_sigstate_init (struct __pthread *thread)
 {
-  /* Nothing to do.  */
+  static int do_init_global;
+
+  /* Mark the thread as a global signal receiver so as to conform with
+     the pthread semantics.  However, we must be careful.  The first
+     pthread created is the main thread, during libpthread initialization.
+     We must not mark it, otherwise the sigprocmask call in
+     __pthread_create would try to access _hurd_global_sigstate,
+     which is not initialized yet.  When glibc runs _hurdsig_init later
+     on, the message thread is created, which must not be marked either.  */
+  if (do_init_global)
+    {
+      struct hurd_sigstate *ss = _hurd_thread_sigstate (thread->kernel_thread);
+      _hurd_sigstate_set_global_rcv (ss);
+    }
+  else if (__pthread_num_threads >= 2)
+    do_init_global = 1;
+
   return 0;
 }
diff --git a/sysdeps/mach/hurd/pt-sysdep.c b/sysdeps/mach/hurd/pt-sysdep.c
index 09f0dcb..fd71aba 100644
--- a/sysdeps/mach/hurd/pt-sysdep.c
+++ b/sysdeps/mach/hurd/pt-sysdep.c
@@ -65,6 +65,11 @@ init_routine (void)
   err = __pthread_create_internal (&thread, 0, 0, 0);
   assert_perror (err);
 
+  /* XXX The caller copies the command line arguments and the environment
+     to the new stack.  Pretend it wasn't allocated so that it remains
+     valid if the main thread terminates.  */
+  thread->stack = 0;
+
   ___pthread_self = thread;
 
   /* Decrease the number of threads, to take into account that the
diff --git a/sysdeps/mach/hurd/pt-sysdep.h b/sysdeps/mach/hurd/pt-sysdep.h
index 89592f9..35912a3 100644
--- a/sysdeps/mach/hurd/pt-sysdep.h
+++ b/sysdeps/mach/hurd/pt-sysdep.h
@@ -30,8 +30,7 @@
 
 #define PTHREAD_SYSDEP_MEMBERS \
   thread_t kernel_thread;      \
-  mach_msg_header_t wakeupmsg; \
-  int have_kernel_resources;
+  mach_msg_header_t wakeupmsg;
 
 extern __thread struct __pthread *___pthread_self;
 #define _pthread_self()                                            \
diff --git a/sysdeps/mach/pt-thread-alloc.c b/sysdeps/mach/pt-thread-alloc.c
index 794f63e..77aa933 100644
--- a/sysdeps/mach/pt-thread-alloc.c
+++ b/sysdeps/mach/pt-thread-alloc.c
@@ -67,44 +67,29 @@ create_wakeupmsg (struct __pthread *thread)
 int
 __pthread_thread_alloc (struct __pthread *thread)
 {
-  if (thread->have_kernel_resources)
-    return 0;
-
+  static int do_create;
   error_t err;
 
   err = create_wakeupmsg (thread);
   if (err)
     return err;
 
-  /* If there are no pthreads in the system then the pthread library
-     is bootstrapping and the main thread must create initialize
-     itself.  The thread itself is already running, it just has not
-     pthread context.  We want to reuse what it already has (including
-     the kernel thread), however, we must determine which thread is
-     the main thread.
-
-     We cannot test if __pthread_total is one as we later decrement
-     before creating the signal thread.  Currently, we check if
-     __pthread_num_threads--the number of allocated thread
-     structures--is one.  __pthread_alloc has already been called in
-     __pthread_create_internal for us.  This predicate could be improved,
-     however, it is sufficient for now.  */
-  if (__pthread_num_threads == 1)
+  if (! do_create)
     {
       assert (__pthread_total == 0);
       thread->kernel_thread = __mach_thread_self ();
-      /* We implicitly hold a reference drop the one that we just
-	 acquired.  */
-      __mach_port_deallocate (__mach_task_self (), thread->kernel_thread);
+      do_create = 1;
     }
   else
     {
       err = __thread_create (__mach_task_self (), &thread->kernel_thread);
       if (err)
-	return EAGAIN;
+	{
+	  __mach_port_destroy (__mach_task_self (),
+			       thread->wakeupmsg.msgh_remote_port);
+	  return EAGAIN;
+	}
     }
 
-  thread->have_kernel_resources = 1;
-
   return 0;
 }
diff --git a/sysdeps/mach/pt-thread-dealloc.c b/sysdeps/mach/pt-thread-dealloc.c
index 55d8c4d..e977c3f 100644
--- a/sysdeps/mach/pt-thread-dealloc.c
+++ b/sysdeps/mach/pt-thread-dealloc.c
@@ -24,8 +24,7 @@
 #include <pt-internal.h>
 
 /* Deallocate any kernel resources associated with THREAD except don't
-   halt the thread itself.  On return, the thread will be marked as
-   dead and __pthread_halt will be called.  */
+   terminate the thread itself.  */
 void
 __pthread_thread_dealloc (struct __pthread *thread)
 {
diff --git a/sysdeps/mach/pt-thread-halt.c b/sysdeps/mach/pt-thread-halt.c
deleted file mode 100644
index 973cde1..0000000
--- a/sysdeps/mach/pt-thread-halt.c
+++ /dev/null
@@ -1,37 +0,0 @@
-/* Deallocate the kernel thread resources.  Mach version.
-   Copyright (C) 2000, 2002, 2005 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Library General Public License as
-   published by the Free Software Foundation; either version 2 of the
-   License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
-
-#include <assert.h>
-#include <errno.h>
-#include <mach.h>
-
-#include <pt-internal.h>
-
-/* Stop the kernel thread associated with THREAD.  If NEED_DEALLOC is
-   true, the function must call __pthread_dealloc on THREAD.
-
-   NB: The thread executing this function may be the thread which is
-   being halted, thus the last action should be halting the thread
-   itself.  */
-void
-__pthread_thread_halt (struct __pthread *thread)
-{
-  error_t err = __thread_terminate (thread->kernel_thread);
-  assert_perror (err);
-}
diff --git a/sysdeps/mach/pt-thread-start.c b/sysdeps/mach/pt-thread-start.c
index 11b017f..df490ab 100644
--- a/sysdeps/mach/pt-thread-start.c
+++ b/sysdeps/mach/pt-thread-start.c
@@ -27,9 +27,10 @@
 int
 __pthread_thread_start (struct __pthread *thread)
 {
+  static int do_start;
   error_t err;
 
-  if (__pthread_num_threads == 1)
+  if (! do_start)
     {
       /* The main thread is already running: do nothing.  */
       assert (__pthread_total == 1);
@@ -38,6 +39,7 @@ __pthread_thread_start (struct __pthread *thread)
                  __mach_port_deallocate (__mach_task_self (),
 					 thread->kernel_thread);
 		 ok; }));
+      do_start = 1;
     }
   else
     {
diff --git a/sysdeps/mach/pt-thread-terminate.c b/sysdeps/mach/pt-thread-terminate.c
new file mode 100644
index 0000000..a89e505
--- /dev/null
+++ b/sysdeps/mach/pt-thread-terminate.c
@@ -0,0 +1,72 @@
+/* Deallocate the kernel thread resources.  Mach version.
+   Copyright (C) 2000, 2002, 2005 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Library General Public License as
+   published by the Free Software Foundation; either version 2 of the
+   License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Library General Public License for more details.
+
+   You should have received a copy of the GNU Library General Public
+   License along with the GNU C Library; see the file COPYING.LIB.  If not,
+   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+   Boston, MA 02111-1307, USA.  */
+
+#include <assert.h>
+#include <errno.h>
+#include <mach.h>
+
+#include <mach/mig_support.h>
+
+#include <pt-internal.h>
+
+/* Terminate the kernel thread associated with THREAD, and deallocate its
+   right reference and its stack.  The function also drops a reference
+   on THREAD.  */
+void
+__pthread_thread_terminate (struct __pthread *thread)
+{
+  thread_t kernel_thread, self_ktid;
+  mach_port_t reply_port;
+  void *stackaddr;
+  size_t stacksize;
+  error_t err;
+
+  kernel_thread = thread->kernel_thread;
+
+  if (thread->stack)
+    {
+      stackaddr = thread->stackaddr;
+      stacksize = thread->stacksize;
+    }
+  else
+    {
+      stackaddr = NULL;
+      stacksize = 0;
+    }
+
+  /* Each thread has its own reply port, allocated from MiG stub code calling
+     __mig_get_reply_port.  Destroying it is a bit tricky because the calls
+     involved are also RPCs, causing the creation of a new reply port if
+     currently null. The __thread_terminate_release call is actually a one way
+     simple routine designed not to require a reply port.  */
+  self_ktid = __mach_thread_self ();
+  reply_port = (self_ktid == kernel_thread)
+	       ? __mig_get_reply_port ()
+	       : MACH_PORT_NULL;
+  __mach_port_deallocate (__mach_task_self (), self_ktid);
+
+  /* Finally done with the thread structure.  */
+  __pthread_dealloc (thread);
+
+  /* Terminate and release all that's left.  */
+  err = __thread_terminate_release (kernel_thread, mach_task_self (),
+				    kernel_thread, reply_port,
+				    stackaddr, stacksize);
+  assert_perror (err);
+}