1 files changed, 146 insertions, 47 deletions

diff --git a/malloc/arena.c b/malloc/arena.c
index 21ecc5a137..1edb4d4d35 100644
--- a/malloc/arena.c
+++ b/malloc/arena.c
@@ -1,5 +1,5 @@
 /* Malloc implementation for multiple threads without lock contention.
-   Copyright (C) 2001-2015 Free Software Foundation, Inc.
+   Copyright (C) 2001-2016 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Wolfram Gloger <wg@malloc.de>, 2001.
 
@@ -64,13 +64,33 @@ extern int sanity_check_heap_info_alignment[(sizeof (heap_info)
                                              + 2 * SIZE_SZ) % MALLOC_ALIGNMENT
                                             ? -1 : 1];
 
-/* Thread specific data */
+/* Thread specific data.  */
 
-static tsd_key_t arena_key;
-static mutex_t list_lock = MUTEX_INITIALIZER;
+static __thread mstate thread_arena attribute_tls_model_ie;
+
+/* Arena free list.  free_list_lock synchronizes access to the
+   free_list variable below, and the next_free and attached_threads
+   members of struct malloc_state objects.  No other locks must be
+   acquired after free_list_lock has been acquired.  */
+
+static mutex_t free_list_lock = _LIBC_LOCK_INITIALIZER;
 static size_t narenas = 1;
 static mstate free_list;
 
+/* list_lock prevents concurrent writes to the next member of struct
+   malloc_state objects.
+
+   Read access to the next member is supposed to synchronize with the
+   atomic_write_barrier and the write to the next member in
+   _int_new_arena.  This suffers from data races; see the FIXME
+   comments in _int_new_arena and reused_arena.
+
+   list_lock also prevents concurrent forks.  At the time list_lock is
+   acquired, no arena lock must have been acquired, but it is
+   permitted to acquire arena locks subsequently, while list_lock is
+   acquired.  */
+static mutex_t list_lock = _LIBC_LOCK_INITIALIZER;
+
 /* Mapped memory in non-main arenas (reliable only for NO_THREADS). */
 static unsigned long arena_mem;
 
@@ -89,20 +109,15 @@ int __malloc_initialized = -1;
    in the new arena. */
 
 #define arena_get(ptr, size) do { \
-      arena_lookup (ptr);						      \
+      ptr = thread_arena;						      \
       arena_lock (ptr, size);						      \
   } while (0)
 
-#define arena_lookup(ptr) do { \
-      void *vptr = NULL;						      \
-      ptr = (mstate) tsd_getspecific (arena_key, vptr);			      \
-  } while (0)
-
 #define arena_lock(ptr, size) do {					      \
       if (ptr && !arena_is_corrupt (ptr))				      \
         (void) mutex_lock (&ptr->mutex);				      \
       else								      \
-        ptr = arena_get2 (ptr, (size), NULL);				      \
+        ptr = arena_get2 ((size), NULL);				      \
   } while (0)
 
 /* find the heap and corresponding arena for a given ptr */
@@ -138,11 +153,9 @@ ATFORK_MEM;
 static void *
 malloc_atfork (size_t sz, const void *caller)
 {
-  void *vptr = NULL;
   void *victim;
 
-  tsd_getspecific (arena_key, vptr);
-  if (vptr == ATFORK_ARENA_PTR)
+  if (thread_arena == ATFORK_ARENA_PTR)
     {
       /* We are the only thread that may allocate at all.  */
       if (save_malloc_hook != malloc_check)
@@ -172,7 +185,6 @@ malloc_atfork (size_t sz, const void *caller)
 static void
 free_atfork (void *mem, const void *caller)
 {
-  void *vptr = NULL;
   mstate ar_ptr;
   mchunkptr p;                          /* chunk corresponding to mem */
 
@@ -188,8 +200,7 @@ free_atfork (void *mem, const void *caller)
     }
 
   ar_ptr = arena_for_chunk (p);
-  tsd_getspecific (arena_key, vptr);
-  _int_free (ar_ptr, p, vptr == ATFORK_ARENA_PTR);
+  _int_free (ar_ptr, p, thread_arena == ATFORK_ARENA_PTR);
 }
 
 
@@ -210,11 +221,12 @@ ptmalloc_lock_all (void)
   if (__malloc_initialized < 1)
     return;
 
+  /* We do not acquire free_list_lock here because we completely
+     reconstruct free_list in ptmalloc_unlock_all2.  */
+
   if (mutex_trylock (&list_lock))
     {
-      void *my_arena;
-      tsd_getspecific (arena_key, my_arena);
-      if (my_arena == ATFORK_ARENA_PTR)
+      if (thread_arena == ATFORK_ARENA_PTR)
         /* This is the same thread which already locks the global list.
            Just bump the counter.  */
         goto out;
@@ -233,9 +245,12 @@ ptmalloc_lock_all (void)
   save_free_hook = __free_hook;
   __malloc_hook = malloc_atfork;
   __free_hook = free_atfork;
-  /* Only the current thread may perform malloc/free calls now. */
-  tsd_getspecific (arena_key, save_arena);
-  tsd_setspecific (arena_key, ATFORK_ARENA_PTR);
+  /* Only the current thread may perform malloc/free calls now.
+     save_arena will be reattached to the current thread, in
+     ptmalloc_lock_all, so save_arena->attached_threads is not
+     updated.  */
+  save_arena = thread_arena;
+  thread_arena = ATFORK_ARENA_PTR;
 out:
   ++atfork_recursive_cntr;
 }
@@ -251,7 +266,10 @@ ptmalloc_unlock_all (void)
   if (--atfork_recursive_cntr != 0)
     return;
 
-  tsd_setspecific (arena_key, save_arena);
+  /* Replace ATFORK_ARENA_PTR with save_arena.
+     save_arena->attached_threads was not changed in ptmalloc_lock_all
+     and is still correct.  */
+  thread_arena = save_arena;
   __malloc_hook = save_malloc_hook;
   __free_hook = save_free_hook;
   for (ar_ptr = &main_arena;; )
@@ -279,15 +297,23 @@ ptmalloc_unlock_all2 (void)
   if (__malloc_initialized < 1)
     return;
 
-  tsd_setspecific (arena_key, save_arena);
+  thread_arena = save_arena;
   __malloc_hook = save_malloc_hook;
   __free_hook = save_free_hook;
+
+  /* Push all arenas to the free list, except save_arena, which is
+     attached to the current thread.  */
+  mutex_init (&free_list_lock);
+  if (save_arena != NULL)
+    ((mstate) save_arena)->attached_threads = 1;
   free_list = NULL;
   for (ar_ptr = &main_arena;; )
     {
       mutex_init (&ar_ptr->mutex);
       if (ar_ptr != save_arena)
         {
+	  /* This arena is no longer attached to any thread.  */
+	  ar_ptr->attached_threads = 0;
           ar_ptr->next_free = free_list;
           free_list = ar_ptr;
         }
@@ -295,6 +321,7 @@ ptmalloc_unlock_all2 (void)
       if (ar_ptr == &main_arena)
         break;
     }
+
   mutex_init (&list_lock);
   atfork_recursive_cntr = 0;
 }
@@ -372,8 +399,7 @@ ptmalloc_init (void)
     __morecore = __failing_morecore;
 #endif
 
-  tsd_key_create (&arena_key, NULL);
-  tsd_setspecific (arena_key, (void *) &main_arena);
+  thread_arena = &main_arena;
   thread_atfork (ptmalloc_lock_all, ptmalloc_unlock_all, ptmalloc_unlock_all2);
   const char *s = NULL;
   if (__glibc_likely (_environ != NULL))
@@ -696,14 +722,20 @@ heap_trim (heap_info *heap, size_t pad)
     }
 
   /* Uses similar logic for per-thread arenas as the main arena with systrim
-     by preserving the top pad and at least a page.  */
+     and _int_free by preserving the top pad and rounding down to the nearest
+     page.  */
   top_size = chunksize (top_chunk);
+  if ((unsigned long)(top_size) <
+      (unsigned long)(mp_.trim_threshold))
+    return 0;
+
   top_area = top_size - MINSIZE - 1;
   if (top_area < 0 || (size_t) top_area <= pad)
     return 0;
 
+  /* Release in pagesize units and round down to the nearest page.  */
   extra = ALIGN_DOWN(top_area - pad, pagesz);
-  if ((unsigned long) extra < mp_.trim_threshold)
+  if (extra == 0)
     return 0;
 
   /* Try to shrink. */
@@ -721,6 +753,22 @@ heap_trim (heap_info *heap, size_t pad)
 
 /* Create a new arena with initial size "size".  */
 
+/* If REPLACED_ARENA is not NULL, detach it from this thread.  Must be
+   called while free_list_lock is held.  */
+static void
+detach_arena (mstate replaced_arena)
+{
+  if (replaced_arena != NULL)
+    {
+      assert (replaced_arena->attached_threads > 0);
+      /* The current implementation only detaches from main_arena in
+	 case of allocation failure.  This means that it is likely not
+	 beneficial to put the arena on free_list even if the
+	 reference count reaches zero.  */
+      --replaced_arena->attached_threads;
+    }
+}
+
 static mstate
 _int_new_arena (size_t size)
 {
@@ -742,6 +790,7 @@ _int_new_arena (size_t size)
     }
   a = h->ar_ptr = (mstate) (h + 1);
   malloc_init_state (a);
+  a->attached_threads = 1;
   /*a->next = NULL;*/
   a->system_mem = a->max_system_mem = h->size;
   arena_mem += h->size;
@@ -755,40 +804,68 @@ _int_new_arena (size_t size)
   set_head (top (a), (((char *) h + h->size) - ptr) | PREV_INUSE);
 
   LIBC_PROBE (memory_arena_new, 2, a, size);
-  tsd_setspecific (arena_key, (void *) a);
+  mstate replaced_arena = thread_arena;
+  thread_arena = a;
   mutex_init (&a->mutex);
-  (void) mutex_lock (&a->mutex);
 
   (void) mutex_lock (&list_lock);
 
   /* Add the new arena to the global list.  */
   a->next = main_arena.next;
+  /* FIXME: The barrier is an attempt to synchronize with read access
+     in reused_arena, which does not acquire list_lock while
+     traversing the list.  */
   atomic_write_barrier ();
   main_arena.next = a;
 
   (void) mutex_unlock (&list_lock);
 
+  (void) mutex_lock (&free_list_lock);
+  detach_arena (replaced_arena);
+  (void) mutex_unlock (&free_list_lock);
+
+  /* Lock this arena.  NB: Another thread may have been attached to
+     this arena because the arena is now accessible from the
+     main_arena.next list and could have been picked by reused_arena.
+     This can only happen for the last arena created (before the arena
+     limit is reached).  At this point, some arena has to be attached
+     to two threads.  We could acquire the arena lock before list_lock
+     to make it less likely that reused_arena picks this new arena,
+     but this could result in a deadlock with ptmalloc_lock_all.  */
+
+  (void) mutex_lock (&a->mutex);
+
   return a;
 }
 
 
+/* Remove an arena from free_list.  The arena may be in use because it
+   was attached concurrently to a thread by reused_arena below.  */
 static mstate
 get_free_list (void)
 {
+  mstate replaced_arena = thread_arena;
   mstate result = free_list;
   if (result != NULL)
     {
-      (void) mutex_lock (&list_lock);
+      (void) mutex_lock (&free_list_lock);
       result = free_list;
       if (result != NULL)
-        free_list = result->next_free;
-      (void) mutex_unlock (&list_lock);
+	{
+	  free_list = result->next_free;
+
+	  /* The arena will be attached to this thread.  */
+	  ++result->attached_threads;
+
+	  detach_arena (replaced_arena);
+	}
+      (void) mutex_unlock (&free_list_lock);
 
       if (result != NULL)
         {
           LIBC_PROBE (memory_arena_reuse_free_list, 1, result);
           (void) mutex_lock (&result->mutex);
-          tsd_setspecific (arena_key, (void *) result);
+	  thread_arena = result;
         }
     }
 
@@ -802,16 +879,20 @@ static mstate
 reused_arena (mstate avoid_arena)
 {
   mstate result;
+  /* FIXME: Access to next_to_use suffers from data races.  */
   static mstate next_to_use;
   if (next_to_use == NULL)
     next_to_use = &main_arena;
 
+  /* Iterate over all arenas (including those linked from
+     free_list).  */
   result = next_to_use;
   do
     {
       if (!arena_is_corrupt (result) && !mutex_trylock (&result->mutex))
         goto out;
 
+      /* FIXME: This is a data race, see _int_new_arena.  */
       result = result->next;
     }
   while (result != next_to_use);
@@ -840,8 +921,19 @@ reused_arena (mstate avoid_arena)
   (void) mutex_lock (&result->mutex);
 
 out:
+  /* Attach the arena to the current thread.  Note that we may have
+     selected an arena which was on free_list.  */
+  {
+    /* Update the arena thread attachment counters.   */
+    mstate replaced_arena = thread_arena;
+    (void) mutex_lock (&free_list_lock);
+    detach_arena (replaced_arena);
+    ++result->attached_threads;
+    (void) mutex_unlock (&free_list_lock);
+  }
+
   LIBC_PROBE (memory_arena_reuse, 2, result, avoid_arena);
-  tsd_setspecific (arena_key, (void *) result);
+  thread_arena = result;
   next_to_use = result->next;
 
   return result;
@@ -849,7 +941,7 @@ out:
 
 static mstate
 internal_function
-arena_get2 (mstate a_tsd, size_t size, mstate avoid_arena)
+arena_get2 (size_t size, mstate avoid_arena)
 {
   mstate a;
 
@@ -909,15 +1001,17 @@ arena_get_retry (mstate ar_ptr, size_t bytes)
   if (ar_ptr != &main_arena)
     {
       (void) mutex_unlock (&ar_ptr->mutex);
+      /* Don't touch the main arena if it is corrupt.  */
+      if (arena_is_corrupt (&main_arena))
+	return NULL;
+
       ar_ptr = &main_arena;
       (void) mutex_lock (&ar_ptr->mutex);
     }
   else
     {
-      /* Grab ar_ptr->next prior to releasing its lock.  */
-      mstate prev = ar_ptr->next ? ar_ptr : 0;
       (void) mutex_unlock (&ar_ptr->mutex);
-      ar_ptr = arena_get2 (prev, bytes, ar_ptr);
+      ar_ptr = arena_get2 (bytes, ar_ptr);
     }
 
   return ar_ptr;
@@ -926,16 +1020,21 @@ arena_get_retry (mstate ar_ptr, size_t bytes)
 static void __attribute__ ((section ("__libc_thread_freeres_fn")))
 arena_thread_freeres (void)
 {
-  void *vptr = NULL;
-  mstate a = tsd_getspecific (arena_key, vptr);
-  tsd_setspecific (arena_key, NULL);
+  mstate a = thread_arena;
+  thread_arena = NULL;
 
   if (a != NULL)
     {
-      (void) mutex_lock (&list_lock);
-      a->next_free = free_list;
-      free_list = a;
-      (void) mutex_unlock (&list_lock);
+      (void) mutex_lock (&free_list_lock);
+      /* If this was the last attached thread for this arena, put the
+	 arena on the free list.  */
+      assert (a->attached_threads > 0);
+      if (--a->attached_threads == 0)
+	{
+	  a->next_free = free_list;
+	  free_list = a;
+	}
+      (void) mutex_unlock (&free_list_lock);
     }
 }
 text_set_element (__libc_thread_subfreeres, arena_thread_freeres);