diff options
Diffstat (limited to 'malloc/arena.c')
-rw-r--r-- | malloc/arena.c | 193 |
1 files changed, 146 insertions, 47 deletions
diff --git a/malloc/arena.c b/malloc/arena.c index 21ecc5a137..1edb4d4d35 100644 --- a/malloc/arena.c +++ b/malloc/arena.c @@ -1,5 +1,5 @@ /* Malloc implementation for multiple threads without lock contention. - Copyright (C) 2001-2015 Free Software Foundation, Inc. + Copyright (C) 2001-2016 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Wolfram Gloger <wg@malloc.de>, 2001. @@ -64,13 +64,33 @@ extern int sanity_check_heap_info_alignment[(sizeof (heap_info) + 2 * SIZE_SZ) % MALLOC_ALIGNMENT ? -1 : 1]; -/* Thread specific data */ +/* Thread specific data. */ -static tsd_key_t arena_key; -static mutex_t list_lock = MUTEX_INITIALIZER; +static __thread mstate thread_arena attribute_tls_model_ie; + +/* Arena free list. free_list_lock synchronizes access to the + free_list variable below, and the next_free and attached_threads + members of struct malloc_state objects. No other locks must be + acquired after free_list_lock has been acquired. */ + +static mutex_t free_list_lock = _LIBC_LOCK_INITIALIZER; static size_t narenas = 1; static mstate free_list; +/* list_lock prevents concurrent writes to the next member of struct + malloc_state objects. + + Read access to the next member is supposed to synchronize with the + atomic_write_barrier and the write to the next member in + _int_new_arena. This suffers from data races; see the FIXME + comments in _int_new_arena and reused_arena. + + list_lock also prevents concurrent forks. At the time list_lock is + acquired, no arena lock must have been acquired, but it is + permitted to acquire arena locks subsequently, while list_lock is + acquired. */ +static mutex_t list_lock = _LIBC_LOCK_INITIALIZER; + /* Mapped memory in non-main arenas (reliable only for NO_THREADS). */ static unsigned long arena_mem; @@ -89,20 +109,15 @@ int __malloc_initialized = -1; in the new arena. */ #define arena_get(ptr, size) do { \ - arena_lookup (ptr); \ + ptr = thread_arena; \ arena_lock (ptr, size); \ } while (0) -#define arena_lookup(ptr) do { \ - void *vptr = NULL; \ - ptr = (mstate) tsd_getspecific (arena_key, vptr); \ - } while (0) - #define arena_lock(ptr, size) do { \ if (ptr && !arena_is_corrupt (ptr)) \ (void) mutex_lock (&ptr->mutex); \ else \ - ptr = arena_get2 (ptr, (size), NULL); \ + ptr = arena_get2 ((size), NULL); \ } while (0) /* find the heap and corresponding arena for a given ptr */ @@ -138,11 +153,9 @@ ATFORK_MEM; static void * malloc_atfork (size_t sz, const void *caller) { - void *vptr = NULL; void *victim; - tsd_getspecific (arena_key, vptr); - if (vptr == ATFORK_ARENA_PTR) + if (thread_arena == ATFORK_ARENA_PTR) { /* We are the only thread that may allocate at all. */ if (save_malloc_hook != malloc_check) @@ -172,7 +185,6 @@ malloc_atfork (size_t sz, const void *caller) static void free_atfork (void *mem, const void *caller) { - void *vptr = NULL; mstate ar_ptr; mchunkptr p; /* chunk corresponding to mem */ @@ -188,8 +200,7 @@ free_atfork (void *mem, const void *caller) } ar_ptr = arena_for_chunk (p); - tsd_getspecific (arena_key, vptr); - _int_free (ar_ptr, p, vptr == ATFORK_ARENA_PTR); + _int_free (ar_ptr, p, thread_arena == ATFORK_ARENA_PTR); } @@ -210,11 +221,12 @@ ptmalloc_lock_all (void) if (__malloc_initialized < 1) return; + /* We do not acquire free_list_lock here because we completely + reconstruct free_list in ptmalloc_unlock_all2. */ + if (mutex_trylock (&list_lock)) { - void *my_arena; - tsd_getspecific (arena_key, my_arena); - if (my_arena == ATFORK_ARENA_PTR) + if (thread_arena == ATFORK_ARENA_PTR) /* This is the same thread which already locks the global list. Just bump the counter. */ goto out; @@ -233,9 +245,12 @@ ptmalloc_lock_all (void) save_free_hook = __free_hook; __malloc_hook = malloc_atfork; __free_hook = free_atfork; - /* Only the current thread may perform malloc/free calls now. */ - tsd_getspecific (arena_key, save_arena); - tsd_setspecific (arena_key, ATFORK_ARENA_PTR); + /* Only the current thread may perform malloc/free calls now. + save_arena will be reattached to the current thread, in + ptmalloc_lock_all, so save_arena->attached_threads is not + updated. */ + save_arena = thread_arena; + thread_arena = ATFORK_ARENA_PTR; out: ++atfork_recursive_cntr; } @@ -251,7 +266,10 @@ ptmalloc_unlock_all (void) if (--atfork_recursive_cntr != 0) return; - tsd_setspecific (arena_key, save_arena); + /* Replace ATFORK_ARENA_PTR with save_arena. + save_arena->attached_threads was not changed in ptmalloc_lock_all + and is still correct. */ + thread_arena = save_arena; __malloc_hook = save_malloc_hook; __free_hook = save_free_hook; for (ar_ptr = &main_arena;; ) @@ -279,15 +297,23 @@ ptmalloc_unlock_all2 (void) if (__malloc_initialized < 1) return; - tsd_setspecific (arena_key, save_arena); + thread_arena = save_arena; __malloc_hook = save_malloc_hook; __free_hook = save_free_hook; + + /* Push all arenas to the free list, except save_arena, which is + attached to the current thread. */ + mutex_init (&free_list_lock); + if (save_arena != NULL) + ((mstate) save_arena)->attached_threads = 1; free_list = NULL; for (ar_ptr = &main_arena;; ) { mutex_init (&ar_ptr->mutex); if (ar_ptr != save_arena) { + /* This arena is no longer attached to any thread. */ + ar_ptr->attached_threads = 0; ar_ptr->next_free = free_list; free_list = ar_ptr; } @@ -295,6 +321,7 @@ ptmalloc_unlock_all2 (void) if (ar_ptr == &main_arena) break; } + mutex_init (&list_lock); atfork_recursive_cntr = 0; } @@ -372,8 +399,7 @@ ptmalloc_init (void) __morecore = __failing_morecore; #endif - tsd_key_create (&arena_key, NULL); - tsd_setspecific (arena_key, (void *) &main_arena); + thread_arena = &main_arena; thread_atfork (ptmalloc_lock_all, ptmalloc_unlock_all, ptmalloc_unlock_all2); const char *s = NULL; if (__glibc_likely (_environ != NULL)) @@ -696,14 +722,20 @@ heap_trim (heap_info *heap, size_t pad) } /* Uses similar logic for per-thread arenas as the main arena with systrim - by preserving the top pad and at least a page. */ + and _int_free by preserving the top pad and rounding down to the nearest + page. */ top_size = chunksize (top_chunk); + if ((unsigned long)(top_size) < + (unsigned long)(mp_.trim_threshold)) + return 0; + top_area = top_size - MINSIZE - 1; if (top_area < 0 || (size_t) top_area <= pad) return 0; + /* Release in pagesize units and round down to the nearest page. */ extra = ALIGN_DOWN(top_area - pad, pagesz); - if ((unsigned long) extra < mp_.trim_threshold) + if (extra == 0) return 0; /* Try to shrink. */ @@ -721,6 +753,22 @@ heap_trim (heap_info *heap, size_t pad) /* Create a new arena with initial size "size". */ +/* If REPLACED_ARENA is not NULL, detach it from this thread. Must be + called while free_list_lock is held. */ +static void +detach_arena (mstate replaced_arena) +{ + if (replaced_arena != NULL) + { + assert (replaced_arena->attached_threads > 0); + /* The current implementation only detaches from main_arena in + case of allocation failure. This means that it is likely not + beneficial to put the arena on free_list even if the + reference count reaches zero. */ + --replaced_arena->attached_threads; + } +} + static mstate _int_new_arena (size_t size) { @@ -742,6 +790,7 @@ _int_new_arena (size_t size) } a = h->ar_ptr = (mstate) (h + 1); malloc_init_state (a); + a->attached_threads = 1; /*a->next = NULL;*/ a->system_mem = a->max_system_mem = h->size; arena_mem += h->size; @@ -755,40 +804,68 @@ _int_new_arena (size_t size) set_head (top (a), (((char *) h + h->size) - ptr) | PREV_INUSE); LIBC_PROBE (memory_arena_new, 2, a, size); - tsd_setspecific (arena_key, (void *) a); + mstate replaced_arena = thread_arena; + thread_arena = a; mutex_init (&a->mutex); - (void) mutex_lock (&a->mutex); (void) mutex_lock (&list_lock); /* Add the new arena to the global list. */ a->next = main_arena.next; + /* FIXME: The barrier is an attempt to synchronize with read access + in reused_arena, which does not acquire list_lock while + traversing the list. */ atomic_write_barrier (); main_arena.next = a; (void) mutex_unlock (&list_lock); + (void) mutex_lock (&free_list_lock); + detach_arena (replaced_arena); + (void) mutex_unlock (&free_list_lock); + + /* Lock this arena. NB: Another thread may have been attached to + this arena because the arena is now accessible from the + main_arena.next list and could have been picked by reused_arena. + This can only happen for the last arena created (before the arena + limit is reached). At this point, some arena has to be attached + to two threads. We could acquire the arena lock before list_lock + to make it less likely that reused_arena picks this new arena, + but this could result in a deadlock with ptmalloc_lock_all. */ + + (void) mutex_lock (&a->mutex); + return a; } +/* Remove an arena from free_list. The arena may be in use because it + was attached concurrently to a thread by reused_arena below. */ static mstate get_free_list (void) { + mstate replaced_arena = thread_arena; mstate result = free_list; if (result != NULL) { - (void) mutex_lock (&list_lock); + (void) mutex_lock (&free_list_lock); result = free_list; if (result != NULL) - free_list = result->next_free; - (void) mutex_unlock (&list_lock); + { + free_list = result->next_free; + + /* The arena will be attached to this thread. */ + ++result->attached_threads; + + detach_arena (replaced_arena); + } + (void) mutex_unlock (&free_list_lock); if (result != NULL) { LIBC_PROBE (memory_arena_reuse_free_list, 1, result); (void) mutex_lock (&result->mutex); - tsd_setspecific (arena_key, (void *) result); + thread_arena = result; } } @@ -802,16 +879,20 @@ static mstate reused_arena (mstate avoid_arena) { mstate result; + /* FIXME: Access to next_to_use suffers from data races. */ static mstate next_to_use; if (next_to_use == NULL) next_to_use = &main_arena; + /* Iterate over all arenas (including those linked from + free_list). */ result = next_to_use; do { if (!arena_is_corrupt (result) && !mutex_trylock (&result->mutex)) goto out; + /* FIXME: This is a data race, see _int_new_arena. */ result = result->next; } while (result != next_to_use); @@ -840,8 +921,19 @@ reused_arena (mstate avoid_arena) (void) mutex_lock (&result->mutex); out: + /* Attach the arena to the current thread. Note that we may have + selected an arena which was on free_list. */ + { + /* Update the arena thread attachment counters. */ + mstate replaced_arena = thread_arena; + (void) mutex_lock (&free_list_lock); + detach_arena (replaced_arena); + ++result->attached_threads; + (void) mutex_unlock (&free_list_lock); + } + LIBC_PROBE (memory_arena_reuse, 2, result, avoid_arena); - tsd_setspecific (arena_key, (void *) result); + thread_arena = result; next_to_use = result->next; return result; @@ -849,7 +941,7 @@ out: static mstate internal_function -arena_get2 (mstate a_tsd, size_t size, mstate avoid_arena) +arena_get2 (size_t size, mstate avoid_arena) { mstate a; @@ -909,15 +1001,17 @@ arena_get_retry (mstate ar_ptr, size_t bytes) if (ar_ptr != &main_arena) { (void) mutex_unlock (&ar_ptr->mutex); + /* Don't touch the main arena if it is corrupt. */ + if (arena_is_corrupt (&main_arena)) + return NULL; + ar_ptr = &main_arena; (void) mutex_lock (&ar_ptr->mutex); } else { - /* Grab ar_ptr->next prior to releasing its lock. */ - mstate prev = ar_ptr->next ? ar_ptr : 0; (void) mutex_unlock (&ar_ptr->mutex); - ar_ptr = arena_get2 (prev, bytes, ar_ptr); + ar_ptr = arena_get2 (bytes, ar_ptr); } return ar_ptr; @@ -926,16 +1020,21 @@ arena_get_retry (mstate ar_ptr, size_t bytes) static void __attribute__ ((section ("__libc_thread_freeres_fn"))) arena_thread_freeres (void) { - void *vptr = NULL; - mstate a = tsd_getspecific (arena_key, vptr); - tsd_setspecific (arena_key, NULL); + mstate a = thread_arena; + thread_arena = NULL; if (a != NULL) { - (void) mutex_lock (&list_lock); - a->next_free = free_list; - free_list = a; - (void) mutex_unlock (&list_lock); + (void) mutex_lock (&free_list_lock); + /* If this was the last attached thread for this arena, put the + arena on the free list. */ + assert (a->attached_threads > 0); + if (--a->attached_threads == 0) + { + a->next_free = free_list; + free_list = a; + } + (void) mutex_unlock (&free_list_lock); } } text_set_element (__libc_thread_subfreeres, arena_thread_freeres); |