diff --git a/mark-sweep.h b/mark-sweep.h index 0dc20e550..4e10ba858 100644 --- a/mark-sweep.h +++ b/mark-sweep.h @@ -104,13 +104,6 @@ struct gcobj { }; struct mark_space { - pthread_mutex_t lock; - pthread_cond_t collector_cond; - pthread_cond_t mutator_cond; - int collecting; - int multithreaded; - size_t active_mutator_count; - size_t mutator_count; struct gcobj_freelists small_objects; // Unordered list of medium objects. struct gcobj_free_medium *medium_objects; @@ -119,16 +112,24 @@ struct mark_space { uintptr_t heap_base; size_t heap_size; uintptr_t sweep; - struct handle *global_roots; - struct mutator_mark_buf *mutator_roots; void *mem; size_t mem_size; - long count; - struct mutator *deactivated_mutators; }; struct heap { struct mark_space mark_space; + pthread_mutex_t lock; + pthread_cond_t collector_cond; + pthread_cond_t mutator_cond; + size_t size; + int collecting; + int multithreaded; + size_t active_mutator_count; + size_t mutator_count; + struct handle *global_roots; + struct mutator_mark_buf *mutator_roots; + long count; + struct mutator *deactivated_mutators; struct tracer tracer; }; @@ -157,9 +158,6 @@ static inline struct mark_space* heap_mark_space(struct heap *heap) { static inline struct heap* mutator_heap(struct mutator *mutator) { return mutator->heap; } -static inline struct mark_space* mutator_mark_space(struct mutator *mutator) { - return heap_mark_space(mutator_heap(mutator)); -} static inline struct gcobj_free** get_small_object_freelist(struct gcobj_freelists *freelists, @@ -216,61 +214,59 @@ static void clear_global_freelists(struct mark_space *space) { space->medium_objects = NULL; } -static int space_has_multiple_mutators(struct mark_space *space) { - return atomic_load_explicit(&space->multithreaded, memory_order_relaxed); +static int heap_has_multiple_mutators(struct heap *heap) { + return atomic_load_explicit(&heap->multithreaded, memory_order_relaxed); } -static int mutators_are_stopping(struct mark_space *space) { - return atomic_load_explicit(&space->collecting, memory_order_relaxed); +static int mutators_are_stopping(struct heap *heap) { + return atomic_load_explicit(&heap->collecting, memory_order_relaxed); } -static inline void mark_space_lock(struct mark_space *space) { - pthread_mutex_lock(&space->lock); +static inline void heap_lock(struct heap *heap) { + pthread_mutex_lock(&heap->lock); } -static inline void mark_space_unlock(struct mark_space *space) { - pthread_mutex_unlock(&space->lock); +static inline void heap_unlock(struct heap *heap) { + pthread_mutex_unlock(&heap->lock); } static void add_mutator(struct heap *heap, struct mutator *mut) { mut->heap = heap; - struct mark_space *space = heap_mark_space(heap); - mark_space_lock(space); + heap_lock(heap); // We have no roots. If there is a GC currently in progress, we have // nothing to add. Just wait until it's done. - while (mutators_are_stopping(space)) - pthread_cond_wait(&space->mutator_cond, &space->lock); - if (space->mutator_count == 1) - space->multithreaded = 1; - space->active_mutator_count++; - space->mutator_count++; - mark_space_unlock(space); + while (mutators_are_stopping(heap)) + pthread_cond_wait(&heap->mutator_cond, &heap->lock); + if (heap->mutator_count == 1) + heap->multithreaded = 1; + heap->active_mutator_count++; + heap->mutator_count++; + heap_unlock(heap); } static void remove_mutator(struct heap *heap, struct mutator *mut) { mut->heap = NULL; - struct mark_space *space = heap_mark_space(heap); - mark_space_lock(space); - space->active_mutator_count--; - space->mutator_count--; + heap_lock(heap); + heap->active_mutator_count--; + heap->mutator_count--; // We have no roots. If there is a GC stop currently in progress, // maybe tell the controller it can continue. - if (mutators_are_stopping(space) && space->active_mutator_count == 0) - pthread_cond_signal(&space->collector_cond); - mark_space_unlock(space); + if (mutators_are_stopping(heap) && heap->active_mutator_count == 0) + pthread_cond_signal(&heap->collector_cond); + heap_unlock(heap); } -static void request_mutators_to_stop(struct mark_space *space) { - ASSERT(!mutators_are_stopping(space)); - atomic_store_explicit(&space->collecting, 1, memory_order_relaxed); +static void request_mutators_to_stop(struct heap *heap) { + ASSERT(!mutators_are_stopping(heap)); + atomic_store_explicit(&heap->collecting, 1, memory_order_relaxed); } -static void allow_mutators_to_continue(struct mark_space *space) { - ASSERT(mutators_are_stopping(space)); - ASSERT(space->active_mutator_count == 0); - space->active_mutator_count++; - atomic_store_explicit(&space->collecting, 0, memory_order_relaxed); - ASSERT(!mutators_are_stopping(space)); - pthread_cond_broadcast(&space->mutator_cond); +static void allow_mutators_to_continue(struct heap *heap) { + ASSERT(mutators_are_stopping(heap)); + ASSERT(heap->active_mutator_count == 0); + heap->active_mutator_count++; + atomic_store_explicit(&heap->collecting, 0, memory_order_relaxed); + ASSERT(!mutators_are_stopping(heap)); + pthread_cond_broadcast(&heap->mutator_cond); } static void mutator_mark_buf_grow(struct mutator_mark_buf *buf) { @@ -318,7 +314,6 @@ static void mutator_mark_buf_destroy(struct mutator_mark_buf *buf) { // enqueue them directly, so we send them to the controller in a buffer. static void mark_stopping_mutator_roots(struct mutator *mut) { struct heap *heap = mutator_heap(mut); - struct mark_space *space = heap_mark_space(heap); struct mutator_mark_buf *local_roots = &mut->mark_buf; for (struct handle *h = mut->roots; h; h = h->next) { struct gcobj *root = h->v; @@ -328,10 +323,10 @@ static void mark_stopping_mutator_roots(struct mutator *mut) { // Post to global linked-list of thread roots. struct mutator_mark_buf *next = - atomic_load_explicit(&space->mutator_roots, memory_order_acquire); + atomic_load_explicit(&heap->mutator_roots, memory_order_acquire); do { local_roots->next = next; - } while (!atomic_compare_exchange_weak(&space->mutator_roots, + } while (!atomic_compare_exchange_weak(&heap->mutator_roots, &next, local_roots)); } @@ -349,77 +344,76 @@ static void release_stopping_mutator_roots(struct mutator *mut) { mutator_mark_buf_release(&mut->mark_buf); } -static void wait_for_mutators_to_stop(struct mark_space *space) { - space->active_mutator_count--; - while (space->active_mutator_count) - pthread_cond_wait(&space->collector_cond, &space->lock); +static void wait_for_mutators_to_stop(struct heap *heap) { + heap->active_mutator_count--; + while (heap->active_mutator_count) + pthread_cond_wait(&heap->collector_cond, &heap->lock); } -static void mark_inactive_mutators(struct mark_space *space) { - for (struct mutator *mut = space->deactivated_mutators; mut; mut = mut->next) +static void mark_inactive_mutators(struct heap *heap) { + for (struct mutator *mut = heap->deactivated_mutators; mut; mut = mut->next) mark_controlling_mutator_roots(mut); } static void mark_global_roots(struct heap *heap) { - struct mark_space *space = heap_mark_space(heap); - for (struct handle *h = space->global_roots; h; h = h->next) { + for (struct handle *h = heap->global_roots; h; h = h->next) { struct gcobj *obj = h->v; if (obj && trace_object(heap, obj)) tracer_enqueue_root(&heap->tracer, obj); } - struct mutator_mark_buf *roots = atomic_load(&space->mutator_roots); + struct mutator_mark_buf *roots = atomic_load(&heap->mutator_roots); for (; roots; roots = roots->next) tracer_enqueue_roots(&heap->tracer, roots->objects, roots->size); - atomic_store(&space->mutator_roots, NULL); + atomic_store(&heap->mutator_roots, NULL); } -static void pause_mutator_for_collection(struct mark_space *space) NEVER_INLINE; -static void pause_mutator_for_collection(struct mark_space *space) { - ASSERT(mutators_are_stopping(space)); - ASSERT(space->active_mutator_count); - space->active_mutator_count--; - if (space->active_mutator_count == 0) - pthread_cond_signal(&space->collector_cond); +static void pause_mutator_for_collection(struct heap *heap) NEVER_INLINE; +static void pause_mutator_for_collection(struct heap *heap) { + ASSERT(mutators_are_stopping(heap)); + ASSERT(heap->active_mutator_count); + heap->active_mutator_count--; + if (heap->active_mutator_count == 0) + pthread_cond_signal(&heap->collector_cond); // Go to sleep and wake up when the collector is done. Note, // however, that it may be that some other mutator manages to // trigger collection before we wake up. In that case we need to // mark roots, not just sleep again. To detect a wakeup on this // collection vs a future collection, we use the global GC count. - // This is safe because the count is protected by the space lock, + // This is safe because the count is protected by the heap lock, // which we hold. - long epoch = space->count; + long epoch = heap->count; do - pthread_cond_wait(&space->mutator_cond, &space->lock); - while (mutators_are_stopping(space) && space->count == epoch); + pthread_cond_wait(&heap->mutator_cond, &heap->lock); + while (mutators_are_stopping(heap) && heap->count == epoch); - space->active_mutator_count++; + heap->active_mutator_count++; } static void pause_mutator_for_collection_with_lock(struct mutator *mut) NEVER_INLINE; static void pause_mutator_for_collection_with_lock(struct mutator *mut) { - struct mark_space *space = mutator_mark_space(mut); - ASSERT(mutators_are_stopping(space)); + struct heap *heap = mutator_heap(mut); + ASSERT(mutators_are_stopping(heap)); mark_controlling_mutator_roots(mut); - pause_mutator_for_collection(space); + pause_mutator_for_collection(heap); clear_mutator_freelists(mut); } static void pause_mutator_for_collection_without_lock(struct mutator *mut) NEVER_INLINE; static void pause_mutator_for_collection_without_lock(struct mutator *mut) { - struct mark_space *space = mutator_mark_space(mut); - ASSERT(mutators_are_stopping(space)); + struct heap *heap = mutator_heap(mut); + ASSERT(mutators_are_stopping(heap)); mark_stopping_mutator_roots(mut); - mark_space_lock(space); - pause_mutator_for_collection(space); - mark_space_unlock(space); + heap_lock(heap); + pause_mutator_for_collection(heap); + heap_unlock(heap); release_stopping_mutator_roots(mut); clear_mutator_freelists(mut); } static inline void maybe_pause_mutator_for_collection(struct mutator *mut) { - while (mutators_are_stopping(mutator_mark_space(mut))) + while (mutators_are_stopping(mutator_heap(mut))) pause_mutator_for_collection_without_lock(mut); } @@ -429,19 +423,19 @@ static void reset_sweeper(struct mark_space *space) { static void collect(struct heap *heap, struct mutator *mut) { struct mark_space *space = heap_mark_space(heap); - DEBUG("start collect #%ld:\n", space->count); + DEBUG("start collect #%ld:\n", heap->count); tracer_prepare(heap); - request_mutators_to_stop(space); + request_mutators_to_stop(heap); mark_controlling_mutator_roots(mut); - wait_for_mutators_to_stop(space); - mark_inactive_mutators(space); + wait_for_mutators_to_stop(heap); + mark_inactive_mutators(heap); mark_global_roots(heap); tracer_trace(heap); tracer_release(heap); clear_global_freelists(space); reset_sweeper(space); - space->count++; - allow_mutators_to_continue(space); + heap->count++; + allow_mutators_to_continue(heap); clear_mutator_freelists(mut); DEBUG("collect done\n"); } @@ -618,15 +612,16 @@ static int sweep(struct mark_space *space, static void* allocate_medium(struct mutator *mut, enum alloc_kind kind, size_t granules) { - struct mark_space *space = mutator_mark_space(mut); - struct gcobj_freelists *small_objects = space_has_multiple_mutators(space) ? + struct heap *heap = mutator_heap(mut); + struct mark_space *space = heap_mark_space(heap); + struct gcobj_freelists *small_objects = heap_has_multiple_mutators(heap) ? &space->small_objects : &mut->small_objects; maybe_pause_mutator_for_collection(mut); - mark_space_lock(space); + heap_lock(heap); - while (mutators_are_stopping(space)) + while (mutators_are_stopping(heap)) pause_mutator_for_collection_with_lock(mut); int swept_from_beginning = 0; @@ -640,7 +635,7 @@ static void* allocate_medium(struct mutator *mut, enum alloc_kind kind, if (medium->granules >= granules) { unlink_medium_object(prev, medium); split_medium_object(space, small_objects, medium, granules); - mark_space_unlock(space); + heap_unlock(heap); struct gcobj *obj = (struct gcobj *)medium; obj->tag = tag_live(kind); return medium; @@ -651,7 +646,7 @@ static void* allocate_medium(struct mutator *mut, enum alloc_kind kind, // No medium object, and we swept across the whole heap. Collect. if (swept_from_beginning) { - fprintf(stderr, "ran out of space, heap size %zu\n", space->heap_size); + fprintf(stderr, "ran out of space, heap size %zu\n", heap->size); abort(); } else { collect(mutator_heap(mut), mut); @@ -682,7 +677,7 @@ static int fill_small_from_local(struct gcobj_freelists *small_objects, return 0; } -// with space lock +// with heap lock static int fill_small_from_medium(struct mark_space *space, struct gcobj_freelists *small_objects, enum small_object_size kind) { @@ -719,13 +714,14 @@ static void fill_small_from_global(struct mutator *mut, static void fill_small_from_global(struct mutator *mut, enum small_object_size kind) { struct gcobj_freelists *small_objects = &mut->small_objects; - struct mark_space *space = mutator_mark_space(mut); + struct heap *heap = mutator_heap(mut); + struct mark_space *space = heap_mark_space(heap); maybe_pause_mutator_for_collection(mut); - mark_space_lock(space); + heap_lock(heap); - while (mutators_are_stopping(space)) + while (mutators_are_stopping(heap)) pause_mutator_for_collection_with_lock(mut); int swept_from_beginning = 0; @@ -739,7 +735,7 @@ static void fill_small_from_global(struct mutator *mut, // By default, pull in 16 kB of data at a time. if (!sweep(space, small_objects, kind, 0)) { if (swept_from_beginning) { - fprintf(stderr, "ran out of space, heap size %zu\n", space->heap_size); + fprintf(stderr, "ran out of space, heap size %zu\n", heap->size); abort(); } else { collect(mutator_heap(mut), mut); @@ -750,7 +746,7 @@ static void fill_small_from_global(struct mutator *mut, if (*get_small_object_freelist(small_objects, kind)) break; } - mark_space_unlock(space); + heap_unlock(heap); } static void fill_small(struct mutator *mut, enum small_object_size kind) { @@ -799,18 +795,8 @@ static inline void* get_field(void **addr) { return *addr; } -static int initialize_gc(size_t size, struct heap **heap, - struct mutator **mut) { -#define SMALL_OBJECT_GRANULE_SIZE(i) \ - ASSERT_EQ(SMALL_OBJECT_##i, small_object_sizes_for_granules[i]); \ - ASSERT_EQ(SMALL_OBJECT_##i + 1, small_object_sizes_for_granules[i+1]); - FOR_EACH_SMALL_OBJECT_GRANULES(SMALL_OBJECT_GRANULE_SIZE); -#undef SMALL_OBJECT_GRANULE_SIZE - - ASSERT_EQ(SMALL_OBJECT_SIZES - 1, - small_object_sizes_for_granules[MEDIUM_OBJECT_GRANULE_THRESHOLD]); - - size = align_up(size, getpagesize()); +static int mark_space_init(struct mark_space *space, struct heap *heap) { + size_t size = align_up(heap->size, getpagesize()); void *mem = mmap(NULL, size, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); @@ -819,9 +805,6 @@ static int initialize_gc(size_t size, struct heap **heap, return 0; } - *heap = calloc(1, sizeof(struct heap)); - if (!*heap) abort(); - struct mark_space *space = heap_mark_space(*heap); space->mem = mem; space->mem_size = size; // If there is 1 mark byte per granule, and SIZE bytes available for @@ -834,18 +817,43 @@ static int initialize_gc(size_t size, struct heap **heap, size_t mark_bytes_size = (size + GRANULE_SIZE) / (GRANULE_SIZE + 1); size_t overhead = align_up(mark_bytes_size, GRANULE_SIZE); - pthread_mutex_init(&space->lock, NULL); - pthread_cond_init(&space->mutator_cond, NULL); - pthread_cond_init(&space->collector_cond, NULL); - space->heap_base = ((uintptr_t) mem) + overhead; space->heap_size = size - overhead; space->sweep = space->heap_base + space->heap_size; - if (!tracer_init(*heap)) - abort(); reclaim(space, NULL, NOT_SMALL_OBJECT, (void*)space->heap_base, size_to_granules(space->heap_size)); + return 1; +} +static int initialize_gc(size_t size, struct heap **heap, + struct mutator **mut) { +#define SMALL_OBJECT_GRANULE_SIZE(i) \ + ASSERT_EQ(SMALL_OBJECT_##i, small_object_sizes_for_granules[i]); \ + ASSERT_EQ(SMALL_OBJECT_##i + 1, small_object_sizes_for_granules[i+1]); + FOR_EACH_SMALL_OBJECT_GRANULES(SMALL_OBJECT_GRANULE_SIZE); +#undef SMALL_OBJECT_GRANULE_SIZE + + ASSERT_EQ(SMALL_OBJECT_SIZES - 1, + small_object_sizes_for_granules[MEDIUM_OBJECT_GRANULE_THRESHOLD]); + + *heap = calloc(1, sizeof(struct heap)); + if (!*heap) abort(); + + pthread_mutex_init(&(*heap)->lock, NULL); + pthread_cond_init(&(*heap)->mutator_cond, NULL); + pthread_cond_init(&(*heap)->collector_cond, NULL); + (*heap)->size = size; + + if (!tracer_init(*heap)) + abort(); + + struct mark_space *space = heap_mark_space(*heap); + if (!mark_space_init(space, *heap)) { + free(*heap); + *heap = NULL; + return 0; + } + *mut = calloc(1, sizeof(struct mutator)); if (!*mut) abort(); add_mutator(*heap, *mut); @@ -867,28 +875,28 @@ static void finish_gc_for_thread(struct mutator *mut) { free(mut); } -static void deactivate_mutator(struct mark_space *space, struct mutator *mut) { +static void deactivate_mutator(struct heap *heap, struct mutator *mut) { ASSERT(mut->next == NULL); - mark_space_lock(space); - mut->next = space->deactivated_mutators; - space->deactivated_mutators = mut; - space->active_mutator_count--; - if (!space->active_mutator_count && mutators_are_stopping(space)) - pthread_cond_signal(&space->collector_cond); - mark_space_unlock(space); + heap_lock(heap); + mut->next = heap->deactivated_mutators; + heap->deactivated_mutators = mut; + heap->active_mutator_count--; + if (!heap->active_mutator_count && mutators_are_stopping(heap)) + pthread_cond_signal(&heap->collector_cond); + heap_unlock(heap); } -static void reactivate_mutator(struct mark_space *space, struct mutator *mut) { - mark_space_lock(space); - while (mutators_are_stopping(space)) - pthread_cond_wait(&space->mutator_cond, &space->lock); - struct mutator **prev = &space->deactivated_mutators; +static void reactivate_mutator(struct heap *heap, struct mutator *mut) { + heap_lock(heap); + while (mutators_are_stopping(heap)) + pthread_cond_wait(&heap->mutator_cond, &heap->lock); + struct mutator **prev = &heap->deactivated_mutators; while (*prev != mut) prev = &(*prev)->next; *prev = mut->next; mut->next = NULL; - space->active_mutator_count++; - mark_space_unlock(space); + heap->active_mutator_count++; + heap_unlock(heap); } static void* call_without_gc(struct mutator *mut, void* (*f)(void*), @@ -896,10 +904,10 @@ static void* call_without_gc(struct mutator *mut, void* (*f)(void*), static void* call_without_gc(struct mutator *mut, void* (*f)(void*), void *data) { - struct mark_space *space = mutator_mark_space(mut); - deactivate_mutator(space, mut); + struct heap *heap = mutator_heap(mut); + deactivate_mutator(heap, mut); void *ret = f(data); - reactivate_mutator(space, mut); + reactivate_mutator(heap, mut); return ret; } @@ -907,6 +915,6 @@ static inline void print_start_gc_stats(struct heap *heap) { } static inline void print_end_gc_stats(struct heap *heap) { - printf("Completed %ld collections\n", heap_mark_space(heap)->count); - printf("Heap size with overhead is %zd\n", heap_mark_space(heap)->mem_size); + printf("Completed %ld collections\n", heap->count); + printf("Heap size with overhead is %zd\n", heap->size); }