From deed415a0631a12566e1d4d6bb6200edc81e2e8b Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 21 Sep 2022 10:33:31 +0200 Subject: [PATCH] Whippet captures stack when stopping mutators This is part of work to enable conservative GC. --- conservative-roots-embedder.h | 24 +++++--------- gc-embedder-api.h | 12 ++----- precise-roots-embedder.h | 21 +++++------- whippet.c | 60 ++++++++++++++++++++++++----------- 4 files changed, 60 insertions(+), 57 deletions(-) diff --git a/conservative-roots-embedder.h b/conservative-roots-embedder.h index 5b3d6fba9..2ac2d2b78 100644 --- a/conservative-roots-embedder.h +++ b/conservative-roots-embedder.h @@ -1,35 +1,27 @@ #ifndef CONSERVATIVE_ROOTS_EMBEDDER_H #define CONSERVATIVE_ROOTS_EMBEDDER_H -#include "gc-assert.h" -#include "conservative-roots-types.h" +#include "gc-embedder-api.h" -static inline int gc_has_conservative_roots(void) { +static inline int gc_has_mutator_conservative_roots(void) { + return 1; +} +static inline int gc_mutator_conservative_roots_may_be_interior(void) { + return 1; +} +static inline int gc_has_global_conservative_roots(void) { return 1; } static inline int gc_has_conservative_intraheap_edges(void) { - // FIXME: Implement both ways. return 0; } -static inline void gc_trace_conservative_mutator_roots(struct gc_mutator_roots *roots, - void (*trace_ref)(struct gc_ref edge, - void *trace_data), - void *trace_data) { -} - static inline void gc_trace_precise_mutator_roots(struct gc_mutator_roots *roots, void (*trace_edge)(struct gc_edge edge, void *trace_data), void *trace_data) { } -static inline void gc_trace_conservative_heap_roots(struct gc_heap_roots *roots, - void (*trace_ref)(struct gc_ref ref, - void *trace_data), - void *trace_data) { -} - static inline void gc_trace_precise_heap_roots(struct gc_heap_roots *roots, void (*trace_edge)(struct gc_edge edge, void *trace_data), diff --git a/gc-embedder-api.h b/gc-embedder-api.h index 2e0029e97..b24483245 100644 --- a/gc-embedder-api.h +++ b/gc-embedder-api.h @@ -12,18 +12,16 @@ struct gc_mutator_roots; struct gc_heap_roots; struct gc_atomic_forward; -GC_EMBEDDER_API inline int gc_has_conservative_roots(void); +GC_EMBEDDER_API inline int gc_has_mutator_conservative_roots(void); +GC_EMBEDDER_API inline int gc_has_global_conservative_roots(void); GC_EMBEDDER_API inline int gc_has_conservative_intraheap_edges(void); +GC_EMBEDDER_API inline int gc_mutator_conservative_roots_may_be_interior(void); GC_EMBEDDER_API inline void gc_trace_object(struct gc_ref ref, void (*trace_edge)(struct gc_edge edge, void *trace_data), void *trace_data, size_t *size) GC_ALWAYS_INLINE; -GC_EMBEDDER_API inline void gc_trace_conservative_mutator_roots(struct gc_mutator_roots *roots, - void (*trace_ref)(struct gc_ref edge, - void *trace_data), - void *trace_data); GC_EMBEDDER_API inline void gc_trace_precise_mutator_roots(struct gc_mutator_roots *roots, void (*trace_edge)(struct gc_edge edge, void *trace_data), @@ -32,10 +30,6 @@ GC_EMBEDDER_API inline void gc_trace_precise_heap_roots(struct gc_heap_roots *ro void (*trace_edge)(struct gc_edge edge, void *trace_data), void *trace_data); -GC_EMBEDDER_API inline void gc_trace_conservative_heap_roots(struct gc_heap_roots *roots, - void (*trace_ref)(struct gc_ref ref, - void *trace_data), - void *trace_data); GC_EMBEDDER_API inline uintptr_t gc_object_forwarded_nonatomic(struct gc_ref ref); GC_EMBEDDER_API inline void gc_object_forward_nonatomic(struct gc_ref ref, diff --git a/precise-roots-embedder.h b/precise-roots-embedder.h index 8b4deb481..cf649e14d 100644 --- a/precise-roots-embedder.h +++ b/precise-roots-embedder.h @@ -2,9 +2,16 @@ #define PRECISE_ROOTS_EMBEDDER_H #include "gc-edge.h" +#include "gc-embedder-api.h" #include "precise-roots-types.h" -static inline int gc_has_conservative_roots(void) { +static inline int gc_has_mutator_conservative_roots(void) { + return 0; +} +static inline int gc_mutator_conservative_roots_may_be_interior(void) { + return 0; +} +static inline int gc_has_global_conservative_roots(void) { return 0; } static inline int gc_has_conservative_intraheap_edges(void) { @@ -19,12 +26,6 @@ static inline void visit_roots(struct handle *roots, trace_edge(gc_edge(&h->v), trace_data); } -static inline void gc_trace_conservative_mutator_roots(struct gc_mutator_roots *roots, - void (*trace_ref)(struct gc_ref edge, - void *trace_data), - void *trace_data) { -} - static inline void gc_trace_precise_mutator_roots(struct gc_mutator_roots *roots, void (*trace_edge)(struct gc_edge edge, void *trace_data), @@ -33,12 +34,6 @@ static inline void gc_trace_precise_mutator_roots(struct gc_mutator_roots *roots visit_roots(roots->roots, trace_edge, trace_data); } -static inline void gc_trace_conservative_heap_roots(struct gc_heap_roots *roots, - void (*trace_ref)(struct gc_ref ref, - void *trace_data), - void *trace_data) { -} - static inline void gc_trace_precise_heap_roots(struct gc_heap_roots *roots, void (*trace_edge)(struct gc_edge edge, void *trace_data), diff --git a/whippet.c b/whippet.c index dccb2a470..ad1e02260 100644 --- a/whippet.c +++ b/whippet.c @@ -15,6 +15,7 @@ #include "debug.h" #include "gc-align.h" #include "gc-inline.h" +#include "gc-stack.h" #include "large-object-space.h" #if GC_PARALLEL #include "parallel-tracer.h" @@ -27,7 +28,7 @@ #if GC_PRECISE #include "precise-roots-embedder.h" #else -#error whippet only currently implements precise collection +#include "conservative-roots-embedder.h" #endif #define GRANULE_SIZE 16 @@ -340,6 +341,7 @@ struct gc_mutator { uintptr_t sweep; uintptr_t block; struct gc_heap *heap; + struct gc_stack stack; struct gc_mutator_roots *roots; struct gc_mutator_mark_buf mark_buf; // Three uses for this in-object linked-list pointer: @@ -914,8 +916,11 @@ static void trace_ref_and_enqueue_globally(struct gc_ref ref, void *data) { // enqueue them directly, so we send them to the controller in a buffer. static void trace_stopping_mutator_roots(struct gc_mutator *mut) { GC_ASSERT(mutator_should_mark_while_stopping(mut)); - gc_trace_conservative_mutator_roots(mut->roots, trace_ref_and_enqueue_locally, - mut); + /* + trace_mutator_conservative_roots(mut, + mark_and_locally_enqueue_conservative_roots, + mut); + */ gc_trace_precise_mutator_roots(mut->roots, trace_and_enqueue_locally, mut); } @@ -924,18 +929,21 @@ static void trace_precise_mutator_roots_with_lock(struct gc_mutator *mut) { mutator_heap(mut)); } -static void trace_conservative_mutator_roots_with_lock(struct gc_mutator *mut) { - gc_trace_conservative_mutator_roots(mut->roots, - trace_ref_and_enqueue_globally, - mutator_heap(mut)); +static void trace_mutator_conservative_roots_with_lock(struct gc_mutator *mut) { + /* + trace_mutator_conservative_roots(mut, + mark_and_globally_enqueue_conservative_roots, + mutator_heap(mut)); + */ } static void trace_mutator_roots_with_lock(struct gc_mutator *mut) { - trace_conservative_mutator_roots_with_lock(mut); + trace_mutator_conservative_roots_with_lock(mut); trace_precise_mutator_roots_with_lock(mut); } static void trace_mutator_roots_with_lock_before_stop(struct gc_mutator *mut) { + gc_stack_capture_hot(&mut->stack); if (mutator_should_mark_while_stopping(mut)) trace_mutator_roots_with_lock(mut); else @@ -955,19 +963,19 @@ static void wait_for_mutators_to_stop(struct gc_heap *heap) { static void finish_sweeping(struct gc_mutator *mut); static void finish_sweeping_in_block(struct gc_mutator *mut); -static void trace_conservative_mutator_roots_after_stop(struct gc_heap *heap) { +static void trace_mutator_conservative_roots_after_stop(struct gc_heap *heap) { int active_mutators_already_marked = heap_should_mark_while_stopping(heap); if (!active_mutators_already_marked) { for (struct gc_mutator *mut = atomic_load(&heap->mutator_trace_list); mut; mut = mut->next) - trace_conservative_mutator_roots_with_lock(mut); + trace_mutator_conservative_roots_with_lock(mut); } for (struct gc_mutator *mut = heap->deactivated_mutators; mut; mut = mut->next) - trace_conservative_mutator_roots_with_lock(mut); + trace_mutator_conservative_roots_with_lock(mut); } static void trace_precise_mutator_roots_after_stop(struct gc_heap *heap) { @@ -998,8 +1006,12 @@ static void trace_precise_global_roots(struct gc_heap *heap) { gc_trace_precise_heap_roots(heap->roots, trace_and_enqueue_globally, heap); } -static void trace_conservative_global_roots(struct gc_heap *heap) { - gc_trace_conservative_heap_roots(heap->roots, trace_ref_and_enqueue_globally, heap); +static void trace_global_conservative_roots(struct gc_heap *heap) { + /* + if (gc_has_global_conservative_roots()) + gc_platform_visit_global_conservative_roots + (mark_and_globally_enqueue_conservative_roots, heap); + */ } static inline uint64_t load_eight_aligned_bytes(uint8_t *mark) { @@ -1111,6 +1123,7 @@ static void pause_mutator_for_collection_with_lock(struct gc_mutator *mut) { struct gc_heap *heap = mutator_heap(mut); GC_ASSERT(mutators_are_stopping(heap)); finish_sweeping_in_block(mut); + gc_stack_capture_hot(&mut->stack); if (mutator_should_mark_while_stopping(mut)) // No need to collect results in mark buf; we can enqueue roots directly. trace_mutator_roots_with_lock(mut); @@ -1124,6 +1137,7 @@ static void pause_mutator_for_collection_without_lock(struct gc_mutator *mut) { struct gc_heap *heap = mutator_heap(mut); GC_ASSERT(mutators_are_stopping(heap)); finish_sweeping(mut); + gc_stack_capture_hot(&mut->stack); if (mutator_should_mark_while_stopping(mut)) trace_stopping_mutator_roots(mut); enqueue_mutator_for_tracing(mut); @@ -1228,6 +1242,11 @@ static double clamp_major_gc_yield_threshold(struct gc_heap *heap, return threshold; } +static inline int has_conservative_roots(void) { + return gc_has_mutator_conservative_roots() || + gc_has_global_conservative_roots(); +} + static enum gc_kind determine_collection_kind(struct gc_heap *heap) { struct mark_space *mark_space = heap_mark_space(heap); enum gc_kind previous_gc_kind = atomic_load(&heap->gc_kind); @@ -1257,7 +1276,7 @@ static enum gc_kind determine_collection_kind(struct gc_heap *heap) { // blocks, try to avoid any pinning caused by the ragged-stop // marking. Of course if the mutator has conservative roots we will // have pinning anyway and might as well allow ragged stops. - mark_while_stopping = gc_has_conservative_roots(); + mark_while_stopping = has_conservative_roots(); } else if (previous_gc_kind == GC_KIND_MAJOR_EVACUATING && fragmentation >= heap->fragmentation_low_threshold) { DEBUG("continuing evacuation due to fragmentation %.2f%% > %.2f%%\n", @@ -1426,15 +1445,15 @@ static void prepare_for_evacuation(struct gc_heap *heap) { static void trace_conservative_roots_after_stop(struct gc_heap *heap) { GC_ASSERT(!heap_mark_space(heap)->evacuating); - GC_ASSERT(gc_has_conservative_roots()); - trace_conservative_mutator_roots_after_stop(heap); - trace_conservative_global_roots(heap); + if (gc_has_mutator_conservative_roots()) + trace_mutator_conservative_roots_after_stop(heap); + if (gc_has_global_conservative_roots()) + trace_global_conservative_roots(heap); } static void trace_pinned_roots_after_stop(struct gc_heap *heap) { GC_ASSERT(!heap_mark_space(heap)->evacuating); - if (gc_has_conservative_roots()) - trace_conservative_roots_after_stop(heap); + trace_conservative_roots_after_stop(heap); } static void trace_precise_roots_after_stop(struct gc_heap *heap) { @@ -2034,6 +2053,7 @@ int gc_init(int argc, struct gc_option argv[], *mut = calloc(1, sizeof(struct gc_mutator)); if (!*mut) GC_CRASH(); + gc_stack_init(&(*mut)->stack, stack_base); add_mutator(*heap, *mut); return 1; } @@ -2043,6 +2063,7 @@ struct gc_mutator* gc_init_for_thread(struct gc_stack_addr *stack_base, struct gc_mutator *ret = calloc(1, sizeof(struct gc_mutator)); if (!ret) GC_CRASH(); + gc_stack_init(&ret->stack, stack_base); add_mutator(heap, ret); return ret; } @@ -2059,6 +2080,7 @@ static void deactivate_mutator(struct gc_heap *heap, struct gc_mutator *mut) { mut->next = heap->deactivated_mutators; heap->deactivated_mutators = mut; heap->active_mutator_count--; + gc_stack_capture_hot(&mut->stack); if (!heap->active_mutator_count && mutators_are_stopping(heap)) pthread_cond_signal(&heap->collector_cond); heap_unlock(heap);