From 41de2dfd91d6301c271da605055632edef41ce74 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Wed, 21 May 2025 14:27:17 +0200 Subject: [PATCH] Update conservative roots embedder interface --- api/gc-embedder-api.h | 38 +++++++++++--------- benchmarks/simple-gc-embedder.h | 38 +++++++++++--------- src/bdw.c | 20 +++++++---- src/mmc.c | 63 +++++++++++++++++++++++++++------ src/nofl-space.h | 13 +++++++ src/root.h | 12 +++---- 6 files changed, 129 insertions(+), 55 deletions(-) diff --git a/api/gc-embedder-api.h b/api/gc-embedder-api.h index ec630c26a..012ec5bd6 100644 --- a/api/gc-embedder-api.h +++ b/api/gc-embedder-api.h @@ -50,23 +50,29 @@ GC_EMBEDDER_API inline void gc_trace_heap_roots(struct gc_heap_roots *roots, void *trace_data); GC_EMBEDDER_API inline void -gc_trace_mutator_conservative_roots(struct gc_mutator_roots *roots, - void (*trace_range)(uintptr_t start, - uintptr_t end, - int possibly_interior, - struct gc_heap *heap, - void *data), - struct gc_heap *heap, - void *data); +gc_trace_mutator_pinned_roots(struct gc_mutator_roots *roots, + void (*trace_pinned)(struct gc_ref ref, + struct gc_heap *heap, + void *data), + void (*trace_ambiguous)(uintptr_t start, + uintptr_t end, + int possibly_interior, + struct gc_heap *heap, + void *data), + struct gc_heap *heap, + void *data); GC_EMBEDDER_API inline void -gc_trace_heap_conservative_roots(struct gc_heap_roots *roots, - void (*trace_range)(uintptr_t start, - uintptr_t end, - int possibly_interior, - struct gc_heap *heap, - void *data), - struct gc_heap *heap, - void *data); +gc_trace_heap_pinned_roots(struct gc_heap_roots *roots, + void (*trace_pinned)(struct gc_ref ref, + struct gc_heap *heap, + void *data), + void (*trace_ambiguous)(uintptr_t start, + uintptr_t end, + int possibly_interior, + struct gc_heap *heap, + void *data), + struct gc_heap *heap, + void *data); GC_EMBEDDER_API inline uintptr_t gc_object_forwarded_nonatomic(struct gc_ref ref); GC_EMBEDDER_API inline void gc_object_forward_nonatomic(struct gc_ref ref, diff --git a/benchmarks/simple-gc-embedder.h b/benchmarks/simple-gc-embedder.h index 4ad7bb8b2..bf9f4e91f 100644 --- a/benchmarks/simple-gc-embedder.h +++ b/benchmarks/simple-gc-embedder.h @@ -92,24 +92,30 @@ static inline void gc_trace_heap_roots(struct gc_heap_roots *roots, } static inline void -gc_trace_mutator_conservative_roots(struct gc_mutator_roots *roots, - void (*trace_range)(uintptr_t start, - uintptr_t end, - int possibly_interior, - struct gc_heap *heap, - void *data), - struct gc_heap *heap, - void *data) {} +gc_trace_mutator_pinned_roots(struct gc_mutator_roots *roots, + void (*trace_pinned)(struct gc_ref ref, + struct gc_heap *heap, + void *data), + void (*trace_ambiguous)(uintptr_t start, + uintptr_t end, + int possibly_interior, + struct gc_heap *heap, + void *data), + struct gc_heap *heap, + void *data) {} static inline void -gc_trace_heap_conservative_roots(struct gc_heap_roots *roots, - void (*trace_range)(uintptr_t start, - uintptr_t end, - int possibly_interior, - struct gc_heap *heap, - void *data), - struct gc_heap *heap, - void *data) {} +gc_trace_heap_pinned_roots(struct gc_heap_roots *roots, + void (*trace_pinned)(struct gc_ref ref, + struct gc_heap *heap, + void *data), + void (*trace_ambiguous)(uintptr_t start, + uintptr_t end, + int possibly_interior, + struct gc_heap *heap, + void *data), + struct gc_heap *heap, + void *data) {} static inline uintptr_t gc_object_forwarded_nonatomic(struct gc_ref ref) { uintptr_t tag = *tag_word(ref); diff --git a/src/bdw.c b/src/bdw.c index 8cee58a73..980626f6b 100644 --- a/src/bdw.c +++ b/src/bdw.c @@ -5,6 +5,7 @@ #define GC_IMPL 1 +#include "gc-align.h" #include "gc-api.h" #include "gc-ephemeron.h" #include "gc-tracepoint.h" @@ -218,16 +219,20 @@ struct bdw_mark_state { struct GC_ms_entry *mark_stack_limit; }; -static void bdw_mark_edge(struct gc_edge edge, struct gc_heap *heap, - void *visit_data) { +static void bdw_mark(struct gc_ref ref, struct gc_heap *heap, + void *visit_data) { struct bdw_mark_state *state = visit_data; - uintptr_t addr = gc_ref_value(gc_edge_ref(edge)); - state->mark_stack_ptr = GC_MARK_AND_PUSH ((void *) addr, + state->mark_stack_ptr = GC_MARK_AND_PUSH ((void *) gc_ref_value(ref), state->mark_stack_ptr, state->mark_stack_limit, NULL); } +static void bdw_mark_edge(struct gc_edge edge, struct gc_heap *heap, + void *visit_data) { + bdw_mark(gc_edge_ref(edge), heap, visit_data); +} + static void bdw_mark_range(uintptr_t lo, uintptr_t hi, int possibly_interior, struct gc_heap *heap, void *visit_data) { struct bdw_mark_state *state = visit_data; @@ -395,7 +400,8 @@ mark_heap(GC_word *addr, struct GC_ms_entry *mark_stack_ptr, return state.mark_stack_ptr; if (heap->roots) { - gc_trace_heap_conservative_roots(heap->roots, bdw_mark_range, heap, &state); + gc_trace_heap_pinned_roots(heap->roots, bdw_mark, bdw_mark_range, + heap, &state); gc_trace_heap_roots(heap->roots, bdw_mark_edge, heap, &state); } @@ -432,8 +438,8 @@ mark_mutator(GC_word *addr, struct GC_ms_entry *mark_stack_ptr, memset(mut->freelists, 0, sizeof(void*) * GC_INLINE_FREELIST_COUNT); if (mut->roots) { - gc_trace_mutator_conservative_roots(mut->roots, bdw_mark_range, - mut->heap, &state); + gc_trace_mutator_pinned_roots(mut->roots, bdw_mark, bdw_mark_range, + mut->heap, &state); gc_trace_mutator_roots(mut->roots, bdw_mark_edge, mut->heap, &state); } diff --git a/src/mmc.c b/src/mmc.c index 86b949f03..4bbfed019 100644 --- a/src/mmc.c +++ b/src/mmc.c @@ -166,6 +166,34 @@ trace_edge(struct gc_heap *heap, struct gc_edge edge, return is_new; } +static inline int +do_trace_pinned(struct gc_heap *heap, struct gc_ref ref, + struct gc_trace_worker_data *data) { + if (GC_LIKELY(nofl_space_contains(heap_nofl_space(heap), ref))) + return nofl_space_mark_object(heap_nofl_space(heap), ref, &data->allocator); + else if (large_object_space_contains_with_lock(heap_large_object_space(heap), + ref)) + return large_object_space_mark(heap_large_object_space(heap), ref); + else + return gc_extern_space_visit(heap_extern_space(heap), ref); +} + +static inline int +trace_pinned_edge(struct gc_heap *heap, struct gc_ref ref, + struct gc_trace_worker_data *data) { + if (gc_ref_is_null(ref) || gc_ref_is_immediate(ref)) + return 0; + + int is_new = do_trace_pinned(heap, ref, data); + + if (is_new && + GC_UNLIKELY(atomic_load_explicit(&heap->check_pending_ephemerons, + memory_order_relaxed))) + gc_resolve_pending_ephemerons(ref, heap); + + return is_new; +} + int gc_visit_ephemeron_key(struct gc_edge edge, struct gc_heap *heap) { struct gc_ref ref = gc_edge_ref(edge); @@ -274,6 +302,17 @@ tracer_visit(struct gc_edge edge, struct gc_heap *heap, void *trace_data) { gc_trace_worker_enqueue(worker, gc_edge_ref(edge)); } +static inline void +tracer_visit_pinned_root(struct gc_ref ref, struct gc_heap *heap, + void *trace_data) GC_ALWAYS_INLINE; +static inline void +tracer_visit_pinned_root(struct gc_ref ref, struct gc_heap *heap, + void *trace_data) { + struct gc_trace_worker *worker = trace_data; + if (trace_pinned_edge(heap, ref, gc_trace_worker_data(worker))) + gc_trace_worker_enqueue(worker, ref); +} + static inline int trace_remembered_edge(struct gc_edge edge, struct gc_heap *heap, void *trace_data) { tracer_visit(edge, heap, trace_data); @@ -331,8 +370,8 @@ static inline void trace_conservative_edges(uintptr_t low, uintptr_t high, int possibly_interior, struct gc_heap *heap, void *data) { struct gc_trace_worker *worker = data; - GC_ASSERT(low == align_down(low, sizeof(uintptr_t))); - GC_ASSERT(high == align_down(high, sizeof(uintptr_t))); + GC_ASSERT_EQ(low, align_down(low, sizeof(uintptr_t))); + GC_ASSERT_EQ(high, align_down(high, sizeof(uintptr_t))); for (uintptr_t addr = low; addr < high; addr += sizeof(uintptr_t)) tracer_trace_conservative_ref(load_conservative_ref(addr), heap, worker, possibly_interior); @@ -404,13 +443,17 @@ trace_root(struct gc_root root, struct gc_heap *heap, gc_field_set_visit_edge_buffer(&heap->remembered_set, root.edge_buffer, trace_remembered_edge, heap, worker); break; - case GC_ROOT_KIND_HEAP_CONSERVATIVE_ROOTS: - gc_trace_heap_conservative_roots(root.heap->roots, trace_conservative_edges, - heap, worker); + case GC_ROOT_KIND_HEAP_PINNED_ROOTS: + gc_trace_heap_pinned_roots(root.heap->roots, + tracer_visit_pinned_root, + trace_conservative_edges, + heap, worker); break; - case GC_ROOT_KIND_MUTATOR_CONSERVATIVE_ROOTS: - gc_trace_mutator_conservative_roots(root.mutator->roots, - trace_conservative_edges, heap, worker); + case GC_ROOT_KIND_MUTATOR_PINNED_ROOTS: + gc_trace_mutator_pinned_roots(root.mutator->roots, + tracer_visit_pinned_root, + trace_conservative_edges, + heap, worker); break; default: GC_CRASH(); @@ -671,7 +714,7 @@ enqueue_mutator_conservative_roots(struct gc_heap *heap) { &possibly_interior); if (mut->roots) gc_tracer_add_root(&heap->tracer, - gc_root_mutator_conservative_roots(mut)); + gc_root_mutator_pinned_roots(mut)); } return 1; } @@ -685,7 +728,7 @@ enqueue_global_conservative_roots(struct gc_heap *heap) { gc_platform_visit_global_conservative_roots (enqueue_conservative_roots, heap, &possibly_interior); if (heap->roots) - gc_tracer_add_root(&heap->tracer, gc_root_heap_conservative_roots(heap)); + gc_tracer_add_root(&heap->tracer, gc_root_heap_pinned_roots(heap)); return 1; } return 0; diff --git a/src/nofl-space.h b/src/nofl-space.h index 1c0f36d5d..49068e941 100644 --- a/src/nofl-space.h +++ b/src/nofl-space.h @@ -1617,6 +1617,19 @@ nofl_space_evacuate_or_mark_object(struct nofl_space *space, return nofl_space_set_nonempty_mark(space, metadata, byte, old_ref); } +static inline int +nofl_space_mark_object(struct nofl_space *space, struct gc_ref ref, + struct nofl_allocator *evacuate) { + uint8_t *metadata = nofl_metadata_byte_for_object(ref); + uint8_t byte = *metadata; + if (nofl_metadata_byte_has_mark(byte, space->current_mark)) + return 0; + + GC_ASSERT(!nofl_space_should_evacuate(space, byte, ref)); + + return nofl_space_set_nonempty_mark(space, metadata, byte, ref); +} + static inline int nofl_space_forward_if_evacuated(struct nofl_space *space, struct gc_edge edge, diff --git a/src/root.h b/src/root.h index 597e8d840..8bea07408 100644 --- a/src/root.h +++ b/src/root.h @@ -18,8 +18,8 @@ enum gc_root_kind { GC_ROOT_KIND_RESOLVED_EPHEMERONS, GC_ROOT_KIND_EDGE, GC_ROOT_KIND_EDGE_BUFFER, - GC_ROOT_KIND_HEAP_CONSERVATIVE_ROOTS, - GC_ROOT_KIND_MUTATOR_CONSERVATIVE_ROOTS, + GC_ROOT_KIND_HEAP_PINNED_ROOTS, + GC_ROOT_KIND_MUTATOR_PINNED_ROOTS, }; struct gc_root { @@ -81,15 +81,15 @@ gc_root_edge_buffer(struct gc_edge_buffer *buf) { } static inline struct gc_root -gc_root_heap_conservative_roots(struct gc_heap* heap) { - struct gc_root ret = { GC_ROOT_KIND_HEAP_CONSERVATIVE_ROOTS }; +gc_root_heap_pinned_roots(struct gc_heap* heap) { + struct gc_root ret = { GC_ROOT_KIND_HEAP_PINNED_ROOTS }; ret.heap = heap; return ret; } static inline struct gc_root -gc_root_mutator_conservative_roots(struct gc_mutator* mutator) { - struct gc_root ret = { GC_ROOT_KIND_MUTATOR_CONSERVATIVE_ROOTS }; +gc_root_mutator_pinned_roots(struct gc_mutator* mutator) { + struct gc_root ret = { GC_ROOT_KIND_MUTATOR_PINNED_ROOTS }; ret.mutator = mutator; return ret; }