From c49a372da757ef4801f57c5a2b486dacf27ba388 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 27 May 2025 15:50:28 +0200 Subject: [PATCH] Add gc_resolve_conservative_ref API --- api/gc-api.h | 4 +++ src/bdw.c | 19 ++++++++++++ src/large-object-space.h | 33 +++++++++++++++++---- src/mmc.c | 17 +++++++++++ src/nofl-space.h | 64 ++++++++++++++++++++++++++++++---------- src/pcc.c | 6 ++++ src/semi.c | 6 ++++ 7 files changed, 129 insertions(+), 20 deletions(-) diff --git a/api/gc-api.h b/api/gc-api.h index 6d4f9cbe1..b5d279443 100644 --- a/api/gc-api.h +++ b/api/gc-api.h @@ -6,6 +6,7 @@ #include "gc-assert.h" #include "gc-attrs.h" #include "gc-collection-kind.h" +#include "gc-conservative-ref.h" #include "gc-edge.h" #include "gc-event-listener.h" #include "gc-inline.h" @@ -283,6 +284,9 @@ static inline void gc_write_barrier(struct gc_mutator *mut, struct gc_ref obj, gc_write_barrier_slow(mut, obj, obj_size, edge, new_val); } +GC_API_ struct gc_ref gc_resolve_conservative_ref(struct gc_heap *heap, + struct gc_conservative_ref ref, + int possibly_interior); GC_API_ void gc_pin_object(struct gc_mutator *mut, struct gc_ref obj); GC_API_ void gc_safepoint_slow(struct gc_mutator *mut) GC_NEVER_INLINE; diff --git a/src/bdw.c b/src/bdw.c index 980626f6b..7709b16de 100644 --- a/src/bdw.c +++ b/src/bdw.c @@ -8,6 +8,7 @@ #include "gc-align.h" #include "gc-api.h" #include "gc-ephemeron.h" +#include "gc-trace.h" #include "gc-tracepoint.h" #include "gc-internal.h" @@ -168,6 +169,24 @@ void gc_pin_object(struct gc_mutator *mut, struct gc_ref ref) { // Nothing to do. } +struct gc_ref gc_resolve_conservative_ref(struct gc_heap *heap, + struct gc_conservative_ref ref, + int possibly_interior) { + if (!gc_conservative_ref_might_be_a_heap_object(ref, possibly_interior)) + return gc_ref_null(); + + uintptr_t start = align_down(gc_conservative_ref_value(ref), + GC_INLINE_GRANULE_BYTES); + uintptr_t base = (uintptr_t)GC_base((void*)start); + + if (!base) + return gc_ref_null(); + if (possibly_interior || start == base) + return gc_ref(base); + else + return gc_ref_null(); +} + void gc_collect(struct gc_mutator *mut, enum gc_collection_kind requested_kind) { switch (requested_kind) { diff --git a/src/large-object-space.h b/src/large-object-space.h index cdd798343..999839e57 100644 --- a/src/large-object-space.h +++ b/src/large-object-space.h @@ -385,10 +385,10 @@ large_object_space_add_to_allocation_counter(struct large_object_space *space, *counter += pages << space->page_size_log2; } -static inline struct gc_ref -large_object_space_mark_conservative_ref(struct large_object_space *space, - struct gc_conservative_ref ref, - int possibly_interior) { +static inline struct large_object_node* +large_object_space_lookup_conservative_ref(struct large_object_space *space, + struct gc_conservative_ref ref, + int possibly_interior) { uintptr_t addr = gc_conservative_ref_value(ref); if (!possibly_interior) { @@ -396,7 +396,7 @@ large_object_space_mark_conservative_ref(struct large_object_space *space, // Otherwise strip the displacement to obtain the true base address. uintptr_t displacement = addr & (space->page_size - 1); if (!gc_is_valid_conservative_ref_displacement(displacement)) - return gc_ref_null(); + return NULL; addr -= displacement; } @@ -409,6 +409,29 @@ large_object_space_mark_conservative_ref(struct large_object_space *space, node = large_object_space_lookup(space, gc_ref(addr)); } + return node; +} + +static inline struct gc_ref +large_object_space_resolve_conservative_ref(struct large_object_space *space, + struct gc_conservative_ref ref, + int possibly_interior) { + struct large_object_node *node = + large_object_space_lookup_conservative_ref(space, ref, possibly_interior); + + if (node && node->value.is_live) + return gc_ref(node->key.addr); + + return gc_ref_null(); +} + +static inline struct gc_ref +large_object_space_mark_conservative_ref(struct large_object_space *space, + struct gc_conservative_ref ref, + int possibly_interior) { + struct large_object_node *node = + large_object_space_lookup_conservative_ref(space, ref, possibly_interior); + if (node && node->value.is_live && large_object_space_mark(space, gc_ref(node->key.addr))) return gc_ref(node->key.addr); diff --git a/src/mmc.c b/src/mmc.c index e0868bd76..279f56278 100644 --- a/src/mmc.c +++ b/src/mmc.c @@ -1112,6 +1112,23 @@ gc_pin_object(struct gc_mutator *mut, struct gc_ref ref) { // Otherwise if it's a large or external object, it won't move. } +struct gc_ref +gc_resolve_conservative_ref(struct gc_heap *heap, + struct gc_conservative_ref ref, + int possibly_interior) +{ + if (!gc_conservative_ref_might_be_a_heap_object(ref, possibly_interior)) + return gc_ref_null(); + + struct nofl_space *nofl_space = heap_nofl_space(heap); + if (GC_LIKELY(nofl_space_contains_conservative_ref(nofl_space, ref))) + return nofl_space_resolve_conservative_ref(nofl_space, ref, possibly_interior); + + struct large_object_space *lospace = heap_large_object_space(heap); + return large_object_space_resolve_conservative_ref(lospace, ref, + possibly_interior); +} + int gc_object_is_old_generation_slow(struct gc_mutator *mut, struct gc_ref obj) { if (!GC_GENERATIONAL) diff --git a/src/nofl-space.h b/src/nofl-space.h index dd7ed9071..04e6ec649 100644 --- a/src/nofl-space.h +++ b/src/nofl-space.h @@ -1714,11 +1714,18 @@ nofl_space_forward_or_mark_if_traced(struct nofl_space *space, return nofl_space_forward_if_evacuated(space, edge, ref); } -static inline struct gc_ref -nofl_space_mark_conservative_ref(struct nofl_space *space, - struct gc_conservative_ref ref, - int possibly_interior) { +struct nofl_resolved_conservative_ref { + uintptr_t addr; + uint8_t *metadata; + uint8_t byte; +}; + +static inline struct nofl_resolved_conservative_ref +nofl_space_resolve_conservative_ref_with_metadata(struct nofl_space *space, + struct gc_conservative_ref ref, + int possibly_interior) { uintptr_t addr = gc_conservative_ref_value(ref); + struct nofl_resolved_conservative_ref not_an_object = { 0, }; if (possibly_interior) { addr = align_down(addr, NOFL_GRANULE_SIZE); @@ -1726,17 +1733,17 @@ nofl_space_mark_conservative_ref(struct nofl_space *space, // Addr not an aligned granule? Not an object. uintptr_t displacement = addr & (NOFL_GRANULE_SIZE - 1); if (!gc_is_valid_conservative_ref_displacement(displacement)) - return gc_ref_null(); + return not_an_object; addr -= displacement; } // Addr in meta block? Not an object. if ((addr & (NOFL_SLAB_SIZE - 1)) < NOFL_META_BLOCKS_PER_SLAB * NOFL_BLOCK_SIZE) - return gc_ref_null(); + return not_an_object; // Addr in block that has been paged out? Not an object. if (nofl_block_has_flag(nofl_block_for_addr(addr), NOFL_BLOCK_UNAVAILABLE)) - return gc_ref_null(); + return not_an_object; uint8_t *loc = nofl_metadata_byte_for_addr(addr); uint8_t byte = atomic_load_explicit(loc, memory_order_relaxed); @@ -1745,7 +1752,7 @@ nofl_space_mark_conservative_ref(struct nofl_space *space, // is possibly interior, otherwise bail. if ((byte & NOFL_METADATA_BYTE_MARK_MASK) == 0) { if (!possibly_interior) - return gc_ref_null(); + return not_an_object; uintptr_t block_base = align_down(addr, NOFL_BLOCK_SIZE); uint8_t *loc_base = nofl_metadata_byte_for_addr(block_base); @@ -1753,27 +1760,54 @@ nofl_space_mark_conservative_ref(struct nofl_space *space, loc = scan_backwards_for_byte_with_bits(loc, loc_base, mask); if (!loc) - return gc_ref_null(); + return not_an_object; byte = atomic_load_explicit(loc, memory_order_relaxed); GC_ASSERT(byte & mask); // Ran into the end of some other allocation? Not an object, then. if (byte & NOFL_METADATA_BYTE_END) - return gc_ref_null(); + return not_an_object; // Found object start, and object is unmarked; adjust addr. addr = block_base + (loc - loc_base) * NOFL_GRANULE_SIZE; } - // Object already marked? Nothing to do. - if (nofl_metadata_byte_has_mark(byte, space->current_mark)) + return (struct nofl_resolved_conservative_ref) {addr, loc, byte}; +} + +static inline struct gc_ref +nofl_space_resolve_conservative_ref(struct nofl_space *space, + struct gc_conservative_ref ref, + int possibly_interior) { + struct nofl_resolved_conservative_ref resolved = + nofl_space_resolve_conservative_ref_with_metadata(space, ref, + possibly_interior); + + // Possibly null. + return gc_ref(resolved.addr); +} + +static inline struct gc_ref +nofl_space_mark_conservative_ref(struct nofl_space *space, + struct gc_conservative_ref ref, + int possibly_interior) { + struct nofl_resolved_conservative_ref resolved = + nofl_space_resolve_conservative_ref_with_metadata(space, ref, + possibly_interior); + + if (!resolved.addr) return gc_ref_null(); - GC_ASSERT(nofl_metadata_byte_is_young_or_has_mark(byte, + // Object already marked? Nothing to do. + if (nofl_metadata_byte_has_mark(resolved.byte, space->current_mark)) + return gc_ref_null(); + + GC_ASSERT(nofl_metadata_byte_is_young_or_has_mark(resolved.byte, space->survivor_mark)); - nofl_space_set_nonempty_mark(space, loc, byte, gc_ref(addr)); + nofl_space_set_nonempty_mark(space, resolved.metadata, resolved.byte, + gc_ref(resolved.addr)); - return gc_ref(addr); + return gc_ref(resolved.addr); } static inline size_t diff --git a/src/pcc.c b/src/pcc.c index d5eb50cd1..b78cb50f7 100644 --- a/src/pcc.c +++ b/src/pcc.c @@ -1037,6 +1037,12 @@ void gc_pin_object(struct gc_mutator *mut, struct gc_ref ref) { GC_CRASH(); } +struct gc_ref gc_resolve_conservative_ref(struct gc_heap *heap, + struct gc_conservative_ref ref, + int possibly_interior) { + GC_CRASH(); +} + int gc_object_is_old_generation_slow(struct gc_mutator *mut, struct gc_ref obj) { if (!GC_GENERATIONAL) diff --git a/src/semi.c b/src/semi.c index d449dfc94..53fceae37 100644 --- a/src/semi.c +++ b/src/semi.c @@ -551,6 +551,12 @@ void gc_pin_object(struct gc_mutator *mut, struct gc_ref ref) { GC_CRASH(); } +struct gc_ref gc_resolve_conservative_ref(struct gc_heap *heap, + struct gc_conservative_ref ref, + int possibly_interior) { + GC_CRASH(); +} + struct gc_ephemeron* gc_allocate_ephemeron(struct gc_mutator *mut) { return gc_allocate(mut, gc_ephemeron_size(), GC_ALLOCATION_TAGGED); }