From 4fe4177d7cbde85839365b81ccecdc4bf9fb22ce Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 1 Jul 2025 12:45:41 +0200 Subject: [PATCH 1/3] Split inline function definitions out to separate headers Users will need to include gc-allocate.h, gc-safepoint.h, and gc-barrier.h explicitly. --- api/gc-allocate.h | 119 ++++++++++++++++++++ api/gc-api.h | 205 +--------------------------------- api/gc-barrier.h | 79 +++++++++++++ api/gc-safepoint.h | 35 ++++++ benchmarks/ephemerons.c | 1 + benchmarks/finalizers.c | 1 + benchmarks/mt-gcbench.c | 1 + benchmarks/quads.c | 1 + benchmarks/simple-allocator.h | 1 + embed.am | 3 + src/gc-internal.h | 3 + 11 files changed, 248 insertions(+), 201 deletions(-) create mode 100644 api/gc-allocate.h create mode 100644 api/gc-barrier.h create mode 100644 api/gc-safepoint.h diff --git a/api/gc-allocate.h b/api/gc-allocate.h new file mode 100644 index 000000000..ba04b74d4 --- /dev/null +++ b/api/gc-allocate.h @@ -0,0 +1,119 @@ +#ifndef GC_ALLOCATE_H_ +#define GC_ALLOCATE_H_ + +#include "gc-api.h" + +struct gc_heap; +struct gc_mutator; + +static inline void gc_update_alloc_table(struct gc_ref obj, size_t size, + enum gc_allocation_kind kind) { + size_t alignment = gc_allocator_alloc_table_alignment(); + if (!alignment) return; + + uintptr_t addr = gc_ref_value(obj); + uintptr_t base = addr & ~(alignment - 1); + size_t granule_size = gc_allocator_small_granule_size(); + uintptr_t granule = (addr & (alignment - 1)) / granule_size; + uint8_t *alloc = (uint8_t*)(base + granule); + + uint8_t begin_pattern = gc_allocator_alloc_table_begin_pattern(kind); + uint8_t end_pattern = gc_allocator_alloc_table_end_pattern(); + if (end_pattern) { + size_t granules = size / granule_size; + if (granules == 1) { + alloc[0] = begin_pattern | end_pattern; + } else { + alloc[0] = begin_pattern; + alloc[granules - 1] = end_pattern; + } + } else { + alloc[0] = begin_pattern; + } +} + +static inline void* gc_allocate_small_fast_bump_pointer(struct gc_mutator *mut, + size_t size, + enum gc_allocation_kind kind) { + GC_ASSERT(size <= gc_allocator_large_threshold()); + + size_t granule_size = gc_allocator_small_granule_size(); + size_t hp_offset = gc_allocator_allocation_pointer_offset(); + size_t limit_offset = gc_allocator_allocation_limit_offset(); + + uintptr_t base_addr = (uintptr_t)mut; + uintptr_t *hp_loc = (uintptr_t*)(base_addr + hp_offset); + uintptr_t *limit_loc = (uintptr_t*)(base_addr + limit_offset); + + size = (size + granule_size - 1) & ~(granule_size - 1); + uintptr_t hp = *hp_loc; + uintptr_t limit = *limit_loc; + uintptr_t new_hp = hp + size; + + if (GC_UNLIKELY (new_hp > limit)) + return NULL; + + *hp_loc = new_hp; + + gc_update_alloc_table(gc_ref(hp), size, kind); + + return (void*)hp; +} + +static inline void* gc_allocate_small_fast_freelist(struct gc_mutator *mut, + size_t size, + enum gc_allocation_kind kind) { + GC_ASSERT(size <= gc_allocator_large_threshold()); + + size_t freelist_offset = gc_allocator_freelist_offset(size, kind); + uintptr_t base_addr = (uintptr_t)mut; + void **freelist_loc = (void**)(base_addr + freelist_offset); + + void *head = *freelist_loc; + if (GC_UNLIKELY(!head)) + return NULL; + + *freelist_loc = *(void**)head; + *(void**)head = NULL; + + gc_update_alloc_table(gc_ref_from_heap_object(head), size, kind); + + return head; +} + +static inline void* gc_allocate_small_fast(struct gc_mutator *mut, size_t size, + enum gc_allocation_kind kind) { + GC_ASSERT(size != 0); + GC_ASSERT(size <= gc_allocator_large_threshold()); + + switch (gc_inline_allocator_kind(kind)) { + case GC_INLINE_ALLOCATOR_BUMP_POINTER: + return gc_allocate_small_fast_bump_pointer(mut, size, kind); + case GC_INLINE_ALLOCATOR_FREELIST: + return gc_allocate_small_fast_freelist(mut, size, kind); + case GC_INLINE_ALLOCATOR_NONE: + return NULL; + default: + GC_CRASH(); + } +} + +static inline void* gc_allocate_fast(struct gc_mutator *mut, size_t size, + enum gc_allocation_kind kind) { + GC_ASSERT(size != 0); + if (size > gc_allocator_large_threshold()) + return NULL; + + return gc_allocate_small_fast(mut, size, kind); +} + +static inline void* gc_allocate(struct gc_mutator *mut, size_t size, + enum gc_allocation_kind kind) { + void *ret = gc_allocate_fast(mut, size, kind); + if (GC_LIKELY(ret != NULL)) + return ret; + + return gc_allocate_slow(mut, size, kind); +} + +#endif // GC_ALLOCATE_H_ diff --git a/api/gc-api.h b/api/gc-api.h index b5d279443..809a01751 100644 --- a/api/gc-api.h +++ b/api/gc-api.h @@ -70,31 +70,6 @@ GC_API_ int gc_heap_contains(struct gc_heap *heap, struct gc_ref ref); static inline void gc_update_alloc_table(struct gc_ref obj, size_t size, enum gc_allocation_kind kind) GC_ALWAYS_INLINE; -static inline void gc_update_alloc_table(struct gc_ref obj, size_t size, - enum gc_allocation_kind kind) { - size_t alignment = gc_allocator_alloc_table_alignment(); - if (!alignment) return; - - uintptr_t addr = gc_ref_value(obj); - uintptr_t base = addr & ~(alignment - 1); - size_t granule_size = gc_allocator_small_granule_size(); - uintptr_t granule = (addr & (alignment - 1)) / granule_size; - uint8_t *alloc = (uint8_t*)(base + granule); - - uint8_t begin_pattern = gc_allocator_alloc_table_begin_pattern(kind); - uint8_t end_pattern = gc_allocator_alloc_table_end_pattern(); - if (end_pattern) { - size_t granules = size / granule_size; - if (granules == 1) { - alloc[0] = begin_pattern | end_pattern; - } else { - alloc[0] = begin_pattern; - alloc[granules - 1] = end_pattern; - } - } else { - alloc[0] = begin_pattern; - } -} GC_API_ void* gc_allocate_slow(struct gc_mutator *mut, size_t bytes, enum gc_allocation_kind kind) GC_NEVER_INLINE; @@ -102,98 +77,19 @@ GC_API_ void* gc_allocate_slow(struct gc_mutator *mut, size_t bytes, static inline void* gc_allocate_small_fast_bump_pointer(struct gc_mutator *mut, size_t size, enum gc_allocation_kind kind) GC_ALWAYS_INLINE; -static inline void* gc_allocate_small_fast_bump_pointer(struct gc_mutator *mut, - size_t size, - enum gc_allocation_kind kind) { - GC_ASSERT(size <= gc_allocator_large_threshold()); - - size_t granule_size = gc_allocator_small_granule_size(); - size_t hp_offset = gc_allocator_allocation_pointer_offset(); - size_t limit_offset = gc_allocator_allocation_limit_offset(); - - uintptr_t base_addr = (uintptr_t)mut; - uintptr_t *hp_loc = (uintptr_t*)(base_addr + hp_offset); - uintptr_t *limit_loc = (uintptr_t*)(base_addr + limit_offset); - - size = (size + granule_size - 1) & ~(granule_size - 1); - uintptr_t hp = *hp_loc; - uintptr_t limit = *limit_loc; - uintptr_t new_hp = hp + size; - - if (GC_UNLIKELY (new_hp > limit)) - return NULL; - - *hp_loc = new_hp; - - gc_update_alloc_table(gc_ref(hp), size, kind); - - return (void*)hp; -} static inline void* gc_allocate_small_fast_freelist(struct gc_mutator *mut, size_t size, enum gc_allocation_kind kind) GC_ALWAYS_INLINE; -static inline void* gc_allocate_small_fast_freelist(struct gc_mutator *mut, - size_t size, - enum gc_allocation_kind kind) { - GC_ASSERT(size <= gc_allocator_large_threshold()); - - size_t freelist_offset = gc_allocator_freelist_offset(size, kind); - uintptr_t base_addr = (uintptr_t)mut; - void **freelist_loc = (void**)(base_addr + freelist_offset); - - void *head = *freelist_loc; - if (GC_UNLIKELY(!head)) - return NULL; - - *freelist_loc = *(void**)head; - *(void**)head = NULL; - - gc_update_alloc_table(gc_ref_from_heap_object(head), size, kind); - - return head; -} static inline void* gc_allocate_small_fast(struct gc_mutator *mut, size_t size, enum gc_allocation_kind kind) GC_ALWAYS_INLINE; -static inline void* gc_allocate_small_fast(struct gc_mutator *mut, size_t size, - enum gc_allocation_kind kind) { - GC_ASSERT(size != 0); - GC_ASSERT(size <= gc_allocator_large_threshold()); - - switch (gc_inline_allocator_kind(kind)) { - case GC_INLINE_ALLOCATOR_BUMP_POINTER: - return gc_allocate_small_fast_bump_pointer(mut, size, kind); - case GC_INLINE_ALLOCATOR_FREELIST: - return gc_allocate_small_fast_freelist(mut, size, kind); - case GC_INLINE_ALLOCATOR_NONE: - return NULL; - default: - GC_CRASH(); - } -} static inline void* gc_allocate_fast(struct gc_mutator *mut, size_t size, enum gc_allocation_kind kind) GC_ALWAYS_INLINE; -static inline void* gc_allocate_fast(struct gc_mutator *mut, size_t size, - enum gc_allocation_kind kind) { - GC_ASSERT(size != 0); - if (size > gc_allocator_large_threshold()) - return NULL; - - return gc_allocate_small_fast(mut, size, kind); -} static inline void* gc_allocate(struct gc_mutator *mut, size_t size, - enum gc_allocation_kind kind) GC_ALWAYS_INLINE; -static inline void* gc_allocate(struct gc_mutator *mut, size_t size, - enum gc_allocation_kind kind) { - void *ret = gc_allocate_fast(mut, size, kind); - if (GC_LIKELY(ret != NULL)) - return ret; - - return gc_allocate_slow(mut, size, kind); -} + enum gc_allocation_kind kind) GC_ALWAYS_INLINE; GC_API_ int gc_object_is_old_generation_slow(struct gc_mutator *mut, struct gc_ref obj) GC_NEVER_INLINE; @@ -201,40 +97,6 @@ GC_API_ int gc_object_is_old_generation_slow(struct gc_mutator *mut, static inline int gc_object_is_old_generation(struct gc_mutator *mut, struct gc_ref obj, size_t obj_size) GC_ALWAYS_INLINE; -static inline int gc_object_is_old_generation(struct gc_mutator *mut, - struct gc_ref obj, - size_t obj_size) { - switch (gc_old_generation_check_kind(obj_size)) { - case GC_OLD_GENERATION_CHECK_ALLOC_TABLE: { - size_t alignment = gc_allocator_alloc_table_alignment(); - GC_ASSERT(alignment); - uintptr_t addr = gc_ref_value(obj); - uintptr_t base = addr & ~(alignment - 1); - size_t granule_size = gc_allocator_small_granule_size(); - uintptr_t granule = (addr & (alignment - 1)) / granule_size; - uint8_t *byte_loc = (uint8_t*)(base + granule); - uint8_t byte = atomic_load_explicit(byte_loc, memory_order_relaxed); - uint8_t mask = gc_old_generation_check_alloc_table_tag_mask(); - uint8_t young = gc_old_generation_check_alloc_table_young_tag(); - return (byte & mask) != young; - } - case GC_OLD_GENERATION_CHECK_SMALL_OBJECT_NURSERY: { - struct gc_heap *heap = gc_mutator_heap(mut); - // Note that these addresses are fixed and that the embedder might - // want to store them somewhere or inline them into the output of - // JIT-generated code. They may also be power-of-two aligned. - uintptr_t low_addr = gc_small_object_nursery_low_address(heap); - uintptr_t high_addr = gc_small_object_nursery_high_address(heap); - uintptr_t size = high_addr - low_addr; - uintptr_t addr = gc_ref_value(obj); - return addr - low_addr >= size; - } - case GC_OLD_GENERATION_CHECK_SLOW: - return gc_object_is_old_generation_slow(mut, obj); - default: - GC_CRASH(); - } -} GC_API_ void gc_write_barrier_slow(struct gc_mutator *mut, struct gc_ref obj, size_t obj_size, struct gc_edge edge, @@ -243,46 +105,10 @@ GC_API_ void gc_write_barrier_slow(struct gc_mutator *mut, struct gc_ref obj, static inline int gc_write_barrier_fast(struct gc_mutator *mut, struct gc_ref obj, size_t obj_size, struct gc_edge edge, struct gc_ref new_val) GC_ALWAYS_INLINE; -static inline int gc_write_barrier_fast(struct gc_mutator *mut, struct gc_ref obj, - size_t obj_size, struct gc_edge edge, - struct gc_ref new_val) { - switch (gc_write_barrier_kind(obj_size)) { - case GC_WRITE_BARRIER_NONE: - return 0; - case GC_WRITE_BARRIER_FIELD: { - if (!gc_object_is_old_generation(mut, obj, obj_size)) - return 0; - - size_t field_table_alignment = gc_write_barrier_field_table_alignment(); - size_t fields_per_byte = gc_write_barrier_field_fields_per_byte(); - uint8_t first_bit_pattern = gc_write_barrier_field_first_bit_pattern(); - ssize_t table_offset = gc_write_barrier_field_table_offset(); - - uintptr_t addr = gc_edge_address(edge); - uintptr_t base = addr & ~(field_table_alignment - 1); - uintptr_t field = (addr & (field_table_alignment - 1)) / sizeof(uintptr_t); - uintptr_t log_byte = field / fields_per_byte; - uint8_t log_bit = first_bit_pattern << (field % fields_per_byte); - uint8_t *byte_loc = (uint8_t*)(base + table_offset + log_byte); - uint8_t byte = atomic_load_explicit(byte_loc, memory_order_relaxed); - return !(byte & log_bit); - } - case GC_WRITE_BARRIER_SLOW: - return 1; - default: - GC_CRASH(); - } -} static inline void gc_write_barrier(struct gc_mutator *mut, struct gc_ref obj, size_t obj_size, struct gc_edge edge, struct gc_ref new_val) GC_ALWAYS_INLINE; -static inline void gc_write_barrier(struct gc_mutator *mut, struct gc_ref obj, - size_t obj_size, struct gc_edge edge, - struct gc_ref new_val) { - if (GC_UNLIKELY(gc_write_barrier_fast(mut, obj, obj_size, edge, new_val))) - gc_write_barrier_slow(mut, obj, obj_size, edge, new_val); -} GC_API_ struct gc_ref gc_resolve_conservative_ref(struct gc_heap *heap, struct gc_conservative_ref ref, @@ -291,36 +117,13 @@ GC_API_ void gc_pin_object(struct gc_mutator *mut, struct gc_ref obj); GC_API_ void gc_safepoint_slow(struct gc_mutator *mut) GC_NEVER_INLINE; GC_API_ int* gc_safepoint_flag_loc(struct gc_mutator *mut); -static inline int gc_should_stop_for_safepoint(struct gc_mutator *mut) { - switch (gc_cooperative_safepoint_kind()) { - case GC_COOPERATIVE_SAFEPOINT_NONE: - return 0; - case GC_COOPERATIVE_SAFEPOINT_MUTATOR_FLAG: - case GC_COOPERATIVE_SAFEPOINT_HEAP_FLAG: { - return atomic_load_explicit(gc_safepoint_flag_loc(mut), - memory_order_relaxed); - } - default: - GC_CRASH(); - } -} -static inline void gc_safepoint(struct gc_mutator *mut) { - if (GC_UNLIKELY(gc_should_stop_for_safepoint(mut))) - gc_safepoint_slow(mut); -} +static inline void gc_safepoint(struct gc_mutator *mut) GC_ALWAYS_INLINE; GC_API_ int gc_safepoint_signal_number(void); GC_API_ void gc_safepoint_signal_inhibit(struct gc_mutator *mut); GC_API_ void gc_safepoint_signal_reallow(struct gc_mutator *mut); -static inline void gc_inhibit_preemption(struct gc_mutator *mut) { - if (gc_safepoint_mechanism() == GC_SAFEPOINT_MECHANISM_SIGNAL) - gc_safepoint_signal_inhibit(mut); -} - -static inline void gc_reallow_preemption(struct gc_mutator *mut) { - if (gc_safepoint_mechanism() == GC_SAFEPOINT_MECHANISM_SIGNAL) - gc_safepoint_signal_reallow(mut); -} +static inline void gc_inhibit_preemption(struct gc_mutator *mut) GC_ALWAYS_INLINE; +static inline void gc_reallow_preemption(struct gc_mutator *mut) GC_ALWAYS_INLINE; #endif // GC_API_H_ diff --git a/api/gc-barrier.h b/api/gc-barrier.h new file mode 100644 index 000000000..a1ff2c196 --- /dev/null +++ b/api/gc-barrier.h @@ -0,0 +1,79 @@ +#ifndef GC_BARRIER_H_ +#define GC_BARRIER_H_ + +#include "gc-api.h" + +static inline int gc_object_is_old_generation(struct gc_mutator *mut, + struct gc_ref obj, + size_t obj_size) { + switch (gc_old_generation_check_kind(obj_size)) { + case GC_OLD_GENERATION_CHECK_ALLOC_TABLE: { + size_t alignment = gc_allocator_alloc_table_alignment(); + GC_ASSERT(alignment); + uintptr_t addr = gc_ref_value(obj); + uintptr_t base = addr & ~(alignment - 1); + size_t granule_size = gc_allocator_small_granule_size(); + uintptr_t granule = (addr & (alignment - 1)) / granule_size; + uint8_t *byte_loc = (uint8_t*)(base + granule); + uint8_t byte = atomic_load_explicit(byte_loc, memory_order_relaxed); + uint8_t mask = gc_old_generation_check_alloc_table_tag_mask(); + uint8_t young = gc_old_generation_check_alloc_table_young_tag(); + return (byte & mask) != young; + } + case GC_OLD_GENERATION_CHECK_SMALL_OBJECT_NURSERY: { + struct gc_heap *heap = gc_mutator_heap(mut); + // Note that these addresses are fixed and that the embedder might + // want to store them somewhere or inline them into the output of + // JIT-generated code. They may also be power-of-two aligned. + uintptr_t low_addr = gc_small_object_nursery_low_address(heap); + uintptr_t high_addr = gc_small_object_nursery_high_address(heap); + uintptr_t size = high_addr - low_addr; + uintptr_t addr = gc_ref_value(obj); + return addr - low_addr >= size; + } + case GC_OLD_GENERATION_CHECK_SLOW: + return gc_object_is_old_generation_slow(mut, obj); + default: + GC_CRASH(); + } +} + +static inline int gc_write_barrier_fast(struct gc_mutator *mut, struct gc_ref obj, + size_t obj_size, struct gc_edge edge, + struct gc_ref new_val) { + switch (gc_write_barrier_kind(obj_size)) { + case GC_WRITE_BARRIER_NONE: + return 0; + case GC_WRITE_BARRIER_FIELD: { + if (!gc_object_is_old_generation(mut, obj, obj_size)) + return 0; + + size_t field_table_alignment = gc_write_barrier_field_table_alignment(); + size_t fields_per_byte = gc_write_barrier_field_fields_per_byte(); + uint8_t first_bit_pattern = gc_write_barrier_field_first_bit_pattern(); + ssize_t table_offset = gc_write_barrier_field_table_offset(); + + uintptr_t addr = gc_edge_address(edge); + uintptr_t base = addr & ~(field_table_alignment - 1); + uintptr_t field = (addr & (field_table_alignment - 1)) / sizeof(uintptr_t); + uintptr_t log_byte = field / fields_per_byte; + uint8_t log_bit = first_bit_pattern << (field % fields_per_byte); + uint8_t *byte_loc = (uint8_t*)(base + table_offset + log_byte); + uint8_t byte = atomic_load_explicit(byte_loc, memory_order_relaxed); + return !(byte & log_bit); + } + case GC_WRITE_BARRIER_SLOW: + return 1; + default: + GC_CRASH(); + } +} + +static inline void gc_write_barrier(struct gc_mutator *mut, struct gc_ref obj, + size_t obj_size, struct gc_edge edge, + struct gc_ref new_val) { + if (GC_UNLIKELY(gc_write_barrier_fast(mut, obj, obj_size, edge, new_val))) + gc_write_barrier_slow(mut, obj, obj_size, edge, new_val); +} + +#endif // GC_BARRIER_H_ diff --git a/api/gc-safepoint.h b/api/gc-safepoint.h new file mode 100644 index 000000000..41b666375 --- /dev/null +++ b/api/gc-safepoint.h @@ -0,0 +1,35 @@ +#ifndef GC_SAFEPOINT_H_ +#define GC_SAFEPOINT_H_ + +#include "gc-api.h" + +static inline int gc_should_stop_for_safepoint(struct gc_mutator *mut) { + switch (gc_cooperative_safepoint_kind()) { + case GC_COOPERATIVE_SAFEPOINT_NONE: + return 0; + case GC_COOPERATIVE_SAFEPOINT_MUTATOR_FLAG: + case GC_COOPERATIVE_SAFEPOINT_HEAP_FLAG: { + return atomic_load_explicit(gc_safepoint_flag_loc(mut), + memory_order_relaxed); + } + default: + GC_CRASH(); + } +} + +static inline void gc_safepoint(struct gc_mutator *mut) { + if (GC_UNLIKELY(gc_should_stop_for_safepoint(mut))) + gc_safepoint_slow(mut); +} + +static inline void gc_inhibit_preemption(struct gc_mutator *mut) { + if (gc_safepoint_mechanism() == GC_SAFEPOINT_MECHANISM_SIGNAL) + gc_safepoint_signal_inhibit(mut); +} + +static inline void gc_reallow_preemption(struct gc_mutator *mut) { + if (gc_safepoint_mechanism() == GC_SAFEPOINT_MECHANISM_SIGNAL) + gc_safepoint_signal_reallow(mut); +} + +#endif // GC_SAFEPOINT_H_ diff --git a/benchmarks/ephemerons.c b/benchmarks/ephemerons.c index f0459dab4..e42deb721 100644 --- a/benchmarks/ephemerons.c +++ b/benchmarks/ephemerons.c @@ -7,6 +7,7 @@ #include "assert.h" #include "gc-api.h" +#include "gc-barrier.h" #include "gc-basic-stats.h" #include "gc-ephemeron.h" #include "simple-roots-api.h" diff --git a/benchmarks/finalizers.c b/benchmarks/finalizers.c index 8d4b5c25a..03ba9f46f 100644 --- a/benchmarks/finalizers.c +++ b/benchmarks/finalizers.c @@ -7,6 +7,7 @@ #include "assert.h" #include "gc-api.h" +#include "gc-barrier.h" #include "gc-basic-stats.h" #include "gc-finalizer.h" #include "simple-roots-api.h" diff --git a/benchmarks/mt-gcbench.c b/benchmarks/mt-gcbench.c index e705702bb..5ae916c09 100644 --- a/benchmarks/mt-gcbench.c +++ b/benchmarks/mt-gcbench.c @@ -46,6 +46,7 @@ #include "assert.h" #include "gc-api.h" +#include "gc-barrier.h" #include "gc-basic-stats.h" #include "mt-gcbench-types.h" #include "simple-roots-api.h" diff --git a/benchmarks/quads.c b/benchmarks/quads.c index 51bcca870..8faa0c1c5 100644 --- a/benchmarks/quads.c +++ b/benchmarks/quads.c @@ -5,6 +5,7 @@ #include "assert.h" #include "gc-api.h" +#include "gc-barrier.h" #include "gc-basic-stats.h" #include "simple-roots-api.h" #include "quads-types.h" diff --git a/benchmarks/simple-allocator.h b/benchmarks/simple-allocator.h index 09ed8f3be..493c9b3d2 100644 --- a/benchmarks/simple-allocator.h +++ b/benchmarks/simple-allocator.h @@ -3,6 +3,7 @@ #include "simple-tagging-scheme.h" #include "gc-api.h" +#include "gc-allocate.h" static inline void* gc_allocate_with_kind(struct gc_mutator *mut, enum alloc_kind kind, size_t bytes) { diff --git a/embed.am b/embed.am index 7bf3a7120..0ced83f37 100644 --- a/embed.am +++ b/embed.am @@ -46,10 +46,12 @@ noinst_LTLIBRARIES += libwhippet.la libwhippet_la_SOURCES = \ %D%/api/bdw-attrs.h \ + %D%/api/gc-allocate.h \ %D%/api/gc-allocation-kind.h \ %D%/api/gc-api.h \ %D%/api/gc-assert.h \ %D%/api/gc-attrs.h \ + %D%/api/gc-barrier.h \ %D%/api/gc-basic-stats.h \ %D%/api/gc-collection-kind.h \ %D%/api/gc-config.h \ @@ -67,6 +69,7 @@ libwhippet_la_SOURCES = \ %D%/api/gc-null-event-listener.h \ %D%/api/gc-options.h \ %D%/api/gc-ref.h \ + %D%/api/gc-safepoint.h \ %D%/api/gc-stack-addr.h \ %D%/api/gc-tracepoint.h \ %D%/api/gc-visibility.h \ diff --git a/src/gc-internal.h b/src/gc-internal.h index fe1cdf66c..46f0f0064 100644 --- a/src/gc-internal.h +++ b/src/gc-internal.h @@ -9,6 +9,9 @@ #include "gc-ephemeron-internal.h" #include "gc-finalizer-internal.h" #include "gc-options-internal.h" +#include "gc-allocate.h" +#include "gc-barrier.h" +#include "gc-safepoint.h" uint64_t gc_heap_total_bytes_allocated(struct gc_heap *heap); void gc_mutator_adjust_heap_size(struct gc_mutator *mut, uint64_t new_size); From 36043468e85bd45852b2bdbb4c22c4c7335cd2d9 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 1 Jul 2025 13:00:12 +0200 Subject: [PATCH 2/3] Mark always-inline functions as maybe-unused --- api/gc-inline.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/gc-inline.h b/api/gc-inline.h index 30eac54f3..eba8c66c7 100644 --- a/api/gc-inline.h +++ b/api/gc-inline.h @@ -1,7 +1,7 @@ #ifndef GC_INLINE_H_ #define GC_INLINE_H_ -#define GC_ALWAYS_INLINE __attribute__((always_inline)) +#define GC_ALWAYS_INLINE __attribute__((always_inline)) __attribute__((unused)) #define GC_NEVER_INLINE __attribute__((noinline)) #endif // GC_INLINE_H_ From 86baf260cca95bf71ccf6e9dcd317b50124e229f Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 1 Jul 2025 13:09:46 +0200 Subject: [PATCH 3/3] Move inline function decls to their impl headers --- api/gc-allocate.h | 23 ++++++++++++++++++++ api/gc-api.h | 53 ---------------------------------------------- api/gc-barrier.h | 19 +++++++++++++++++ api/gc-inline.h | 2 +- api/gc-safepoint.h | 11 ++++++++++ 5 files changed, 54 insertions(+), 54 deletions(-) diff --git a/api/gc-allocate.h b/api/gc-allocate.h index ba04b74d4..f6c76ebc5 100644 --- a/api/gc-allocate.h +++ b/api/gc-allocate.h @@ -6,6 +6,29 @@ struct gc_heap; struct gc_mutator; +static inline void gc_update_alloc_table(struct gc_ref obj, size_t size, + enum gc_allocation_kind kind) GC_ALWAYS_INLINE; + +GC_API_ void* gc_allocate_slow(struct gc_mutator *mut, size_t bytes, + enum gc_allocation_kind kind) GC_NEVER_INLINE; + +static inline void* +gc_allocate_small_fast_bump_pointer(struct gc_mutator *mut, size_t size, + enum gc_allocation_kind kind) GC_ALWAYS_INLINE; + +static inline void* gc_allocate_small_fast_freelist(struct gc_mutator *mut, + size_t size, + enum gc_allocation_kind kind) GC_ALWAYS_INLINE; + +static inline void* gc_allocate_small_fast(struct gc_mutator *mut, size_t size, + enum gc_allocation_kind kind) GC_ALWAYS_INLINE; + +static inline void* gc_allocate_fast(struct gc_mutator *mut, size_t size, + enum gc_allocation_kind kind) GC_ALWAYS_INLINE; + +static inline void* gc_allocate(struct gc_mutator *mut, size_t size, + enum gc_allocation_kind kind) GC_ALWAYS_INLINE; + static inline void gc_update_alloc_table(struct gc_ref obj, size_t size, enum gc_allocation_kind kind) { size_t alignment = gc_allocator_alloc_table_alignment(); diff --git a/api/gc-api.h b/api/gc-api.h index 809a01751..484cab89b 100644 --- a/api/gc-api.h +++ b/api/gc-api.h @@ -68,62 +68,9 @@ GC_API_ void gc_collect(struct gc_mutator *mut, GC_API_ int gc_heap_contains(struct gc_heap *heap, struct gc_ref ref); -static inline void gc_update_alloc_table(struct gc_ref obj, size_t size, - enum gc_allocation_kind kind) GC_ALWAYS_INLINE; - -GC_API_ void* gc_allocate_slow(struct gc_mutator *mut, size_t bytes, - enum gc_allocation_kind kind) GC_NEVER_INLINE; - -static inline void* -gc_allocate_small_fast_bump_pointer(struct gc_mutator *mut, size_t size, - enum gc_allocation_kind kind) GC_ALWAYS_INLINE; - -static inline void* gc_allocate_small_fast_freelist(struct gc_mutator *mut, - size_t size, - enum gc_allocation_kind kind) GC_ALWAYS_INLINE; - -static inline void* gc_allocate_small_fast(struct gc_mutator *mut, size_t size, - enum gc_allocation_kind kind) GC_ALWAYS_INLINE; - -static inline void* gc_allocate_fast(struct gc_mutator *mut, size_t size, - enum gc_allocation_kind kind) GC_ALWAYS_INLINE; - -static inline void* gc_allocate(struct gc_mutator *mut, size_t size, - enum gc_allocation_kind kind) GC_ALWAYS_INLINE; - -GC_API_ int gc_object_is_old_generation_slow(struct gc_mutator *mut, - struct gc_ref obj) GC_NEVER_INLINE; - -static inline int gc_object_is_old_generation(struct gc_mutator *mut, - struct gc_ref obj, - size_t obj_size) GC_ALWAYS_INLINE; - -GC_API_ void gc_write_barrier_slow(struct gc_mutator *mut, struct gc_ref obj, - size_t obj_size, struct gc_edge edge, - struct gc_ref new_val) GC_NEVER_INLINE; - -static inline int gc_write_barrier_fast(struct gc_mutator *mut, struct gc_ref obj, - size_t obj_size, struct gc_edge edge, - struct gc_ref new_val) GC_ALWAYS_INLINE; - -static inline void gc_write_barrier(struct gc_mutator *mut, struct gc_ref obj, - size_t obj_size, struct gc_edge edge, - struct gc_ref new_val) GC_ALWAYS_INLINE; - GC_API_ struct gc_ref gc_resolve_conservative_ref(struct gc_heap *heap, struct gc_conservative_ref ref, int possibly_interior); GC_API_ void gc_pin_object(struct gc_mutator *mut, struct gc_ref obj); -GC_API_ void gc_safepoint_slow(struct gc_mutator *mut) GC_NEVER_INLINE; -GC_API_ int* gc_safepoint_flag_loc(struct gc_mutator *mut); -static inline void gc_safepoint(struct gc_mutator *mut) GC_ALWAYS_INLINE; - -GC_API_ int gc_safepoint_signal_number(void); -GC_API_ void gc_safepoint_signal_inhibit(struct gc_mutator *mut); -GC_API_ void gc_safepoint_signal_reallow(struct gc_mutator *mut); - -static inline void gc_inhibit_preemption(struct gc_mutator *mut) GC_ALWAYS_INLINE; -static inline void gc_reallow_preemption(struct gc_mutator *mut) GC_ALWAYS_INLINE; - #endif // GC_API_H_ diff --git a/api/gc-barrier.h b/api/gc-barrier.h index a1ff2c196..9995793ed 100644 --- a/api/gc-barrier.h +++ b/api/gc-barrier.h @@ -3,6 +3,25 @@ #include "gc-api.h" +GC_API_ int gc_object_is_old_generation_slow(struct gc_mutator *mut, + struct gc_ref obj) GC_NEVER_INLINE; + +static inline int gc_object_is_old_generation(struct gc_mutator *mut, + struct gc_ref obj, + size_t obj_size) GC_ALWAYS_INLINE; + +GC_API_ void gc_write_barrier_slow(struct gc_mutator *mut, struct gc_ref obj, + size_t obj_size, struct gc_edge edge, + struct gc_ref new_val) GC_NEVER_INLINE; + +static inline int gc_write_barrier_fast(struct gc_mutator *mut, struct gc_ref obj, + size_t obj_size, struct gc_edge edge, + struct gc_ref new_val) GC_ALWAYS_INLINE; + +static inline void gc_write_barrier(struct gc_mutator *mut, struct gc_ref obj, + size_t obj_size, struct gc_edge edge, + struct gc_ref new_val) GC_ALWAYS_INLINE; + static inline int gc_object_is_old_generation(struct gc_mutator *mut, struct gc_ref obj, size_t obj_size) { diff --git a/api/gc-inline.h b/api/gc-inline.h index eba8c66c7..30eac54f3 100644 --- a/api/gc-inline.h +++ b/api/gc-inline.h @@ -1,7 +1,7 @@ #ifndef GC_INLINE_H_ #define GC_INLINE_H_ -#define GC_ALWAYS_INLINE __attribute__((always_inline)) __attribute__((unused)) +#define GC_ALWAYS_INLINE __attribute__((always_inline)) #define GC_NEVER_INLINE __attribute__((noinline)) #endif // GC_INLINE_H_ diff --git a/api/gc-safepoint.h b/api/gc-safepoint.h index 41b666375..c0e24b285 100644 --- a/api/gc-safepoint.h +++ b/api/gc-safepoint.h @@ -3,6 +3,17 @@ #include "gc-api.h" +GC_API_ void gc_safepoint_slow(struct gc_mutator *mut) GC_NEVER_INLINE; +GC_API_ int* gc_safepoint_flag_loc(struct gc_mutator *mut); +static inline void gc_safepoint(struct gc_mutator *mut) GC_ALWAYS_INLINE; + +GC_API_ int gc_safepoint_signal_number(void); +GC_API_ void gc_safepoint_signal_inhibit(struct gc_mutator *mut); +GC_API_ void gc_safepoint_signal_reallow(struct gc_mutator *mut); + +static inline void gc_inhibit_preemption(struct gc_mutator *mut) GC_ALWAYS_INLINE; +static inline void gc_reallow_preemption(struct gc_mutator *mut) GC_ALWAYS_INLINE; + static inline int gc_should_stop_for_safepoint(struct gc_mutator *mut) { switch (gc_cooperative_safepoint_kind()) { case GC_COOPERATIVE_SAFEPOINT_NONE: