From f6057184e1490e6ba6ad983f363350db4dc2de4b Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 23 Jul 2024 22:32:57 +0200 Subject: [PATCH] Add finalizers --- Makefile | 4 +- README.md | 2 +- api/gc-basic-stats.h | 2 + api/gc-embedder-api.h | 1 - api/gc-event-listener-chain.h | 6 + api/gc-event-listener.h | 1 + api/gc-finalizer.h | 81 +++++++++ api/gc-null-event-listener.h | 2 + benchmarks/simple-gc-embedder.h | 3 + embed.mk | 4 +- src/bdw.c | 72 ++++++++ src/gc-finalizer-internal.h | 65 +++++++ src/gc-finalizer.c | 307 ++++++++++++++++++++++++++++++++ src/gc-internal.h | 1 + src/pcc.c | 67 ++++++- src/root.h | 13 +- src/semi.c | 74 ++++++-- src/whippet.c | 79 +++++++- 18 files changed, 756 insertions(+), 28 deletions(-) create mode 100644 api/gc-finalizer.h create mode 100644 src/gc-finalizer-internal.h create mode 100644 src/gc-finalizer.c diff --git a/Makefile b/Makefile index 0ea60d022..c1ba15f43 100644 --- a/Makefile +++ b/Makefile @@ -52,6 +52,8 @@ obj/gc-options.o: src/gc-options.c | .deps obj $(COMPILE) -c $< obj/%.gc-ephemeron.o: src/gc-ephemeron.c | .deps obj $(COMPILE) -include benchmarks/$*-embedder.h -c $< +obj/%.gc-finalizer.o: src/gc-finalizer.c | .deps obj + $(COMPILE) -include benchmarks/$*-embedder.h -c $< GC_STEM_bdw = bdw GC_CFLAGS_bdw = -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 @@ -99,7 +101,7 @@ obj/$(1).$(2).gc.o: src/$(call gc_impl,$(2)) | .deps obj $$(COMPILE) $(call gc_cflags,$(2)) $(call gc_impl_cflags,$(2)) -include benchmarks/$(1)-embedder.h -c $$< obj/$(1).$(2).o: benchmarks/$(1).c | .deps obj $$(COMPILE) $(call gc_cflags,$(2)) -include api/$(call gc_attrs,$(2)) -c $$< -bin/$(1).$(2): obj/$(1).$(2).gc.o obj/$(1).$(2).o obj/gc-stack.o obj/gc-options.o obj/gc-platform.o obj/$(1).gc-ephemeron.o | bin +bin/$(1).$(2): obj/$(1).$(2).gc.o obj/$(1).$(2).o obj/gc-stack.o obj/gc-options.o obj/gc-platform.o obj/$(1).gc-ephemeron.o obj/$(1).gc-finalizer.o | bin $$(LINK) $$^ $(call gc_libs,$(2)) endef diff --git a/README.md b/README.md index e1ac66150..e62b0661a 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ a talk given at FOSDEM 2023. - [X] Conservative data segments - [ ] Heap growth/shrinking - [ ] Debugging/tracing - - [ ] Finalizers + - [X] Finalizers - [X] Weak references / weak maps ### Features that would improve Whippet performance diff --git a/api/gc-basic-stats.h b/api/gc-basic-stats.h index af8cd4243..40cdbcb3e 100644 --- a/api/gc-basic-stats.h +++ b/api/gc-basic-stats.h @@ -58,6 +58,7 @@ static inline void gc_basic_stats_mutators_stopped(void *data) {} static inline void gc_basic_stats_roots_traced(void *data) {} static inline void gc_basic_stats_heap_traced(void *data) {} static inline void gc_basic_stats_ephemerons_traced(void *data) {} +static inline void gc_basic_stats_finalizers_traced(void *data) {} static inline void gc_basic_stats_restarting_mutators(void *data) { struct gc_basic_stats *stats = data; @@ -100,6 +101,7 @@ static inline void gc_basic_stats_live_data_size(void *data, size_t size) { gc_basic_stats_roots_traced, \ gc_basic_stats_heap_traced, \ gc_basic_stats_ephemerons_traced, \ + gc_basic_stats_finalizers_traced, \ gc_basic_stats_restarting_mutators, \ gc_basic_stats_mutator_added, \ gc_basic_stats_mutator_cause_gc, \ diff --git a/api/gc-embedder-api.h b/api/gc-embedder-api.h index 30ba62946..bb091caeb 100644 --- a/api/gc-embedder-api.h +++ b/api/gc-embedder-api.h @@ -16,7 +16,6 @@ struct gc_mutator_roots; struct gc_heap_roots; struct gc_atomic_forward; struct gc_heap; -struct gc_ephemeron; struct gc_extern_space; GC_EMBEDDER_API inline int gc_is_valid_conservative_ref_displacement(uintptr_t displacement); diff --git a/api/gc-event-listener-chain.h b/api/gc-event-listener-chain.h index 3bebf3531..96a7356a8 100644 --- a/api/gc-event-listener-chain.h +++ b/api/gc-event-listener-chain.h @@ -57,6 +57,11 @@ static inline void gc_event_listener_chain_ephemerons_traced(void *data) { chain->head.ephemerons_traced(chain->head_data); chain->tail.ephemerons_traced(chain->tail_data); } +static inline void gc_event_listener_chain_finalizers_traced(void *data) { + struct gc_event_listener_chain *chain = data; + chain->head.finalizers_traced(chain->head_data); + chain->tail.finalizers_traced(chain->tail_data); +} static inline void gc_event_listener_chain_restarting_mutators(void *data) { struct gc_event_listener_chain *chain = data; @@ -123,6 +128,7 @@ static inline void gc_event_listener_chain_live_data_size(void *data, size_t siz gc_event_listener_chain_roots_traced, \ gc_event_listener_chain_heap_traced, \ gc_event_listener_chain_ephemerons_traced, \ + gc_event_listener_chain_finalizers_traced, \ gc_event_listener_chain_restarting_mutators, \ gc_event_listener_chain_mutator_added, \ gc_event_listener_chain_mutator_cause_gc, \ diff --git a/api/gc-event-listener.h b/api/gc-event-listener.h index 25558838b..f5d8180f6 100644 --- a/api/gc-event-listener.h +++ b/api/gc-event-listener.h @@ -12,6 +12,7 @@ struct gc_event_listener { void (*roots_traced)(void *data); void (*heap_traced)(void *data); void (*ephemerons_traced)(void *data); + void (*finalizers_traced)(void *data); void (*restarting_mutators)(void *data); void* (*mutator_added)(void *data); diff --git a/api/gc-finalizer.h b/api/gc-finalizer.h new file mode 100644 index 000000000..1dcb0fb2f --- /dev/null +++ b/api/gc-finalizer.h @@ -0,0 +1,81 @@ +#ifndef GC_FINALIZER_H_ +#define GC_FINALIZER_H_ + +#include "gc-edge.h" +#include "gc-ref.h" +#include "gc-visibility.h" + +// A finalizer allows the embedder to be notified when an object becomes +// unreachable. +// +// A finalizer has a priority. When the heap is created, the embedder +// should declare how many priorities there are. Lower-numbered +// priorities take precedence; if an object has a priority-0 finalizer +// outstanding, that will prevent any finalizer at level 1 (or 2, ...) +// from firing until no priority-0 finalizer remains. +// +// Call gc_attach_finalizer to attach a finalizer to an object. +// +// A finalizer also references an associated GC-managed closure object. +// A finalizer's reference to the closure object is strong: if a +// finalizer's closure closure references its finalizable object, +// directly or indirectly, the finalizer will never fire. +// +// When an object with a finalizer becomes unreachable, it is added to a +// queue. The embedder can call gc_pop_finalizable to get the next +// finalizable object and its associated closure. At that point the +// embedder can do anything with the object, including keeping it alive. +// Ephemeron associations will still be present while the finalizable +// object is live. Note however that any objects referenced by the +// finalizable object may themselves be already finalized; finalizers +// are enqueued for objects when they become unreachable, which can +// concern whole subgraphs of objects at once. +// +// The usual way for an embedder to know when the queue of finalizable +// object is non-empty is to call gc_set_finalizer_callback to +// provide a function that will be invoked when there are pending +// finalizers. +// +// Arranging to call gc_pop_finalizable and doing something with the +// finalizable object and closure is the responsibility of the embedder. +// The embedder's finalization action can end up invoking arbitrary +// code, so unless the embedder imposes some kind of restriction on what +// finalizers can do, generally speaking finalizers should be run in a +// dedicated thread instead of recursively from within whatever mutator +// thread caused GC. Setting up such a thread is the responsibility of +// the mutator. gc_pop_finalizable is thread-safe, allowing multiple +// finalization threads if that is appropriate. +// +// gc_allocate_finalizer returns a finalizer, which is a fresh +// GC-managed heap object. The mutator should then directly attach it +// to an object using gc_finalizer_attach. When the finalizer is fired, +// it becomes available to the mutator via gc_pop_finalizable. + +struct gc_heap; +struct gc_mutator; +struct gc_finalizer; + +GC_API_ size_t gc_finalizer_size(void); +GC_API_ struct gc_finalizer* gc_allocate_finalizer(struct gc_mutator *mut); +GC_API_ void gc_finalizer_attach(struct gc_mutator *mut, + struct gc_finalizer *finalizer, + unsigned priority, + struct gc_ref object, struct gc_ref closure); + +GC_API_ struct gc_ref gc_finalizer_object(struct gc_finalizer *finalizer); +GC_API_ struct gc_ref gc_finalizer_closure(struct gc_finalizer *finalizer); + +GC_API_ struct gc_finalizer* gc_pop_finalizable(struct gc_mutator *mut); + +typedef void (*gc_finalizer_callback)(struct gc_heap *heap, size_t count); +GC_API_ void gc_set_finalizer_callback(struct gc_heap *heap, + gc_finalizer_callback callback); + +GC_API_ void gc_trace_finalizer(struct gc_finalizer *finalizer, + void (*visit)(struct gc_edge edge, + struct gc_heap *heap, + void *visit_data), + struct gc_heap *heap, + void *trace_data); + +#endif // GC_FINALIZER_H_ diff --git a/api/gc-null-event-listener.h b/api/gc-null-event-listener.h index 5ca17975e..7563c3a46 100644 --- a/api/gc-null-event-listener.h +++ b/api/gc-null-event-listener.h @@ -12,6 +12,7 @@ static inline void gc_null_event_listener_mutators_stopped(void *data) {} static inline void gc_null_event_listener_roots_traced(void *data) {} static inline void gc_null_event_listener_heap_traced(void *data) {} static inline void gc_null_event_listener_ephemerons_traced(void *data) {} +static inline void gc_null_event_listener_finalizers_traced(void *data) {} static inline void gc_null_event_listener_restarting_mutators(void *data) {} static inline void* gc_null_event_listener_mutator_added(void *data) {} @@ -34,6 +35,7 @@ static inline void gc_null_event_listener_live_data_size(void *, size_t) {} gc_null_event_listener_roots_traced, \ gc_null_event_listener_heap_traced, \ gc_null_event_listener_ephemerons_traced, \ + gc_null_event_listener_finalizers_traced, \ gc_null_event_listener_restarting_mutators, \ gc_null_event_listener_mutator_added, \ gc_null_event_listener_mutator_cause_gc, \ diff --git a/benchmarks/simple-gc-embedder.h b/benchmarks/simple-gc-embedder.h index 23fc54a5d..683cc15ca 100644 --- a/benchmarks/simple-gc-embedder.h +++ b/benchmarks/simple-gc-embedder.h @@ -6,6 +6,9 @@ #include "gc-embedder-api.h" #define GC_EMBEDDER_EPHEMERON_HEADER struct gc_header header; +#define GC_EMBEDDER_FINALIZER_HEADER struct gc_header header; + +static inline size_t gc_finalizer_priority_count(void) { return 2; } static inline int gc_is_valid_conservative_ref_displacement(uintptr_t displacement) { diff --git a/embed.mk b/embed.mk index 49a7b7347..9284781e0 100644 --- a/embed.mk +++ b/embed.mk @@ -31,6 +31,8 @@ $(GC_OBJDIR)gc-options.o: $(WHIPPET)src/gc-options.c $(GC_COMPILE) -c $< $(GC_OBJDIR)gc-ephemeron.o: $(WHIPPET)src/gc-ephemeron.c $(GC_COMPILE) $(EMBEDDER_TO_GC_CFLAGS) -c $< +$(GC_OBJDIR)gc-finalizer.o: $(WHIPPET)src/gc-finalizer.c + $(GC_COMPILE) $(EMBEDDER_TO_GC_CFLAGS) -c $< GC_STEM_bdw = bdw GC_CFLAGS_bdw = -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 @@ -83,4 +85,4 @@ GC_LIBS = $(call gc_libs,$(GC_COLLECTOR)) $(GC_OBJDIR)gc-impl.o: $(WHIPPET)src/$(call gc_impl,$(GC_COLLECTOR)) $(GC_COMPILE) $(GC_IMPL_CFLAGS) $(EMBEDDER_TO_GC_CFLAGS) -c $< -GC_OBJS=$(foreach O,gc-platform.o gc-stack.o gc-options.o gc-ephemeron.o gc-impl.o,$(GC_OBJDIR)$(O)) +GC_OBJS=$(foreach O,gc-platform.o gc-stack.o gc-options.o gc-ephemeron.o gc-finalizer.o gc-impl.o,$(GC_OBJDIR)$(O)) diff --git a/src/bdw.c b/src/bdw.c index f429b43c2..e9b7cb9f5 100644 --- a/src/bdw.c +++ b/src/bdw.c @@ -55,6 +55,8 @@ struct gc_heap { struct gc_heap_roots *roots; struct gc_mutator *mutators; struct gc_event_listener event_listener; + struct gc_finalizer_state *finalizer_state; + gc_finalizer_callback have_finalizers; void *event_listener_data; }; @@ -165,6 +167,7 @@ static void bdw_mark_edge(struct gc_edge edge, struct gc_heap *heap, static int heap_gc_kind; static int mutator_gc_kind; static int ephemeron_gc_kind; +static int finalizer_gc_kind; // In BDW-GC, we can't hook into the mark phase to call // gc_trace_ephemerons_for_object, so the advertised ephemeron strategy @@ -199,6 +202,46 @@ int gc_visit_ephemeron_key(struct gc_edge edge, struct gc_heap *heap) { return 1; } +struct gc_finalizer* gc_allocate_finalizer(struct gc_mutator *mut) { + return GC_generic_malloc(gc_finalizer_size(), finalizer_gc_kind); +} + +static void finalize_object(void *obj, void *data) { + struct gc_finalizer *f = data; + gc_finalizer_externally_fired(__the_bdw_gc_heap->finalizer_state, f); +} + +void gc_finalizer_attach(struct gc_mutator *mut, struct gc_finalizer *finalizer, + unsigned priority, struct gc_ref object, + struct gc_ref closure) { + // Don't bother much about the actual finalizer; just delegate to BDW-GC. + GC_finalization_proc prev = NULL; + void *prev_data = NULL; + gc_finalizer_init_internal(finalizer, object, closure); + gc_finalizer_externally_activated(finalizer); + GC_REGISTER_FINALIZER_NO_ORDER (gc_ref_heap_object(object), finalize_object, + finalizer, &prev, &prev_data); + // FIXME: Allow multiple finalizers per object. + GC_ASSERT(prev == NULL); + GC_ASSERT(prev_data == NULL); +} + +struct gc_finalizer* gc_finalizer_pop(struct gc_mutator *mut) { + GC_invoke_finalizers(); + return gc_finalizer_state_pop(mut->heap->finalizer_state); +} + +void gc_set_finalizer_callback(struct gc_heap *heap, + gc_finalizer_callback callback) { + heap->have_finalizers = callback; +} + +static void have_finalizers(void) { + struct gc_heap *heap = __the_bdw_gc_heap; + if (heap->have_finalizers) + heap->have_finalizers(heap, 1); +} + static struct GC_ms_entry * mark_ephemeron(GC_word *addr, struct GC_ms_entry *mark_stack_ptr, struct GC_ms_entry *mark_stack_limit, GC_word env) { @@ -228,6 +271,29 @@ mark_ephemeron(GC_word *addr, struct GC_ms_entry *mark_stack_ptr, return state.mark_stack_ptr; } +static struct GC_ms_entry * +mark_finalizer(GC_word *addr, struct GC_ms_entry *mark_stack_ptr, + struct GC_ms_entry *mark_stack_limit, GC_word env) { + + struct bdw_mark_state state = { + mark_stack_ptr, + mark_stack_limit, + }; + + struct gc_finalizer *finalizer = (struct gc_finalizer*) addr; + + // If this ephemeron is on a freelist, its first word will be a + // freelist link and everything else will be NULL. + if (!gc_ref_value(gc_finalizer_object(finalizer))) { + bdw_mark_edge(gc_edge(addr), NULL, &state); + return state.mark_stack_ptr; + } + + gc_trace_finalizer(finalizer, bdw_mark_edge, NULL, &state); + + return state.mark_stack_ptr; +} + static struct GC_ms_entry * mark_heap(GC_word *addr, struct GC_ms_entry *mark_stack_ptr, struct GC_ms_entry *mark_stack_limit, GC_word env) { @@ -428,6 +494,8 @@ int gc_init(const struct gc_options *options, struct gc_stack_addr *stack_base, } GC_set_all_interior_pointers (0); + GC_set_finalize_on_demand (1); + GC_set_finalizer_notifier(have_finalizers); // Not part of 7.3, sigh. Have to set an env var. // GC_set_markers_count(options->common.parallelism); @@ -453,6 +521,9 @@ int gc_init(const struct gc_options *options, struct gc_stack_addr *stack_base, ephemeron_gc_kind = GC_new_kind(GC_new_free_list(), GC_MAKE_PROC(GC_new_proc(mark_ephemeron), 0), add_size_to_descriptor, clear_memory); + finalizer_gc_kind = GC_new_kind(GC_new_free_list(), + GC_MAKE_PROC(GC_new_proc(mark_finalizer), 0), + add_size_to_descriptor, clear_memory); } *heap = GC_generic_malloc(sizeof(struct gc_heap), heap_gc_kind); @@ -460,6 +531,7 @@ int gc_init(const struct gc_options *options, struct gc_stack_addr *stack_base, (*heap)->event_listener = event_listener; (*heap)->event_listener_data = event_listener_data; + (*heap)->finalizer_state = gc_make_finalizer_state(); __the_bdw_gc_heap = *heap; HEAP_EVENT(init, GC_get_heap_size()); diff --git a/src/gc-finalizer-internal.h b/src/gc-finalizer-internal.h new file mode 100644 index 000000000..529a087ee --- /dev/null +++ b/src/gc-finalizer-internal.h @@ -0,0 +1,65 @@ +#ifndef GC_FINALIZER_INTERNAL_H +#define GC_FINALIZER_INTERNAL_H + +#ifndef GC_IMPL +#error internal header file, not part of API +#endif + +#include "gc-finalizer.h" +#include "root.h" + +struct gc_finalizer_state; + +GC_INTERNAL +struct gc_finalizer_state* gc_make_finalizer_state(void); + +GC_INTERNAL +void gc_finalizer_init_internal(struct gc_finalizer *f, + struct gc_ref object, + struct gc_ref closure); + +GC_INTERNAL +void gc_finalizer_attach_internal(struct gc_finalizer_state *state, + struct gc_finalizer *f, + unsigned priority); + +GC_INTERNAL +void gc_finalizer_externally_activated(struct gc_finalizer *f); + +GC_INTERNAL +void gc_finalizer_externally_fired(struct gc_finalizer_state *state, + struct gc_finalizer *finalizer); + +GC_INTERNAL +struct gc_finalizer* gc_finalizer_state_pop(struct gc_finalizer_state *state); + +GC_INTERNAL +void gc_finalizer_fire(struct gc_finalizer **fired_list_loc, + struct gc_finalizer *finalizer); + +GC_INTERNAL +void gc_finalizer_state_set_callback(struct gc_finalizer_state *state, + gc_finalizer_callback callback); + +GC_INTERNAL +size_t gc_visit_finalizer_roots(struct gc_finalizer_state *state, + void (*visit)(struct gc_edge edge, + struct gc_heap *heap, + void *visit_data), + struct gc_heap *heap, + void *visit_data); + +GC_INTERNAL +size_t gc_resolve_finalizers(struct gc_finalizer_state *state, + size_t priority, + void (*visit)(struct gc_edge edge, + struct gc_heap *heap, + void *visit_data), + struct gc_heap *heap, + void *visit_data); + +GC_INTERNAL +void gc_notify_finalizers(struct gc_finalizer_state *state, + struct gc_heap *heap); + +#endif // GC_FINALIZER_INTERNAL_H diff --git a/src/gc-finalizer.c b/src/gc-finalizer.c new file mode 100644 index 000000000..ed9c39173 --- /dev/null +++ b/src/gc-finalizer.c @@ -0,0 +1,307 @@ +#include +#include +#include +#include + +#define GC_IMPL 1 + +#include "debug.h" +#include "gc-embedder-api.h" +#include "gc-ephemeron-internal.h" // for gc_visit_ephemeron_key +#include "gc-finalizer-internal.h" + +// # Overview +// +// See gc-finalizer.h for a overview of finalizers from the user and +// embedder point of view. +// +// ## Tracing +// +// From the perspecive of the collector implementation, finalizers are +// GC-managed objects, allowing their size to be accounted for within +// the heap size. They get traced during collection, allowing for +// relocation of their object references, and allowing the finalizer +// object itself to be evacuated if appropriate. +// +// The collector holds on to outstanding finalizers in a *finalizer +// state*, which holds one *finalizer table* for each priority. We +// don't need to look up finalizers by object, so we could just hold +// them in a big list, but to facilitate parallelism we slice them +// across some number of shards, where the "next" pointer is part of the +// finalizer object. +// +// There are a number of ways you could imagine integrating finalizers +// into a system. The way Whippet does it goes like this. See +// https://wingolog.org/archives/2022/10/31/ephemerons-and-finalizers +// and +// https://wingolog.org/archives/2024/07/22/finalizers-guardians-phantom-references-et-cetera +// for some further discussion. +// +// 1. The collector should begin a cycle by adding all shards from all +// priorities to the root set. When the embedder comes across a +// finalizer (as it will, because we added them to the root set), +// it traces it via gc_trace_finalizer(), which will visit the +// finalizer's closure and its "next" pointer. +// +// 2. After the full trace, and then the fix-point on pending +// ephemerons, for each priority from 0 upwards: +// +// i. Visit each finalizable object in the table. If the object +// was as-yet unvisited, then it is unreachable and thus +// finalizable; the finalizer is added to the global "fired" +// list, and changes state from "attached" to "fired". +// Otherwise it is re-added to the finalizer table. +// +// ii. If any finalizer was added to the fired list, then those +// objects were also added to the grey worklist; run tracing +// again until the grey set is empty, including ephemerons. +// +// 3. Finally, call the finalizer callback if the list of fired finalizers is +// nonempty. +// +// ## Concurrency +// +// The finalizer table is wait-free. It keeps a count of active finalizers, and +// chooses a bucket based on the count modulo the number of buckets. Adding a +// finalizer to the table is an atomic push on a linked list. The table is +// completely rebuilt during the GC pause, redistributing survivor entries +// across the buckets, and pushing all finalizable entries onto the single +// "fired" linked list. +// +// The fired list is also wait-free. As noted above, it is built +// during the pause, and mutators pop items off of it atomically. +// +// ## Generations +// +// It would be ideal if a young generation had its own finalizer table. +// Promoting an object would require promoting its finalizer to the old +// finalizer table. Not yet implemented (but would be nice). + +#ifndef GC_EMBEDDER_FINALIZER_HEADER +#error Embedder should define GC_EMBEDDER_FINALIZER_HEADER +#endif + +enum finalizer_state { + FINALIZER_STATE_INIT = 0, // Finalizer is newborn. + FINALIZER_STATE_ACTIVE, // Finalizer is ours and in the finalizer table. + FINALIZER_STATE_FIRED, // Finalizer is handed back to mutator. +}; + +struct gc_finalizer { + GC_EMBEDDER_FINALIZER_HEADER + enum finalizer_state state; + struct gc_ref object; + struct gc_ref closure; + struct gc_finalizer *next; +}; + +// Enough buckets to parallelize closure marking. No need to look up a +// finalizer for a given object. +#define BUCKET_COUNT 32 + +struct gc_finalizer_table { + size_t finalizer_count; + struct gc_finalizer* buckets[BUCKET_COUNT]; +}; + +struct gc_finalizer_state { + gc_finalizer_callback have_finalizers; + struct gc_finalizer *fired; + size_t fired_this_cycle; + size_t table_count; + struct gc_finalizer_table tables[0]; +}; + +// public +size_t gc_finalizer_size(void) { return sizeof(struct gc_finalizer); } +struct gc_ref gc_finalizer_object(struct gc_finalizer *f) { return f->object; } +struct gc_ref gc_finalizer_closure(struct gc_finalizer *f) { return f->closure; } + +// internal +struct gc_finalizer_state* gc_make_finalizer_state(void) { + size_t ntables = gc_finalizer_priority_count(); + size_t size = (sizeof(struct gc_finalizer_state) + + sizeof(struct gc_finalizer_table) * ntables); + struct gc_finalizer_state *ret = malloc(size); + if (!ret) + return NULL; + memset(ret, 0, size); + ret->table_count = ntables; + return ret; +} + +static void finalizer_list_push(struct gc_finalizer **loc, + struct gc_finalizer *head) { + struct gc_finalizer *tail = atomic_load_explicit(loc, memory_order_acquire); + do { + head->next = tail; + } while (!atomic_compare_exchange_weak(loc, &tail, head)); +} + +static struct gc_finalizer* finalizer_list_pop(struct gc_finalizer **loc) { + struct gc_finalizer *head = atomic_load_explicit(loc, memory_order_acquire); + do { + if (!head) return NULL; + } while (!atomic_compare_exchange_weak(loc, &head, head->next)); + head->next = NULL; + return head; +} + +static void add_finalizer_to_table(struct gc_finalizer_table *table, + struct gc_finalizer *f) { + size_t count = atomic_fetch_add_explicit(&table->finalizer_count, 1, + memory_order_relaxed); + struct gc_finalizer **loc = &table->buckets[count % BUCKET_COUNT]; + finalizer_list_push(loc, f); +} + +// internal +void gc_finalizer_init_internal(struct gc_finalizer *f, + struct gc_ref object, + struct gc_ref closure) { + // Caller responsible for any write barrier, though really the + // assumption is that the finalizer is younger than the key and the + // value. + if (f->state != FINALIZER_STATE_INIT) + GC_CRASH(); + if (gc_ref_is_heap_object(f->object)) + GC_CRASH(); + f->object = object; + f->closure = closure; +} + +// internal +void gc_finalizer_attach_internal(struct gc_finalizer_state *state, + struct gc_finalizer *f, + unsigned priority) { + // Caller responsible for any write barrier, though really the + // assumption is that the finalizer is younger than the key and the + // value. + if (f->state != FINALIZER_STATE_INIT) + GC_CRASH(); + if (!gc_ref_is_heap_object(f->object)) + GC_CRASH(); + + f->state = FINALIZER_STATE_ACTIVE; + + GC_ASSERT(priority < state->table_count); + add_finalizer_to_table(&state->tables[priority], f); +} + +// internal +struct gc_finalizer* gc_finalizer_state_pop(struct gc_finalizer_state *state) { + return finalizer_list_pop(&state->fired); +} + +static void +add_fired_finalizer(struct gc_finalizer_state *state, + struct gc_finalizer *f) { + if (f->state != FINALIZER_STATE_ACTIVE) + GC_CRASH(); + f->state = FINALIZER_STATE_FIRED; + finalizer_list_push(&state->fired, f); +} + +// internal +void +gc_finalizer_externally_activated(struct gc_finalizer *f) { + if (f->state != FINALIZER_STATE_INIT) + GC_CRASH(); + f->state = FINALIZER_STATE_ACTIVE; +} + +// internal +void +gc_finalizer_externally_fired(struct gc_finalizer_state *state, + struct gc_finalizer *f) { + add_fired_finalizer(state, f); +} + +// internal +size_t gc_visit_finalizer_roots(struct gc_finalizer_state *state, + void (*visit)(struct gc_edge, + struct gc_heap*, + void *), + struct gc_heap *heap, + void *visit_data) { + size_t count; + for (size_t tidx = 0; tidx < state->table_count; tidx++) { + struct gc_finalizer_table *table = &state->tables[tidx]; + if (table->finalizer_count) { + count += table->finalizer_count; + for (size_t bidx = 0; bidx < BUCKET_COUNT; bidx++) + visit(gc_edge(&table->buckets[bidx]), heap, visit_data); + } + } + return count; +} + +// public +void gc_trace_finalizer(struct gc_finalizer *f, + void (*visit)(struct gc_edge edge, + struct gc_heap *heap, + void *visit_data), + struct gc_heap *heap, + void *trace_data) { + if (f->state != FINALIZER_STATE_ACTIVE) + visit(gc_edge(&f->object), heap, trace_data); + visit(gc_edge(&f->closure), heap, trace_data); + visit(gc_edge(&f->next), heap, trace_data); +} + +// Sweeping is currently serial. It could run in parallel but we want to +// resolve all finalizers before shading any additional node. Perhaps we should +// relax this restriction though; if the user attaches two finalizers to the +// same object, it's probably OK to only have one finalizer fire per cycle. + +// internal +size_t gc_resolve_finalizers(struct gc_finalizer_state *state, + size_t priority, + void (*visit)(struct gc_edge edge, + struct gc_heap *heap, + void *visit_data), + struct gc_heap *heap, + void *visit_data) { + GC_ASSERT(priority < state->table_count); + struct gc_finalizer_table *table = &state->tables[priority]; + size_t finalizers_fired = 0; + // Visit each finalizer in the table. If its object was already visited, + // re-add the finalizer to the table. Otherwise enqueue its object edge for + // tracing and mark the finalizer as fired. + if (table->finalizer_count) { + struct gc_finalizer_table scratch = { 0, }; + for (size_t bidx = 0; bidx < BUCKET_COUNT; bidx++) { + struct gc_finalizer *next; + for (struct gc_finalizer *f = table->buckets[bidx]; f; f = next) { + next = f->next; + f->next = NULL; + struct gc_edge edge = gc_edge(&f->object); + if (gc_visit_ephemeron_key(edge, heap)) { + add_finalizer_to_table(&scratch, f); + } else { + finalizers_fired++; + visit(edge, heap, visit_data); + add_fired_finalizer(state, f); + } + } + } + memcpy(table, &scratch, sizeof(*table)); + } + state->fired_this_cycle += finalizers_fired; + return finalizers_fired; +} + +// internal +void gc_notify_finalizers(struct gc_finalizer_state *state, + struct gc_heap *heap) { + if (state->fired_this_cycle && state->have_finalizers) { + state->have_finalizers(heap, state->fired_this_cycle); + state->fired_this_cycle = 0; + } +} + +// internal +void gc_finalizer_state_set_callback(struct gc_finalizer_state *state, + gc_finalizer_callback callback) { + state->have_finalizers = callback; +} diff --git a/src/gc-internal.h b/src/gc-internal.h index abc9bd83a..7cbb79f58 100644 --- a/src/gc-internal.h +++ b/src/gc-internal.h @@ -6,6 +6,7 @@ #endif #include "gc-ephemeron-internal.h" +#include "gc-finalizer-internal.h" #include "gc-options-internal.h" #endif // GC_INTERNAL_H diff --git a/src/pcc.c b/src/pcc.c index c71f2d04e..fa7342e4d 100644 --- a/src/pcc.c +++ b/src/pcc.c @@ -131,6 +131,7 @@ struct gc_heap { int collecting; int check_pending_ephemerons; struct gc_pending_ephemerons *pending_ephemerons; + struct gc_finalizer_state *finalizer_state; size_t mutator_count; size_t paused_mutator_count; size_t inactive_mutator_count; @@ -649,6 +650,9 @@ static inline void trace_root(struct gc_root root, struct gc_heap *heap, gc_trace_resolved_ephemerons(root.resolved_ephemerons, tracer_visit, heap, worker); break; + case GC_ROOT_KIND_EDGE: + tracer_visit(root.edge, heap, worker); + break; default: GC_CRASH(); } @@ -712,10 +716,16 @@ static int maybe_grow_heap(struct gc_heap *heap) { return 0; } +static void visit_root_edge(struct gc_edge edge, struct gc_heap *heap, + void *unused) { + gc_tracer_add_root(&heap->tracer, gc_root_edge(edge)); +} + static void add_roots(struct gc_heap *heap) { for (struct gc_mutator *mut = heap->mutators; mut; mut = mut->next) gc_tracer_add_root(&heap->tracer, gc_root_mutator(mut)); gc_tracer_add_root(&heap->tracer, gc_root_heap(heap)); + gc_visit_finalizer_roots(heap->finalizer_state, visit_root_edge, heap, NULL); } static void resolve_ephemerons_lazily(struct gc_heap *heap) { @@ -729,12 +739,26 @@ static void resolve_ephemerons_eagerly(struct gc_heap *heap) { gc_scan_pending_ephemerons(heap->pending_ephemerons, heap, 0, 1); } -static int enqueue_resolved_ephemerons(struct gc_heap *heap) { - struct gc_ephemeron *resolved = gc_pop_resolved_ephemerons(heap); - if (!resolved) - return 0; - gc_tracer_add_root(&heap->tracer, gc_root_resolved_ephemerons(resolved)); - return 1; +static void trace_resolved_ephemerons(struct gc_heap *heap) { + for (struct gc_ephemeron *resolved = gc_pop_resolved_ephemerons(heap); + resolved; + resolved = gc_pop_resolved_ephemerons(heap)) { + gc_tracer_add_root(&heap->tracer, gc_root_resolved_ephemerons(resolved)); + gc_tracer_trace(&heap->tracer); + } +} + +static void resolve_finalizers(struct gc_heap *heap) { + for (size_t priority = 0; + priority < gc_finalizer_priority_count(); + priority++) { + if (gc_resolve_finalizers(heap->finalizer_state, priority, + visit_root_edge, heap, NULL)) { + gc_tracer_trace(&heap->tracer); + trace_resolved_ephemerons(heap); + } + } + gc_notify_finalizers(heap->finalizer_state, heap); } static void sweep_ephemerons(struct gc_heap *heap) { @@ -765,9 +789,10 @@ static void collect(struct gc_mutator *mut) { gc_tracer_trace(&heap->tracer); HEAP_EVENT(heap, heap_traced); resolve_ephemerons_eagerly(heap); - while (enqueue_resolved_ephemerons(heap)) - gc_tracer_trace(&heap->tracer); + trace_resolved_ephemerons(heap); HEAP_EVENT(heap, ephemerons_traced); + resolve_finalizers(heap); + HEAP_EVENT(heap, finalizers_traced); sweep_ephemerons(heap); gc_tracer_release(&heap->tracer); pcc_space_finish_gc(cspace); @@ -891,6 +916,28 @@ unsigned gc_heap_ephemeron_trace_epoch(struct gc_heap *heap) { return heap->count; } +struct gc_finalizer* gc_allocate_finalizer(struct gc_mutator *mut) { + return gc_allocate(mut, gc_finalizer_size()); +} + +void gc_finalizer_attach(struct gc_mutator *mut, struct gc_finalizer *finalizer, + unsigned priority, struct gc_ref object, + struct gc_ref closure) { + gc_finalizer_init_internal(finalizer, object, closure); + gc_finalizer_attach_internal(mutator_heap(mut)->finalizer_state, + finalizer, priority); + // No write barrier. +} + +struct gc_finalizer* gc_finalizer_pop(struct gc_mutator *mut) { + return gc_finalizer_state_pop(mutator_heap(mut)->finalizer_state); +} + +void gc_set_finalizer_callback(struct gc_heap *heap, + gc_finalizer_callback callback) { + gc_finalizer_state_set_callback(heap->finalizer_state, callback); +} + static struct pcc_slab* allocate_slabs(size_t nslabs) { size_t size = nslabs * SLAB_SIZE; size_t extent = size + SLAB_SIZE; @@ -969,6 +1016,10 @@ static int heap_init(struct gc_heap *heap, const struct gc_options *options) { if (!heap_prepare_pending_ephemerons(heap)) GC_CRASH(); + heap->finalizer_state = gc_make_finalizer_state(); + if (!heap->finalizer_state) + GC_CRASH(); + return 1; } diff --git a/src/root.h b/src/root.h index a6a91f987..5228dcb4f 100644 --- a/src/root.h +++ b/src/root.h @@ -1,6 +1,8 @@ #ifndef ROOT_H #define ROOT_H +#include "gc-edge.h" + struct gc_ephemeron; struct gc_heap; struct gc_mutator; @@ -9,7 +11,8 @@ enum gc_root_kind { GC_ROOT_KIND_NONE, GC_ROOT_KIND_HEAP, GC_ROOT_KIND_MUTATOR, - GC_ROOT_KIND_RESOLVED_EPHEMERONS + GC_ROOT_KIND_RESOLVED_EPHEMERONS, + GC_ROOT_KIND_EDGE, }; struct gc_root { @@ -18,6 +21,7 @@ struct gc_root { struct gc_heap *heap; struct gc_mutator *mutator; struct gc_ephemeron *resolved_ephemerons; + struct gc_edge edge; }; }; @@ -40,4 +44,11 @@ gc_root_resolved_ephemerons(struct gc_ephemeron* resolved) { return ret; } +static inline struct gc_root +gc_root_edge(struct gc_edge edge) { + struct gc_root ret = { GC_ROOT_KIND_EDGE }; + ret.edge = edge; + return ret; +} + #endif // ROOT_H diff --git a/src/semi.c b/src/semi.c index af9134bd7..739a21d75 100644 --- a/src/semi.c +++ b/src/semi.c @@ -37,6 +37,7 @@ struct gc_heap { struct semi_space semi_space; struct large_object_space large_object_space; struct gc_pending_ephemerons *pending_ephemerons; + struct gc_finalizer_state *finalizer_state; struct gc_extern_space *extern_space; double pending_ephemerons_size_factor; double pending_ephemerons_size_slop; @@ -350,6 +351,37 @@ static void adjust_heap_size_and_limits(struct gc_heap *heap, semi->limit = new_limit; } +static uintptr_t trace_closure(struct gc_heap *heap, struct semi_space *semi, + uintptr_t grey) { + while(grey < semi->hp) + grey = scan(heap, gc_ref(grey)); + return grey; +} + +static uintptr_t resolve_ephemerons(struct gc_heap *heap, uintptr_t grey) { + for (struct gc_ephemeron *resolved = gc_pop_resolved_ephemerons(heap); + resolved; + resolved = gc_pop_resolved_ephemerons(heap)) { + gc_trace_resolved_ephemerons(resolved, trace, heap, NULL); + grey = trace_closure(heap, heap_semi_space(heap), grey); + } + return grey; +} + +static uintptr_t resolve_finalizers(struct gc_heap *heap, uintptr_t grey) { + for (size_t priority = 0; + priority < gc_finalizer_priority_count(); + priority++) { + if (gc_resolve_finalizers(heap->finalizer_state, priority, + trace, heap, NULL)) { + grey = trace_closure(heap, heap_semi_space(heap), grey); + grey = resolve_ephemerons(heap, grey); + } + } + gc_notify_finalizers(heap->finalizer_state, heap); + return grey; +} + static void collect(struct gc_mutator *mut, size_t for_alloc) { struct gc_heap *heap = mutator_heap(mut); int is_minor = 0; @@ -373,22 +405,17 @@ static void collect(struct gc_mutator *mut, size_t for_alloc) { gc_trace_heap_roots(heap->roots, trace, heap, NULL); if (mut->roots) gc_trace_mutator_roots(mut->roots, trace, heap, NULL); + gc_visit_finalizer_roots(heap->finalizer_state, trace, heap, NULL); HEAP_EVENT(heap, roots_traced); // fprintf(stderr, "pushed %zd bytes in roots\n", space->hp - grey); - while(grey < semi->hp) - grey = scan(heap, gc_ref(grey)); + grey = trace_closure(heap, semi, grey); HEAP_EVENT(heap, heap_traced); gc_scan_pending_ephemerons(heap->pending_ephemerons, heap, 0, 1); heap->check_pending_ephemerons = 1; - do { - struct gc_ephemeron *resolved = gc_pop_resolved_ephemerons(heap); - if (!resolved) - break; - gc_trace_resolved_ephemerons(resolved, trace, heap, NULL); - while(grey < semi->hp) - grey = scan(heap, gc_ref(grey)); - } while (1); + grey = resolve_ephemerons(heap, grey); HEAP_EVENT(heap, ephemerons_traced); + grey = resolve_finalizers(heap, grey); + HEAP_EVENT(heap, finalizers_traced); large_object_space_finish_gc(large, 0); gc_extern_space_finish_gc(heap->extern_space, 0); semi_space_finish_gc(semi, large->live_pages_at_last_collection); @@ -486,6 +513,28 @@ void gc_ephemeron_init(struct gc_mutator *mut, struct gc_ephemeron *ephemeron, gc_ephemeron_init_internal(mutator_heap(mut), ephemeron, key, value); } +struct gc_finalizer* gc_allocate_finalizer(struct gc_mutator *mut) { + return gc_allocate(mut, gc_finalizer_size()); +} + +void gc_finalizer_attach(struct gc_mutator *mut, struct gc_finalizer *finalizer, + unsigned priority, struct gc_ref object, + struct gc_ref closure) { + gc_finalizer_init_internal(finalizer, object, closure); + gc_finalizer_attach_internal(mutator_heap(mut)->finalizer_state, + finalizer, priority); + // No write barrier. +} + +struct gc_finalizer* gc_finalizer_pop(struct gc_mutator *mut) { + return gc_finalizer_state_pop(mutator_heap(mut)->finalizer_state); +} + +void gc_set_finalizer_callback(struct gc_heap *heap, + gc_finalizer_callback callback) { + gc_finalizer_state_set_callback(heap->finalizer_state, callback); +} + static int region_init(struct region *region, size_t size) { region->base = 0; region->active_size = 0; @@ -542,8 +591,11 @@ static int heap_init(struct gc_heap *heap, const struct gc_options *options) { heap->options = options; heap->size = options->common.heap_size; heap->roots = NULL; + heap->finalizer_state = gc_make_finalizer_state(); + if (!heap->finalizer_state) + GC_CRASH(); - return heap_prepare_pending_ephemerons(heap); +return heap_prepare_pending_ephemerons(heap); } int gc_option_from_string(const char *str) { diff --git a/src/whippet.c b/src/whippet.c index 3e17f5422..3771babde 100644 --- a/src/whippet.c +++ b/src/whippet.c @@ -301,6 +301,7 @@ struct gc_heap { int mark_while_stopping; int check_pending_ephemerons; struct gc_pending_ephemerons *pending_ephemerons; + struct gc_finalizer_state *finalizer_state; enum gc_collection_kind gc_kind; int multithreaded; size_t mutator_count; @@ -1231,8 +1232,28 @@ static inline void trace_one(struct gc_ref ref, struct gc_heap *heap, static inline void trace_root(struct gc_root root, struct gc_heap *heap, struct gc_trace_worker *worker) { - // We don't use parallel root tracing yet. - GC_CRASH(); + switch (root.kind) { + case GC_ROOT_KIND_HEAP: + gc_trace_heap_roots(root.heap->roots, tracer_visit, heap, worker); + break; + case GC_ROOT_KIND_MUTATOR: + gc_trace_mutator_roots(root.mutator->roots, tracer_visit, heap, worker); + break; + case GC_ROOT_KIND_RESOLVED_EPHEMERONS: + gc_trace_resolved_ephemerons(root.resolved_ephemerons, tracer_visit, + heap, worker); + break; + case GC_ROOT_KIND_EDGE: + tracer_visit(root.edge, heap, worker); + break; + default: + GC_CRASH(); + } +} + +static void visit_root_edge(struct gc_edge edge, struct gc_heap *heap, + void *unused) { + gc_tracer_add_root(&heap->tracer, gc_root_edge(edge)); } static void @@ -1823,6 +1844,7 @@ static void trace_pinned_roots_after_stop(struct gc_heap *heap) { static void trace_roots_after_stop(struct gc_heap *heap) { trace_mutator_roots_after_stop(heap); gc_trace_heap_roots(heap->roots, trace_and_enqueue_globally, heap, NULL); + gc_visit_finalizer_roots(heap->finalizer_state, visit_root_edge, heap, NULL); trace_generational_roots(heap); } @@ -1890,6 +1912,28 @@ static int enqueue_resolved_ephemerons(struct gc_heap *heap) { return 1; } +static void trace_resolved_ephemerons(struct gc_heap *heap) { + for (struct gc_ephemeron *resolved = gc_pop_resolved_ephemerons(heap); + resolved; + resolved = gc_pop_resolved_ephemerons(heap)) { + gc_tracer_add_root(&heap->tracer, gc_root_resolved_ephemerons(resolved)); + gc_tracer_trace(&heap->tracer); + } +} + +static void resolve_finalizers(struct gc_heap *heap) { + for (size_t priority = 0; + priority < gc_finalizer_priority_count(); + priority++) { + if (gc_resolve_finalizers(heap->finalizer_state, priority, + visit_root_edge, heap, NULL)) { + gc_tracer_trace(&heap->tracer); + trace_resolved_ephemerons(heap); + } + } + gc_notify_finalizers(heap->finalizer_state, heap); +} + static void sweep_ephemerons(struct gc_heap *heap) { return gc_sweep_pending_ephemerons(heap->pending_ephemerons, 0, 1); } @@ -1934,9 +1978,10 @@ static void collect(struct gc_mutator *mut, gc_tracer_trace(&heap->tracer); HEAP_EVENT(heap, heap_traced); resolve_ephemerons_eagerly(heap); - while (enqueue_resolved_ephemerons(heap)) - gc_tracer_trace(&heap->tracer); + trace_resolved_ephemerons(heap); HEAP_EVENT(heap, ephemerons_traced); + resolve_finalizers(heap); + HEAP_EVENT(heap, finalizers_traced); sweep_ephemerons(heap); gc_tracer_release(&heap->tracer); mark_space_finish_gc(space, gc_kind); @@ -2322,6 +2367,28 @@ unsigned gc_heap_ephemeron_trace_epoch(struct gc_heap *heap) { return heap->count; } +struct gc_finalizer* gc_allocate_finalizer(struct gc_mutator *mut) { + return gc_allocate(mut, gc_finalizer_size()); +} + +void gc_finalizer_attach(struct gc_mutator *mut, struct gc_finalizer *finalizer, + unsigned priority, struct gc_ref object, + struct gc_ref closure) { + gc_finalizer_init_internal(finalizer, object, closure); + gc_finalizer_attach_internal(mutator_heap(mut)->finalizer_state, + finalizer, priority); + // No write barrier. +} + +struct gc_finalizer* gc_finalizer_pop(struct gc_mutator *mut) { + return gc_finalizer_state_pop(mutator_heap(mut)->finalizer_state); +} + +void gc_set_finalizer_callback(struct gc_heap *heap, + gc_finalizer_callback callback) { + gc_finalizer_state_set_callback(heap->finalizer_state, callback); +} + static struct slab* allocate_slabs(size_t nslabs) { size_t size = nslabs * SLAB_SIZE; size_t extent = size + SLAB_SIZE; @@ -2406,6 +2473,10 @@ static int heap_init(struct gc_heap *heap, const struct gc_options *options) { if (!heap_prepare_pending_ephemerons(heap)) GC_CRASH(); + heap->finalizer_state = gc_make_finalizer_state(); + if (!heap->finalizer_state) + GC_CRASH(); + return 1; }