Add finalizers

Makefile

@@ -52,6 +52,8 @@ obj/gc-options.o: src/gc-options.c | .deps obj
 	$(COMPILE) -c $<
 obj/%.gc-ephemeron.o: src/gc-ephemeron.c | .deps obj
 	$(COMPILE) -include benchmarks/$*-embedder.h -c $<
+obj/%.gc-finalizer.o: src/gc-finalizer.c | .deps obj
+	$(COMPILE) -include benchmarks/$*-embedder.h -c $<
 
 GC_STEM_bdw   	   = bdw
 GC_CFLAGS_bdw 	   = -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1
@@ -99,7 +101,7 @@ obj/$(1).$(2).gc.o: src/$(call gc_impl,$(2)) | .deps obj
 	$$(COMPILE) $(call gc_cflags,$(2)) $(call gc_impl_cflags,$(2)) -include benchmarks/$(1)-embedder.h -c $$<
 obj/$(1).$(2).o: benchmarks/$(1).c | .deps obj
 	$$(COMPILE) $(call gc_cflags,$(2)) -include api/$(call gc_attrs,$(2)) -c $$<
-bin/$(1).$(2): obj/$(1).$(2).gc.o obj/$(1).$(2).o obj/gc-stack.o obj/gc-options.o obj/gc-platform.o obj/$(1).gc-ephemeron.o | bin
+bin/$(1).$(2): obj/$(1).$(2).gc.o obj/$(1).$(2).o obj/gc-stack.o obj/gc-options.o obj/gc-platform.o obj/$(1).gc-ephemeron.o obj/$(1).gc-finalizer.o | bin
 	$$(LINK) $$^ $(call gc_libs,$(2))
 endef
 

README.md

@@ -45,7 +45,7 @@ a talk given at FOSDEM 2023.
  - [X] Conservative data segments
  - [ ] Heap growth/shrinking
  - [ ] Debugging/tracing
- - [ ] Finalizers
+ - [X] Finalizers
  - [X] Weak references / weak maps
 
 ### Features that would improve Whippet performance

api/gc-basic-stats.h

@@ -58,6 +58,7 @@ static inline void gc_basic_stats_mutators_stopped(void *data) {}
 static inline void gc_basic_stats_roots_traced(void *data) {}
 static inline void gc_basic_stats_heap_traced(void *data) {}
 static inline void gc_basic_stats_ephemerons_traced(void *data) {}
+static inline void gc_basic_stats_finalizers_traced(void *data) {}
 
 static inline void gc_basic_stats_restarting_mutators(void *data) {
   struct gc_basic_stats *stats = data;
@@ -100,6 +101,7 @@ static inline void gc_basic_stats_live_data_size(void *data, size_t size) {
     gc_basic_stats_roots_traced,                                        \
     gc_basic_stats_heap_traced,                                         \
     gc_basic_stats_ephemerons_traced,                                   \
+    gc_basic_stats_finalizers_traced,                                   \
     gc_basic_stats_restarting_mutators,                                 \
     gc_basic_stats_mutator_added,                                       \
     gc_basic_stats_mutator_cause_gc,                                    \

api/gc-embedder-api.h

@@ -16,7 +16,6 @@ struct gc_mutator_roots;
 struct gc_heap_roots;
 struct gc_atomic_forward;
 struct gc_heap;
-struct gc_ephemeron;
 struct gc_extern_space;
 
 GC_EMBEDDER_API inline int gc_is_valid_conservative_ref_displacement(uintptr_t displacement);

api/gc-event-listener-chain.h

@@ -57,6 +57,11 @@ static inline void gc_event_listener_chain_ephemerons_traced(void *data) {
   chain->head.ephemerons_traced(chain->head_data);
   chain->tail.ephemerons_traced(chain->tail_data);
 }
+static inline void gc_event_listener_chain_finalizers_traced(void *data) {
+  struct gc_event_listener_chain *chain = data;
+  chain->head.finalizers_traced(chain->head_data);
+  chain->tail.finalizers_traced(chain->tail_data);
+}
 
 static inline void gc_event_listener_chain_restarting_mutators(void *data) {
   struct gc_event_listener_chain *chain = data;
@@ -123,6 +128,7 @@ static inline void gc_event_listener_chain_live_data_size(void *data, size_t siz
     gc_event_listener_chain_roots_traced,                               \
     gc_event_listener_chain_heap_traced,                                \
     gc_event_listener_chain_ephemerons_traced,                          \
+    gc_event_listener_chain_finalizers_traced,                          \
     gc_event_listener_chain_restarting_mutators,                        \
     gc_event_listener_chain_mutator_added,                              \
     gc_event_listener_chain_mutator_cause_gc,                           \

api/gc-event-listener.h

@@ -12,6 +12,7 @@ struct gc_event_listener {
   void (*roots_traced)(void *data);
   void (*heap_traced)(void *data);
   void (*ephemerons_traced)(void *data);
+  void (*finalizers_traced)(void *data);
   void (*restarting_mutators)(void *data);
 
   void* (*mutator_added)(void *data);

api/gc-finalizer.h

@@ -0,0 +1,81 @@
+#ifndef GC_FINALIZER_H_
+#define GC_FINALIZER_H_
+
+#include "gc-edge.h"
+#include "gc-ref.h"
+#include "gc-visibility.h"
+
+// A finalizer allows the embedder to be notified when an object becomes
+// unreachable.
+//
+// A finalizer has a priority.  When the heap is created, the embedder
+// should declare how many priorities there are.  Lower-numbered
+// priorities take precedence; if an object has a priority-0 finalizer
+// outstanding, that will prevent any finalizer at level 1 (or 2, ...)
+// from firing until no priority-0 finalizer remains.
+//
+// Call gc_attach_finalizer to attach a finalizer to an object.
+//
+// A finalizer also references an associated GC-managed closure object.
+// A finalizer's reference to the closure object is strong:  if a
+// finalizer's closure closure references its finalizable object,
+// directly or indirectly, the finalizer will never fire.
+//
+// When an object with a finalizer becomes unreachable, it is added to a
+// queue.  The embedder can call gc_pop_finalizable to get the next
+// finalizable object and its associated closure.  At that point the
+// embedder can do anything with the object, including keeping it alive.
+// Ephemeron associations will still be present while the finalizable
+// object is live.  Note however that any objects referenced by the
+// finalizable object may themselves be already finalized; finalizers
+// are enqueued for objects when they become unreachable, which can
+// concern whole subgraphs of objects at once.
+//
+// The usual way for an embedder to know when the queue of finalizable
+// object is non-empty is to call gc_set_finalizer_callback to
+// provide a function that will be invoked when there are pending
+// finalizers.
+//
+// Arranging to call gc_pop_finalizable and doing something with the
+// finalizable object and closure is the responsibility of the embedder.
+// The embedder's finalization action can end up invoking arbitrary
+// code, so unless the embedder imposes some kind of restriction on what
+// finalizers can do, generally speaking finalizers should be run in a
+// dedicated thread instead of recursively from within whatever mutator
+// thread caused GC.  Setting up such a thread is the responsibility of
+// the mutator.  gc_pop_finalizable is thread-safe, allowing multiple
+// finalization threads if that is appropriate.
+//
+// gc_allocate_finalizer returns a finalizer, which is a fresh
+// GC-managed heap object.  The mutator should then directly attach it
+// to an object using gc_finalizer_attach.  When the finalizer is fired,
+// it becomes available to the mutator via gc_pop_finalizable.
+
+struct gc_heap;
+struct gc_mutator;
+struct gc_finalizer;
+
+GC_API_ size_t gc_finalizer_size(void);
+GC_API_ struct gc_finalizer* gc_allocate_finalizer(struct gc_mutator *mut);
+GC_API_ void gc_finalizer_attach(struct gc_mutator *mut,
+                                 struct gc_finalizer *finalizer,
+                                 unsigned priority,
+                                 struct gc_ref object, struct gc_ref closure);
+
+GC_API_ struct gc_ref gc_finalizer_object(struct gc_finalizer *finalizer);
+GC_API_ struct gc_ref gc_finalizer_closure(struct gc_finalizer *finalizer);
+
+GC_API_ struct gc_finalizer* gc_pop_finalizable(struct gc_mutator *mut);
+
+typedef void (*gc_finalizer_callback)(struct gc_heap *heap, size_t count);
+GC_API_ void gc_set_finalizer_callback(struct gc_heap *heap,
+                                       gc_finalizer_callback callback);
+
+GC_API_ void gc_trace_finalizer(struct gc_finalizer *finalizer,
+                                void (*visit)(struct gc_edge edge,
+                                              struct gc_heap *heap,
+                                              void *visit_data),
+                                struct gc_heap *heap,
+                                void *trace_data);
+
+#endif // GC_FINALIZER_H_

api/gc-null-event-listener.h

@@ -12,6 +12,7 @@ static inline void gc_null_event_listener_mutators_stopped(void *data) {}
 static inline void gc_null_event_listener_roots_traced(void *data) {}
 static inline void gc_null_event_listener_heap_traced(void *data) {}
 static inline void gc_null_event_listener_ephemerons_traced(void *data) {}
+static inline void gc_null_event_listener_finalizers_traced(void *data) {}
 static inline void gc_null_event_listener_restarting_mutators(void *data) {}
 
 static inline void* gc_null_event_listener_mutator_added(void *data) {}
@@ -34,6 +35,7 @@ static inline void gc_null_event_listener_live_data_size(void *, size_t) {}
     gc_null_event_listener_roots_traced,                               \
     gc_null_event_listener_heap_traced,                                \
     gc_null_event_listener_ephemerons_traced,                          \
+    gc_null_event_listener_finalizers_traced,                          \
     gc_null_event_listener_restarting_mutators,                        \
     gc_null_event_listener_mutator_added,                              \
     gc_null_event_listener_mutator_cause_gc,                           \

benchmarks/simple-gc-embedder.h

@@ -6,6 +6,9 @@
 #include "gc-embedder-api.h"
 
 #define GC_EMBEDDER_EPHEMERON_HEADER struct gc_header header;
+#define GC_EMBEDDER_FINALIZER_HEADER struct gc_header header;
+
+static inline size_t gc_finalizer_priority_count(void) { return 2; }
 
 static inline int
 gc_is_valid_conservative_ref_displacement(uintptr_t displacement) {

embed.mk

@@ -31,6 +31,8 @@ $(GC_OBJDIR)gc-options.o: $(WHIPPET)src/gc-options.c
 	$(GC_COMPILE) -c $<
 $(GC_OBJDIR)gc-ephemeron.o: $(WHIPPET)src/gc-ephemeron.c
 	$(GC_COMPILE) $(EMBEDDER_TO_GC_CFLAGS) -c $<
+$(GC_OBJDIR)gc-finalizer.o: $(WHIPPET)src/gc-finalizer.c
+	$(GC_COMPILE) $(EMBEDDER_TO_GC_CFLAGS) -c $<
 
 GC_STEM_bdw   	   = bdw
 GC_CFLAGS_bdw 	   = -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1
@@ -83,4 +85,4 @@ GC_LIBS             = $(call gc_libs,$(GC_COLLECTOR))
 $(GC_OBJDIR)gc-impl.o: $(WHIPPET)src/$(call gc_impl,$(GC_COLLECTOR))
 	$(GC_COMPILE) $(GC_IMPL_CFLAGS) $(EMBEDDER_TO_GC_CFLAGS) -c $<
 
-GC_OBJS=$(foreach O,gc-platform.o gc-stack.o gc-options.o gc-ephemeron.o gc-impl.o,$(GC_OBJDIR)$(O))
+GC_OBJS=$(foreach O,gc-platform.o gc-stack.o gc-options.o gc-ephemeron.o gc-finalizer.o gc-impl.o,$(GC_OBJDIR)$(O))

src/bdw.c

@@ -55,6 +55,8 @@ struct gc_heap {
   struct gc_heap_roots *roots;
   struct gc_mutator *mutators;
   struct gc_event_listener event_listener;
+  struct gc_finalizer_state *finalizer_state;
+  gc_finalizer_callback have_finalizers;
   void *event_listener_data;
 };
 
@@ -165,6 +167,7 @@ static void bdw_mark_edge(struct gc_edge edge, struct gc_heap *heap,
 static int heap_gc_kind;
 static int mutator_gc_kind;
 static int ephemeron_gc_kind;
+static int finalizer_gc_kind;
 
 // In BDW-GC, we can't hook into the mark phase to call
 // gc_trace_ephemerons_for_object, so the advertised ephemeron strategy
@@ -199,6 +202,46 @@ int gc_visit_ephemeron_key(struct gc_edge edge, struct gc_heap *heap) {
   return 1;
 }
 
+struct gc_finalizer* gc_allocate_finalizer(struct gc_mutator *mut) {
+  return GC_generic_malloc(gc_finalizer_size(), finalizer_gc_kind);
+}
+
+static void finalize_object(void *obj, void *data) {
+  struct gc_finalizer *f = data;
+  gc_finalizer_externally_fired(__the_bdw_gc_heap->finalizer_state, f);
+}
+
+void gc_finalizer_attach(struct gc_mutator *mut, struct gc_finalizer *finalizer,
+                         unsigned priority, struct gc_ref object,
+                         struct gc_ref closure) {
+  // Don't bother much about the actual finalizer; just delegate to BDW-GC.
+  GC_finalization_proc prev = NULL;
+  void *prev_data = NULL;
+  gc_finalizer_init_internal(finalizer, object, closure);
+  gc_finalizer_externally_activated(finalizer);
+  GC_REGISTER_FINALIZER_NO_ORDER (gc_ref_heap_object(object), finalize_object,
+                                  finalizer, &prev, &prev_data);
+  // FIXME: Allow multiple finalizers per object.
+  GC_ASSERT(prev == NULL);
+  GC_ASSERT(prev_data == NULL);
+}
+
+struct gc_finalizer* gc_finalizer_pop(struct gc_mutator *mut) {
+  GC_invoke_finalizers();
+  return gc_finalizer_state_pop(mut->heap->finalizer_state);
+}
+
+void gc_set_finalizer_callback(struct gc_heap *heap,
+                               gc_finalizer_callback callback) {
+  heap->have_finalizers = callback;
+}
+
+static void have_finalizers(void) {
+  struct gc_heap *heap = __the_bdw_gc_heap;
+  if (heap->have_finalizers)
+    heap->have_finalizers(heap, 1);
+}
+
 static struct GC_ms_entry *
 mark_ephemeron(GC_word *addr, struct GC_ms_entry *mark_stack_ptr,
                struct GC_ms_entry *mark_stack_limit, GC_word env) {
@@ -228,6 +271,29 @@ mark_ephemeron(GC_word *addr, struct GC_ms_entry *mark_stack_ptr,
   return state.mark_stack_ptr;
 }
 
+static struct GC_ms_entry *
+mark_finalizer(GC_word *addr, struct GC_ms_entry *mark_stack_ptr,
+               struct GC_ms_entry *mark_stack_limit, GC_word env) {
+
+  struct bdw_mark_state state = {
+    mark_stack_ptr,
+    mark_stack_limit,
+  };
+  
+  struct gc_finalizer *finalizer = (struct gc_finalizer*) addr;
+
+  // If this ephemeron is on a freelist, its first word will be a
+  // freelist link and everything else will be NULL.
+  if (!gc_ref_value(gc_finalizer_object(finalizer))) {
+    bdw_mark_edge(gc_edge(addr), NULL, &state);
+    return state.mark_stack_ptr;
+  }
+
+  gc_trace_finalizer(finalizer, bdw_mark_edge, NULL, &state);
+
+  return state.mark_stack_ptr;
+}
+
 static struct GC_ms_entry *
 mark_heap(GC_word *addr, struct GC_ms_entry *mark_stack_ptr,
           struct GC_ms_entry *mark_stack_limit, GC_word env) {
@@ -428,6 +494,8 @@ int gc_init(const struct gc_options *options, struct gc_stack_addr *stack_base,
   }
 
   GC_set_all_interior_pointers (0);
+  GC_set_finalize_on_demand (1);
+  GC_set_finalizer_notifier(have_finalizers);
 
   // Not part of 7.3, sigh.  Have to set an env var.
   // GC_set_markers_count(options->common.parallelism);
@@ -453,6 +521,9 @@ int gc_init(const struct gc_options *options, struct gc_stack_addr *stack_base,
     ephemeron_gc_kind = GC_new_kind(GC_new_free_list(),
                                     GC_MAKE_PROC(GC_new_proc(mark_ephemeron), 0),
                                     add_size_to_descriptor, clear_memory);
+    finalizer_gc_kind = GC_new_kind(GC_new_free_list(),
+                                    GC_MAKE_PROC(GC_new_proc(mark_finalizer), 0),
+                                    add_size_to_descriptor, clear_memory);
   }
 
   *heap = GC_generic_malloc(sizeof(struct gc_heap), heap_gc_kind);
@@ -460,6 +531,7 @@ int gc_init(const struct gc_options *options, struct gc_stack_addr *stack_base,
 
   (*heap)->event_listener = event_listener;
   (*heap)->event_listener_data = event_listener_data;
+  (*heap)->finalizer_state = gc_make_finalizer_state();
 
   __the_bdw_gc_heap = *heap;
   HEAP_EVENT(init, GC_get_heap_size());

src/gc-finalizer-internal.h

@@ -0,0 +1,65 @@
+#ifndef GC_FINALIZER_INTERNAL_H
+#define GC_FINALIZER_INTERNAL_H
+
+#ifndef GC_IMPL
+#error internal header file, not part of API
+#endif
+
+#include "gc-finalizer.h"
+#include "root.h"
+
+struct gc_finalizer_state;
+
+GC_INTERNAL
+struct gc_finalizer_state* gc_make_finalizer_state(void);
+
+GC_INTERNAL
+void gc_finalizer_init_internal(struct gc_finalizer *f,
+                                struct gc_ref object,
+                                struct gc_ref closure);
+
+GC_INTERNAL
+void gc_finalizer_attach_internal(struct gc_finalizer_state *state,
+                                  struct gc_finalizer *f,
+                                  unsigned priority);
+
+GC_INTERNAL
+void gc_finalizer_externally_activated(struct gc_finalizer *f);
+
+GC_INTERNAL
+void gc_finalizer_externally_fired(struct gc_finalizer_state *state,
+                                   struct gc_finalizer *finalizer);
+
+GC_INTERNAL
+struct gc_finalizer* gc_finalizer_state_pop(struct gc_finalizer_state *state);
+
+GC_INTERNAL
+void gc_finalizer_fire(struct gc_finalizer **fired_list_loc,
+                       struct gc_finalizer *finalizer);
+
+GC_INTERNAL
+void gc_finalizer_state_set_callback(struct gc_finalizer_state *state,
+                                     gc_finalizer_callback callback);
+
+GC_INTERNAL
+size_t gc_visit_finalizer_roots(struct gc_finalizer_state *state,
+                                void (*visit)(struct gc_edge edge,
+                                              struct gc_heap *heap,
+                                              void *visit_data),
+                                struct gc_heap *heap,
+                                void *visit_data);
+
+GC_INTERNAL
+size_t gc_resolve_finalizers(struct gc_finalizer_state *state,
+                             size_t priority,
+                             void (*visit)(struct gc_edge edge,
+                                           struct gc_heap *heap,
+                                           void *visit_data),
+                             struct gc_heap *heap,
+                             void *visit_data);
+
+GC_INTERNAL
+void gc_notify_finalizers(struct gc_finalizer_state *state,
+                          struct gc_heap *heap);
+
+#endif // GC_FINALIZER_INTERNAL_H

src/gc-finalizer.c

@@ -0,0 +1,307 @@
+#include <math.h>
+#include <stdatomic.h>
+#include <stdlib.h>
+#include <string.h>
+
+#define GC_IMPL 1
+
+#include "debug.h"
+#include "gc-embedder-api.h"
+#include "gc-ephemeron-internal.h" // for gc_visit_ephemeron_key
+#include "gc-finalizer-internal.h"
+
+// # Overview
+//
+// See gc-finalizer.h for a overview of finalizers from the user and
+// embedder point of view.
+//
+// ## Tracing
+//
+// From the perspecive of the collector implementation, finalizers are
+// GC-managed objects, allowing their size to be accounted for within
+// the heap size.  They get traced during collection, allowing for
+// relocation of their object references, and allowing the finalizer
+// object itself to be evacuated if appropriate.
+//
+// The collector holds on to outstanding finalizers in a *finalizer
+// state*, which holds one *finalizer table* for each priority.  We
+// don't need to look up finalizers by object, so we could just hold
+// them in a big list, but to facilitate parallelism we slice them
+// across some number of shards, where the "next" pointer is part of the
+// finalizer object.
+//
+// There are a number of ways you could imagine integrating finalizers
+// into a system.  The way Whippet does it goes like this.  See
+// https://wingolog.org/archives/2022/10/31/ephemerons-and-finalizers
+// and
+// https://wingolog.org/archives/2024/07/22/finalizers-guardians-phantom-references-et-cetera
+// for some further discussion.
+//
+//   1. The collector should begin a cycle by adding all shards from all
+//      priorities to the root set.  When the embedder comes across a
+//      finalizer (as it will, because we added them to the root set),
+//      it traces it via gc_trace_finalizer(), which will visit the
+//      finalizer's closure and its "next" pointer.
+//
+//   2. After the full trace, and then the fix-point on pending
+//      ephemerons, for each priority from 0 upwards:
+//
+//      i. Visit each finalizable object in the table.  If the object
+//         was as-yet unvisited, then it is unreachable and thus
+//         finalizable; the finalizer is added to the global "fired"
+//         list, and changes state from "attached" to "fired".
+//         Otherwise it is re-added to the finalizer table.
+//
+//     ii. If any finalizer was added to the fired list, then those
+//         objects were also added to the grey worklist; run tracing
+//         again until the grey set is empty, including ephemerons.
+//
+//   3. Finally, call the finalizer callback if the list of fired finalizers is
+//      nonempty.
+//
+// ## Concurrency
+//
+// The finalizer table is wait-free.  It keeps a count of active finalizers, and
+// chooses a bucket based on the count modulo the number of buckets.  Adding a
+// finalizer to the table is an atomic push on a linked list.  The table is
+// completely rebuilt during the GC pause, redistributing survivor entries
+// across the buckets, and pushing all finalizable entries onto the single
+// "fired" linked list.
+//
+// The fired list is also wait-free.  As noted above, it is built
+// during the pause, and mutators pop items off of it atomically.
+//
+// ## Generations
+//
+// It would be ideal if a young generation had its own finalizer table.
+// Promoting an object would require promoting its finalizer to the old
+// finalizer table.  Not yet implemented (but would be nice).
+
+#ifndef GC_EMBEDDER_FINALIZER_HEADER
+#error Embedder should define GC_EMBEDDER_FINALIZER_HEADER
+#endif
+
+enum finalizer_state {
+  FINALIZER_STATE_INIT = 0, // Finalizer is newborn.
+  FINALIZER_STATE_ACTIVE,   // Finalizer is ours and in the finalizer table.
+  FINALIZER_STATE_FIRED,    // Finalizer is handed back to mutator.
+};
+
+struct gc_finalizer {
+  GC_EMBEDDER_FINALIZER_HEADER
+  enum finalizer_state state;
+  struct gc_ref object;
+  struct gc_ref closure;
+  struct gc_finalizer *next;
+};
+
+// Enough buckets to parallelize closure marking.  No need to look up a
+// finalizer for a given object.
+#define BUCKET_COUNT 32
+
+struct gc_finalizer_table {
+  size_t finalizer_count;
+  struct gc_finalizer* buckets[BUCKET_COUNT];
+};
+
+struct gc_finalizer_state {
+  gc_finalizer_callback have_finalizers;
+  struct gc_finalizer *fired;
+  size_t fired_this_cycle;
+  size_t table_count;
+  struct gc_finalizer_table tables[0];
+};
+
+// public
+size_t gc_finalizer_size(void) { return sizeof(struct gc_finalizer); }
+struct gc_ref gc_finalizer_object(struct gc_finalizer *f) { return f->object; }
+struct gc_ref gc_finalizer_closure(struct gc_finalizer *f) { return f->closure; }
+
+// internal
+struct gc_finalizer_state* gc_make_finalizer_state(void) {
+  size_t ntables = gc_finalizer_priority_count();
+  size_t size = (sizeof(struct gc_finalizer_state) +
+                 sizeof(struct gc_finalizer_table) * ntables);
+  struct gc_finalizer_state *ret = malloc(size);
+  if (!ret)
+    return NULL;
+  memset(ret, 0, size);
+  ret->table_count = ntables;
+  return ret;
+}
+
+static void finalizer_list_push(struct gc_finalizer **loc,
+                                struct gc_finalizer *head) {
+  struct gc_finalizer *tail = atomic_load_explicit(loc, memory_order_acquire);
+  do {
+    head->next = tail;
+  } while (!atomic_compare_exchange_weak(loc, &tail, head));
+}
+
+static struct gc_finalizer* finalizer_list_pop(struct gc_finalizer **loc) {
+  struct gc_finalizer *head = atomic_load_explicit(loc, memory_order_acquire);
+  do {
+    if (!head) return NULL;
+  } while (!atomic_compare_exchange_weak(loc, &head, head->next));
+  head->next = NULL;
+  return head;
+}
+
+static void add_finalizer_to_table(struct gc_finalizer_table *table,
+                                   struct gc_finalizer *f) {
+  size_t count = atomic_fetch_add_explicit(&table->finalizer_count, 1,
+                                           memory_order_relaxed);
+  struct gc_finalizer **loc = &table->buckets[count % BUCKET_COUNT];
+  finalizer_list_push(loc, f);
+}
+
+// internal
+void gc_finalizer_init_internal(struct gc_finalizer *f,
+                                struct gc_ref object,
+                                struct gc_ref closure) {
+  // Caller responsible for any write barrier, though really the
+  // assumption is that the finalizer is younger than the key and the
+  // value.
+  if (f->state != FINALIZER_STATE_INIT)
+    GC_CRASH();
+  if (gc_ref_is_heap_object(f->object))
+    GC_CRASH();
+  f->object = object;
+  f->closure = closure;
+}
+
+// internal
+void gc_finalizer_attach_internal(struct gc_finalizer_state *state,
+                                  struct gc_finalizer *f,
+                                  unsigned priority) {
+  // Caller responsible for any write barrier, though really the
+  // assumption is that the finalizer is younger than the key and the
+  // value.
+  if (f->state != FINALIZER_STATE_INIT)
+    GC_CRASH();
+  if (!gc_ref_is_heap_object(f->object))
+    GC_CRASH();
+
+  f->state = FINALIZER_STATE_ACTIVE;
+
+  GC_ASSERT(priority < state->table_count);
+  add_finalizer_to_table(&state->tables[priority], f);
+}
+
+// internal
+struct gc_finalizer* gc_finalizer_state_pop(struct gc_finalizer_state *state) {
+  return finalizer_list_pop(&state->fired);
+}
+
+static void
+add_fired_finalizer(struct gc_finalizer_state *state,
+                    struct gc_finalizer *f) {
+  if (f->state != FINALIZER_STATE_ACTIVE)
+    GC_CRASH();
+  f->state = FINALIZER_STATE_FIRED;
+  finalizer_list_push(&state->fired, f);
+}
+
+// internal
+void
+gc_finalizer_externally_activated(struct gc_finalizer *f) {
+  if (f->state != FINALIZER_STATE_INIT)
+    GC_CRASH();
+  f->state = FINALIZER_STATE_ACTIVE;
+}
+
+// internal
+void
+gc_finalizer_externally_fired(struct gc_finalizer_state *state,
+                              struct gc_finalizer *f) {
+  add_fired_finalizer(state, f);
+}
+
+// internal
+size_t gc_visit_finalizer_roots(struct gc_finalizer_state *state,
+                                void (*visit)(struct gc_edge,
+                                              struct gc_heap*,
+                                              void *),
+                                struct gc_heap *heap,
+                                void *visit_data) {
+  size_t count;
+  for (size_t tidx = 0; tidx < state->table_count; tidx++) {
+    struct gc_finalizer_table *table = &state->tables[tidx];
+    if (table->finalizer_count) {
+      count += table->finalizer_count;
+      for (size_t bidx = 0; bidx < BUCKET_COUNT; bidx++)
+        visit(gc_edge(&table->buckets[bidx]), heap, visit_data);
+    }
+  }
+  return count;
+}
+
+// public
+void gc_trace_finalizer(struct gc_finalizer *f,
+                        void (*visit)(struct gc_edge edge,
+                                      struct gc_heap *heap,
+                                      void *visit_data),
+                        struct gc_heap *heap,
+                        void *trace_data) {
+  if (f->state != FINALIZER_STATE_ACTIVE)
+    visit(gc_edge(&f->object), heap, trace_data);
+  visit(gc_edge(&f->closure), heap, trace_data);
+  visit(gc_edge(&f->next), heap, trace_data);
+}
+
+// Sweeping is currently serial.  It could run in parallel but we want to
+// resolve all finalizers before shading any additional node.  Perhaps we should
+// relax this restriction though; if the user attaches two finalizers to the
+// same object, it's probably OK to only have one finalizer fire per cycle.
+
+// internal
+size_t gc_resolve_finalizers(struct gc_finalizer_state *state,
+                             size_t priority,
+                             void (*visit)(struct gc_edge edge,
+                                           struct gc_heap *heap,
+                                           void *visit_data),
+                             struct gc_heap *heap,
+                             void *visit_data) {
+  GC_ASSERT(priority < state->table_count);
+  struct gc_finalizer_table *table = &state->tables[priority];
+  size_t finalizers_fired = 0;
+  // Visit each finalizer in the table.  If its object was already visited,
+  // re-add the finalizer to the table.  Otherwise enqueue its object edge for
+  // tracing and mark the finalizer as fired.
+  if (table->finalizer_count) {
+    struct gc_finalizer_table scratch = { 0, };
+    for (size_t bidx = 0; bidx < BUCKET_COUNT; bidx++) {
+      struct gc_finalizer *next;
+      for (struct gc_finalizer *f = table->buckets[bidx]; f; f = next) {
+        next = f->next;
+        f->next = NULL;
+        struct gc_edge edge = gc_edge(&f->object);
+        if (gc_visit_ephemeron_key(edge, heap)) {
+          add_finalizer_to_table(&scratch, f);
+        } else {
+          finalizers_fired++;
+          visit(edge, heap, visit_data);
+          add_fired_finalizer(state, f);
+        }
+      }
+    }
+    memcpy(table, &scratch, sizeof(*table));
+  }
+  state->fired_this_cycle += finalizers_fired;
+  return finalizers_fired;
+}
+
+// internal
+void gc_notify_finalizers(struct gc_finalizer_state *state,
+                          struct gc_heap *heap) {
+  if (state->fired_this_cycle && state->have_finalizers) {
+    state->have_finalizers(heap, state->fired_this_cycle);
+    state->fired_this_cycle = 0;
+  }
+}
+
+// internal
+void gc_finalizer_state_set_callback(struct gc_finalizer_state *state,
+                                     gc_finalizer_callback callback) {
+  state->have_finalizers = callback;
+}

src/gc-internal.h

@@ -6,6 +6,7 @@
 #endif
 
 #include "gc-ephemeron-internal.h"
+#include "gc-finalizer-internal.h"
 #include "gc-options-internal.h"
 
 #endif // GC_INTERNAL_H

src/pcc.c

@@ -131,6 +131,7 @@ struct gc_heap {
   int collecting;
   int check_pending_ephemerons;
   struct gc_pending_ephemerons *pending_ephemerons;
+  struct gc_finalizer_state *finalizer_state;
   size_t mutator_count;
   size_t paused_mutator_count;
   size_t inactive_mutator_count;
@@ -649,6 +650,9 @@ static inline void trace_root(struct gc_root root, struct gc_heap *heap,
     gc_trace_resolved_ephemerons(root.resolved_ephemerons, tracer_visit,
                                  heap, worker);
     break;
+  case GC_ROOT_KIND_EDGE:
+    tracer_visit(root.edge, heap, worker);
+    break;
   default:
     GC_CRASH();
   }
@@ -712,10 +716,16 @@ static int maybe_grow_heap(struct gc_heap *heap) {
   return 0;
 }
 
+static void visit_root_edge(struct gc_edge edge, struct gc_heap *heap,
+                            void *unused) {
+  gc_tracer_add_root(&heap->tracer, gc_root_edge(edge));
+}
+
 static void add_roots(struct gc_heap *heap) {
   for (struct gc_mutator *mut = heap->mutators; mut; mut = mut->next)
     gc_tracer_add_root(&heap->tracer, gc_root_mutator(mut));
   gc_tracer_add_root(&heap->tracer, gc_root_heap(heap));
+  gc_visit_finalizer_roots(heap->finalizer_state, visit_root_edge, heap, NULL);
 }
 
 static void resolve_ephemerons_lazily(struct gc_heap *heap) {
@@ -729,12 +739,26 @@ static void resolve_ephemerons_eagerly(struct gc_heap *heap) {
   gc_scan_pending_ephemerons(heap->pending_ephemerons, heap, 0, 1);
 }
 
-static int enqueue_resolved_ephemerons(struct gc_heap *heap) {
+static void trace_resolved_ephemerons(struct gc_heap *heap) {
-  struct gc_ephemeron *resolved = gc_pop_resolved_ephemerons(heap);
+  for (struct gc_ephemeron *resolved = gc_pop_resolved_ephemerons(heap);
-  if (!resolved)
+       resolved;
-    return 0;
+       resolved = gc_pop_resolved_ephemerons(heap)) {
-  gc_tracer_add_root(&heap->tracer, gc_root_resolved_ephemerons(resolved));
+    gc_tracer_add_root(&heap->tracer, gc_root_resolved_ephemerons(resolved));
-  return 1;
+    gc_tracer_trace(&heap->tracer);
+  }
+}
+
+static void resolve_finalizers(struct gc_heap *heap) {
+  for (size_t priority = 0;
+       priority < gc_finalizer_priority_count();
+       priority++) {
+    if (gc_resolve_finalizers(heap->finalizer_state, priority,
+                              visit_root_edge, heap, NULL)) {
+      gc_tracer_trace(&heap->tracer);
+      trace_resolved_ephemerons(heap);
+    }
+  }
+  gc_notify_finalizers(heap->finalizer_state, heap);
 }
 
 static void sweep_ephemerons(struct gc_heap *heap) {
@@ -765,9 +789,10 @@ static void collect(struct gc_mutator *mut) {
   gc_tracer_trace(&heap->tracer);
   HEAP_EVENT(heap, heap_traced);
   resolve_ephemerons_eagerly(heap);
-  while (enqueue_resolved_ephemerons(heap))
+  trace_resolved_ephemerons(heap);
-    gc_tracer_trace(&heap->tracer);
   HEAP_EVENT(heap, ephemerons_traced);
+  resolve_finalizers(heap);
+  HEAP_EVENT(heap, finalizers_traced);
   sweep_ephemerons(heap);
   gc_tracer_release(&heap->tracer);
   pcc_space_finish_gc(cspace);
@@ -891,6 +916,28 @@ unsigned gc_heap_ephemeron_trace_epoch(struct gc_heap *heap) {
   return heap->count;
 }
 
+struct gc_finalizer* gc_allocate_finalizer(struct gc_mutator *mut) {
+  return gc_allocate(mut, gc_finalizer_size());
+}
+
+void gc_finalizer_attach(struct gc_mutator *mut, struct gc_finalizer *finalizer,
+                         unsigned priority, struct gc_ref object,
+                         struct gc_ref closure) {
+  gc_finalizer_init_internal(finalizer, object, closure);
+  gc_finalizer_attach_internal(mutator_heap(mut)->finalizer_state,
+                               finalizer, priority);
+  // No write barrier.
+}
+
+struct gc_finalizer* gc_finalizer_pop(struct gc_mutator *mut) {
+  return gc_finalizer_state_pop(mutator_heap(mut)->finalizer_state);
+}
+
+void gc_set_finalizer_callback(struct gc_heap *heap,
+                               gc_finalizer_callback callback) {
+  gc_finalizer_state_set_callback(heap->finalizer_state, callback);
+}
+
 static struct pcc_slab* allocate_slabs(size_t nslabs) {
   size_t size = nslabs * SLAB_SIZE;
   size_t extent = size + SLAB_SIZE;
@@ -969,6 +1016,10 @@ static int heap_init(struct gc_heap *heap, const struct gc_options *options) {
   if (!heap_prepare_pending_ephemerons(heap))
     GC_CRASH();
 
+  heap->finalizer_state = gc_make_finalizer_state();
+  if (!heap->finalizer_state)
+    GC_CRASH();
+
   return 1;
 }
 

src/root.h

@@ -1,6 +1,8 @@
 #ifndef ROOT_H
 #define ROOT_H
 
+#include "gc-edge.h"
+
 struct gc_ephemeron;
 struct gc_heap;
 struct gc_mutator;
@@ -9,7 +11,8 @@ enum gc_root_kind {
   GC_ROOT_KIND_NONE,
   GC_ROOT_KIND_HEAP,
   GC_ROOT_KIND_MUTATOR,
-  GC_ROOT_KIND_RESOLVED_EPHEMERONS
+  GC_ROOT_KIND_RESOLVED_EPHEMERONS,
+  GC_ROOT_KIND_EDGE,
 };
 
 struct gc_root {
@@ -18,6 +21,7 @@ struct gc_root {
     struct gc_heap *heap;
     struct gc_mutator *mutator;
     struct gc_ephemeron *resolved_ephemerons;
+    struct gc_edge edge;
   };
 };
 
@@ -40,4 +44,11 @@ gc_root_resolved_ephemerons(struct gc_ephemeron* resolved) {
   return ret;
 }
 
+static inline struct gc_root
+gc_root_edge(struct gc_edge edge) {
+  struct gc_root ret = { GC_ROOT_KIND_EDGE };
+  ret.edge = edge;
+  return ret;
+}
+
 #endif // ROOT_H

src/semi.c

@@ -37,6 +37,7 @@ struct gc_heap {
   struct semi_space semi_space;
   struct large_object_space large_object_space;
   struct gc_pending_ephemerons *pending_ephemerons;
+  struct gc_finalizer_state *finalizer_state;
   struct gc_extern_space *extern_space;
   double pending_ephemerons_size_factor;
   double pending_ephemerons_size_slop;
@@ -350,6 +351,37 @@ static void adjust_heap_size_and_limits(struct gc_heap *heap,
   semi->limit = new_limit;
 }
 
+static uintptr_t trace_closure(struct gc_heap *heap, struct semi_space *semi,
+                               uintptr_t grey) {
+  while(grey < semi->hp)
+    grey = scan(heap, gc_ref(grey));
+  return grey;
+}
+
+static uintptr_t resolve_ephemerons(struct gc_heap *heap, uintptr_t grey) {
+  for (struct gc_ephemeron *resolved = gc_pop_resolved_ephemerons(heap);
+       resolved;
+       resolved = gc_pop_resolved_ephemerons(heap)) {
+    gc_trace_resolved_ephemerons(resolved, trace, heap, NULL);
+    grey = trace_closure(heap, heap_semi_space(heap), grey);
+  }
+  return grey;
+}
+
+static uintptr_t resolve_finalizers(struct gc_heap *heap, uintptr_t grey) {
+  for (size_t priority = 0;
+       priority < gc_finalizer_priority_count();
+       priority++) {
+    if (gc_resolve_finalizers(heap->finalizer_state, priority,
+                              trace, heap, NULL)) {
+      grey = trace_closure(heap, heap_semi_space(heap), grey);
+      grey = resolve_ephemerons(heap, grey);
+    }
+  }
+  gc_notify_finalizers(heap->finalizer_state, heap);
+  return grey;
+}
+
 static void collect(struct gc_mutator *mut, size_t for_alloc) {
   struct gc_heap *heap = mutator_heap(mut);
   int is_minor = 0;
@@ -373,22 +405,17 @@ static void collect(struct gc_mutator *mut, size_t for_alloc) {
     gc_trace_heap_roots(heap->roots, trace, heap, NULL);
   if (mut->roots)
     gc_trace_mutator_roots(mut->roots, trace, heap, NULL);
+  gc_visit_finalizer_roots(heap->finalizer_state, trace, heap, NULL);
   HEAP_EVENT(heap, roots_traced);
   // fprintf(stderr, "pushed %zd bytes in roots\n", space->hp - grey);
-  while(grey < semi->hp)
+  grey = trace_closure(heap, semi, grey);
-    grey = scan(heap, gc_ref(grey));
   HEAP_EVENT(heap, heap_traced);
   gc_scan_pending_ephemerons(heap->pending_ephemerons, heap, 0, 1);
   heap->check_pending_ephemerons = 1;
-  do {
+  grey = resolve_ephemerons(heap, grey);
-    struct gc_ephemeron *resolved = gc_pop_resolved_ephemerons(heap);
-    if (!resolved)
-      break;
-    gc_trace_resolved_ephemerons(resolved, trace, heap, NULL);
-    while(grey < semi->hp)
-      grey = scan(heap, gc_ref(grey));
-  } while (1);
   HEAP_EVENT(heap, ephemerons_traced);
+  grey = resolve_finalizers(heap, grey);
+  HEAP_EVENT(heap, finalizers_traced);
   large_object_space_finish_gc(large, 0);
   gc_extern_space_finish_gc(heap->extern_space, 0);
   semi_space_finish_gc(semi, large->live_pages_at_last_collection);
@@ -486,6 +513,28 @@ void gc_ephemeron_init(struct gc_mutator *mut, struct gc_ephemeron *ephemeron,
   gc_ephemeron_init_internal(mutator_heap(mut), ephemeron, key, value);
 }
 
+struct gc_finalizer* gc_allocate_finalizer(struct gc_mutator *mut) {
+  return gc_allocate(mut, gc_finalizer_size());
+}
+
+void gc_finalizer_attach(struct gc_mutator *mut, struct gc_finalizer *finalizer,
+                         unsigned priority, struct gc_ref object,
+                         struct gc_ref closure) {
+  gc_finalizer_init_internal(finalizer, object, closure);
+  gc_finalizer_attach_internal(mutator_heap(mut)->finalizer_state,
+                               finalizer, priority);
+  // No write barrier.
+}
+
+struct gc_finalizer* gc_finalizer_pop(struct gc_mutator *mut) {
+  return gc_finalizer_state_pop(mutator_heap(mut)->finalizer_state);
+}
+
+void gc_set_finalizer_callback(struct gc_heap *heap,
+                               gc_finalizer_callback callback) {
+  gc_finalizer_state_set_callback(heap->finalizer_state, callback);
+}
+
 static int region_init(struct region *region, size_t size) {
   region->base = 0;
   region->active_size = 0;
@@ -542,8 +591,11 @@ static int heap_init(struct gc_heap *heap, const struct gc_options *options) {
   heap->options = options;
   heap->size = options->common.heap_size;
   heap->roots = NULL;
+  heap->finalizer_state = gc_make_finalizer_state();
+  if (!heap->finalizer_state)
+    GC_CRASH();
 
-  return heap_prepare_pending_ephemerons(heap);
+return heap_prepare_pending_ephemerons(heap);
 }
 
 int gc_option_from_string(const char *str) {

src/whippet.c

@@ -301,6 +301,7 @@ struct gc_heap {
   int mark_while_stopping;
   int check_pending_ephemerons;
   struct gc_pending_ephemerons *pending_ephemerons;
+  struct gc_finalizer_state *finalizer_state;
   enum gc_collection_kind gc_kind;
   int multithreaded;
   size_t mutator_count;
@@ -1231,8 +1232,28 @@ static inline void trace_one(struct gc_ref ref, struct gc_heap *heap,
 static inline void trace_root(struct gc_root root,
                               struct gc_heap *heap,
                               struct gc_trace_worker *worker) {
-  // We don't use parallel root tracing yet.
+  switch (root.kind) {
-  GC_CRASH();
+  case GC_ROOT_KIND_HEAP:
+    gc_trace_heap_roots(root.heap->roots, tracer_visit, heap, worker);
+    break;
+  case GC_ROOT_KIND_MUTATOR:
+    gc_trace_mutator_roots(root.mutator->roots, tracer_visit, heap, worker);
+    break;
+  case GC_ROOT_KIND_RESOLVED_EPHEMERONS:
+    gc_trace_resolved_ephemerons(root.resolved_ephemerons, tracer_visit,
+                                 heap, worker);
+    break;
+  case GC_ROOT_KIND_EDGE:
+    tracer_visit(root.edge, heap, worker);
+    break;
+  default:
+    GC_CRASH();
+  }
+}
+
+static void visit_root_edge(struct gc_edge edge, struct gc_heap *heap,
+                            void *unused) {
+  gc_tracer_add_root(&heap->tracer, gc_root_edge(edge));
 }
 
 static void
@@ -1823,6 +1844,7 @@ static void trace_pinned_roots_after_stop(struct gc_heap *heap) {
 static void trace_roots_after_stop(struct gc_heap *heap) {
   trace_mutator_roots_after_stop(heap);
   gc_trace_heap_roots(heap->roots, trace_and_enqueue_globally, heap, NULL);
+  gc_visit_finalizer_roots(heap->finalizer_state, visit_root_edge, heap, NULL);
   trace_generational_roots(heap);
 }
 
@@ -1890,6 +1912,28 @@ static int enqueue_resolved_ephemerons(struct gc_heap *heap) {
   return 1;
 }
 
+static void trace_resolved_ephemerons(struct gc_heap *heap) {
+  for (struct gc_ephemeron *resolved = gc_pop_resolved_ephemerons(heap);
+       resolved;
+       resolved = gc_pop_resolved_ephemerons(heap)) {
+    gc_tracer_add_root(&heap->tracer, gc_root_resolved_ephemerons(resolved));
+    gc_tracer_trace(&heap->tracer);
+  }
+}
+
+static void resolve_finalizers(struct gc_heap *heap) {
+  for (size_t priority = 0;
+       priority < gc_finalizer_priority_count();
+       priority++) {
+    if (gc_resolve_finalizers(heap->finalizer_state, priority,
+                              visit_root_edge, heap, NULL)) {
+      gc_tracer_trace(&heap->tracer);
+      trace_resolved_ephemerons(heap);
+    }
+  }
+  gc_notify_finalizers(heap->finalizer_state, heap);
+}
+
 static void sweep_ephemerons(struct gc_heap *heap) {
   return gc_sweep_pending_ephemerons(heap->pending_ephemerons, 0, 1);
 }
@@ -1934,9 +1978,10 @@ static void collect(struct gc_mutator *mut,
   gc_tracer_trace(&heap->tracer);
   HEAP_EVENT(heap, heap_traced);
   resolve_ephemerons_eagerly(heap);
-  while (enqueue_resolved_ephemerons(heap))
+  trace_resolved_ephemerons(heap);
-    gc_tracer_trace(&heap->tracer);
   HEAP_EVENT(heap, ephemerons_traced);
+  resolve_finalizers(heap);
+  HEAP_EVENT(heap, finalizers_traced);
   sweep_ephemerons(heap);
   gc_tracer_release(&heap->tracer);
   mark_space_finish_gc(space, gc_kind);
@@ -2322,6 +2367,28 @@ unsigned gc_heap_ephemeron_trace_epoch(struct gc_heap *heap) {
   return heap->count;
 }
 
+struct gc_finalizer* gc_allocate_finalizer(struct gc_mutator *mut) {
+  return gc_allocate(mut, gc_finalizer_size());
+}
+
+void gc_finalizer_attach(struct gc_mutator *mut, struct gc_finalizer *finalizer,
+                         unsigned priority, struct gc_ref object,
+                         struct gc_ref closure) {
+  gc_finalizer_init_internal(finalizer, object, closure);
+  gc_finalizer_attach_internal(mutator_heap(mut)->finalizer_state,
+                               finalizer, priority);
+  // No write barrier.
+}
+
+struct gc_finalizer* gc_finalizer_pop(struct gc_mutator *mut) {
+  return gc_finalizer_state_pop(mutator_heap(mut)->finalizer_state);
+}
+
+void gc_set_finalizer_callback(struct gc_heap *heap,
+                               gc_finalizer_callback callback) {
+  gc_finalizer_state_set_callback(heap->finalizer_state, callback);
+}
+
 static struct slab* allocate_slabs(size_t nslabs) {
   size_t size = nslabs * SLAB_SIZE;
   size_t extent = size + SLAB_SIZE;
@@ -2406,6 +2473,10 @@ static int heap_init(struct gc_heap *heap, const struct gc_options *options) {
   if (!heap_prepare_pending_ephemerons(heap))
     GC_CRASH();
 
+  heap->finalizer_state = gc_make_finalizer_state();
+  if (!heap->finalizer_state)
+    GC_CRASH();
+
   return 1;
 }