Update conservative roots embedder interface

api/gc-embedder-api.h

@@ -50,8 +50,11 @@ GC_EMBEDDER_API inline void gc_trace_heap_roots(struct gc_heap_roots *roots,
                                                 void *trace_data);
 
 GC_EMBEDDER_API inline void
-gc_trace_mutator_conservative_roots(struct gc_mutator_roots *roots,
-                                    void (*trace_range)(uintptr_t start,
+gc_trace_mutator_pinned_roots(struct gc_mutator_roots *roots,
+                              void (*trace_pinned)(struct gc_ref ref,
+                                                   struct gc_heap *heap,
+                                                   void *data),
+                              void (*trace_ambiguous)(uintptr_t start,
                                                       uintptr_t end,
                                                       int possibly_interior,
                                                       struct gc_heap *heap,
@@ -59,8 +62,11 @@ gc_trace_mutator_conservative_roots(struct gc_mutator_roots *roots,
                               struct gc_heap *heap,
                               void *data);
 GC_EMBEDDER_API inline void
-gc_trace_heap_conservative_roots(struct gc_heap_roots *roots,
-                                 void (*trace_range)(uintptr_t start,
+gc_trace_heap_pinned_roots(struct gc_heap_roots *roots,
+                           void (*trace_pinned)(struct gc_ref ref,
+                                                struct gc_heap *heap,
+                                                void *data),
+                           void (*trace_ambiguous)(uintptr_t start,
                                                    uintptr_t end,
                                                    int possibly_interior,
                                                    struct gc_heap *heap,

benchmarks/simple-gc-embedder.h

@@ -92,8 +92,11 @@ static inline void gc_trace_heap_roots(struct gc_heap_roots *roots,
 }
 
 static inline void
-gc_trace_mutator_conservative_roots(struct gc_mutator_roots *roots,
-                                    void (*trace_range)(uintptr_t start,
+gc_trace_mutator_pinned_roots(struct gc_mutator_roots *roots,
+                              void (*trace_pinned)(struct gc_ref ref,
+                                                   struct gc_heap *heap,
+                                                   void *data),
+                              void (*trace_ambiguous)(uintptr_t start,
                                                       uintptr_t end,
                                                       int possibly_interior,
                                                       struct gc_heap *heap,
@@ -102,8 +105,11 @@ gc_trace_mutator_conservative_roots(struct gc_mutator_roots *roots,
                               void *data) {}
 
 static inline void
-gc_trace_heap_conservative_roots(struct gc_heap_roots *roots,
-                                 void (*trace_range)(uintptr_t start,
+gc_trace_heap_pinned_roots(struct gc_heap_roots *roots,
+                           void (*trace_pinned)(struct gc_ref ref,
+                                                struct gc_heap *heap,
+                                                void *data),
+                           void (*trace_ambiguous)(uintptr_t start,
                                                    uintptr_t end,
                                                    int possibly_interior,
                                                    struct gc_heap *heap,

src/bdw.c

@@ -5,6 +5,7 @@
 
 #define GC_IMPL 1
 
+#include "gc-align.h"
 #include "gc-api.h"
 #include "gc-ephemeron.h"
 #include "gc-tracepoint.h"
@@ -218,16 +219,20 @@ struct bdw_mark_state {
   struct GC_ms_entry *mark_stack_limit;
 };
 
-static void bdw_mark_edge(struct gc_edge edge, struct gc_heap *heap,
+static void bdw_mark(struct gc_ref ref, struct gc_heap *heap,
                      void *visit_data) {
   struct bdw_mark_state *state = visit_data;
-  uintptr_t addr = gc_ref_value(gc_edge_ref(edge));
-  state->mark_stack_ptr = GC_MARK_AND_PUSH ((void *) addr,
+  state->mark_stack_ptr = GC_MARK_AND_PUSH ((void *) gc_ref_value(ref),
                                             state->mark_stack_ptr,
                                             state->mark_stack_limit,
                                             NULL);
 }
 
+static void bdw_mark_edge(struct gc_edge edge, struct gc_heap *heap,
+                          void *visit_data) {
+  bdw_mark(gc_edge_ref(edge), heap, visit_data);
+}
+
 static void bdw_mark_range(uintptr_t lo, uintptr_t hi, int possibly_interior,
                            struct gc_heap *heap, void *visit_data) {
   struct bdw_mark_state *state = visit_data;
@@ -395,7 +400,8 @@ mark_heap(GC_word *addr, struct GC_ms_entry *mark_stack_ptr,
     return state.mark_stack_ptr;
 
   if (heap->roots) {
-    gc_trace_heap_conservative_roots(heap->roots, bdw_mark_range, heap, &state);
+    gc_trace_heap_pinned_roots(heap->roots, bdw_mark, bdw_mark_range,
+                               heap, &state);
     gc_trace_heap_roots(heap->roots, bdw_mark_edge, heap, &state);
   }
 
@@ -432,7 +438,7 @@ mark_mutator(GC_word *addr, struct GC_ms_entry *mark_stack_ptr,
   memset(mut->freelists, 0, sizeof(void*) * GC_INLINE_FREELIST_COUNT);
 
   if (mut->roots) {
-    gc_trace_mutator_conservative_roots(mut->roots, bdw_mark_range,
+    gc_trace_mutator_pinned_roots(mut->roots, bdw_mark, bdw_mark_range,
                                   mut->heap, &state);
     gc_trace_mutator_roots(mut->roots, bdw_mark_edge, mut->heap, &state);
   }

src/mmc.c

@@ -166,6 +166,34 @@ trace_edge(struct gc_heap *heap, struct gc_edge edge,
   return is_new;
 }
 
+static inline int
+do_trace_pinned(struct gc_heap *heap, struct gc_ref ref,
+                struct gc_trace_worker_data *data) {
+  if (GC_LIKELY(nofl_space_contains(heap_nofl_space(heap), ref)))
+    return nofl_space_mark_object(heap_nofl_space(heap), ref, &data->allocator);
+  else if (large_object_space_contains_with_lock(heap_large_object_space(heap),
+                                                 ref))
+    return large_object_space_mark(heap_large_object_space(heap), ref);
+  else
+    return gc_extern_space_visit(heap_extern_space(heap), ref);
+}
+
+static inline int
+trace_pinned_edge(struct gc_heap *heap, struct gc_ref ref,
+                  struct gc_trace_worker_data *data) {
+  if (gc_ref_is_null(ref) || gc_ref_is_immediate(ref))
+    return 0;
+
+  int is_new = do_trace_pinned(heap, ref, data);
+
+  if (is_new &&
+      GC_UNLIKELY(atomic_load_explicit(&heap->check_pending_ephemerons,
+                                       memory_order_relaxed)))
+    gc_resolve_pending_ephemerons(ref, heap);
+
+  return is_new;
+}
+
 int
 gc_visit_ephemeron_key(struct gc_edge edge, struct gc_heap *heap) {
   struct gc_ref ref = gc_edge_ref(edge);
@@ -274,6 +302,17 @@ tracer_visit(struct gc_edge edge, struct gc_heap *heap, void *trace_data) {
     gc_trace_worker_enqueue(worker, gc_edge_ref(edge));
 }
 
+static inline void
+tracer_visit_pinned_root(struct gc_ref ref, struct gc_heap *heap,
+                         void *trace_data) GC_ALWAYS_INLINE;
+static inline void
+tracer_visit_pinned_root(struct gc_ref ref, struct gc_heap *heap,
+                         void *trace_data) {
+  struct gc_trace_worker *worker = trace_data;
+  if (trace_pinned_edge(heap, ref, gc_trace_worker_data(worker)))
+    gc_trace_worker_enqueue(worker, ref);
+}
+
 static inline int
 trace_remembered_edge(struct gc_edge edge, struct gc_heap *heap, void *trace_data) {
   tracer_visit(edge, heap, trace_data);
@@ -331,8 +370,8 @@ static inline void
 trace_conservative_edges(uintptr_t low, uintptr_t high, int possibly_interior,
                          struct gc_heap *heap, void *data) {
   struct gc_trace_worker *worker = data;
-  GC_ASSERT(low == align_down(low, sizeof(uintptr_t)));
-  GC_ASSERT(high == align_down(high, sizeof(uintptr_t)));
+  GC_ASSERT_EQ(low, align_down(low, sizeof(uintptr_t)));
+  GC_ASSERT_EQ(high, align_down(high, sizeof(uintptr_t)));
   for (uintptr_t addr = low; addr < high; addr += sizeof(uintptr_t))
     tracer_trace_conservative_ref(load_conservative_ref(addr), heap, worker,
                                   possibly_interior);
@@ -404,13 +443,17 @@ trace_root(struct gc_root root, struct gc_heap *heap,
     gc_field_set_visit_edge_buffer(&heap->remembered_set, root.edge_buffer,
                                    trace_remembered_edge, heap, worker);
     break;
-  case GC_ROOT_KIND_HEAP_CONSERVATIVE_ROOTS:
-    gc_trace_heap_conservative_roots(root.heap->roots, trace_conservative_edges,
+  case GC_ROOT_KIND_HEAP_PINNED_ROOTS:
+    gc_trace_heap_pinned_roots(root.heap->roots,
+                               tracer_visit_pinned_root,
+                               trace_conservative_edges,
                                heap, worker);
     break;
-  case GC_ROOT_KIND_MUTATOR_CONSERVATIVE_ROOTS:
-    gc_trace_mutator_conservative_roots(root.mutator->roots,
-                                        trace_conservative_edges, heap, worker);
+  case GC_ROOT_KIND_MUTATOR_PINNED_ROOTS:
+    gc_trace_mutator_pinned_roots(root.mutator->roots,
+                                  tracer_visit_pinned_root,
+                                  trace_conservative_edges,
+                                  heap, worker);
     break;
   default:
     GC_CRASH();
@@ -671,7 +714,7 @@ enqueue_mutator_conservative_roots(struct gc_heap *heap) {
                      &possibly_interior);
       if (mut->roots)
         gc_tracer_add_root(&heap->tracer,
-                           gc_root_mutator_conservative_roots(mut));
+                           gc_root_mutator_pinned_roots(mut));
     }
     return 1;
   }
@@ -685,7 +728,7 @@ enqueue_global_conservative_roots(struct gc_heap *heap) {
     gc_platform_visit_global_conservative_roots
       (enqueue_conservative_roots, heap, &possibly_interior);
     if (heap->roots)
-      gc_tracer_add_root(&heap->tracer, gc_root_heap_conservative_roots(heap));
+      gc_tracer_add_root(&heap->tracer, gc_root_heap_pinned_roots(heap));
     return 1;
   }
   return 0;

src/nofl-space.h

@@ -1617,6 +1617,19 @@ nofl_space_evacuate_or_mark_object(struct nofl_space *space,
   return nofl_space_set_nonempty_mark(space, metadata, byte, old_ref);
 }
 
+static inline int
+nofl_space_mark_object(struct nofl_space *space, struct gc_ref ref,
+                       struct nofl_allocator *evacuate) {
+  uint8_t *metadata = nofl_metadata_byte_for_object(ref);
+  uint8_t byte = *metadata;
+  if (nofl_metadata_byte_has_mark(byte, space->current_mark))
+    return 0;
+
+  GC_ASSERT(!nofl_space_should_evacuate(space, byte, ref));
+
+  return nofl_space_set_nonempty_mark(space, metadata, byte, ref);
+}
+
 static inline int
 nofl_space_forward_if_evacuated(struct nofl_space *space,
                                 struct gc_edge edge,

src/root.h

@@ -18,8 +18,8 @@ enum gc_root_kind {
   GC_ROOT_KIND_RESOLVED_EPHEMERONS,
   GC_ROOT_KIND_EDGE,
   GC_ROOT_KIND_EDGE_BUFFER,
-  GC_ROOT_KIND_HEAP_CONSERVATIVE_ROOTS,
-  GC_ROOT_KIND_MUTATOR_CONSERVATIVE_ROOTS,
+  GC_ROOT_KIND_HEAP_PINNED_ROOTS,
+  GC_ROOT_KIND_MUTATOR_PINNED_ROOTS,
 };
 
 struct gc_root {
@@ -81,15 +81,15 @@ gc_root_edge_buffer(struct gc_edge_buffer *buf) {
 }
 
 static inline struct gc_root
-gc_root_heap_conservative_roots(struct gc_heap* heap) {
-  struct gc_root ret = { GC_ROOT_KIND_HEAP_CONSERVATIVE_ROOTS };
+gc_root_heap_pinned_roots(struct gc_heap* heap) {
+  struct gc_root ret = { GC_ROOT_KIND_HEAP_PINNED_ROOTS };
   ret.heap = heap;
   return ret;
 }
 
 static inline struct gc_root
-gc_root_mutator_conservative_roots(struct gc_mutator* mutator) {
-  struct gc_root ret = { GC_ROOT_KIND_MUTATOR_CONSERVATIVE_ROOTS };
+gc_root_mutator_pinned_roots(struct gc_mutator* mutator) {
+  struct gc_root ret = { GC_ROOT_KIND_MUTATOR_PINNED_ROOTS };
   ret.mutator = mutator;
   return ret;
 }