Add conservative heap tracing (not just roots)

Also accelerate mark_space_live_object_granules.

Makefile

@@ -1,5 +1,23 @@
 TESTS=quads mt-gcbench # MT_GCBench MT_GCBench2
-COLLECTORS=bdw semi whippet conservative-whippet parallel-whippet conservative-parallel-whippet generational-whippet conservative-generational-whippet parallel-generational-whippet conservative-parallel-generational-whippet
+COLLECTORS= \
+	bdw \
+	semi \
+	\
+	whippet \
+	conservative-whippet \
+	fully-conservative-whippet \
+	\
+	parallel-whippet \
+	conservative-parallel-whippet \
+	fully-conservative-parallel-whippet \
+	\
+	generational-whippet \
+	conservative-generational-whippet \
+	fully-conservative-generational-whippet \
+	\
+	parallel-generational-whippet \
+	conservative-parallel-generational-whippet \
+	fully-conservative-parallel-generational-whippet
 
 CC=gcc
 CFLAGS=-Wall -O2 -g -flto -fno-strict-aliasing -fvisibility=hidden -Wno-unused -DNDEBUG
@@ -18,57 +36,77 @@ gc-platform.o: gc-platform.h gc-platform-$(PLATFORM).c gc-visibility.h
 gc-stack.o: gc-stack.c
 	$(COMPILE) -o $@ -c $<
 
-bdw-%-gc.o: semi.c %-embedder.h %.c
+bdw-%-gc.o: bdw.c %-embedder.h %.c
-	$(COMPILE) `pkg-config --cflags bdw-gc` -include $*-embedder.h -o $@ -c bdw.c
+	$(COMPILE) -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 `pkg-config --cflags bdw-gc` -include $*-embedder.h -o $@ -c bdw.c
-bdw-%.o: semi.c %.c
+bdw-%.o: bdw.c %.c
-	$(COMPILE) -include bdw-attrs.h -o $@ -c $*.c
+	$(COMPILE) -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 -include bdw-attrs.h -o $@ -c $*.c
 bdw-%: bdw-%.o bdw-%-gc.o gc-stack.o gc-platform.o
 	$(CC) $(LDFLAGS) `pkg-config --libs bdw-gc` -o $@ $^
 
 semi-%-gc.o: semi.c %-embedder.h large-object-space.h assert.h debug.h %.c
-	$(COMPILE) -DGC_PRECISE=1 -include $*-embedder.h -o $@ -c semi.c
+	$(COMPILE) -DGC_PRECISE_ROOTS=1 -include $*-embedder.h -o $@ -c semi.c
 semi-%.o: semi.c %.c
-	$(COMPILE) -DGC_PRECISE=1 -include semi-attrs.h -o $@ -c $*.c
+	$(COMPILE) -DGC_PRECISE_ROOTS=1 -include semi-attrs.h -o $@ -c $*.c
 
 whippet-%-gc.o: whippet.c %-embedder.h large-object-space.h serial-tracer.h assert.h debug.h heap-objects.h %.c
-	$(COMPILE) -DGC_PRECISE=1 -include $*-embedder.h -o $@ -c whippet.c
+	$(COMPILE) -DGC_PRECISE_ROOTS=1 -include $*-embedder.h -o $@ -c whippet.c
 whippet-%.o: whippet.c %.c
-	$(COMPILE) -DGC_PRECISE=1 -include whippet-attrs.h -o $@ -c $*.c
+	$(COMPILE) -DGC_PRECISE_ROOTS=1 -include whippet-attrs.h -o $@ -c $*.c
 
 conservative-whippet-%-gc.o: whippet.c %-embedder.h large-object-space.h serial-tracer.h assert.h debug.h heap-objects.h %.c
-	$(COMPILE) -DGC_PRECISE=0 -include $*-embedder.h -o $@ -c whippet.c
+	$(COMPILE) -DGC_CONSERVATIVE_ROOTS=1 -include $*-embedder.h -o $@ -c whippet.c
 conservative-whippet-%.o: whippet.c %.c
-	$(COMPILE) -DGC_PRECISE=0 -include whippet-attrs.h -o $@ -c $*.c
+	$(COMPILE) -DGC_CONSERVATIVE_ROOTS=1 -include whippet-attrs.h -o $@ -c $*.c
+
+fully-conservative-whippet-%-gc.o: whippet.c %-embedder.h large-object-space.h serial-tracer.h assert.h debug.h heap-objects.h %.c
+	$(COMPILE) -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 -include $*-embedder.h -o $@ -c whippet.c
+fully-conservative-whippet-%.o: whippet.c %.c
+	$(COMPILE) -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 -include whippet-attrs.h -o $@ -c $*.c
 
 parallel-whippet-%-gc.o: whippet.c %-embedder.h large-object-space.h parallel-tracer.h assert.h debug.h heap-objects.h %.c
-	$(COMPILE) -DGC_PARALLEL=1 -DGC_PRECISE=1 -include $*-embedder.h -o $@ -c whippet.c
+	$(COMPILE) -DGC_PARALLEL=1 -DGC_PRECISE_ROOTS=1 -include $*-embedder.h -o $@ -c whippet.c
 parallel-whippet-%.o: whippet.c %.c
-	$(COMPILE) -DGC_PARALLEL=1 -DGC_PRECISE=1 -include whippet-attrs.h -o $@ -c $*.c
+	$(COMPILE) -DGC_PARALLEL=1 -DGC_PRECISE_ROOTS=1 -include whippet-attrs.h -o $@ -c $*.c
 
 conservative-parallel-whippet-%-gc.o: whippet.c %-embedder.h large-object-space.h serial-tracer.h assert.h debug.h heap-objects.h %.c
-	$(COMPILE) -DGC_PARALLEL=1 -DGC_PRECISE=0 -include $*-embedder.h -o $@ -c whippet.c
+	$(COMPILE) -DGC_PARALLEL=1 -DGC_CONSERVATIVE_ROOTS=1 -include $*-embedder.h -o $@ -c whippet.c
 conservative-parallel-whippet-%.o: whippet.c %.c
-	$(COMPILE) -DGC_PARALLEL=1 -DGC_PRECISE=0 -include whippet-attrs.h -o $@ -c $*.c
+	$(COMPILE) -DGC_PARALLEL=1 -DGC_CONSERVATIVE_ROOTS=1 -include whippet-attrs.h -o $@ -c $*.c
+
+fully-conservative-parallel-whippet-%-gc.o: whippet.c %-embedder.h large-object-space.h serial-tracer.h assert.h debug.h heap-objects.h %.c
+	$(COMPILE) -DGC_PARALLEL=1 -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 -include $*-embedder.h -o $@ -c whippet.c
+fully-conservative-parallel-whippet-%.o: whippet.c %.c
+	$(COMPILE) -DGC_PARALLEL=1 -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 -DGC_FULLY_CONSERVATIVE=1 -include whippet-attrs.h -o $@ -c $*.c
 
 generational-whippet-%-gc.o: whippet.c %-embedder.h large-object-space.h serial-tracer.h assert.h debug.h heap-objects.h %.c
-	$(COMPILE) -DGC_GENERATIONAL=1 -DGC_PRECISE=1 -include $*-embedder.h -o $@ -c whippet.c
+	$(COMPILE) -DGC_GENERATIONAL=1 -DGC_PRECISE_ROOTS=1 -include $*-embedder.h -o $@ -c whippet.c
 generational-whippet-%.o: whippet.c %.c
-	$(COMPILE) -DGC_GENERATIONAL=1 -DGC_PRECISE=1 -include whippet-attrs.h -o $@ -c $*.c
+	$(COMPILE) -DGC_GENERATIONAL=1 -DGC_PRECISE_ROOTS=1 -include whippet-attrs.h -o $@ -c $*.c
 
 conservative-generational-whippet-%-gc.o: whippet.c %-embedder.h large-object-space.h serial-tracer.h assert.h debug.h heap-objects.h %.c
-	$(COMPILE) -DGC_GENERATIONAL=1 -DGC_PRECISE=0 -include $*-embedder.h -o $@ -c whippet.c
+	$(COMPILE) -DGC_GENERATIONAL=1 -DGC_CONSERVATIVE_ROOTS=1 -include $*-embedder.h -o $@ -c whippet.c
 conservative-generational-whippet-%.o: whippet.c %.c
-	$(COMPILE) -DGC_GENERATIONAL=1 -DGC_PRECISE=0 -include whippet-attrs.h -o $@ -c $*.c
+	$(COMPILE) -DGC_GENERATIONAL=1 -DGC_CONSERVATIVE_ROOTS=1 -include whippet-attrs.h -o $@ -c $*.c
+
+fully-conservative-generational-whippet-%-gc.o: whippet.c %-embedder.h large-object-space.h serial-tracer.h assert.h debug.h heap-objects.h %.c
+	$(COMPILE) -DGC_GENERATIONAL=1 -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 -include $*-embedder.h -o $@ -c whippet.c
+fully-conservative-generational-whippet-%.o: whippet.c %.c
+	$(COMPILE) -DGC_GENERATIONAL=1 -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 -include whippet-attrs.h -o $@ -c $*.c
 
 parallel-generational-whippet-%-gc.o: whippet.c %-embedder.h large-object-space.h parallel-tracer.h assert.h debug.h heap-objects.h %.c
-	$(COMPILE) -DGC_PARALLEL=1 -DGC_GENERATIONAL=1 -DGC_PRECISE=1 -include $*-embedder.h -o $@ -c whippet.c
+	$(COMPILE) -DGC_PARALLEL=1 -DGC_GENERATIONAL=1 -DGC_PRECISE_ROOTS=1 -include $*-embedder.h -o $@ -c whippet.c
 parallel-generational-whippet-%.o: whippet.c %.c
-	$(COMPILE) -DGC_PARALLEL=1 -DGC_GENERATIONAL=1 -DGC_PRECISE=1 -include whippet-attrs.h -o $@ -c $*.c
+	$(COMPILE) -DGC_PARALLEL=1 -DGC_GENERATIONAL=1 -DGC_PRECISE_ROOTS=1 -include whippet-attrs.h -o $@ -c $*.c
 
 conservative-parallel-generational-whippet-%-gc.o: whippet.c %-embedder.h large-object-space.h parallel-tracer.h assert.h debug.h heap-objects.h %.c
-	$(COMPILE) -DGC_PARALLEL=1 -DGC_GENERATIONAL=1 -DGC_PRECISE=0 -include $*-embedder.h -o $@ -c whippet.c
+	$(COMPILE) -DGC_PARALLEL=1 -DGC_GENERATIONAL=1 -DGC_CONSERVATIVE_ROOTS=1 -include $*-embedder.h -o $@ -c whippet.c
 conservative-parallel-generational-whippet-%.o: whippet.c %.c
-	$(COMPILE) -DGC_PARALLEL=1 -DGC_GENERATIONAL=1 -DGC_PRECISE=0 -include whippet-attrs.h -o $@ -c $*.c
+	$(COMPILE) -DGC_PARALLEL=1 -DGC_GENERATIONAL=1 -DGC_CONSERVATIVE_ROOTS=1 -include whippet-attrs.h -o $@ -c $*.c
+
+fully-conservative-parallel-generational-whippet-%-gc.o: whippet.c %-embedder.h large-object-space.h parallel-tracer.h assert.h debug.h heap-objects.h %.c
+	$(COMPILE) -DGC_PARALLEL=1 -DGC_GENERATIONAL=1 -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 -include $*-embedder.h -o $@ -c whippet.c
+fully-conservative-parallel-generational-whippet-%.o: whippet.c %.c
+	$(COMPILE) -DGC_PARALLEL=1 -DGC_GENERATIONAL=1 -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 -include whippet-attrs.h -o $@ -c $*.c
 
 %: %.o %-gc.o gc-platform.o gc-stack.o
 	$(CC) $(LDFLAGS) $($*_LDFLAGS) -o $@ $^

bdw.c

@@ -8,12 +8,20 @@
 
 #include "bdw-attrs.h"
 
-#if GC_PRECISE
+#if GC_PRECISE_ROOTS
 #error bdw-gc is a conservative collector
-#else
-#include "conservative-roots-embedder.h"
 #endif
 
+#if !GC_CONSERVATIVE_ROOTS
+#error bdw-gc is a conservative collector
+#endif
+
+#if !GC_CONSERVATIVE_TRACE
+#error bdw-gc is a conservative collector
+#endif
+
+#include "conservative-roots-embedder.h"
+
 // When pthreads are used, let `libgc' know about it and redirect
 // allocation calls such as `GC_MALLOC ()' to (contention-free, faster)
 // thread-local allocation.

conservative-roots-embedder.h

@@ -1,6 +1,7 @@
 #ifndef CONSERVATIVE_ROOTS_EMBEDDER_H
 #define CONSERVATIVE_ROOTS_EMBEDDER_H
 
+#include "gc-config.h"
 #include "gc-embedder-api.h"
 
 static inline int gc_has_mutator_conservative_roots(void) {
@@ -13,7 +14,7 @@ static inline int gc_has_global_conservative_roots(void) {
   return 1;
 }
 static inline int gc_has_conservative_intraheap_edges(void) {
-  return 0;
+  return GC_CONSERVATIVE_TRACE;
 }
 
 static inline int

gc-config.h

@@ -13,8 +13,20 @@
 #define GC_GENERATIONAL 0
 #endif
 
-#ifndef GC_PRECISE
+// Though you normally wouldn't configure things this way, it's possible
-#define GC_PRECISE 0
+// to have both precise and conservative roots.  However we have to
+// either have precise or conservative tracing; not a mix.
+
+#ifndef GC_PRECISE_ROOTS
+#define GC_PRECISE_ROOTS 0
+#endif
+
+#ifndef GC_CONSERVATIVE_ROOTS
+#define GC_CONSERVATIVE_ROOTS 0
+#endif
+
+#ifndef GC_CONSERVATIVE_TRACE
+#define GC_CONSERVATIVE_TRACE 0
 #endif
 
 #endif // GC_CONFIG_H

large-object-space.h

@@ -9,6 +9,7 @@
 #include <sys/mman.h>
 #include <unistd.h>
 
+#include "gc-assert.h"
 #include "gc-ref.h"
 #include "gc-conservative-ref.h"
 #include "address-map.h"
@@ -96,6 +97,14 @@ static int large_object_space_mark_object(struct large_object_space *space,
   return large_object_space_copy(space, ref);
 }
 
+static inline size_t large_object_space_object_size(struct large_object_space *space,
+                                                    struct gc_ref ref) {
+  size_t npages = address_map_lookup(&space->object_pages,
+                                     gc_ref_value(ref), 0);
+  GC_ASSERT(npages != 0);
+  return npages * space->page_size;
+}
+
 static void large_object_space_reclaim_one(uintptr_t addr, void *data) {
   struct large_object_space *space = data;
   size_t npages = address_map_lookup(&space->object_pages, addr, 0);

mt-gcbench-embedder.h

@@ -1,6 +1,7 @@
 #ifndef MT_GCBENCH_EMBEDDER_H
 #define MT_GCBENCH_EMBEDDER_H
 
+#include "gc-config.h"
 #include "mt-gcbench-types.h"
 
 struct gc_heap;
@@ -44,9 +45,8 @@ visit_hole_fields(Hole *obj,
                   void (*visit)(struct gc_edge edge,
                                 struct gc_heap *heap, void *visit_data),
                   struct gc_heap *heap, void *visit_data) {
-#if GC_PRECISE
+  if (GC_PRECISE_ROOTS)
-  GC_CRASH();
+    GC_CRASH();
-#endif
 }
 
 #include "simple-gc-embedder.h"

mt-gcbench.c

@@ -47,9 +47,10 @@
 #include "assert.h"
 #include "gc-api.h"
 #include "mt-gcbench-types.h"
-#if GC_PRECISE
+#if GC_PRECISE_ROOTS
 #include "precise-roots-api.h"
-#else
+#endif
+#if GC_CONSERVATIVE_ROOTS
 #include "conservative-roots-api.h"
 #endif
 #include "mt-gcbench-types.h"

parallel-tracer.h

@@ -450,6 +450,8 @@ static void tracer_release(struct gc_heap *heap) {
 
 static inline void tracer_visit(struct gc_edge edge, struct gc_heap *heap,
                                 void *trace_data) GC_ALWAYS_INLINE;
+static inline void tracer_enqueue(struct gc_ref ref, struct gc_heap *heap,
+                                  void *trace_data) GC_ALWAYS_INLINE;
 static inline void trace_one(struct gc_ref ref, struct gc_heap *heap,
                              void *trace_data) GC_ALWAYS_INLINE;
 static inline int trace_edge(struct gc_heap *heap,
@@ -462,8 +464,22 @@ tracer_share(struct local_tracer *trace) {
     trace_deque_push(trace->share_deque, local_trace_queue_pop(&trace->local));
 }
 
+static inline void
+tracer_enqueue(struct gc_ref ref, struct gc_heap *heap, void *trace_data) {
+  struct local_tracer *trace = trace_data;
+  if (local_trace_queue_full(&trace->local))
+    tracer_share(trace);
+  local_trace_queue_push(&trace->local, ref);
+}
+
 static inline void
 tracer_visit(struct gc_edge edge, struct gc_heap *heap, void *trace_data) {
+  if (trace_edge(heap, edge))
+    tracer_enqueue(gc_edge_ref(edge), heap, trace_data);
+}
+
+static inline void
+tracer_visit_(struct gc_edge edge, struct gc_heap *heap, void *trace_data) {
   if (trace_edge(heap, edge)) {
     struct local_tracer *trace = trace_data;
     if (local_trace_queue_full(&trace->local))

quads.c

@@ -5,9 +5,10 @@
 
 #include "assert.h"
 #include "gc-api.h"
-#if GC_PRECISE
+#if GC_PRECISE_ROOTS
 #include "precise-roots-api.h"
-#else
+#endif
+#if GC_CONSERVATIVE_ROOTS
 #include "conservative-roots-api.h"
 #endif
 #include "quads-types.h"

semi.c

@@ -11,9 +11,11 @@
 #include "semi-attrs.h"
 #include "large-object-space.h"
 
-#if GC_PRECISE
+#if GC_PRECISE_ROOTS
 #include "precise-roots-embedder.h"
-#else
+#endif
+
+#if GC_CONSERVATIVE_ROOTS
 #error semi is a precise collector
 #endif
 

serial-tracer.h

@@ -137,6 +137,8 @@ static void tracer_release(struct gc_heap *heap) {
 
 static inline void tracer_visit(struct gc_edge edge, struct gc_heap *heap,
                                 void *trace_data) GC_ALWAYS_INLINE;
+static inline void tracer_enqueue(struct gc_ref ref, struct gc_heap *heap,
+                                  void *trace_data) GC_ALWAYS_INLINE;
 static inline void trace_one(struct gc_ref ref, struct gc_heap *heap,
                              void *trace_data) GC_ALWAYS_INLINE;
 static inline int trace_edge(struct gc_heap *heap,
@@ -152,9 +154,13 @@ tracer_enqueue_roots(struct tracer *tracer, struct gc_ref *objs,
   trace_queue_push_many(&tracer->queue, objs, count);
 }
 static inline void
+tracer_enqueue(struct gc_ref ref, struct gc_heap *heap, void *trace_data) {
+  tracer_enqueue_root(heap_tracer(heap), ref);
+}
+static inline void
 tracer_visit(struct gc_edge edge, struct gc_heap *heap, void *trace_data) {
   if (trace_edge(heap, edge))
-    tracer_enqueue_root(heap_tracer(heap), gc_edge_ref(edge));
+    tracer_enqueue(gc_edge_ref(edge), heap, trace_data);
 }
 static inline void
 tracer_trace(struct gc_heap *heap) {

simple-gc-embedder.h

@@ -26,9 +26,11 @@ static inline void gc_trace_object(struct gc_ref ref,
   }
 }
 
-#if GC_PRECISE
+#if GC_PRECISE_ROOTS
 #include "precise-roots-embedder.h"
-#else
+#endif
+
+#if GC_CONSERVATIVE_ROOTS
 #include "conservative-roots-embedder.h"
 #endif
 

whippet.c

@@ -26,9 +26,11 @@
 #include "spin.h"
 #include "whippet-attrs.h"
 
-#if GC_PRECISE
+#if GC_PRECISE_ROOTS
 #include "precise-roots-embedder.h"
-#else
+#endif
+
+#if GC_CONSERVATIVE_ROOTS
 #include "conservative-roots-embedder.h"
 #endif
 
@@ -371,11 +373,52 @@ static inline void clear_memory(uintptr_t addr, size_t size) {
 
 static void collect(struct gc_mutator *mut) GC_NEVER_INLINE;
 
-static size_t mark_space_live_object_granules(uint8_t *metadata) {
+static inline uint64_t load_eight_aligned_bytes(uint8_t *mark) {
+  GC_ASSERT(((uintptr_t)mark & 7) == 0);
+  uint8_t * __attribute__((aligned(8))) aligned_mark = mark;
+  uint64_t word;
+  memcpy(&word, aligned_mark, 8);
+#ifdef WORDS_BIGENDIAN
+  word = __builtin_bswap64(word);
+#endif
+  return word;
+}
+
+static inline size_t count_zero_bytes(uint64_t bytes) {
+  return bytes ? (__builtin_ctzll(bytes) / 8) : sizeof(bytes);
+}
+
+static uint64_t broadcast_byte(uint8_t byte) {
+  uint64_t result = byte;
+  return result * 0x0101010101010101ULL;
+}
+
+static size_t next_mark(uint8_t *mark, size_t limit, uint64_t sweep_mask) {
   size_t n = 0;
-  while ((metadata[n] & METADATA_BYTE_END) == 0)
+  // If we have a hole, it is likely to be more that 8 granules long.
-    n++;
+  // Assuming that it's better to make aligned loads, first we align the
-  return n + 1;
+  // sweep pointer, then we load aligned mark words.
+  size_t unaligned = ((uintptr_t) mark) & 7;
+  if (unaligned) {
+    uint64_t bytes = load_eight_aligned_bytes(mark - unaligned) >> (unaligned * 8);
+    bytes &= sweep_mask;
+    if (bytes)
+      return count_zero_bytes(bytes);
+    n += 8 - unaligned;
+  }
+
+  for(; n < limit; n += 8) {
+    uint64_t bytes = load_eight_aligned_bytes(mark + n);
+    bytes &= sweep_mask;
+    if (bytes)
+      return n + count_zero_bytes(bytes);
+  }
+
+  return limit;
+}
+
+static size_t mark_space_live_object_granules(uint8_t *metadata) {
+  return next_mark(metadata, -1, broadcast_byte(METADATA_BYTE_END)) + 1;
 }
 
 static inline int mark_space_mark_object(struct mark_space *space,
@@ -710,9 +753,18 @@ static inline struct gc_ref trace_conservative_ref(struct gc_heap *heap,
                                                     ref, possibly_interior);
 }
 
-static inline void trace_one(struct gc_ref ref, struct gc_heap *heap,
+static inline size_t mark_space_object_size(struct mark_space *space,
-                             void *mark_data) {
+                                            struct gc_ref ref) {
-  gc_trace_object(ref, tracer_visit, heap, mark_data, NULL);
+  uint8_t *loc = metadata_byte_for_object(ref);
+  size_t granules = mark_space_live_object_granules(loc);
+  return granules * GRANULE_SIZE;
+}
+
+static inline size_t gc_object_allocation_size(struct gc_heap *heap,
+                                               struct gc_ref ref) {
+  if (GC_LIKELY(mark_space_contains(heap_mark_space(heap), ref)))
+    return mark_space_object_size(heap_mark_space(heap), ref);
+  return large_object_space_object_size(heap_large_object_space(heap), ref);
 }
 
 static int heap_has_multiple_mutators(struct gc_heap *heap) {
@@ -1037,6 +1089,29 @@ trace_conservative_edges(uintptr_t low,
     trace(load_conservative_ref(addr), heap, data);
 }
 
+static inline void tracer_trace_conservative_ref(struct gc_conservative_ref ref,
+                                                 struct gc_heap *heap,
+                                                 void *data) {
+  int possibly_interior = 0;
+  struct gc_ref resolved = trace_conservative_ref(heap, ref, possibly_interior);
+  if (gc_ref_is_heap_object(resolved))
+    tracer_enqueue(resolved, heap, data);
+}
+
+static inline void trace_one(struct gc_ref ref, struct gc_heap *heap,
+                             void *mark_data) {
+  if (gc_has_conservative_intraheap_edges()) {
+    size_t bytes = GC_LIKELY(mark_space_contains(heap_mark_space(heap), ref))
+      ? mark_space_object_size(heap_mark_space(heap), ref)
+      : large_object_space_object_size(heap_large_object_space(heap), ref);
+    trace_conservative_edges(gc_ref_value(ref),
+                             gc_ref_value(ref) + bytes,
+                             tracer_trace_conservative_ref, heap, mark_data);
+  } else {
+    gc_trace_object(ref, tracer_visit, heap, mark_data, NULL);
+  }
+}
+
 static void
 mark_and_globally_enqueue_mutator_conservative_roots(uintptr_t low,
                                                      uintptr_t high,
@@ -1172,26 +1247,6 @@ static void trace_global_conservative_roots(struct gc_heap *heap) {
       (mark_and_globally_enqueue_heap_conservative_roots, heap, NULL);
 }
 
-static inline uint64_t load_eight_aligned_bytes(uint8_t *mark) {
-  GC_ASSERT(((uintptr_t)mark & 7) == 0);
-  uint8_t * __attribute__((aligned(8))) aligned_mark = mark;
-  uint64_t word;
-  memcpy(&word, aligned_mark, 8);
-#ifdef WORDS_BIGENDIAN
-  word = __builtin_bswap64(word);
-#endif
-  return word;
-}
-
-static inline size_t count_zero_bytes(uint64_t bytes) {
-  return bytes ? (__builtin_ctzll(bytes) / 8) : sizeof(bytes);
-}
-
-static uint64_t broadcast_byte(uint8_t byte) {
-  uint64_t result = byte;
-  return result * 0x0101010101010101ULL;
-}
-
 // Note that it's quite possible (and even likely) that any given remset
 // byte doesn't hold any roots, if all stores were to nursery objects.
 STATIC_ASSERT_EQ(GRANULES_PER_REMSET_BYTE % 8, 0);
@@ -1690,30 +1745,6 @@ static int sweep_word(uintptr_t *loc, uintptr_t sweep_mask) {
   return 0;
 }
 
-static size_t next_mark(uint8_t *mark, size_t limit, uint64_t sweep_mask) {
-  size_t n = 0;
-  // If we have a hole, it is likely to be more that 8 granules long.
-  // Assuming that it's better to make aligned loads, first we align the
-  // sweep pointer, then we load aligned mark words.
-  size_t unaligned = ((uintptr_t) mark) & 7;
-  if (unaligned) {
-    uint64_t bytes = load_eight_aligned_bytes(mark - unaligned) >> (unaligned * 8);
-    bytes &= sweep_mask;
-    if (bytes)
-      return count_zero_bytes(bytes);
-    n += 8 - unaligned;
-  }
-
-  for(; n < limit; n += 8) {
-    uint64_t bytes = load_eight_aligned_bytes(mark + n);
-    bytes &= sweep_mask;
-    if (bytes)
-      return n + count_zero_bytes(bytes);
-  }
-
-  return limit;
-}
-
 static uintptr_t mark_space_next_block_to_sweep(struct mark_space *space) {
   uintptr_t block = atomic_load_explicit(&space->next_block,
                                          memory_order_acquire);