Get handles out of collectors

bdw.h

@@ -231,6 +231,12 @@ static void* gc_call_without_gc(struct mutator *mut,
   return GC_do_blocking(f, data);
 }
 
+static void gc_mutator_set_roots(struct mutator *mut,
+                                 struct gc_mutator_roots *roots) {
+}
+static void gc_heap_set_roots(struct heap *heap, struct gc_heap_roots *roots) {
+}
+
 static void gc_print_stats(struct heap *heap) {
   printf("Completed %ld collections\n", (long)GC_get_gc_no());
   printf("Heap size is %ld\n", (long)GC_get_heap_size());

conservative-roots.h

@@ -5,3 +5,17 @@ struct handle { void *unused; };
 #define HANDLE_SET(h,val) do { h.v = val; } while (0)
 #define PUSH_HANDLE(cx, h) do { (void) &h; } while (0)
 #define POP_HANDLE(cx) do { } while (0)
+
+static inline void visit_thread_roots(void *thread_roots,
+                                      void (*trace_edge)(struct gc_edge edge,
+                                                         void *trace_data),
+                                      void *trace_data) {
+  abort();
+}
+
+static inline void visit_roots(struct handle *roots,
+                               void (*trace_edge)(struct gc_edge edge,
+                                                  void *trace_data),
+                               void *trace_data) {
+  GC_ASSERT(!roots);
+}

gc-api.h

@@ -38,6 +38,12 @@ GC_API_ int gc_option_from_string(const char *str);
 GC_API_ int gc_init(int argc, struct gc_option argv[],
                     struct heap **heap, struct mutator **mutator);
 
+struct gc_mutator_roots;
+struct gc_heap_roots;
+GC_API_ void gc_mutator_set_roots(struct mutator *mut,
+                                  struct gc_mutator_roots *roots);
+GC_API_ void gc_heap_set_roots(struct heap *heap, struct gc_heap_roots *roots);
+
 GC_API_ struct mutator* gc_init_for_thread(uintptr_t *stack_base,
                                            struct heap *heap);
 GC_API_ void gc_finish_for_thread(struct mutator *mut);

gc-embedder-api.h

@@ -13,6 +13,14 @@ GC_EMBEDDER_API inline void gc_trace_object(void *object,
                                                                void *trace_data),
                                             void *trace_data,
                                             size_t *size) GC_ALWAYS_INLINE;
+GC_EMBEDDER_API inline void gc_trace_mutator_roots(struct gc_mutator_roots *roots,
+                                                   void (*trace_edge)(struct gc_edge edge,
+                                                                      void *trace_data),
+                                                   void *trace_data);
+GC_EMBEDDER_API inline void gc_trace_heap_roots(struct gc_heap_roots *roots,
+                                                void (*trace_edge)(struct gc_edge edge,
+                                                                   void *trace_data),
+                                                void *trace_data);
 
 GC_EMBEDDER_API inline uintptr_t gc_object_forwarded_nonatomic(void *object);
 GC_EMBEDDER_API inline void gc_object_forward_nonatomic(void *object, uintptr_t new_addr);
@@ -25,4 +33,5 @@ GC_EMBEDDER_API inline void gc_atomic_forward_commit(struct gc_atomic_forward *,
                                                      uintptr_t new_addr);
 GC_EMBEDDER_API inline uintptr_t gc_atomic_forward_address(struct gc_atomic_forward *);
 
+
 #endif // GC_EMBEDDER_API_H

mt-gcbench.c

@@ -185,6 +185,7 @@ static size_t power_law(size_t *counter) {
 
 struct thread {
   struct mutator *mut;
+  struct gc_mutator_roots roots;
   size_t counter;
 };
 
@@ -209,13 +210,13 @@ static void populate(struct thread *t, int depth, Node *node) {
     return;
 
   NodeHandle self = { node };
-  PUSH_HANDLE(mut, self);
+  PUSH_HANDLE(t, self);
   allocate_garbage(t);
   NodeHandle l = { allocate_node(mut) };
-  PUSH_HANDLE(mut, l);
+  PUSH_HANDLE(t, l);
   allocate_garbage(t);
   NodeHandle r = { allocate_node(mut) };
-  PUSH_HANDLE(mut, r);
+  PUSH_HANDLE(t, r);
 
   set_field(HANDLE_REF(self), &HANDLE_REF(self)->left, HANDLE_REF(l));
   set_field(HANDLE_REF(self), &HANDLE_REF(self)->right, HANDLE_REF(r));
@@ -225,9 +226,9 @@ static void populate(struct thread *t, int depth, Node *node) {
   populate(t, depth-1, HANDLE_REF(self)->left);
   populate(t, depth-1, HANDLE_REF(self)->right);
 
-  POP_HANDLE(mut);
+  POP_HANDLE(t);
-  POP_HANDLE(mut);
+  POP_HANDLE(t);
-  POP_HANDLE(mut);
+  POP_HANDLE(t);
 }
 
 // Build tree bottom-up
@@ -237,9 +238,9 @@ static Node* make_tree(struct thread *t, int depth) {
     return allocate_node(mut);
 
   NodeHandle left = { make_tree(t, depth-1) };
-  PUSH_HANDLE(mut, left);
+  PUSH_HANDLE(t, left);
   NodeHandle right = { make_tree(t, depth-1) };
-  PUSH_HANDLE(mut, right);
+  PUSH_HANDLE(t, right);
 
   allocate_garbage(t);
   Node *result = allocate_node(mut);
@@ -248,8 +249,8 @@ static Node* make_tree(struct thread *t, int depth) {
   // i is 0 because the memory is zeroed.
   result->j = depth;
 
-  POP_HANDLE(mut);
+  POP_HANDLE(t);
-  POP_HANDLE(mut);
+  POP_HANDLE(t);
 
   return result;
 }
@@ -274,7 +275,7 @@ static void time_construction(struct thread *t, int depth) {
   struct mutator *mut = t->mut;
   int num_iters = compute_num_iters(depth);
   NodeHandle temp_tree = { NULL };
-  PUSH_HANDLE(mut, temp_tree);
+  PUSH_HANDLE(t, temp_tree);
 
   printf("Creating %d trees of depth %d\n", num_iters, depth);
 
@@ -301,7 +302,7 @@ static void time_construction(struct thread *t, int depth) {
            elapsed_millis(start));
   }
 
-  POP_HANDLE(mut);
+  POP_HANDLE(t);
 }
 
 static void* call_with_stack_base(void* (*)(uintptr_t*, void*), void*) GC_NEVER_INLINE;
@@ -337,11 +338,12 @@ static void* run_one_test(struct mutator *mut) {
   NodeHandle long_lived_tree = { NULL };
   NodeHandle temp_tree = { NULL };
   DoubleArrayHandle array = { NULL };
-  struct thread t = { mut, 0 };
+  struct thread t = { mut, };
+  gc_mutator_set_roots(mut, &t.roots);
 
-  PUSH_HANDLE(mut, long_lived_tree);
+  PUSH_HANDLE(&t, long_lived_tree);
-  PUSH_HANDLE(mut, temp_tree);
+  PUSH_HANDLE(&t, temp_tree);
-  PUSH_HANDLE(mut, array);
+  PUSH_HANDLE(&t, array);
 
   // Create a long lived object
   printf(" Creating a long-lived binary tree of depth %d\n",
@@ -368,9 +370,10 @@ static void* run_one_test(struct mutator *mut) {
       || HANDLE_REF(array)->values[1000] != 1.0/1000)
     fprintf(stderr, "Failed\n");
 
-  POP_HANDLE(mut);
+  POP_HANDLE(&t);
-  POP_HANDLE(mut);
+  POP_HANDLE(&t);
-  POP_HANDLE(mut);
+  POP_HANDLE(&t);
+  gc_mutator_set_roots(mut, NULL);
   return NULL;
 }
 

precise-roots.h

@@ -6,8 +6,8 @@ struct handle {
 #define HANDLE_TO(T) union { T* v; struct handle handle; }
 #define HANDLE_REF(h) h.v
 #define HANDLE_SET(h,val) do { h.v = val; } while (0)
-#define PUSH_HANDLE(cx, h) push_handle(&cx->roots, &h.handle)
+#define PUSH_HANDLE(cx, h) push_handle(&(cx)->roots.roots, &h.handle)
-#define POP_HANDLE(cx) pop_handle(&cx->roots)
+#define POP_HANDLE(cx) pop_handle(&(cx)->roots.roots)
 
 static inline void push_handle(struct handle **roots, struct handle *handle) {
   handle->next = *roots;
@@ -17,3 +17,11 @@ static inline void push_handle(struct handle **roots, struct handle *handle) {
 static inline void pop_handle(struct handle **roots) {
   *roots = (*roots)->next;
 }
+
+static inline void visit_roots(struct handle *roots,
+                               void (*trace_edge)(struct gc_edge edge,
+                                                  void *trace_data),
+                               void *trace_data) {
+  for (struct handle *h = roots; h; h = h->next)
+    trace_edge(gc_edge(&h->v), trace_data);
+}

quads.c

@@ -38,23 +38,29 @@ static unsigned long current_time(void)
   return t.tv_sec * 1000 * 1000 + t.tv_usec;
 }
 
+struct thread {
+  struct mutator *mut;
+  struct gc_mutator_roots roots;
+  size_t counter;
+};
+
 // Build tree bottom-up
-static Quad* make_tree(struct mutator *mut, int depth) {
+static Quad* make_tree(struct thread *t, int depth) {
   if (depth<=0) {
-    return allocate_quad(mut);
+    return allocate_quad(t->mut);
   } else {
     QuadHandle kids[4] = { { NULL }, };
     for (size_t i = 0; i < 4; i++) {
-      HANDLE_SET(kids[i], make_tree(mut, depth-1));
+      HANDLE_SET(kids[i], make_tree(t, depth-1));
-      PUSH_HANDLE(mut, kids[i]);
+      PUSH_HANDLE(t, kids[i]);
     }
 
-    Quad *result = allocate_quad(mut);
+    Quad *result = allocate_quad(t->mut);
     for (size_t i = 0; i < 4; i++)
       result->kids[i] = HANDLE_REF(kids[i]);
 
     for (size_t i = 0; i < 4; i++)
-      POP_HANDLE(mut);
+      POP_HANDLE(t);
 
     return result;
   }
@@ -144,15 +150,17 @@ int main(int argc, char *argv[]) {
             heap_size);
     return 1;
   }
+  struct thread t = { mut, };
+  gc_mutator_set_roots(mut, &t.roots);
 
   QuadHandle quad = { NULL };
 
-  PUSH_HANDLE(mut, quad);
+  PUSH_HANDLE(&t, quad);
 
   printf("Making quad tree of depth %zu (%zu nodes).  Total size %.3fGB.\n",
          depth, nquads, (nquads * sizeof(Quad)) / 1e9);
   unsigned long start = current_time();
-  HANDLE_SET(quad, make_tree(mut, depth));
+  HANDLE_SET(quad, make_tree(&t, depth));
   print_elapsed("construction", start);
 
   validate_tree(HANDLE_REF(quad), depth);
@@ -165,7 +173,7 @@ int main(int argc, char *argv[]) {
     size_t garbage_depth = 3;
     start = current_time();
     for (size_t i = garbage_step/(tree_size(garbage_depth)*4*sizeof(Quad*)); i; i--)
-      make_tree(mut, garbage_depth);
+      make_tree(&t, garbage_depth);
     print_elapsed("allocating garbage", start);
 
     start = current_time();
@@ -176,7 +184,7 @@ int main(int argc, char *argv[]) {
 
   gc_print_stats(heap);
 
-  POP_HANDLE(mut);
+  POP_HANDLE(&t);
   return 0;
 }
 

semi.h

@@ -27,7 +27,7 @@ struct heap {
 // One mutator per space, can just store the heap in the mutator.
 struct mutator {
   struct heap heap;
-  struct handle *roots;
+  struct gc_mutator_roots *roots;
 };
 
 
@@ -152,8 +152,8 @@ static void collect(struct mutator *mut) {
   large_object_space_start_gc(large, 0);
   flip(semi);
   uintptr_t grey = semi->hp;
-  for (struct handle *h = mut->roots; h; h = h->next)
+  if (mut->roots)
-    visit(gc_edge(&h->v), heap);
+    gc_trace_mutator_roots(mut->roots, visit, heap);
   // fprintf(stderr, "pushed %zd bytes in roots\n", space->hp - grey);
   while(grey < semi->hp)
     grey = scan(heap, grey);
@@ -328,6 +328,14 @@ static int gc_init(int argc, struct gc_option argv[],
   return 1;
 }
 
+static void gc_mutator_set_roots(struct mutator *mut,
+                                 struct gc_mutator_roots *roots) {
+  mut->roots = roots;
+}
+static void gc_heap_set_roots(struct heap *heap, struct gc_heap_roots *roots) {
+  abort();
+}
+
 static struct mutator* gc_init_for_thread(uintptr_t *stack_base,
                                           struct heap *heap) {
   fprintf(stderr,

simple-gc-embedder.h

@@ -23,6 +23,31 @@ static inline void gc_trace_object(void *object,
   }
 }
 
+struct handle;
+struct gc_heap_roots { struct handle *roots; };
+struct gc_mutator_roots { struct handle *roots; };
+
+static inline void visit_roots(struct handle *roots,
+                               void (*trace_edge)(struct gc_edge edge,
+                                                  void *trace_data),
+                               void *trace_data);
+
+static inline void gc_trace_mutator_roots(struct gc_mutator_roots *roots,
+                                          void (*trace_edge)(struct gc_edge edge,
+                                                             void *trace_data),
+                                          void *trace_data) {
+  if (roots)
+    visit_roots(roots->roots, trace_edge, trace_data);
+}
+
+static inline void gc_trace_heap_roots(struct gc_heap_roots *roots,
+                                       void (*trace_edge)(struct gc_edge edge,
+                                                          void *trace_data),
+                                       void *trace_data) {
+  if (roots)
+    visit_roots(roots->roots, trace_edge, trace_data);
+}
+
 static inline uintptr_t gc_object_forwarded_nonatomic(void *object) {
   uintptr_t tag = *tag_word(object);
   return (tag & gcobj_not_forwarded_bit) ? 0 : tag;

whippet.h

@@ -322,7 +322,7 @@ struct heap {
   int allow_pinning;
   size_t active_mutator_count;
   size_t mutator_count;
-  struct handle *global_roots;
+  struct gc_heap_roots *roots;
   struct mutator *mutator_trace_list;
   long count;
   long minor_count;
@@ -348,7 +348,7 @@ struct mutator {
   uintptr_t sweep;
   uintptr_t block;
   struct heap *heap;
-  struct handle *roots;
+  struct gc_mutator_roots *roots;
   struct mutator_mark_buf mark_buf;
   // Three uses for this in-object linked-list pointer:
   //  - inactive (blocked in syscall) mutators
@@ -905,29 +905,39 @@ static int mutator_should_mark_while_stopping(struct mutator *mut) {
   return heap_should_mark_while_stopping(mutator_heap(mut));
 }
 
+static void gc_mutator_set_roots(struct mutator *mut,
+                                 struct gc_mutator_roots *roots) {
+  mut->roots = roots;
+}
+static void gc_heap_set_roots(struct heap *heap, struct gc_heap_roots *roots) {
+  heap->roots = roots;
+}
+
+static void trace_and_enqueue_locally(struct gc_edge edge, void *data) {
+  struct mutator *mut = data;
+  if (trace_edge(mutator_heap(mut), edge))
+    mutator_mark_buf_push(&mut->mark_buf,
+                          gc_ref_heap_object(gc_edge_ref(edge)));
+}
+
+static void trace_and_enqueue_globally(struct gc_edge edge, void *data) {
+  struct heap *heap = data;
+  if (trace_edge(heap, edge))
+    tracer_enqueue_root(&heap->tracer,
+                        gc_ref_heap_object(gc_edge_ref(edge)));
+}
+
 // Mark the roots of a mutator that is stopping for GC.  We can't
 // enqueue them directly, so we send them to the controller in a buffer.
 static void mark_stopping_mutator_roots(struct mutator *mut) {
   GC_ASSERT(mutator_should_mark_while_stopping(mut));
-  struct heap *heap = mutator_heap(mut);
+  gc_trace_mutator_roots(mut->roots, trace_and_enqueue_locally, mut);
-  struct mutator_mark_buf *local_roots = &mut->mark_buf;
-  for (struct handle *h = mut->roots; h; h = h->next) {
-    struct gc_edge root = gc_edge(&h->v);
-    if (trace_edge(heap, root))
-      mutator_mark_buf_push(local_roots,
-                            gc_ref_heap_object(gc_edge_ref(root)));
-  }
 }
 
 // Precondition: the caller holds the heap lock.
 static void mark_mutator_roots_with_lock(struct mutator *mut) {
-  struct heap *heap = mutator_heap(mut);
+  gc_trace_mutator_roots(mut->roots, trace_and_enqueue_globally,
-  for (struct handle *h = mut->roots; h; h = h->next) {
+                         mutator_heap(mut));
-    struct gc_edge root = gc_edge(&h->v);
-    if (trace_edge(heap, root))
-      tracer_enqueue_root(&heap->tracer,
-                          gc_ref_heap_object(gc_edge_ref(root)));
-  }
 }
 
 static void trace_mutator_roots_with_lock(struct mutator *mut) {
@@ -976,12 +986,7 @@ static void trace_mutator_roots_after_stop(struct heap *heap) {
 }
 
 static void trace_global_roots(struct heap *heap) {
-  for (struct handle *h = heap->global_roots; h; h = h->next) {
+  gc_trace_heap_roots(heap->roots, trace_and_enqueue_globally, heap);
-    struct gc_edge edge = gc_edge(&h->v);
-    if (trace_edge(heap, edge))
-      tracer_enqueue_root(&heap->tracer,
-                          gc_ref_heap_object(gc_edge_ref(edge)));
-  }
 }
 
 static inline int