Add gc_edge data structure

Less casting in user programs, and it's a step on the way to evacuation
in whippet.

gc-types.h

@@ -0,0 +1,26 @@
+#ifndef GC_TYPES_H_
+#define GC_TYPES_H_
+
+struct gc_edge {
+  union {
+    void *addr;
+    void **loc;
+  };
+};
+
+static inline struct gc_edge gc_edge(void* addr) {
+  struct gc_edge edge;
+  edge.addr = addr;
+  return edge;
+}
+static inline struct gc_edge object_field(void* addr) {
+  return gc_edge(addr);
+}
+static inline void* dereference_edge(struct gc_edge edge) {
+  return *edge.loc;
+}
+static inline void update_edge(struct gc_edge edge, void *value) {
+  *edge.loc = value;
+}
+
+#endif // GC_TYPES_H_

gc.h

@@ -1,6 +1,8 @@
 #ifndef GC_H_
 #define GC_H_
 
+#include "gc-types.h"
+
 #if defined(GC_BDW)
 #include "bdw.h"
 #elif defined(GC_SEMI)

heap-objects.h

@@ -2,6 +2,7 @@
 #define HEAP_OBJECTS_H
 
 #include "inline.h"
+#include "gc-types.h"
 
 #define DECLARE_NODE_TYPE(name, Name, NAME) \
   struct Name;                              \
@@ -18,7 +19,7 @@ enum alloc_kind {
 #define DEFINE_METHODS(name, Name, NAME) \
   static inline size_t name##_size(Name *obj) ALWAYS_INLINE; \
   static inline void visit_##name##_fields(Name *obj,\
-                                           void (*visit)(void **loc, void *visit_data), \
+                                           void (*visit)(struct gc_edge edge, void *visit_data), \
                                            void *visit_data) ALWAYS_INLINE;
 FOR_EACH_HEAP_OBJECT_KIND(DEFINE_METHODS)
 #undef DEFINE_METHODS

mt-gcbench.c

@@ -77,14 +77,14 @@ static inline size_t double_array_size(DoubleArray *array) {
 }
 static inline void
 visit_node_fields(Node *node,
-                  void (*visit)(void **loc, void *visit_data),
+                  void (*visit)(struct gc_edge edge, void *visit_data),
                   void *visit_data) {
-  visit((void**)&node->left, visit_data);
-  visit((void**)&node->right, visit_data);
+  visit(object_field(&node->left), visit_data);
+  visit(object_field(&node->right), visit_data);
 }
 static inline void
 visit_double_array_fields(DoubleArray *obj,
-                          void (*visit)(void **loc, void *visit_data),
+                          void (*visit)(struct gc_edge edge, void *visit_data),
                           void *visit_data) {
 }
 

parallel-tracer.h

@@ -452,10 +452,10 @@ static void tracer_release(struct heap *heap) {
 }
 
 struct gcobj;
-static inline void tracer_visit(void **loc, void *trace_data) ALWAYS_INLINE;
+static inline void tracer_visit(struct gc_edge edge, void *trace_data) ALWAYS_INLINE;
 static inline void trace_one(struct gcobj *obj, void *trace_data) ALWAYS_INLINE;
-static inline int trace_object(struct heap *heap,
-                               struct gcobj *obj) ALWAYS_INLINE;
+static inline int trace_edge(struct heap *heap,
+                             struct gc_edge edge) ALWAYS_INLINE;
 
 static inline void
 tracer_share(struct local_tracer *trace) {
@@ -465,13 +465,12 @@ tracer_share(struct local_tracer *trace) {
 }
 
 static inline void
-tracer_visit(void **loc, void *trace_data) {
+tracer_visit(struct gc_edge edge, void *trace_data) {
   struct local_tracer *trace = trace_data;
-  struct gcobj *obj = *loc;
-  if (obj && trace_object(trace->heap, obj)) {
+  if (trace_edge(trace->heap, edge)) {
     if (local_trace_queue_full(&trace->local))
       tracer_share(trace);
-    local_trace_queue_push(&trace->local, obj);
+    local_trace_queue_push(&trace->local, dereference_edge(edge));
   }
 }
 

quads.c

@@ -15,10 +15,10 @@ static inline size_t quad_size(Quad *obj) {
 }
 static inline void
 visit_quad_fields(Quad *quad,
-                  void (*visit)(void **loc, void *visit_data),
+                  void (*visit)(struct gc_edge edge, void *visit_data),
                   void *visit_data) {
   for (size_t i = 0; i < 4; i++)
-    visit((void**)&quad->kids[i], visit_data);
+    visit(object_field(&quad->kids[i]), visit_data);
 }
 typedef HANDLE_TO(Quad) QuadHandle;
 

semi.h

@@ -57,7 +57,7 @@ static inline void clear_memory(uintptr_t addr, size_t size) {
 static void collect(struct mutator *mut) NEVER_INLINE;
 static void collect_for_alloc(struct mutator *mut, size_t bytes) NEVER_INLINE;
 
-static void visit(void **loc, void *visit_data);
+static void visit(struct gc_edge edge, void *visit_data);
 
 static int semi_space_steal_pages(struct semi_space *space, size_t npages) {
   size_t stolen_pages = space->stolen_pages + npages;
@@ -143,13 +143,13 @@ static void* forward(struct semi_space *space, void *obj) {
 }  
 
 static void visit_semi_space(struct heap *heap, struct semi_space *space,
-                             void **loc, void *obj) {
-  *loc = forward(space, obj);
+                             struct gc_edge edge, void *obj) {
+  update_edge(edge, forward(space, obj));
 }
 
 static void visit_large_object_space(struct heap *heap,
                                      struct large_object_space *space,
-                                     void **loc, void *obj) {
+                                     void *obj) {
   if (large_object_space_copy(space, (uintptr_t)obj))
     scan(heap, (uintptr_t)obj);
 }
@@ -158,15 +158,15 @@ static int semi_space_contains(struct semi_space *space, void *obj) {
   return (((uintptr_t)obj) - space->base) < space->size;
 }
 
-static void visit(void **loc, void *visit_data) {
+static void visit(struct gc_edge edge, void *visit_data) {
   struct heap *heap = visit_data;
-  void *obj = *loc;
+  void *obj = dereference_edge(edge);
   if (obj == NULL)
     return;
-  if (semi_space_contains(heap_semi_space(heap), obj))
-    visit_semi_space(heap, heap_semi_space(heap), loc, obj);
+  else if (semi_space_contains(heap_semi_space(heap), obj))
+    visit_semi_space(heap, heap_semi_space(heap), edge, obj);
   else if (large_object_space_contains(heap_large_object_space(heap), obj))
-    visit_large_object_space(heap, heap_large_object_space(heap), loc, obj);
+    visit_large_object_space(heap, heap_large_object_space(heap), obj);
   else
     abort();
 }
@@ -180,7 +180,7 @@ static void collect(struct mutator *mut) {
   flip(semi);
   uintptr_t grey = semi->hp;
   for (struct handle *h = mut->roots; h; h = h->next)
-    visit(&h->v, heap);
+    visit(gc_edge(&h->v), heap);
   // fprintf(stderr, "pushed %zd bytes in roots\n", space->hp - grey);
   while(grey < semi->hp)
     grey = scan(heap, grey);

serial-tracer.h

@@ -6,6 +6,7 @@
 
 #include "assert.h"
 #include "debug.h"
+#include "gc-types.h"
 
 struct gcobj;
 
@@ -137,10 +138,10 @@ static void tracer_release(struct heap *heap) {
 }
 
 struct gcobj;
-static inline void tracer_visit(void **loc, void *trace_data) ALWAYS_INLINE;
+static inline void tracer_visit(struct gc_edge edge, void *trace_data) ALWAYS_INLINE;
 static inline void trace_one(struct gcobj *obj, void *trace_data) ALWAYS_INLINE;
-static inline int trace_object(struct heap *heap,
-                               struct gcobj *obj) ALWAYS_INLINE;
+static inline int trace_edge(struct heap *heap,
+                             struct gc_edge edge) ALWAYS_INLINE;
 
 static inline void
 tracer_enqueue_root(struct tracer *tracer, struct gcobj *obj) {
@@ -152,11 +153,10 @@ tracer_enqueue_roots(struct tracer *tracer, struct gcobj **objs,
   trace_queue_push_many(&tracer->queue, objs, count);
 }
 static inline void
-tracer_visit(void **loc, void *trace_data) {
+tracer_visit(struct gc_edge edge, void *trace_data) {
   struct heap *heap = trace_data;
-  struct gcobj *obj = *loc;
-  if (obj && trace_object(heap, obj))
-    tracer_enqueue_root(heap_tracer(heap), obj);
+  if (trace_edge(heap, edge))
+    tracer_enqueue_root(heap_tracer(heap), dereference_edge(edge));
 }
 static inline void
 tracer_trace(struct heap *heap) {

whippet.h

@@ -350,8 +350,9 @@ static inline uint8_t* mark_byte(struct mark_space *space, struct gcobj *obj) {
   return object_metadata_byte(obj);
 }
 
-static inline int mark_space_trace_object(struct mark_space *space,
-                                          struct gcobj *obj) {
+static inline int mark_space_mark_object(struct mark_space *space,
+                                         struct gc_edge edge) {
+  struct gcobj *obj = dereference_edge(edge);
   uint8_t *loc = object_metadata_byte(obj);
   uint8_t byte = *loc;
   if (byte & space->marked_mask)
@@ -368,16 +369,20 @@ static inline int mark_space_contains(struct mark_space *space,
   return addr - space->low_addr < space->extent;
 }
 
-static inline int large_object_space_trace_object(struct large_object_space *space,
-                                                  struct gcobj *obj) {
+static inline int large_object_space_mark_object(struct large_object_space *space,
+                                                 struct gcobj *obj) {
   return large_object_space_copy(space, (uintptr_t)obj);
 }
 
-static inline int trace_object(struct heap *heap, struct gcobj *obj) {
-  if (LIKELY(mark_space_contains(heap_mark_space(heap), obj)))
-    return mark_space_trace_object(heap_mark_space(heap), obj);
+static inline int trace_edge(struct heap *heap, struct gc_edge edge) {
+  struct gcobj *obj = dereference_edge(edge);
+  if (!obj)
+    return 0;
+  else if (LIKELY(mark_space_contains(heap_mark_space(heap), obj)))
+    return mark_space_mark_object(heap_mark_space(heap), edge);
   else if (large_object_space_contains(heap_large_object_space(heap), obj))
-    return large_object_space_trace_object(heap_large_object_space(heap), obj);
+    return large_object_space_mark_object(heap_large_object_space(heap),
+                                          obj);
   else
     abort();
 }
@@ -584,9 +589,9 @@ static void mark_stopping_mutator_roots(struct mutator *mut) {
   struct heap *heap = mutator_heap(mut);
   struct mutator_mark_buf *local_roots = &mut->mark_buf;
   for (struct handle *h = mut->roots; h; h = h->next) {
-    struct gcobj *root = h->v;
-    if (root && trace_object(heap, root))
-      mutator_mark_buf_push(local_roots, root);
+    struct gc_edge root = gc_edge(&h->v);
+    if (trace_edge(heap, root))
+      mutator_mark_buf_push(local_roots, dereference_edge(root));
   }
 
   // Post to global linked-list of thread roots.
@@ -602,9 +607,9 @@ static void mark_stopping_mutator_roots(struct mutator *mut) {
 static void mark_controlling_mutator_roots(struct mutator *mut) {
   struct heap *heap = mutator_heap(mut);
   for (struct handle *h = mut->roots; h; h = h->next) {
-    struct gcobj *root = h->v;
-    if (root && trace_object(heap, root))
-      tracer_enqueue_root(&heap->tracer, root);
+    struct gc_edge root = gc_edge(&h->v);
+    if (trace_edge(heap, root))
+      tracer_enqueue_root(&heap->tracer, dereference_edge(root));
   }
 }
 
@@ -630,9 +635,9 @@ static void mark_inactive_mutators(struct heap *heap) {
 
 static void mark_global_roots(struct heap *heap) {
   for (struct handle *h = heap->global_roots; h; h = h->next) {
-    struct gcobj *obj = h->v;
-    if (obj && trace_object(heap, obj))
-      tracer_enqueue_root(&heap->tracer, obj);
+    struct gc_edge edge = gc_edge(&h->v);
+    if (trace_edge(heap, edge))
+      tracer_enqueue_root(&heap->tracer, dereference_edge(edge));
   }
 
   struct mutator_mark_buf *roots = atomic_load(&heap->mutator_roots);