Explicitly support immediate values

Because we have to deref edges ourselves, as part of generational
marking, we need to ignore edges that don't point to heap objects.

api/gc-config.h

@@ -5,6 +5,10 @@
 #define GC_DEBUG 0
 #endif
 
+#ifndef GC_HAS_IMMEDIATES
+#define GC_HAS_IMMEDIATES 1
+#endif
+
 #ifndef GC_PARALLEL
 #define GC_PARALLEL 0
 #endif

api/gc-ref.h

@@ -2,6 +2,7 @@
 #define GC_REF_H
 
 #include "gc-assert.h"
+#include "gc-config.h"
 
 #include <stdint.h>
 
@@ -19,8 +20,20 @@ static inline uintptr_t gc_ref_value(struct gc_ref ref) {
 static inline struct gc_ref gc_ref_null(void) {
   return gc_ref(0);
 }
+static inline int gc_ref_is_null(struct gc_ref ref) {
+  return ref.value == 0;
+}
+static inline int gc_ref_is_immediate(struct gc_ref ref) {
+  GC_ASSERT(!gc_ref_is_null(ref));
+  return GC_HAS_IMMEDIATES && (ref.value & (sizeof(void*) - 1));
+}
+static inline struct gc_ref gc_ref_immediate(uintptr_t val) {
+  GC_ASSERT(val & (sizeof(void*) - 1));
+  GC_ASSERT(GC_HAS_IMMEDIATES);
+  return gc_ref(val);
+}
 static inline int gc_ref_is_heap_object(struct gc_ref ref) {
-  return ref.value != 0;
+  return !gc_ref_is_immediate(ref);
 }
 static inline struct gc_ref gc_ref_from_heap_object_or_null(void *obj) {
   return gc_ref((uintptr_t) obj);

src/field-set.h

@@ -162,15 +162,15 @@ gc_field_set_clear(struct gc_field_set *set,
 }
 
 static inline void
-gc_field_set_trace_edge_buffer(struct gc_field_set *set,
+gc_field_set_visit_edge_buffer(struct gc_field_set *set,
                                struct gc_edge_buffer *buf,
-                               void (*tracer_visit)(struct gc_edge,
-                                                    struct gc_heap*,
-                                                    void *data),
+                               void (*visit)(struct gc_edge,
+                                             struct gc_heap*,
+                                             void *data),
                                struct gc_heap *heap,
-                               struct gc_trace_worker *worker) {
+                               void *data) {
   for (size_t i = 0; i < buf->size; i++)
-    tracer_visit(buf->edges[i], heap, worker);
+    visit(buf->edges[i], heap, data);
 }
 
 static void

src/gc-finalizer.c

@@ -164,8 +164,7 @@ void gc_finalizer_init_internal(struct gc_finalizer *f,
   // value.
   if (f->state != FINALIZER_STATE_INIT)
     GC_CRASH();
-  if (gc_ref_is_heap_object(f->object))
-    GC_CRASH();
+  GC_ASSERT(gc_ref_is_null(f->object));
   f->object = object;
   f->closure = closure;
 }
@@ -179,7 +178,7 @@ void gc_finalizer_attach_internal(struct gc_finalizer_state *state,
   // value.
   if (f->state != FINALIZER_STATE_INIT)
     GC_CRASH();
-  if (!gc_ref_is_heap_object(f->object))
+  if (gc_ref_is_null(f->object))
     GC_CRASH();
 
   f->state = FINALIZER_STATE_ACTIVE;

src/mmc.c

@@ -121,8 +121,6 @@ gc_trace_worker_call_with_data(void (*f)(struct gc_tracer *tracer,
 static inline int
 do_trace(struct gc_heap *heap, struct gc_edge edge, struct gc_ref ref,
          struct gc_trace_worker_data *data) {
-  if (!gc_ref_is_heap_object(ref))
-    return 0;
   if (GC_LIKELY(nofl_space_contains(heap_nofl_space(heap), ref)))
     return nofl_space_evacuate_or_mark_object(heap_nofl_space(heap), edge, ref,
                                               &data->allocator);
@@ -137,6 +135,9 @@ static inline int
 trace_edge(struct gc_heap *heap, struct gc_edge edge,
            struct gc_trace_worker_data *data) {
   struct gc_ref ref = gc_edge_ref(edge);
+  if (gc_ref_is_null(ref) || gc_ref_is_immediate(ref))
+    return 0;
+
   int is_new = do_trace(heap, edge, ref, data);
 
   if (is_new &&
@@ -150,9 +151,10 @@ trace_edge(struct gc_heap *heap, struct gc_edge edge,
 int
 gc_visit_ephemeron_key(struct gc_edge edge, struct gc_heap *heap) {
   struct gc_ref ref = gc_edge_ref(edge);
-  if (!gc_ref_is_heap_object(ref))
-    return 0;
-
+  GC_ASSERT(!gc_ref_is_null(ref));
+  if (gc_ref_is_immediate(ref))
+    return 1;
+  GC_ASSERT(gc_ref_is_heap_object(ref));
   struct nofl_space *nofl_space = heap_nofl_space(heap);
   if (GC_LIKELY(nofl_space_contains(nofl_space, ref)))
     return nofl_space_forward_or_mark_if_traced(nofl_space, edge, ref);
@@ -271,11 +273,11 @@ static inline struct gc_ref
 trace_conservative_ref(struct gc_heap *heap, struct gc_conservative_ref ref,
                        int possibly_interior) {
   struct gc_ref ret = do_trace_conservative_ref(heap, ref, possibly_interior);
-
-  if (gc_ref_is_heap_object(ret) &&
-      GC_UNLIKELY(atomic_load_explicit(&heap->check_pending_ephemerons,
-                                       memory_order_relaxed)))
-    gc_resolve_pending_ephemerons(ret, heap);
+  if (!gc_ref_is_null(ret)) {
+    if (GC_UNLIKELY(atomic_load_explicit(&heap->check_pending_ephemerons,
+                                         memory_order_relaxed)))
+      gc_resolve_pending_ephemerons(ret, heap);
+  }
 
   return ret;
 }
@@ -286,7 +288,7 @@ tracer_trace_conservative_ref(struct gc_conservative_ref ref,
                               struct gc_trace_worker *worker,
                               int possibly_interior) {
   struct gc_ref resolved = trace_conservative_ref(heap, ref, possibly_interior);
-  if (gc_ref_is_heap_object(resolved))
+  if (!gc_ref_is_null(resolved))
     gc_trace_worker_enqueue(worker, resolved);
 }
 
@@ -367,7 +369,7 @@ trace_root(struct gc_root root, struct gc_heap *heap,
     tracer_visit(root.edge, heap, worker);
     break;
   case GC_ROOT_KIND_EDGE_BUFFER:
-    gc_field_set_trace_edge_buffer(&heap->remembered_set, root.edge_buffer,
+    gc_field_set_visit_edge_buffer(&heap->remembered_set, root.edge_buffer,
                                    tracer_visit, heap, worker);
     break;
   default:
@@ -910,7 +912,7 @@ void
 gc_write_barrier_slow(struct gc_mutator *mut, struct gc_ref obj,
                       size_t obj_size, struct gc_edge edge,
                       struct gc_ref new_val) {
-  GC_ASSERT(gc_ref_is_heap_object(new_val));
+  GC_ASSERT(!gc_ref_is_null(new_val));
   if (!GC_GENERATIONAL) return;
   if (gc_object_is_old_generation_slow(mut, new_val))
     return;

src/nofl-space.h

@@ -1483,7 +1483,7 @@ nofl_space_evacuate(struct nofl_space *space, uint8_t *metadata, uint8_t byte,
     size_t object_granules = nofl_space_live_object_granules(metadata);
     struct gc_ref new_ref = nofl_evacuation_allocate(evacuate, space,
                                                      object_granules);
-    if (gc_ref_is_heap_object(new_ref)) {
+    if (!gc_ref_is_null(new_ref)) {
       // Copy object contents before committing, as we don't know what
       // part of the object (if any) will be overwritten by the
       // commit.

src/parallel-tracer.h

@@ -207,7 +207,7 @@ trace_worker_steal_from_any(struct gc_trace_worker *worker,
   for (size_t i = 0; i < tracer->worker_count; i++) {
     LOG("tracer #%zu: stealing from #%zu\n", worker->id, worker->steal_id);
     struct gc_ref obj = tracer_steal_from_worker(tracer, worker->steal_id);
-    if (gc_ref_is_heap_object(obj)) {
+    if (!gc_ref_is_null(obj)) {
       LOG("tracer #%zu: stealing got %p\n", worker->id,
             gc_ref_heap_object(obj));
       return obj;
@@ -281,13 +281,13 @@ trace_worker_steal(struct gc_trace_worker *worker) {
   {
     LOG("tracer #%zu: trying to pop worker's own deque\n", worker->id);
     struct gc_ref obj = shared_worklist_try_pop(&worker->shared);
-    if (gc_ref_is_heap_object(obj))
+    if (!gc_ref_is_null(obj))
       return obj;
   }
 
   LOG("tracer #%zu: trying to steal\n", worker->id);
   struct gc_ref obj = trace_worker_steal_from_any(worker, tracer);
-  if (gc_ref_is_heap_object(obj))
+  if (!gc_ref_is_null(obj))
     return obj;
 
   return gc_ref_null();
@@ -337,7 +337,7 @@ trace_with_data(struct gc_tracer *tracer,
           ref = local_worklist_pop(&worker->local);
         } else {
           ref = trace_worker_steal(worker);
-          if (!gc_ref_is_heap_object(ref))
+          if (gc_ref_is_null(ref))
             break;
         }
         trace_one(ref, heap, worker);

src/pcc.c

@@ -103,8 +103,6 @@ gc_trace_worker_call_with_data(void (*f)(struct gc_tracer *tracer,
 static inline int do_trace(struct gc_heap *heap, struct gc_edge edge,
                            struct gc_ref ref,
                            struct gc_trace_worker_data *data) {
-  if (!gc_ref_is_heap_object(ref))
-    return 0;
   if (GC_LIKELY(copy_space_contains(heap_copy_space(heap), ref)))
     return copy_space_forward(heap_copy_space(heap), edge, ref,
                               &data->allocator);
@@ -117,6 +115,8 @@ static inline int do_trace(struct gc_heap *heap, struct gc_edge edge,
 static inline int trace_edge(struct gc_heap *heap, struct gc_edge edge,
                              struct gc_trace_worker *worker) {
   struct gc_ref ref = gc_edge_ref(edge);
+  if (gc_ref_is_null(ref) || gc_ref_is_immediate(ref))
+    return 0;
   struct gc_trace_worker_data *data = gc_trace_worker_data(worker);
   int is_new = do_trace(heap, edge, ref, data);
 
@@ -130,8 +130,10 @@ static inline int trace_edge(struct gc_heap *heap, struct gc_edge edge,
 
 int gc_visit_ephemeron_key(struct gc_edge edge, struct gc_heap *heap) {
   struct gc_ref ref = gc_edge_ref(edge);
-  if (!gc_ref_is_heap_object(ref))
-    return 0;
+  GC_ASSERT(!gc_ref_is_null(ref));
+  if (gc_ref_is_immediate(ref))
+    return 1;
+  GC_ASSERT(gc_ref_is_heap_object(ref));
   if (GC_LIKELY(copy_space_contains(heap_copy_space(heap), ref)))
     return copy_space_forward_if_traced(heap_copy_space(heap), edge, ref);
   if (large_object_space_contains(heap_large_object_space(heap), ref))

src/semi.c

@@ -233,7 +233,7 @@ static void visit_external_object(struct gc_heap *heap,
 
 static void visit(struct gc_edge edge, struct gc_heap *heap) {
   struct gc_ref ref = gc_edge_ref(edge);
-  if (!gc_ref_is_heap_object(ref))
+  if (gc_ref_is_null(ref) || gc_ref_is_immediate(ref))
     return;
   if (semi_space_contains(heap_semi_space(heap), ref))
     visit_semi_space(heap, heap_semi_space(heap), edge, ref);
@@ -250,6 +250,9 @@ gc_heap_pending_ephemerons(struct gc_heap *heap) {
 
 int gc_visit_ephemeron_key(struct gc_edge edge, struct gc_heap *heap) {
   struct gc_ref ref = gc_edge_ref(edge);
+  GC_ASSERT(!gc_ref_is_null(ref));
+  if (gc_ref_is_immediate(ref))
+    return 1;
   GC_ASSERT(gc_ref_is_heap_object(ref));
   if (semi_space_contains(heap_semi_space(heap), ref)) {
     uintptr_t forwarded = gc_object_forwarded_nonatomic(ref);

src/serial-tracer.h

@@ -65,7 +65,7 @@ tracer_trace_with_data(struct gc_tracer *tracer, struct gc_heap *heap,
   if (!tracer->trace_roots_only) {
     do {
       struct gc_ref obj = simple_worklist_pop(&tracer->worklist);
-      if (!gc_ref_is_heap_object(obj))
+      if (gc_ref_is_null(obj))
         break;
       trace_one(obj, heap, worker);
     } while (1);