Merge remote-tracking branch 'whippet/main' into wip-whippet

libguile/whippet/src/mmc.c

@@ -379,6 +379,16 @@ trace_conservative_edges(uintptr_t low, uintptr_t high, int possibly_interior,
                                   possibly_interior);
 }
 
+static void
+trace_conservative_edges_wrapper(uintptr_t low, uintptr_t high,
+                                 int possibly_interior,
+                                 struct gc_heap *heap, void *data) {
+  if (possibly_interior)
+    trace_conservative_edges(low, high, 1, heap, data);
+  else
+    trace_conservative_edges(low, high, 0, heap, data);
+}
+
 static inline struct gc_trace_plan
 trace_plan(struct gc_heap *heap, struct gc_ref ref) {
   if (GC_LIKELY(nofl_space_contains(heap_nofl_space(heap), ref))) {
@@ -448,13 +458,13 @@ trace_root(struct gc_root root, struct gc_heap *heap,
   case GC_ROOT_KIND_HEAP_PINNED_ROOTS:
     gc_trace_heap_pinned_roots(root.heap->roots,
                                tracer_visit_pinned_root,
-                               trace_conservative_edges,
+                               trace_conservative_edges_wrapper,
                                heap, worker);
     break;
   case GC_ROOT_KIND_MUTATOR_PINNED_ROOTS:
     gc_trace_mutator_pinned_roots(root.mutator->roots,
                                   tracer_visit_pinned_root,
-                                  trace_conservative_edges,
+                                  trace_conservative_edges_wrapper,
                                   heap, worker);
     break;
   default:

libguile/whippet/src/nofl-space.h

@@ -1741,40 +1741,36 @@ nofl_space_mark_conservative_ref(struct nofl_space *space,
   uint8_t *loc = nofl_metadata_byte_for_addr(addr);
   uint8_t byte = atomic_load_explicit(loc, memory_order_relaxed);
 
-  // Already marked object?  Nothing to do.
-  if (nofl_metadata_byte_has_mark(byte, space->current_mark))
-    return gc_ref_null();
-
-  // Addr is the not start of an unmarked object?  Search backwards if
-  // we have interior pointers, otherwise not an object.
-  if (!nofl_metadata_byte_is_young_or_has_mark(byte, space->survivor_mark)) {
+  // Not pointing to the start of an object?  Scan backwards if the ref
+  // is possibly interior, otherwise bail.
+  if ((byte & NOFL_METADATA_BYTE_MARK_MASK) == 0) {
     if (!possibly_interior)
       return gc_ref_null();
 
     uintptr_t block_base = align_down(addr, NOFL_BLOCK_SIZE);
     uint8_t *loc_base = nofl_metadata_byte_for_addr(block_base);
-    do {
-      // Searched past block?  Not an object.
-      if (loc-- == loc_base)
-        return gc_ref_null();
+    uint8_t mask = NOFL_METADATA_BYTE_MARK_MASK | NOFL_METADATA_BYTE_END;
+    loc = scan_backwards_for_byte_with_bits(loc, loc_base, mask);
 
-      byte = atomic_load_explicit(loc, memory_order_relaxed);
-
-      // Ran into the end of some other allocation?  Not an object, then.
-      if (byte & NOFL_METADATA_BYTE_END)
-        return gc_ref_null();
-      // Object already marked?  Nothing to do.
-      if (nofl_metadata_byte_has_mark(byte, space->current_mark))
-        return gc_ref_null();
-
-      // Continue until we find object start.
-    } while (!nofl_metadata_byte_is_young_or_has_mark(byte, space->survivor_mark));
+    if (!loc)
+      return gc_ref_null();
 
+    byte = atomic_load_explicit(loc, memory_order_relaxed);
+    GC_ASSERT(byte & mask);
+    // Ran into the end of some other allocation?  Not an object, then.
+    if (byte & NOFL_METADATA_BYTE_END)
+      return gc_ref_null();
     // Found object start, and object is unmarked; adjust addr.
     addr = block_base + (loc - loc_base) * NOFL_GRANULE_SIZE;
   }
 
-  GC_ASSERT(*loc & NOFL_METADATA_BYTE_MARK_MASK);
+  // Object already marked?  Nothing to do.
+  if (nofl_metadata_byte_has_mark(byte, space->current_mark))
+    return gc_ref_null();
+
+  GC_ASSERT(nofl_metadata_byte_is_young_or_has_mark(byte,
+                                                    space->survivor_mark));
+
   nofl_space_set_nonempty_mark(space, loc, byte, gc_ref(addr));
 
   return gc_ref(addr);

libguile/whippet/src/swar.h

@@ -53,6 +53,31 @@ scan_for_byte_with_bits(uint8_t *ptr, size_t limit, uint8_t mask) {
   return limit;
 }
 
+static inline uint8_t*
+scan_backwards_for_byte_with_bits(uint8_t *ptr, uint8_t *base, uint8_t mask) {
+  GC_ASSERT_EQ (((uintptr_t)base) & 7, 0);
+
+  size_t unaligned = ((uintptr_t) ptr) & 7;
+  if (unaligned) {
+    uint64_t bytes =
+      load_eight_aligned_bytes(ptr - unaligned) << ((8 - unaligned) * 8);
+    uint64_t match = match_bytes_against_bits(bytes, mask);
+    if (match)
+      return ptr - 1 - __builtin_clzll(match) / 8;
+    ptr -= unaligned;
+  }
+
+  for (; ptr > base; ptr -= 8) {
+    GC_ASSERT(ptr >= base + 8);
+    uint64_t bytes = load_eight_aligned_bytes(ptr - 8);
+    uint64_t match = match_bytes_against_bits(bytes, mask);
+    if (match)
+      return ptr - 1 - __builtin_clzll(match) / 8;
+  }
+
+  return NULL;
+}
+
 static inline uint64_t
 match_bytes_against_tag(uint64_t bytes, uint8_t mask, uint8_t tag) {
   // Precondition: tag within mask.