Factor nofl-space out of whippet

api/gc-api.h

@@ -59,11 +59,9 @@ static inline void gc_clear_fresh_allocation(struct gc_ref obj,
   memset(gc_ref_heap_object(obj), 0, size);
 }
 
-static inline void gc_update_alloc_table(struct gc_mutator *mut,
+static inline void gc_update_alloc_table(struct gc_ref obj,
-                                         struct gc_ref obj,
                                          size_t size) GC_ALWAYS_INLINE;
-static inline void gc_update_alloc_table(struct gc_mutator *mut,
+static inline void gc_update_alloc_table(struct gc_ref obj,
-                                         struct gc_ref obj,
                                          size_t size) {
   size_t alignment = gc_allocator_alloc_table_alignment();
   if (!alignment) return;
@@ -117,7 +115,7 @@ static inline void* gc_allocate_small_fast_bump_pointer(struct gc_mutator *mut,
   *hp_loc = new_hp;
 
   gc_clear_fresh_allocation(gc_ref(hp), size);
-  gc_update_alloc_table(mut, gc_ref(hp), size);
+  gc_update_alloc_table(gc_ref(hp), size);
 
   return (void*)hp;
 }
@@ -138,7 +136,7 @@ static inline void* gc_allocate_small_fast_freelist(struct gc_mutator *mut, size
   *freelist_loc = *(void**)head;
 
   gc_clear_fresh_allocation(gc_ref_from_heap_object(head), size);
-  gc_update_alloc_table(mut, gc_ref_from_heap_object(head), size);
+  gc_update_alloc_table(gc_ref_from_heap_object(head), size);
 
   return head;
 }

src/swar.h

@@ -0,0 +1,51 @@
+#ifndef SWAR_H
+#define SWAR_H
+
+#include <string.h>
+
+static inline size_t
+count_zero_bytes(uint64_t bytes) {
+  return bytes ? (__builtin_ctzll(bytes) / 8) : sizeof(bytes);
+}
+
+static uint64_t
+broadcast_byte(uint8_t byte) {
+  uint64_t result = byte;
+  return result * 0x0101010101010101ULL;
+}
+
+static inline uint64_t
+load_eight_aligned_bytes(uint8_t *ptr) {
+  GC_ASSERT(((uintptr_t)ptr & 7) == 0);
+  uint8_t * __attribute__((aligned(8))) aligned_ptr = ptr;
+  uint64_t word;
+  memcpy(&word, aligned_ptr, 8);
+#ifdef WORDS_BIGENDIAN
+  word = __builtin_bswap64(word);
+#endif
+  return word;
+}
+
+static size_t
+scan_for_byte(uint8_t *ptr, size_t limit, uint64_t mask) {
+  size_t n = 0;
+  size_t unaligned = ((uintptr_t) ptr) & 7;
+  if (unaligned) {
+    uint64_t bytes = load_eight_aligned_bytes(ptr - unaligned) >> (unaligned * 8);
+    bytes &= mask;
+    if (bytes)
+      return count_zero_bytes(bytes);
+    n += 8 - unaligned;
+  }
+
+  for(; n < limit; n += 8) {
+    uint64_t bytes = load_eight_aligned_bytes(ptr + n);
+    bytes &= mask;
+    if (bytes)
+      return n + count_zero_bytes(bytes);
+  }
+
+  return limit;
+}
+
+#endif // SWAR_H