Use block-structured heap for mark-sweep

There are 4 MB aligned slabs, divided into 64 KB pages.  (On 32-bit this
will be 2 MB ad 32 kB).  Then you can get a mark byte per granule by
slab plus granule offset.  The unused slack that would correspond to
mark bytes for the blocks used *by* the mark bytes is used for other
purposes: remembered sets (not yet used), block summaries (not used),
and a slab header (likewise).

mark-sweep.h

@@ -30,6 +30,92 @@ STATIC_ASSERT_EQ(MEDIUM_OBJECT_THRESHOLD,
 STATIC_ASSERT_EQ(LARGE_OBJECT_THRESHOLD,
                  LARGE_OBJECT_GRANULE_THRESHOLD * GRANULE_SIZE);
 
+#define SLAB_SIZE (4 * 1024 * 1024)
+#define BLOCK_SIZE (64 * 1024)
+#define METADATA_BYTES_PER_BLOCK (BLOCK_SIZE / GRANULE_SIZE)
+#define BLOCKS_PER_SLAB (SLAB_SIZE / BLOCK_SIZE)
+#define META_BLOCKS_PER_SLAB (METADATA_BYTES_PER_BLOCK * BLOCKS_PER_SLAB / BLOCK_SIZE)
+#define NONMETA_BLOCKS_PER_SLAB (BLOCKS_PER_SLAB - META_BLOCKS_PER_SLAB)
+#define METADATA_BYTES_PER_SLAB (NONMETA_BLOCKS_PER_SLAB * METADATA_BYTES_PER_BLOCK)
+#define SLACK_METADATA_BYTES_PER_SLAB (META_BLOCKS_PER_SLAB * METADATA_BYTES_PER_BLOCK)
+#define REMSET_BYTES_PER_BLOCK (SLACK_METADATA_BYTES_PER_SLAB / BLOCKS_PER_SLAB)
+#define REMSET_BYTES_PER_SLAB (REMSET_BYTES_PER_BLOCK * NONMETA_BLOCKS_PER_SLAB)
+#define SLACK_REMSET_BYTES_PER_SLAB (REMSET_BYTES_PER_BLOCK * META_BLOCKS_PER_SLAB)
+#define SUMMARY_BYTES_PER_BLOCK (SLACK_REMSET_BYTES_PER_SLAB / BLOCKS_PER_SLAB)
+#define SUMMARY_BYTES_PER_SLAB (SUMMARY_BYTES_PER_BLOCK * NONMETA_BLOCKS_PER_SLAB)
+#define SLACK_SUMMARY_BYTES_PER_SLAB (SUMMARY_BYTES_PER_BLOCK * META_BLOCKS_PER_SLAB)
+#define HEADER_BYTES_PER_SLAB SLACK_SUMMARY_BYTES_PER_SLAB
+
+struct slab;
+
+struct slab_header {
+  union {
+    struct {
+      struct slab *next;
+      struct slab *prev;
+    };
+    uint8_t padding[HEADER_BYTES_PER_SLAB];
+  };
+};
+STATIC_ASSERT_EQ(sizeof(struct slab_header), HEADER_BYTES_PER_SLAB);
+
+struct block_summary {
+  union {
+    struct {
+      uint16_t wasted_granules;
+      uint16_t wasted_spans;
+      uint8_t out_for_thread;
+      uint8_t has_pin;
+      uint8_t paged_out;
+    };
+    uint8_t padding[SUMMARY_BYTES_PER_BLOCK];
+  };
+};
+STATIC_ASSERT_EQ(sizeof(struct block_summary), SUMMARY_BYTES_PER_BLOCK);
+
+struct block {
+  char data[BLOCK_SIZE];
+};
+
+struct slab {
+  struct slab_header header;
+  struct block_summary summaries[NONMETA_BLOCKS_PER_SLAB];
+  uint8_t remsets[REMSET_BYTES_PER_SLAB];
+  uint8_t metadata[METADATA_BYTES_PER_SLAB];
+  struct block blocks[NONMETA_BLOCKS_PER_SLAB];
+};
+STATIC_ASSERT_EQ(sizeof(struct slab), SLAB_SIZE);
+
+static struct slab *object_slab(void *obj) {
+  uintptr_t addr = (uintptr_t) obj;
+  uintptr_t base = addr & ~(SLAB_SIZE - 1);
+  return (struct slab*) base;
+}
+
+static uint8_t *object_metadata_byte(void *obj) {
+  uintptr_t addr = (uintptr_t) obj;
+  uintptr_t base = addr & ~(SLAB_SIZE - 1);
+  uintptr_t granule = (addr & (SLAB_SIZE - 1)) >> GRANULE_SIZE_LOG_2;
+  return (uint8_t*) (base + granule);
+}
+
+#define GRANULES_PER_BLOCK (BLOCK_SIZE / GRANULE_SIZE)
+#define GRANULES_PER_REMSET_BYTE (GRANULES_PER_BLOCK / REMSET_BYTES_PER_BLOCK)
+static uint8_t *object_remset_byte(void *obj) {
+  uintptr_t addr = (uintptr_t) obj;
+  uintptr_t base = addr & ~(SLAB_SIZE - 1);
+  uintptr_t granule = (addr & (SLAB_SIZE - 1)) >> GRANULE_SIZE_LOG_2;
+  uintptr_t remset_byte = granule / GRANULES_PER_REMSET_BYTE;
+  return (uint8_t*) (base + remset_byte);
+}
+
+static struct block_summary* object_block_summary(void *obj) {
+  uintptr_t addr = (uintptr_t) obj;
+  uintptr_t base = addr & ~(SLAB_SIZE - 1);
+  uintptr_t block = (addr & (SLAB_SIZE - 1)) / BLOCK_SIZE;
+  return (struct block_summary*) (base + block * sizeof(struct block_summary));
+}
+
 static uintptr_t align_up(uintptr_t addr, size_t align) {
   return (addr + align - 1) & ~(align-1);
 }
@@ -76,13 +162,12 @@ struct mark_space {
   struct gcobj_freelists small_objects;
   // Unordered list of medium objects.
   struct gcobj_free_medium *medium_objects;
-  uintptr_t base;
+  uintptr_t low_addr;
-  uint8_t *mark_bytes;
+  size_t extent;
-  uintptr_t heap_base;
   size_t heap_size;
   uintptr_t sweep;
-  void *mem;
+  struct slab *slabs;
-  size_t mem_size;
+  size_t nslabs;
 };
 
 struct heap {
@@ -148,10 +233,7 @@ static inline void clear_memory(uintptr_t addr, size_t size) {
 static void collect(struct mutator *mut) NEVER_INLINE;
 
 static inline uint8_t* mark_byte(struct mark_space *space, struct gcobj *obj) {
-  ASSERT(space->heap_base <= (uintptr_t) obj);
+  return object_metadata_byte(obj);
-  ASSERT((uintptr_t) obj < space->heap_base + space->heap_size);
-  uintptr_t granule = (((uintptr_t) obj) - space->heap_base)  / GRANULE_SIZE;
-  return &space->mark_bytes[granule];
 }
 
 static inline int mark_space_trace_object(struct mark_space *space,
@@ -166,7 +248,7 @@ static inline int mark_space_trace_object(struct mark_space *space,
 static inline int mark_space_contains(struct mark_space *space,
                                       struct gcobj *obj) {
   uintptr_t addr = (uintptr_t)obj;
-  return addr - space->heap_base < space->heap_size;
+  return addr - space->low_addr < space->extent;
 }
 
 static inline int large_object_space_trace_object(struct large_object_space *space,
@@ -421,7 +503,7 @@ static inline void maybe_pause_mutator_for_collection(struct mutator *mut) {
 }
 
 static void reset_sweeper(struct mark_space *space) {
-  space->sweep = space->heap_base;
+  space->sweep = (uintptr_t) &space->slabs[0].blocks;
 }
 
 static void collect(struct mutator *mut) {
@@ -560,10 +642,15 @@ static int sweep(struct mark_space *space,
   // end of the heap.
   ssize_t to_reclaim = 32 * 1024 / GRANULE_SIZE;
   uintptr_t sweep = space->sweep;
-  uintptr_t limit = space->heap_base + space->heap_size;
+  uintptr_t limit = align_up(sweep, SLAB_SIZE);
 
-  if (sweep == limit)
+  if (sweep == limit) {
-    return 0;
+    if (sweep == space->low_addr + space->extent)
+      return 0;
+    // Assumes contiguous slabs.  To relax later.
+    sweep += META_BLOCKS_PER_SLAB * BLOCK_SIZE;
+    limit += SLAB_SIZE;
+  }
 
   while (to_reclaim > 0 && sweep < limit) {
     uint8_t* mark = mark_byte(space, (struct gcobj*)sweep);
@@ -813,33 +900,45 @@ static inline void* get_field(void **addr) {
   return *addr;
 }
 
-static int mark_space_init(struct mark_space *space, struct heap *heap) {
+static struct slab* allocate_slabs(size_t nslabs) {
-  size_t size = align_up(heap->size, getpagesize());
+  size_t size = nslabs * SLAB_SIZE;
+  size_t extent = size + SLAB_SIZE;
 
-  void *mem = mmap(NULL, size, PROT_READ|PROT_WRITE,
+  char *mem = mmap(NULL, extent, PROT_READ|PROT_WRITE,
                    MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
   if (mem == MAP_FAILED) {
     perror("mmap failed");
-    return 0;
+    return NULL;
   }
 
-  space->mem = mem;
+  uintptr_t base = (uintptr_t) mem;
-  space->mem_size = size;
+  uintptr_t end = base + extent;
-  // If there is 1 mark byte per granule, and SIZE bytes available for
+  uintptr_t aligned_base = align_up(base, SLAB_SIZE);
-  // HEAP_SIZE + MARK_BYTES, then:
+  uintptr_t aligned_end = aligned_base + size;
-  //
-  //   size = (granule_size + 1) / granule_size * heap_size
-  //   mark_bytes = 1/granule_size * heap_size
-  //   mark_bytes = ceil(heap_size / (granule_size + 1))
-  space->mark_bytes = (uint8_t *) mem;
-  size_t mark_bytes_size = (size + GRANULE_SIZE) / (GRANULE_SIZE + 1);
-  size_t overhead = align_up(mark_bytes_size, GRANULE_SIZE);
 
-  space->heap_base = ((uintptr_t) mem) + overhead;
+  if (aligned_base - base)
-  space->heap_size = size - overhead;
+    munmap((void*)base, aligned_base - base);
-  space->sweep = space->heap_base + space->heap_size;
+  if (end - aligned_end)
-  reclaim(space, NULL, 0, (void*)space->heap_base,
+    munmap((void*)aligned_end, end - aligned_end);
-          size_to_granules(space->heap_size));
+
+  return (struct slab*) aligned_base;
+}
+
+static int mark_space_init(struct mark_space *space, struct heap *heap) {
+  size_t size = align_up(heap->size, SLAB_SIZE);
+  size_t nslabs = size / SLAB_SIZE;
+  struct slab *slabs = allocate_slabs(nslabs);
+  if (!slabs)
+    return 0;
+
+  space->slabs = slabs;
+  space->nslabs = nslabs;
+  space->low_addr = (uintptr_t) slabs;
+  space->extent = size;
+  space->sweep = space->low_addr + space->extent;
+  for (size_t i = 0; i < nslabs; i++)
+    reclaim(space, NULL, 0, &slabs[i].blocks,
+            NONMETA_BLOCKS_PER_SLAB * GRANULES_PER_BLOCK);
   return 1;
 }