Add quads benchmark

Also expand GC interface with "allocate_pointerless".  Limit lazy
sweeping to the allocation size that is causing the sweep, without
adding to fragmentation.

Makefile

@@ -1,4 +1,4 @@
-TESTS=gcbench # MT_GCBench MT_GCBench2
+TESTS=gcbench quads # MT_GCBench MT_GCBench2
 COLLECTORS=bdw semi mark-sweep parallel-mark-sweep
 
 CC=gcc

bdw.h

@@ -1,3 +1,5 @@
+#include <stdint.h>
+
 #include "conservative-roots.h"
 
 // When pthreads are used, let `libgc' know about it and redirect
@@ -20,15 +22,17 @@ struct context {};
 
 static inline void* allocate(struct context *cx, enum alloc_kind kind,
                              size_t size) {
-  switch (kind) {
-  case ALLOC_KIND_NODE:
-    // cleared to 0 by the collector.
-    return GC_malloc(size);
-  case ALLOC_KIND_DOUBLE_ARRAY:
-    // warning: not cleared!
-    return GC_malloc_atomic(size);
-  }
-  abort();
+  return GC_malloc(size);
+}
+
+static inline void*
+allocate_pointerless(struct context *cx, enum alloc_kind kind,
+                     size_t size) {
+  return GC_malloc_atomic(size);
+}
+
+static inline void collect(struct context *cx) {
+  GC_gcollect();
 }
 
 static inline void init_field(void **addr, void *val) {

gcbench.c

@@ -94,9 +94,10 @@ static Node* allocate_node(struct context *cx) {
 
 static struct DoubleArray* allocate_double_array(struct context *cx,
                                                  size_t size) {
-  // note, we might allow the collector to leave this data uninitialized.
-  DoubleArray *ret = allocate(cx, ALLOC_KIND_DOUBLE_ARRAY,
-                              sizeof(DoubleArray) + sizeof (double) * size);
+  // May be uninitialized.
+  DoubleArray *ret =
+    allocate_pointerless(cx, ALLOC_KIND_DOUBLE_ARRAY,
+                         sizeof(DoubleArray) + sizeof (double) * size);
   ret->length = size;
   return ret;
 }

mark-sweep.h

@@ -14,6 +14,8 @@
 #include "serial-marker.h"
 #endif
 
+#define LAZY_SWEEP 1
+
 #define GRANULE_SIZE 8
 #define GRANULE_SIZE_LOG_2 3
 #define LARGE_OBJECT_THRESHOLD 256
@@ -277,17 +279,22 @@ static size_t next_mark(const uint8_t *mark, size_t limit) {
 
 // Sweep some heap to reclaim free space.  Return 1 if there is more
 // heap to sweep, or 0 if we reached the end.
-static int sweep(struct context *cx) {
+static int sweep(struct context *cx, size_t for_granules) {
   // Sweep until we have reclaimed 128 granules (1024 kB), or we reach
   // the end of the heap.
   ssize_t to_reclaim = 128;
   uintptr_t sweep = cx->sweep;
   uintptr_t limit = cx->base + cx->size;
 
+  if (sweep == limit)
+    return 0;
+
   while (to_reclaim > 0 && sweep < limit) {
     uint8_t* mark = mark_byte(cx, (struct gcobj*)sweep);
-    size_t free_granules = next_mark(mark,
-                                     (limit - sweep) >> GRANULE_SIZE_LOG_2);
+    size_t limit_granules = (limit - sweep) >> GRANULE_SIZE_LOG_2;
+    if (limit_granules > for_granules)
+      limit_granules = for_granules;
+    size_t free_granules = next_mark(mark, limit_granules);
     if (free_granules) {
       size_t free_bytes = free_granules * GRANULE_SIZE;
       clear_memory(sweep + GRANULE_SIZE, free_bytes - GRANULE_SIZE);
@@ -296,7 +303,7 @@ static int sweep(struct context *cx) {
       to_reclaim -= free_granules;
 
       mark += free_granules;
-      if (sweep == limit)
+      if (free_granules == limit_granules)
         break;
     }
     // Object survived collection; clear mark and continue sweeping.
@@ -306,7 +313,7 @@ static int sweep(struct context *cx) {
   }
 
   cx->sweep = sweep;
-  return to_reclaim < 128;
+  return 1;
 }
 
 static void* allocate_large(struct context *cx, enum alloc_kind kind,
@@ -328,7 +335,7 @@ static void* allocate_large(struct context *cx, enum alloc_kind kind,
         }
       }
       already_scanned = cx->large_objects;
-    } while (sweep (cx));
+    } while (sweep(cx, granules));
 
     // No large object, and we swept across the whole heap.  Collect.
     if (swept_from_beginning) {
@@ -370,7 +377,7 @@ static void fill_small(struct context *cx, enum small_object_size kind) {
       return;
     }
 
-    if (!sweep(cx)) {
+    if (!sweep(cx, LARGE_OBJECT_GRANULE_THRESHOLD)) {
       if (swept_from_beginning) {
         fprintf(stderr, "ran out of space, heap size %zu\n", cx->size);
         abort();
@@ -402,6 +409,11 @@ static inline void* allocate(struct context *cx, enum alloc_kind kind,
     return allocate_small(cx, kind, granules_to_small_object_size(granules));
   return allocate_large(cx, kind, granules);
 }
+static inline void* allocate_pointerless(struct context *cx,
+                                         enum alloc_kind kind,
+                                         size_t size) {
+  return allocate(cx, kind, size);
+}
 
 static inline void init_field(void **addr, void *val) {
   *addr = val;

quads-types.h

@@ -0,0 +1,9 @@
+#ifndef QUADS_TYPES_H
+#define QUADS_TYPES_H
+
+#define FOR_EACH_HEAP_OBJECT_KIND(M) \
+  M(quad, Quad, QUAD)
+
+#include "heap-objects.h"
+
+#endif // QUADS_TYPES_H

quads.c

@@ -0,0 +1,177 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/time.h>
+
+#include "assert.h"
+#include "quads-types.h"
+#include "gc.h"
+
+typedef struct Quad {
+  GC_HEADER;
+  struct Quad *kids[4];
+} Quad;
+static inline size_t quad_size(Quad *obj) {
+  return sizeof(Quad);
+}
+static inline void
+visit_quad_fields(Quad *quad,
+                  void (*visit)(void **loc, void *visit_data),
+                  void *visit_data) {
+  for (size_t i = 0; i < 4; i++)
+    visit((void**)&quad->kids[i], visit_data);
+}
+typedef HANDLE_TO(Quad) QuadHandle;
+
+static Quad* allocate_quad(struct context *cx) {
+  // memset to 0 by the collector.
+  return allocate(cx, ALLOC_KIND_QUAD, sizeof (Quad));
+}
+
+/* Get the current time in microseconds */
+static unsigned long current_time(void)
+{
+  struct timeval t;
+  if (gettimeofday(&t, NULL) == -1)
+    return 0;
+  return t.tv_sec * 1000 * 1000 + t.tv_usec;
+}
+
+// Build tree bottom-up
+static Quad* make_tree(struct context *cx, int depth) {
+  if (depth<=0) {
+    return allocate_quad(cx);
+  } else {
+    QuadHandle kids[4] = { { NULL }, };
+    for (size_t i = 0; i < 4; i++) {
+      HANDLE_SET(kids[i], make_tree(cx, depth-1));
+      PUSH_HANDLE(cx, kids[i]);
+    }
+
+    Quad *result = allocate_quad(cx);
+    for (size_t i = 0; i < 4; i++)
+      init_field((void**)&result->kids[i], HANDLE_REF(kids[i]));
+
+    for (size_t i = 0; i < 4; i++)
+      POP_HANDLE(cx, kids[3 - i]);
+
+    return result;
+  }
+}
+
+static void validate_tree(Quad *tree, int depth) {
+  for (size_t i = 0; i < 4; i++) {
+    if (depth == 0) {
+      if (tree->kids[i])
+        abort();
+    } else {
+      if (!tree->kids[i])
+        abort();
+      validate_tree(tree->kids[i], depth - 1);
+    }
+  }
+}
+
+static void print_elapsed(const char *what, unsigned long start) {
+  unsigned long end = current_time();
+  unsigned long msec = (end - start) / 1000;
+  unsigned long usec = (end - start) % 1000;
+  printf("Completed %s in %lu.%.3lu msec\n", what, msec, usec);
+}
+
+static size_t parse_size(char *arg, const char *what) {
+  long val = atol(arg);
+  if (val <= 0) {
+    fprintf(stderr, "Failed to parse %s '%s'\n", what, arg);
+    exit(1);
+  }
+  return val;
+}
+
+static size_t tree_size(size_t depth) {
+  size_t nquads = 0;
+  size_t leaf_count = 1;
+  for (size_t i = 0; i <= depth; i++) {
+    if (nquads > ((size_t)-1) - leaf_count) {
+      fprintf(stderr,
+              "error: address space too small for quad tree of depth %zu\n",
+              depth);
+      exit(1);
+    }
+    nquads += leaf_count;
+    leaf_count *= 4;
+  }
+  return nquads;
+}
+
+
+int main(int argc, char *argv[]) {
+  if (argc != 3) {
+    fprintf(stderr, "usage: %s DEPTH MULTIPLIER\n", argv[0]);
+    return 1;
+  }
+
+  size_t depth = parse_size(argv[1], "depth");
+  double multiplier = atof(argv[2]);
+
+  if (!(1.0 < multiplier && multiplier < 100)) {
+    fprintf(stderr, "Failed to parse heap multiplier '%s'\n", argv[2]);
+    return 1;
+  }
+
+  // Compute byte size not counting any header word, so as to compute the same
+  // heap size whether a header word is there or not.
+  size_t nquads = tree_size(depth);
+  size_t tree_bytes = nquads * 4 * sizeof(Quad*);
+  size_t heap_size = tree_bytes * multiplier;
+
+  printf("Allocating heap of %.3fGB (%.2f multiplier of live data).\n",
+         heap_size / 1e9, multiplier);
+  struct context _cx;
+  struct context *cx = &_cx;
+  initialize_gc(cx, heap_size);
+
+  QuadHandle quad = { NULL };
+
+  PUSH_HANDLE(cx, quad);
+
+  print_start_gc_stats(cx);
+
+  printf("Making quad tree of depth %zu (%zu nodes).  Total size %.3fGB.\n",
+         depth, nquads, (nquads * sizeof(Quad)) / 1e9);
+  unsigned long start = current_time();
+  HANDLE_SET(quad, make_tree(cx, depth));
+  print_elapsed("construction", start);
+
+  validate_tree(HANDLE_REF(quad), depth);
+
+  for (size_t i = 0; i < 10; i++) {
+    printf("Allocating 1 GB of garbage.\n");
+    size_t garbage_depth = 3;
+    start = current_time();
+    for (size_t i = 1e9/(tree_size(garbage_depth)*4*sizeof(Quad*)); i; i--)
+      make_tree(cx, garbage_depth);
+    print_elapsed("allocating garbage", start);
+
+#if 0
+#ifdef LAZY_SWEEP
+    start = current_time();
+    do {} while (sweep(cx));
+    print_elapsed("finishing lazy sweep", start);
+#endif
+
+    start = current_time();
+    collect(cx);
+    print_elapsed("collection", start);
+#endif
+
+    start = current_time();
+    validate_tree(HANDLE_REF(quad), depth);
+    print_elapsed("validate tree", start);
+  }
+
+  print_end_gc_stats(cx);
+
+  POP_HANDLE(cx, quad);
+  return 0;
+}
+

semi.h

@@ -27,7 +27,8 @@ static inline void clear_memory(uintptr_t addr, size_t size) {
   memset((char*)addr, 0, size);
 }
 
-static void collect(struct context *cx, size_t bytes) NEVER_INLINE;
+static void collect(struct context *cx) NEVER_INLINE;
+static void collect_for_alloc(struct context *cx, size_t bytes) NEVER_INLINE;
 
 static void visit(void **loc, void *visit_data);
 
@@ -96,7 +97,7 @@ static void visit(void **loc, void *visit_data) {
   if (obj != NULL)
     *loc = forward(cx, obj);
 }
-static void collect(struct context *cx, size_t bytes) {
+static void collect(struct context *cx) {
   // fprintf(stderr, "start collect #%ld:\n", cx->count);
   flip(cx);
   uintptr_t grey = cx->hp;
@@ -107,6 +108,9 @@ static void collect(struct context *cx, size_t bytes) {
     grey = scan(cx, grey);
   // fprintf(stderr, "%zd bytes copied\n", (cx->size>>1)-(cx->limit-cx->hp));
 
+}
+static void collect_for_alloc(struct context *cx, size_t bytes) {
+  collect(cx);
   if (cx->limit - cx->hp < bytes) {
     fprintf(stderr, "ran out of space, heap size %zu\n", cx->size);
     abort();
@@ -119,7 +123,7 @@ static inline void* allocate(struct context *cx, enum alloc_kind kind,
     uintptr_t addr = cx->hp;
     uintptr_t new_hp = align_up (addr + size, ALIGNMENT);
     if (cx->limit < new_hp) {
-      collect(cx, size);
+      collect_for_alloc(cx, size);
       continue;
     }
     cx->hp = new_hp;
@@ -132,6 +136,10 @@ static inline void* allocate(struct context *cx, enum alloc_kind kind,
     return ret;
   }
 }
+static inline void* allocate_pointerless(struct context *cx,
+                                         enum alloc_kind kind, size_t size) {
+  return allocate(cx, kind, size);
+}
 
 static inline void init_field(void **addr, void *val) {
   *addr = val;