Add generational support to pcc

api/pcc-attrs.h

@@ -43,15 +43,23 @@ static inline int gc_allocator_needs_clear(void) {
   return 0;
 }
 
-static inline enum gc_old_generation_check_kind gc_old_generation_check_kind(size_t) {
+static inline enum gc_old_generation_check_kind gc_old_generation_check_kind(size_t size) {
-  return GC_OLD_GENERATION_CHECK_NONE;
+  if (!GC_GENERATIONAL)
+    return GC_OLD_GENERATION_CHECK_NONE;
+  if (size <= gc_allocator_large_threshold())
+    return GC_OLD_GENERATION_CHECK_SMALL_OBJECT_NURSERY;
+  return GC_OLD_GENERATION_CHECK_SLOW;
 }
 static inline uint8_t gc_old_generation_check_alloc_table_bit_pattern(void) {
   GC_CRASH();
 }
 
 static inline enum gc_write_barrier_kind gc_write_barrier_kind(size_t obj_size) {
-  return GC_WRITE_BARRIER_NONE;
+  if (!GC_GENERATIONAL)
+    return GC_WRITE_BARRIER_NONE;
+  if (obj_size <= gc_allocator_large_threshold())
+    return GC_WRITE_BARRIER_FIELD;
+  return GC_WRITE_BARRIER_SLOW;
 }
 static inline size_t gc_write_barrier_card_table_alignment(void) {
   GC_CRASH();
@@ -60,16 +68,20 @@ static inline size_t gc_write_barrier_card_size(void) {
   GC_CRASH();
 }
 static inline size_t gc_write_barrier_field_table_alignment(void) {
-  GC_CRASH();
+  GC_ASSERT(GC_GENERATIONAL);
+  return 64 * 1024 * 1024;
 }
 static inline ptrdiff_t gc_write_barrier_field_table_offset(void) {
-  GC_CRASH();
+  GC_ASSERT(GC_GENERATIONAL);
+  return 128 * 1024;
 }
 static inline size_t gc_write_barrier_field_fields_per_byte(void) {
-  GC_CRASH();
+  GC_ASSERT(GC_GENERATIONAL);
+  return 8;
 }
 static inline uint8_t gc_write_barrier_field_first_bit_pattern(void) {
-  GC_CRASH();
+  GC_ASSERT(GC_GENERATIONAL);
+  return 1;
 }
 
 static inline enum gc_safepoint_mechanism gc_safepoint_mechanism(void) {

src/pcc.c

@@ -12,6 +12,7 @@
 #include "background-thread.h"
 #include "copy-space.h"
 #include "debug.h"
+#include "field-set.h"
 #include "gc-align.h"
 #include "gc-inline.h"
 #include "gc-platform.h"
@@ -27,9 +28,17 @@
 #include "pcc-attrs.h"
 
 struct gc_heap {
-  struct copy_space copy_space;
+#if GC_GENERATIONAL
+  struct copy_space new_space;
+  struct copy_space old_space;
+#else
+  struct copy_space mono_space;
+#endif
   struct large_object_space large_object_space;
   struct gc_extern_space *extern_space;
+#if GC_GENERATIONAL
+  struct gc_field_set remembered_set;
+#endif
   size_t large_object_pages;
   pthread_mutex_t lock;
   pthread_cond_t collector_cond;
@@ -37,6 +46,13 @@ struct gc_heap {
   size_t size;
   size_t total_allocated_bytes_at_last_gc;
   int collecting;
+#if GC_GENERATIONAL
+  int is_minor_collection;
+  size_t per_processor_nursery_size;
+  size_t nursery_size;
+#endif
+  size_t processor_count;
+  size_t max_active_mutator_count;
   int check_pending_ephemerons;
   struct gc_pending_ephemerons *pending_ephemerons;
   struct gc_finalizer_state *finalizer_state;
@@ -62,6 +78,9 @@ struct gc_heap {
 
 struct gc_mutator {
   struct copy_space_allocator allocator;
+#if GC_GENERATIONAL
+  struct gc_field_set_writer logger;
+#endif
   struct gc_heap *heap;
   struct gc_mutator_roots *roots;
   void *event_listener_data;
@@ -70,26 +89,116 @@ struct gc_mutator {
 };
 
 struct gc_trace_worker_data {
+#if GC_GENERATIONAL
+  struct copy_space_allocator new_allocator;
+  struct copy_space_allocator old_allocator;
+  struct gc_field_set_writer logger;
+#else
   struct copy_space_allocator allocator;
+#endif
 };
 
-static inline struct copy_space* heap_copy_space(struct gc_heap *heap) {
+static inline struct copy_space* heap_mono_space(struct gc_heap *heap) {
-  return &heap->copy_space;
+#if GC_GENERATIONAL
+  GC_CRASH();
+#else
+  return &heap->mono_space;
+#endif
 }
+
+static inline struct copy_space* heap_new_space(struct gc_heap *heap) {
+#if GC_GENERATIONAL
+  return &heap->new_space;
+#else
+  GC_CRASH();
+#endif
+}
+
+static inline struct copy_space* heap_old_space(struct gc_heap *heap) {
+#if GC_GENERATIONAL
+  return &heap->old_space;
+#else
+  GC_CRASH();
+#endif
+}
+
+static inline struct gc_field_set* heap_remembered_set(struct gc_heap *heap) {
+#if GC_GENERATIONAL
+  return &heap->remembered_set;
+#else
+  GC_CRASH();
+#endif
+}
+
+static inline struct copy_space_allocator*
+trace_worker_mono_space_allocator(struct gc_trace_worker_data *data) {
+#if GC_GENERATIONAL
+  GC_CRASH();
+#else
+  return &data->allocator;
+#endif
+}
+
+static inline struct copy_space_allocator*
+trace_worker_new_space_allocator(struct gc_trace_worker_data *data) {
+#if GC_GENERATIONAL
+  return &data->new_allocator;
+#else
+  GC_CRASH();
+#endif
+}
+
+static inline struct copy_space_allocator*
+trace_worker_old_space_allocator(struct gc_trace_worker_data *data) {
+#if GC_GENERATIONAL
+  return &data->old_allocator;
+#else
+  GC_CRASH();
+#endif
+}
+
+static inline struct gc_field_set_writer*
+trace_worker_field_logger(struct gc_trace_worker_data *data) {
+#if GC_GENERATIONAL
+  return &data->logger;
+#else
+  GC_CRASH();
+#endif
+}
+
+static inline struct gc_field_set_writer*
+mutator_field_logger(struct gc_mutator *mut) {
+#if GC_GENERATIONAL
+  return &mut->logger;
+#else
+  GC_CRASH();
+#endif
+}
+
+static int is_minor_collection(struct gc_heap *heap) {
+#if GC_GENERATIONAL
+  return heap->is_minor_collection;
+#else
+  GC_CRASH();
+#endif
+}
+
 static inline struct copy_space* heap_allocation_space(struct gc_heap *heap) {
-  // The space into which small objects are allocated.
+  return GC_GENERATIONAL ? heap_new_space(heap) : heap_mono_space(heap);
-  return heap_copy_space(heap);
 }
+
 static inline struct copy_space* heap_resizable_space(struct gc_heap *heap) {
-  // The space that gets resized.
+  return GC_GENERATIONAL ? heap_old_space(heap) : heap_mono_space(heap);
-  return heap_copy_space(heap);
 }
+
 static inline struct large_object_space* heap_large_object_space(struct gc_heap *heap) {
   return &heap->large_object_space;
 }
+
 static inline struct gc_extern_space* heap_extern_space(struct gc_heap *heap) {
   return heap->extern_space;
 }
+
 static inline struct gc_heap* mutator_heap(struct gc_mutator *mutator) {
   return mutator->heap;
 }
@@ -99,9 +208,13 @@ struct gc_heap* gc_mutator_heap(struct gc_mutator *mutator) {
 }
 
 uintptr_t gc_small_object_nursery_low_address(struct gc_heap *heap) {
+  if (GC_GENERATIONAL)
+    return copy_space_low_aligned_address(heap_new_space(heap));
   GC_CRASH();
 }
 uintptr_t gc_small_object_nursery_high_address(struct gc_heap *heap) {
+  if (GC_GENERATIONAL)
+    return copy_space_high_aligned_address(heap_new_space(heap));
   GC_CRASH();
 }
 
@@ -114,18 +227,147 @@ gc_trace_worker_call_with_data(void (*f)(struct gc_tracer *tracer,
                                struct gc_heap *heap,
                                struct gc_trace_worker *worker) {
   struct gc_trace_worker_data data;
-  copy_space_allocator_init(&data.allocator);
+
+  if (GC_GENERATIONAL) {
+    copy_space_allocator_init(trace_worker_new_space_allocator(&data));
+    copy_space_allocator_init(trace_worker_old_space_allocator(&data));
+    gc_field_set_writer_init(trace_worker_field_logger(&data),
+                             heap_remembered_set(heap));
+  } else {
+    copy_space_allocator_init(trace_worker_mono_space_allocator(&data));
+  }
+
   f(tracer, heap, worker, &data);
-  copy_space_allocator_finish(&data.allocator, heap_copy_space(heap));
+
+  if (GC_GENERATIONAL) {
+    copy_space_allocator_finish(trace_worker_new_space_allocator(&data),
+                                heap_new_space(heap));
+    copy_space_allocator_finish(trace_worker_old_space_allocator(&data),
+                                heap_old_space(heap));
+    gc_field_set_writer_release_buffer(trace_worker_field_logger(&data));
+  } else {
+    copy_space_allocator_finish(trace_worker_mono_space_allocator(&data),
+                                heap_mono_space(heap));
+  }
 }
 
+static int new_space_contains_addr(struct gc_heap *heap, uintptr_t addr) {
+  return copy_space_contains_address_aligned(heap_new_space(heap), addr);
+}
+
+static int new_space_contains(struct gc_heap *heap, struct gc_ref ref) {
+  return new_space_contains_addr(heap, gc_ref_value(ref));
+}
+
+static int old_space_contains(struct gc_heap *heap, struct gc_ref ref) {
+  return copy_space_contains(heap_old_space(heap), ref);
+}
+
+static int remember_edge_to_survivor_object(struct gc_heap *heap,
+                                            struct gc_edge edge) {
+  GC_ASSERT(!new_space_contains_addr(heap, gc_edge_address(edge)));
+  GC_ASSERT(new_space_contains(heap, gc_edge_ref(edge)));
+  if (copy_space_contains_edge(heap_old_space(heap), edge))
+    return copy_space_remember_edge(heap_old_space(heap), edge);
+  struct gc_ref large_object =
+    large_object_space_object_containing_edge(heap_large_object_space(heap),
+                                              edge);
+  if (!gc_ref_is_null(large_object))
+    return large_object_space_remember_edge(heap_large_object_space(heap),
+                                            large_object, edge);
+  return 0;
+}
+
+static inline int edge_is_from_survivor(struct gc_heap *heap,
+                                        struct gc_edge edge) {
+  // Currently only the copy-space has survivors.  (A survivor is a live object
+  // which stays in the nursery after collection).  If lospace gains a survivor
+  // stage, we would need to augment this check.
+  GC_ASSERT(is_minor_collection(heap));
+  return copy_space_contains_edge_aligned(heap_new_space(heap), edge);
+}
+
+static inline int do_minor_trace(struct gc_heap *heap, struct gc_edge edge,
+                                 struct gc_ref ref,
+                                 struct gc_trace_worker_data *data) {
+  // Trace EDGE for a minor GC.  We only need to trace edges to young objects.
+  // Young objects are either in the nursery copy space, or in the large object
+  // space.
+
+  if (GC_LIKELY(new_space_contains(heap, ref))) {
+    struct copy_space *new_space = heap_new_space(heap);
+    struct copy_space *old_space = heap_old_space(heap);
+    // We are visiting an edge into newspace.  Either the edge's target will be
+    // promoted to oldspace, or it will stay in newspace as a survivor.
+    //
+    // After the scavenge, we need to preserve the invariant that all old-to-new
+    // edges are part of the remembered set.  So depending on where the edge
+    // comes from and where the object moves to, we may need to add or remove
+    // the edge from the remembered set.  Concretely:
+    //
+    //                   |  survivor dst    |  promoted dst
+    //   ----------------+------------------+-----------------
+    //   survivor src    |    nothing       |    nothing
+    //                   |                  |
+    //   promoted src    |    log edge      |    nothing
+    //                   |                  |
+    //   oldspace src    |    nothing       |   clear log
+    //                   |                  |
+    //   root src        |    nothing       |    nothing
+    //
+    // However, clearing a logged field usually isn't possible, as it's not easy
+    // to go from field address to position in a field set, so instead we lazily
+    // remove old->old edges from the field set during the next minor GC.  (Or,
+    // we will anyway; for now we ignore them.)  So really we only need to log
+    // promoted-to-survivor edges.
+    //
+    // However however, it is hard to distinguish between edges from promoted
+    // objects and edges from old objects, so we mostly just rely on an
+    // idempotent "log if unlogged" operation instead.
+    int promote = copy_space_should_promote(new_space, ref);
+    struct copy_space *dst_space = promote ? old_space : new_space;
+    struct copy_space_allocator *alloc = promote
+      ? trace_worker_old_space_allocator(data)
+      : trace_worker_new_space_allocator(data);
+    // Update the remembered set for promoted-to-survivor edges.
+    if (!promote && !edge_is_from_survivor(heap, edge)
+        && remember_edge_to_survivor_object(heap, edge))
+      gc_field_set_writer_add_edge(trace_worker_field_logger(data), edge);
+    return copy_space_forward(new_space, dst_space, edge, ref, alloc);
+  } else {
+    // Note that although the target of the edge might not be in lospace, this
+    // will do what we want and return 1 if and only if ref is was a young
+    // object in lospace.
+    return large_object_space_copy(heap_large_object_space(heap), ref);
+  }
+}
+
+
 static inline int do_trace(struct gc_heap *heap, struct gc_edge edge,
                            struct gc_ref ref,
                            struct gc_trace_worker_data *data) {
-  if (GC_LIKELY(copy_space_contains(heap_copy_space(heap), ref)))
+  if (GC_GENERATIONAL) {
-    return copy_space_forward(heap_copy_space(heap), heap_copy_space(heap),
+    if (GC_LIKELY(is_minor_collection(heap)))
-                              edge, ref, &data->allocator);
+      return do_minor_trace(heap, edge, ref, data);
-  else if (large_object_space_contains(heap_large_object_space(heap), ref))
+
+    // Major trace: promote all copyspace objects to oldgen.
+    struct copy_space *new_space = heap_new_space(heap);
+    struct copy_space *old_space = heap_old_space(heap);
+    if (new_space_contains(heap, ref))
+      return copy_space_forward(new_space, old_space, edge, ref,
+                                trace_worker_old_space_allocator(data));
+    if (old_space_contains(heap, ref))
+      return copy_space_forward(old_space, old_space, edge, ref,
+                                trace_worker_old_space_allocator(data));
+  } else {
+    if (GC_LIKELY(copy_space_contains(heap_mono_space(heap), ref)))
+      return copy_space_forward(heap_mono_space(heap), heap_mono_space(heap),
+                                edge, ref,
+                                trace_worker_mono_space_allocator(data));
+  }
+
+  // Fall through for objects in large or extern spaces.
+  if (large_object_space_contains(heap_large_object_space(heap), ref))
     return large_object_space_mark_object(heap_large_object_space(heap), ref);
   else
     return gc_extern_space_visit(heap_extern_space(heap), edge, ref);
@@ -153,8 +395,18 @@ int gc_visit_ephemeron_key(struct gc_edge edge, struct gc_heap *heap) {
   if (gc_ref_is_immediate(ref))
     return 1;
   GC_ASSERT(gc_ref_is_heap_object(ref));
-  if (GC_LIKELY(copy_space_contains(heap_copy_space(heap), ref)))
+
-    return copy_space_forward_if_traced(heap_copy_space(heap), edge, ref);
+  if (GC_GENERATIONAL) {
+    if (new_space_contains(heap, ref))
+      return copy_space_forward_if_traced(heap_new_space(heap), edge, ref);
+    if (old_space_contains(heap, ref))
+      return is_minor_collection(heap) ||
+        copy_space_forward_if_traced(heap_old_space(heap), edge, ref);
+  } else {
+    if (copy_space_contains(heap_mono_space(heap), ref))
+      return copy_space_forward_if_traced(heap_mono_space(heap), edge, ref);
+  }
+
   if (large_object_space_contains(heap_large_object_space(heap), ref))
     return large_object_space_is_copied(heap_large_object_space(heap), ref);
   GC_CRASH();
@@ -177,11 +429,21 @@ static inline int all_mutators_stopped(struct gc_heap *heap) {
     heap->paused_mutator_count + heap->inactive_mutator_count;
 }
 
+// with heap lock
+static void maybe_increase_max_active_mutator_count(struct gc_heap *heap) {
+  size_t active_mutators = heap->mutator_count - heap->inactive_mutator_count;
+  if (active_mutators > heap->max_active_mutator_count)
+    heap->max_active_mutator_count = active_mutators;
+}
+
 static void add_mutator(struct gc_heap *heap, struct gc_mutator *mut) {
   mut->heap = heap;
   mut->event_listener_data =
     heap->event_listener.mutator_added(heap->event_listener_data);
   copy_space_allocator_init(&mut->allocator);
+  if (GC_GENERATIONAL)
+    gc_field_set_writer_init(mutator_field_logger(mut),
+                             heap_remembered_set(heap));
   heap_lock(heap);
   // We have no roots.  If there is a GC currently in progress, we have
   // nothing to add.  Just wait until it's done.
@@ -195,11 +457,14 @@ static void add_mutator(struct gc_heap *heap, struct gc_mutator *mut) {
   }
   heap->mutators = mut;
   heap->mutator_count++;
+  maybe_increase_max_active_mutator_count(heap);
   heap_unlock(heap);
 }
 
 static void remove_mutator(struct gc_heap *heap, struct gc_mutator *mut) {
   copy_space_allocator_finish(&mut->allocator, heap_allocation_space(heap));
+  if (GC_GENERATIONAL)
+    gc_field_set_writer_release_buffer(mutator_field_logger(mut));
   MUTATOR_EVENT(mut, mutator_removed);
   mut->heap = NULL;
   heap_lock(heap);
@@ -241,9 +506,20 @@ tracer_visit(struct gc_edge edge, struct gc_heap *heap, void *trace_data) {
 static inline void trace_one(struct gc_ref ref, struct gc_heap *heap,
                              struct gc_trace_worker *worker) {
 #ifdef DEBUG
-  if (copy_space_contains(heap_copy_space(heap), ref))
+  if (GC_GENERATIONAL) {
-    GC_ASSERT(copy_space_object_region(ref) == heap_copy_space(heap)->active_region);
+    if (new_space_contains(heap, ref))
+      GC_ASSERT_EQ(copy_space_object_region(ref),
+                   heap_new_space(heap)->active_region);
+    else if (old_space_contains(heap, ref))
+      GC_ASSERT_EQ(copy_space_object_region(ref),
+                   heap_old_space(heap)->active_region);
+  } else {
+    if (copy_space_contains(heap_mono_space(heap), ref))
+      GC_ASSERT_EQ(copy_space_object_region(ref),
+                   heap_mono_space(heap)->active_region);
+  }
 #endif
+
   gc_trace_object(ref, tracer_visit, heap, worker, NULL);
 }
 
@@ -263,6 +539,10 @@ static inline void trace_root(struct gc_root root, struct gc_heap *heap,
   case GC_ROOT_KIND_EDGE:
     tracer_visit(root.edge, heap, worker);
     break;
+  case GC_ROOT_KIND_EDGE_BUFFER:
+    gc_field_set_visit_edge_buffer(heap_remembered_set(heap), root.edge_buffer,
+                                   tracer_visit, heap, worker);
+    break;
   default:
     GC_CRASH();
   }
@@ -297,12 +577,6 @@ static void wait_for_mutators_to_stop(struct gc_heap *heap) {
     pthread_cond_wait(&heap->collector_cond, &heap->lock);
 }
 
-static int is_minor_collection(struct gc_heap *heap) {
-  if (GC_GENERATIONAL)
-    GC_CRASH();
-  return 0;
-}
-
 static enum gc_collection_kind
 pause_mutator_for_collection(struct gc_heap *heap,
                              struct gc_mutator *mut) GC_NEVER_INLINE;
@@ -313,13 +587,17 @@ pause_mutator_for_collection(struct gc_heap *heap, struct gc_mutator *mut) {
   MUTATOR_EVENT(mut, mutator_stopping);
   MUTATOR_EVENT(mut, mutator_stopped);
   heap->paused_mutator_count++;
-  enum gc_collection_kind collection_kind =
-    is_minor_collection(heap) ? GC_COLLECTION_MINOR : GC_COLLECTION_COMPACTING;
   if (all_mutators_stopped(heap))
     pthread_cond_signal(&heap->collector_cond);
 
+  enum gc_collection_kind collection_kind = GC_COLLECTION_MINOR;
   do {
     pthread_cond_wait(&heap->mutator_cond, &heap->lock);
+    // is_minor_collection is reset before requesting mutators to stop, so this
+    // will pick up either whether the last collection was minor, or whether the
+    // next one will be minor.
+    if (!GC_GENERATIONAL || !is_minor_collection(heap))
+      collection_kind = GC_COLLECTION_COMPACTING;
   } while (mutators_are_stopping(heap));
   heap->paused_mutator_count--;
 
@@ -342,16 +620,83 @@ static void resize_heap(struct gc_heap *heap, size_t new_size) {
   HEAP_EVENT(heap, heap_resized, new_size);
 }
 
+static size_t heap_nursery_size(struct gc_heap *heap) {
+#if GC_GENERATIONAL
+  return heap->nursery_size;
+#else
+  GC_CRASH();
+#endif
+}
+
+static void heap_set_nursery_size(struct gc_heap *heap, size_t size) {
+#if GC_GENERATIONAL
+  GC_ASSERT(size);
+  heap->nursery_size = size;
+#else
+  GC_CRASH();
+#endif
+}
+
+static size_t heap_nursery_size_for_mutator_count(struct gc_heap *heap,
+                                                  size_t count) {
+#if GC_GENERATIONAL
+  return heap->per_processor_nursery_size * count;
+#else
+  GC_CRASH();
+#endif
+}
+
+static void resize_nursery(struct gc_heap *heap, size_t size) {
+  size_t prev_size = heap_nursery_size(heap);
+  if (size < prev_size)
+    copy_space_shrink(heap_new_space(heap), prev_size - size);
+  else
+    copy_space_reacquire_memory(heap_new_space(heap), size - prev_size);
+  heap_set_nursery_size(heap, size);
+}
+
+static void resize_nursery_for_active_mutator_count(struct gc_heap *heap,
+                                                    size_t count) {
+  if (count > heap->processor_count)
+    count = heap->processor_count;
+  size_t prev_size = heap_nursery_size(heap);
+  size_t size = heap_nursery_size_for_mutator_count(heap, count);
+  // If there were more mutator processors this cycle than in the previous,
+  // increase the nursery size.  Otherwise shrink, but with an exponential decay
+  // factor.
+  if (size < prev_size)
+    size = (prev_size + size) / 2;
+  resize_nursery(heap, size);
+}
+
+static void resize_for_active_mutator_count(struct gc_heap *heap) {
+  size_t mutators = heap->max_active_mutator_count;
+  GC_ASSERT(mutators);
+  heap->max_active_mutator_count = 1;
+  maybe_increase_max_active_mutator_count(heap);
+
+  if (GC_GENERATIONAL)
+    resize_nursery_for_active_mutator_count(heap, mutators);
+}
+
 static void visit_root_edge(struct gc_edge edge, struct gc_heap *heap,
                             void *unused) {
   gc_tracer_add_root(&heap->tracer, gc_root_edge(edge));
 }
 
-static void add_roots(struct gc_heap *heap) {
+static void add_roots(struct gc_heap *heap, int is_minor_gc) {
   for (struct gc_mutator *mut = heap->mutators; mut; mut = mut->next)
     gc_tracer_add_root(&heap->tracer, gc_root_mutator(mut));
   gc_tracer_add_root(&heap->tracer, gc_root_heap(heap));
   gc_visit_finalizer_roots(heap->finalizer_state, visit_root_edge, heap, NULL);
+  if (is_minor_gc)
+    gc_field_set_add_roots(heap_remembered_set(heap), &heap->tracer);
+}
+
+static void
+clear_remembered_set(struct gc_heap *heap) {
+  gc_field_set_clear(heap_remembered_set(heap), NULL, NULL);
+  large_object_space_clear_remembered_edges(heap_large_object_space(heap));
 }
 
 static void resolve_ephemerons_lazily(struct gc_heap *heap) {
@@ -394,7 +739,17 @@ static void sweep_ephemerons(struct gc_heap *heap) {
 static int
 heap_can_minor_gc(struct gc_heap *heap) {
   if (!GC_GENERATIONAL) return 0;
-  GC_CRASH();
+  // Invariant: the oldgen always has enough free space to accomodate promoted
+  // objects from the nursery.  This is a precondition for minor GC of course,
+  // but it is also a post-condition: after potentially promoting all nursery
+  // objects, we still need an additional nursery's worth of space in oldgen to
+  // satisfy the invariant.  We ensure the invariant by only doing minor GC if
+  // the copy space can allocate as many bytes as the nursery, which is already
+  // twice the allocatable size because of the copy reserve.
+  struct copy_space *new_space = heap_new_space(heap);
+  struct copy_space *old_space = heap_old_space(heap);
+  size_t nursery_size = heap_nursery_size(heap);
+  return copy_space_can_allocate(old_space, nursery_size);
 }
 
 static enum gc_collection_kind
@@ -405,13 +760,43 @@ determine_collection_kind(struct gc_heap *heap,
   return GC_COLLECTION_COMPACTING;
 }
 
+static void
+copy_spaces_start_gc(struct gc_heap *heap, int is_minor_gc) {
+  if (GC_GENERATIONAL) {
+    copy_space_flip(heap_new_space(heap));
+    if (!is_minor_gc)
+      copy_space_flip(heap_old_space(heap));
+  } else {
+    copy_space_flip(heap_mono_space(heap));
+  }
+}
+
+static void
+copy_spaces_finish_gc(struct gc_heap *heap, int is_minor_gc) {
+  if (GC_GENERATIONAL) {
+    copy_space_finish_gc(heap_new_space(heap));
+    if (!is_minor_gc)
+      copy_space_finish_gc(heap_old_space(heap));
+  } else {
+    copy_space_finish_gc(heap_mono_space(heap));
+  }
+}
+
+static size_t
+copy_spaces_allocated_bytes(struct gc_heap *heap) 
+{
+  return GC_GENERATIONAL
+    ? (heap_new_space(heap)->allocated_bytes_at_last_gc +
+       heap_old_space(heap)->allocated_bytes_at_last_gc)
+    : heap_mono_space(heap)->allocated_bytes_at_last_gc;
+}
+
 static enum gc_collection_kind
 collect(struct gc_mutator *mut,
         enum gc_collection_kind requested_kind) GC_NEVER_INLINE;
 static enum gc_collection_kind
 collect(struct gc_mutator *mut, enum gc_collection_kind requested_kind) {
   struct gc_heap *heap = mutator_heap(mut);
-  struct copy_space *copy_space = heap_copy_space(heap);
   struct large_object_space *lospace = heap_large_object_space(heap);
   struct gc_extern_space *exspace = heap_extern_space(heap);
   uint64_t start_ns = gc_platform_monotonic_nanoseconds();
@@ -422,18 +807,24 @@ collect(struct gc_mutator *mut, enum gc_collection_kind requested_kind) {
   HEAP_EVENT(heap, waiting_for_stop);
   wait_for_mutators_to_stop(heap);
   HEAP_EVENT(heap, mutators_stopped);
-  HEAP_EVENT(heap, prepare_gc, GC_COLLECTION_COMPACTING);
   enum gc_collection_kind gc_kind =
     determine_collection_kind(heap, requested_kind);
+  int is_minor_gc =
+#if GC_GENERATIONAL
+    heap->is_minor_collection =
+#endif
+    GC_GENERATIONAL ? gc_kind == GC_COLLECTION_MINOR : 0;
+  HEAP_EVENT(heap, prepare_gc, gc_kind);
   uint64_t *counter_loc = &heap->total_allocated_bytes_at_last_gc;
-  copy_space_add_to_allocation_counter(copy_space, counter_loc);
+  copy_space_add_to_allocation_counter(heap_allocation_space(heap),
+                                       counter_loc);
   large_object_space_add_to_allocation_counter(lospace, counter_loc);
-  copy_space_flip(copy_space);
+  copy_spaces_start_gc(heap, is_minor_gc);
-  large_object_space_start_gc(lospace, 0);
+  large_object_space_start_gc(lospace, is_minor_gc);
-  gc_extern_space_start_gc(exspace, 0);
+  gc_extern_space_start_gc(exspace, is_minor_gc);
   resolve_ephemerons_lazily(heap);
   gc_tracer_prepare(&heap->tracer);
-  add_roots(heap);
+  add_roots(heap, is_minor_gc);
   HEAP_EVENT(heap, roots_traced);
   gc_tracer_trace(&heap->tracer);
   HEAP_EVENT(heap, heap_traced);
@@ -444,22 +835,27 @@ collect(struct gc_mutator *mut, enum gc_collection_kind requested_kind) {
   HEAP_EVENT(heap, finalizers_traced);
   sweep_ephemerons(heap);
   gc_tracer_release(&heap->tracer);
-  copy_space_finish_gc(copy_space);
+  copy_spaces_finish_gc(heap, is_minor_gc);
-  large_object_space_finish_gc(lospace, 0);
+  large_object_space_finish_gc(lospace, is_minor_gc);
-  gc_extern_space_finish_gc(exspace, 0);
+  gc_extern_space_finish_gc(exspace, is_minor_gc);
+  if (GC_GENERATIONAL && !is_minor_gc)
+    clear_remembered_set(heap);
   heap->count++;
+  resize_for_active_mutator_count(heap);
   heap_reset_large_object_pages(heap, lospace->live_pages_at_last_collection);
-  size_t live_size = (copy_space->allocated_bytes_at_last_gc +
+  size_t live_size = (copy_spaces_allocated_bytes(heap) +
                       large_object_space_size_at_last_collection(lospace));
   uint64_t pause_ns = gc_platform_monotonic_nanoseconds() - start_ns;
   HEAP_EVENT(heap, live_data_size, live_size);
   gc_heap_sizer_on_gc(heap->sizer, heap->size, live_size, pause_ns,
                       resize_heap);
-  if (!copy_space_page_out_blocks_until_memory_released(copy_space)
+  {
-      && heap->sizer.policy == GC_HEAP_SIZE_FIXED) {
+    struct copy_space *space = heap_resizable_space(heap);
-    fprintf(stderr, "ran out of space, heap size %zu (%zu slabs)\n",
+    if (!copy_space_page_out_blocks_until_memory_released(space)
-            heap->size, copy_space->nslabs);
+        && heap->sizer.policy == GC_HEAP_SIZE_FIXED) {
-    GC_CRASH();
+      fprintf(stderr, "ran out of space, heap size %zu\n", heap->size);
+      GC_CRASH();
+    }
   }
   HEAP_EVENT(heap, restarting_mutators);
   allow_mutators_to_continue(heap);
@@ -470,6 +866,8 @@ static void trigger_collection(struct gc_mutator *mut,
                                enum gc_collection_kind requested_kind) {
   struct gc_heap *heap = mutator_heap(mut);
   copy_space_allocator_finish(&mut->allocator, heap_allocation_space(heap));
+  if (GC_GENERATIONAL)
+    gc_field_set_writer_release_buffer(mutator_field_logger(mut));
   heap_lock(heap);
   int prev_kind = -1;
   while (mutators_are_stopping(heap))
@@ -517,11 +915,12 @@ void* gc_allocate_slow(struct gc_mutator *mut, size_t size) {
   if (size > gc_allocator_large_threshold())
     return allocate_large(mut, size);
 
-  struct gc_ref ret = copy_space_allocate(&mut->allocator,
+  struct gc_ref ret =
-                                          heap_copy_space(mutator_heap(mut)),
+    copy_space_allocate(&mut->allocator,
-                                          size,
+                        heap_allocation_space(mutator_heap(mut)),
-                                          get_more_empty_blocks_for_mutator,
+                        size,
-                                          mut);
+                        get_more_empty_blocks_for_mutator,
+                        mut);
   gc_clear_fresh_allocation(ret, size);
   return gc_ref_heap_object(ret);
 }
@@ -536,12 +935,36 @@ void gc_pin_object(struct gc_mutator *mut, struct gc_ref ref) {
 
 int gc_object_is_old_generation_slow(struct gc_mutator *mut,
                                      struct gc_ref obj) {
+  if (!GC_GENERATIONAL)
+    return 0;
+
+  struct gc_heap *heap = mutator_heap(mut);
+
+  if (copy_space_contains(heap_new_space(heap), obj))
+    return 0;
+  if (copy_space_contains(heap_old_space(heap), obj))
+    return 1;
+
+  struct large_object_space *lospace = heap_large_object_space(heap);
+  if (large_object_space_contains(lospace, obj))
+    return large_object_space_is_survivor(lospace, obj);
+
   return 0;
 }
 
 void gc_write_barrier_slow(struct gc_mutator *mut, struct gc_ref obj,
                            size_t obj_size, struct gc_edge edge,
                            struct gc_ref new_val) {
+  GC_ASSERT(!gc_ref_is_null(new_val));
+  if (!GC_GENERATIONAL) return;
+  if (gc_object_is_old_generation_slow(mut, new_val))
+    return;
+  struct gc_heap *heap = mutator_heap(mut);
+  if ((obj_size <= gc_allocator_large_threshold())
+      ? copy_space_remember_edge(heap_old_space(heap), edge)
+      : large_object_space_remember_edge(heap_large_object_space(heap),
+                                         obj, edge))
+    gc_field_set_writer_add_edge(mutator_field_logger(mut), edge);
 }
 
 int* gc_safepoint_flag_loc(struct gc_mutator *mut) {
@@ -551,6 +974,8 @@ int* gc_safepoint_flag_loc(struct gc_mutator *mut) {
 void gc_safepoint_slow(struct gc_mutator *mut) {
   struct gc_heap *heap = mutator_heap(mut);
   copy_space_allocator_finish(&mut->allocator, heap_allocation_space(heap));
+  if (GC_GENERATIONAL)
+    gc_field_set_writer_release_buffer(mutator_field_logger(mut));
   heap_lock(heap);
   while (mutators_are_stopping(mutator_heap(mut)))
     pause_mutator_for_collection(heap, mut);
@@ -636,7 +1061,7 @@ int gc_options_parse_and_set(struct gc_options *options, int option,
 static uint64_t allocation_counter_from_thread(struct gc_heap *heap) {
   uint64_t ret = heap->total_allocated_bytes_at_last_gc;
   if (pthread_mutex_trylock(&heap->lock)) return ret;
-  copy_space_add_to_allocation_counter(heap_copy_space(heap), &ret);
+  copy_space_add_to_allocation_counter(heap_allocation_space(heap), &ret);
   large_object_space_add_to_allocation_counter(heap_large_object_space(heap),
                                                &ret);
   pthread_mutex_unlock(&heap->lock);
@@ -652,10 +1077,20 @@ static void set_heap_size_from_thread(struct gc_heap *heap, size_t size) {
 static int heap_init(struct gc_heap *heap, const struct gc_options *options) {
   // *heap is already initialized to 0.
 
+  if (GC_GENERATIONAL)
+    gc_field_set_init(heap_remembered_set(heap));
   pthread_mutex_init(&heap->lock, NULL);
   pthread_cond_init(&heap->mutator_cond, NULL);
   pthread_cond_init(&heap->collector_cond, NULL);
   heap->size = options->common.heap_size;
+  heap->processor_count = gc_platform_processor_count();
+  // max_active_mutator_count never falls below 1 after this point.
+  heap->max_active_mutator_count = 1;
+
+#if GC_GENERATIONAL
+  // We should add an option to set this, but for now, 2 MB per processor.
+  heap->per_processor_nursery_size = 2 * 1024 * 1024;
+#endif
 
   if (!gc_tracer_init(&heap->tracer, heap, options->common.parallelism))
     GC_CRASH();
@@ -690,6 +1125,12 @@ int gc_init(const struct gc_options *options, struct gc_stack_addr *stack_base,
                offsetof(struct copy_space_allocator, hp));
   GC_ASSERT_EQ(gc_allocator_allocation_limit_offset(),
                offsetof(struct copy_space_allocator, limit));
+  if (GC_GENERATIONAL) {
+    GC_ASSERT_EQ(gc_write_barrier_field_table_alignment(),
+                 COPY_SPACE_SLAB_SIZE);
+    GC_ASSERT_EQ(gc_write_barrier_field_table_offset(),
+                 offsetof(struct copy_space_slab, blocks));
+  }
 
   *heap = calloc(1, sizeof(struct gc_heap));
   if (!*heap) GC_CRASH();
@@ -701,16 +1142,38 @@ int gc_init(const struct gc_options *options, struct gc_stack_addr *stack_base,
   (*heap)->event_listener_data = event_listener_data;
   HEAP_EVENT(*heap, init, (*heap)->size);
 
-  struct copy_space *copy_space = heap_copy_space(*heap);
   {
     uint32_t flags = 0;
     if (options->common.parallelism > 1)
       flags |= COPY_SPACE_ATOMIC_FORWARDING;
-    if (!copy_space_init(copy_space, (*heap)->size, flags,
+    if (GC_GENERATIONAL) {
-                         (*heap)->background_thread)) {
+      size_t nursery_size =
-      free(*heap);
+        heap_nursery_size_for_mutator_count(*heap, (*heap)->processor_count);
-      *heap = NULL;
+      heap_set_nursery_size(*heap, nursery_size);
-      return 0;
+      if (!copy_space_init(heap_new_space(*heap), nursery_size,
+                           flags | COPY_SPACE_ALIGNED,
+                           (*heap)->background_thread)) {
+        free(*heap);
+        *heap = NULL;
+        return 0;
+      }
+      // Initially dimension the nursery for one mutator.
+      resize_nursery(*heap, heap_nursery_size_for_mutator_count(*heap, 1));
+
+      if (!copy_space_init(heap_old_space(*heap), (*heap)->size,
+                           flags | COPY_SPACE_HAS_FIELD_LOGGING_BITS,
+                           (*heap)->background_thread)) {
+        free(*heap);
+        *heap = NULL;
+        return 0;
+      }
+    } else {
+      if (!copy_space_init(heap_mono_space(*heap), (*heap)->size, flags,
+                           (*heap)->background_thread)) {
+        free(*heap);
+        *heap = NULL;
+        return 0;
+      }
     }
   }
   
@@ -744,6 +1207,8 @@ void gc_finish_for_thread(struct gc_mutator *mut) {
 static void deactivate_mutator(struct gc_heap *heap, struct gc_mutator *mut) {
   GC_ASSERT(mut->next == NULL);
   copy_space_allocator_finish(&mut->allocator, heap_allocation_space(heap));
+  if (GC_GENERATIONAL)
+    gc_field_set_writer_release_buffer(mutator_field_logger(mut));
   heap_lock(heap);
   heap->inactive_mutator_count++;
   if (all_mutators_stopped(heap))
@@ -756,6 +1221,7 @@ static void reactivate_mutator(struct gc_heap *heap, struct gc_mutator *mut) {
   while (mutators_are_stopping(heap))
     pthread_cond_wait(&heap->mutator_cond, &heap->lock);
   heap->inactive_mutator_count--;
+  maybe_increase_max_active_mutator_count(heap);
   heap_unlock(heap);
 }