Merge remote-tracking branch 'whippet/main' into wip-whippet

libguile/whippet/api/gc-basic-stats.h

@@ -26,6 +26,7 @@ struct gc_basic_stats {
   size_t max_heap_size;
   size_t live_data_size;
   size_t max_live_data_size;
+  uint64_t allocation_counter_at_last_gc;
   struct gc_latency pause_times;
 };
 
@@ -68,12 +69,14 @@ static inline void gc_basic_stats_waiting_for_stop(void *data) {}
 static inline void gc_basic_stats_mutators_stopped(void *data) {}
 
 static inline void gc_basic_stats_prepare_gc(void *data,
-                                             enum gc_collection_kind kind) {
+                                             enum gc_collection_kind kind,
+                                             uint64_t allocation_counter) {
   struct gc_basic_stats *stats = data;
   if (kind == GC_COLLECTION_MINOR)
     stats->minor_collection_count++;
   else
     stats->major_collection_count++;
+  stats->allocation_counter_at_last_gc = allocation_counter;
 }
 
 static inline void gc_basic_stats_roots_traced(void *data) {}

libguile/whippet/api/gc-event-listener-chain.h

@@ -36,10 +36,11 @@ static inline void gc_event_listener_chain_mutators_stopped(void *data) {
   chain->tail.mutators_stopped(chain->tail_data);
 }
 static inline void
-gc_event_listener_chain_prepare_gc(void *data, enum gc_collection_kind kind) {
+gc_event_listener_chain_prepare_gc(void *data, enum gc_collection_kind kind,
+                                   uint64_t counter) {
   struct gc_event_listener_chain *chain = data;
-  chain->head.prepare_gc(chain->head_data, kind);
+  chain->head.prepare_gc(chain->head_data, kind, counter);
-  chain->tail.prepare_gc(chain->tail_data, kind);
+  chain->tail.prepare_gc(chain->tail_data, kind, counter);
 }
 static inline void gc_event_listener_chain_roots_traced(void *data) {
   struct gc_event_listener_chain *chain = data;

libguile/whippet/api/gc-event-listener.h

@@ -8,7 +8,8 @@ struct gc_event_listener {
   void (*requesting_stop)(void *data);
   void (*waiting_for_stop)(void *data);
   void (*mutators_stopped)(void *data);
-  void (*prepare_gc)(void *data, enum gc_collection_kind kind);
+  void (*prepare_gc)(void *data, enum gc_collection_kind kind,
+                     uint64_t allocation_counter);
   void (*roots_traced)(void *data);
   void (*heap_traced)(void *data);
   void (*ephemerons_traced)(void *data);

libguile/whippet/api/gc-lttng.h

@@ -45,9 +45,10 @@ LTTNG_UST_TRACEPOINT_EVENT_INSTANCE(
   whippet, tracepoint, whippet, mutators_stopped, LTTNG_UST_TP_ARGS())
 LTTNG_UST_TRACEPOINT_EVENT(
   whippet, prepare_gc,
-  LTTNG_UST_TP_ARGS(int, gc_kind),
+  LTTNG_UST_TP_ARGS(int, gc_kind, uint64_t, allocation_counter),
   LTTNG_UST_TP_FIELDS(
-    lttng_ust_field_enum(whippet, gc_kind, int, gc_kind, gc_kind)))
+    lttng_ust_field_enum(whippet, gc_kind, int, gc_kind, gc_kind)
+    lttng_ust_field_integer(uint64_t, allocation_counter, allocation_counter)))
 LTTNG_UST_TRACEPOINT_EVENT_INSTANCE(
   whippet, tracepoint, whippet, roots_traced, LTTNG_UST_TP_ARGS())
 LTTNG_UST_TRACEPOINT_EVENT_INSTANCE(

libguile/whippet/api/gc-null-event-listener.h

@@ -8,7 +8,8 @@ static inline void gc_null_event_listener_requesting_stop(void *data) {}
 static inline void gc_null_event_listener_waiting_for_stop(void *data) {}
 static inline void gc_null_event_listener_mutators_stopped(void *data) {}
 static inline void gc_null_event_listener_prepare_gc(void *data,
-                                                     enum gc_collection_kind) {}
+                                                     enum gc_collection_kind,
+                                                     uint64_t) {}
 static inline void gc_null_event_listener_roots_traced(void *data) {}
 static inline void gc_null_event_listener_heap_traced(void *data) {}
 static inline void gc_null_event_listener_ephemerons_traced(void *data) {}

libguile/whippet/api/mmc-attrs.h

@@ -42,7 +42,6 @@ static inline uint8_t gc_allocator_alloc_table_begin_pattern(enum gc_allocation_
       case GC_ALLOCATION_UNTAGGED_CONSERVATIVE:
         return young | trace_conservatively;
       case GC_ALLOCATION_TAGGED_POINTERLESS:
-        return young | trace_none;
       case GC_ALLOCATION_UNTAGGED_POINTERLESS:
         return young | trace_none;
       default:

libguile/whippet/src/bdw.c

@@ -505,7 +505,7 @@ static void on_collection_event(GC_EventType event) {
   }
   case GC_EVENT_MARK_START:
     HEAP_EVENT(mutators_stopped);
-    HEAP_EVENT(prepare_gc, GC_COLLECTION_MAJOR);
+    HEAP_EVENT(prepare_gc, GC_COLLECTION_MAJOR, GC_get_total_bytes());
     break;
   case GC_EVENT_MARK_END:
     HEAP_EVENT(roots_traced);

libguile/whippet/src/mmc.c

@@ -4,9 +4,10 @@
 #include <stdio.h>
 #include <string.h>
 
+#define GC_IMPL 1
+
 #include "gc-api.h"
 
-#define GC_IMPL 1
 #include "gc-internal.h"
 
 #include "background-thread.h"
@@ -530,6 +531,23 @@ compute_progress(struct gc_heap *heap, uintptr_t allocation_since_last_gc) {
   return allocation_since_last_gc > nofl_space_fragmentation(nofl);
 }
 
+static void
+grow_heap_for_large_allocation_if_necessary(struct gc_heap *heap,
+                                            enum gc_collection_kind gc_kind,
+                                            int progress)
+{
+  if (progress || heap->sizer.policy == GC_HEAP_SIZE_FIXED)
+    return;
+
+  struct nofl_space *nofl = heap_nofl_space(heap);
+  if (nofl_space_shrink (nofl, 0))
+    return;
+
+  ssize_t pending = nofl_space_request_release_memory(nofl, 0);
+  GC_ASSERT (pending > 0);
+  resize_heap(heap, heap->size + pending);
+}
+
 static int
 compute_success(struct gc_heap *heap, enum gc_collection_kind gc_kind,
                 int progress) {
@@ -792,7 +810,7 @@ collect(struct gc_mutator *mut, enum gc_collection_kind requested_kind) {
   enum gc_collection_kind gc_kind =
     determine_collection_kind(heap, requested_kind);
   int is_minor = gc_kind == GC_COLLECTION_MINOR;
-  HEAP_EVENT(heap, prepare_gc, gc_kind);
+  HEAP_EVENT(heap, prepare_gc, gc_kind, heap->total_allocated_bytes_at_last_gc);
   nofl_space_prepare_gc(nofl_space, gc_kind);
   large_object_space_start_gc(lospace, is_minor);
   gc_extern_space_start_gc(exspace, is_minor);
@@ -832,6 +850,7 @@ collect(struct gc_mutator *mut, enum gc_collection_kind requested_kind) {
   DEBUG("--- total live bytes estimate: %zu\n", live_bytes_estimate);
   gc_heap_sizer_on_gc(heap->sizer, heap->size, live_bytes_estimate, pause_ns,
                       resize_heap);
+  grow_heap_for_large_allocation_if_necessary(heap, gc_kind, progress);
   heap->size_at_last_gc = heap->size;
   HEAP_EVENT(heap, restarting_mutators);
   allow_mutators_to_continue(heap);
@@ -1215,16 +1234,18 @@ gc_init(const struct gc_options *options, struct gc_stack_addr stack_base,
   GC_ASSERT_EQ(gc_allocator_allocation_limit_offset(),
                offsetof(struct nofl_allocator, sweep));
   GC_ASSERT_EQ(gc_allocator_alloc_table_alignment(), NOFL_SLAB_SIZE);
-  GC_ASSERT_EQ(gc_allocator_alloc_table_begin_pattern(GC_ALLOCATION_TAGGED),
-               NOFL_METADATA_BYTE_YOUNG | NOFL_METADATA_BYTE_TRACE_PRECISELY);
   GC_ASSERT_EQ(gc_allocator_alloc_table_begin_pattern(GC_ALLOCATION_TAGGED_POINTERLESS),
                NOFL_METADATA_BYTE_YOUNG | NOFL_METADATA_BYTE_TRACE_NONE);
   if (GC_CONSERVATIVE_TRACE) {
+    GC_ASSERT_EQ(gc_allocator_alloc_table_begin_pattern(GC_ALLOCATION_TAGGED),
+                 NOFL_METADATA_BYTE_YOUNG | NOFL_METADATA_BYTE_TRACE_CONSERVATIVELY);
     GC_ASSERT_EQ(gc_allocator_alloc_table_begin_pattern(GC_ALLOCATION_UNTAGGED_CONSERVATIVE),
                  NOFL_METADATA_BYTE_YOUNG | NOFL_METADATA_BYTE_TRACE_CONSERVATIVELY);
     GC_ASSERT_EQ(gc_allocator_alloc_table_begin_pattern(GC_ALLOCATION_UNTAGGED_POINTERLESS),
                  NOFL_METADATA_BYTE_YOUNG | NOFL_METADATA_BYTE_TRACE_NONE);
   } else {
+    GC_ASSERT_EQ(gc_allocator_alloc_table_begin_pattern(GC_ALLOCATION_TAGGED),
+                 NOFL_METADATA_BYTE_YOUNG | NOFL_METADATA_BYTE_TRACE_PRECISELY);
     GC_ASSERT_EQ(gc_allocator_alloc_table_begin_pattern(GC_ALLOCATION_UNTAGGED_POINTERLESS),
                  NOFL_METADATA_BYTE_YOUNG | NOFL_METADATA_BYTE_TRACE_NONE |
                  NOFL_METADATA_BYTE_PINNED);
@@ -1290,7 +1311,7 @@ gc_finish_for_thread(struct gc_mutator *mut) {
 
 static void
 deactivate_mutator(struct gc_heap *heap, struct gc_mutator *mut) {
-  GC_ASSERT(mut->next == NULL);
+  GC_ASSERT(mut->active);
   nofl_allocator_finish(&mut->allocator, heap_nofl_space(heap));
   if (GC_GENERATIONAL)
     gc_field_set_writer_release_buffer(&mut->logger);

libguile/whippet/src/pcc.c

@@ -905,11 +905,11 @@ collect(struct gc_mutator *mut, enum gc_collection_kind requested_kind) {
     heap->is_minor_collection =
 #endif
     GC_GENERATIONAL ? gc_kind == GC_COLLECTION_MINOR : 0;
-  HEAP_EVENT(heap, prepare_gc, gc_kind);
   uint64_t *counter_loc = &heap->total_allocated_bytes_at_last_gc;
   copy_space_add_to_allocation_counter(heap_allocation_space(heap),
                                        counter_loc);
   large_object_space_add_to_allocation_counter(lospace, counter_loc);
+  HEAP_EVENT(heap, prepare_gc, gc_kind, *counter_loc);
   copy_spaces_start_gc(heap, is_minor_gc);
   large_object_space_start_gc(lospace, is_minor_gc);
   gc_extern_space_start_gc(exspace, is_minor_gc);

libguile/whippet/src/semi.c

@@ -395,7 +395,6 @@ static void collect(struct gc_mutator *mut) {
   HEAP_EVENT(heap, requesting_stop);
   HEAP_EVENT(heap, waiting_for_stop);
   HEAP_EVENT(heap, mutators_stopped);
-  HEAP_EVENT(heap, prepare_gc, GC_COLLECTION_COMPACTING);
 
   struct semi_space *semi = heap_semi_space(heap);
   struct large_object_space *large = heap_large_object_space(heap);
@@ -403,6 +402,9 @@ static void collect(struct gc_mutator *mut) {
   uint64_t *counter_loc = &heap->total_allocated_bytes_at_last_gc;
   semi_space_add_to_allocation_counter(semi, counter_loc);
   large_object_space_add_to_allocation_counter(large, counter_loc);
+
+  HEAP_EVENT(heap, prepare_gc, GC_COLLECTION_COMPACTING, *counter_loc);
+
   large_object_space_start_gc(large, 0);
   gc_extern_space_start_gc(heap->extern_space, 0);
   flip(semi);

libguile/whippet/whippet.m4

@@ -176,7 +176,7 @@ END
   fi])
 
 AC_DEFUN([WHIPPET_PKG_DEBUG],
- [AC_ARG_WITH(whippet-debug,
+ [AC_ARG_WITH(gc-debug,
     AS_HELP_STRING([--with-gc-debug],
                    [Compile GC library with debugging support (default: no)]),
     [whippet_with_debug=$withval],