Merge remote-tracking branch 'whippet/main' into HEAD

libguile/whippet/api/gc-ephemeron.h

@@ -26,9 +26,17 @@ GC_API_ void gc_ephemeron_init(struct gc_mutator *mut,
 GC_API_ struct gc_ref gc_ephemeron_key(struct gc_ephemeron *ephemeron);
 GC_API_ struct gc_ref gc_ephemeron_value(struct gc_ephemeron *ephemeron);
 
+GC_API_ struct gc_ref gc_ephemeron_swap_value(struct gc_mutator *mut,
+                                              struct gc_ephemeron *ephemeron,
+                                              struct gc_ref ref);
+
 GC_API_ struct gc_ephemeron* gc_ephemeron_chain_head(struct gc_ephemeron **loc);
+
 GC_API_ void gc_ephemeron_chain_push(struct gc_ephemeron **loc,
                                      struct gc_ephemeron *ephemeron);
+GC_API_ int gc_ephemeron_chain_try_push(struct gc_ephemeron **loc,
+                                        struct gc_ephemeron *ephemeron,
+                                        struct gc_ephemeron **tail);
 GC_API_ struct gc_ephemeron* gc_ephemeron_chain_next(struct gc_ephemeron *ephemeron);
 GC_API_ void gc_ephemeron_mark_dead(struct gc_ephemeron *ephemeron);
 

libguile/whippet/src/bdw.c

@@ -53,7 +53,6 @@
 
 struct gc_heap {
   struct gc_heap *freelist; // see mark_heap
-  pthread_mutex_t lock;
   struct gc_heap_roots *roots;
   struct gc_mutator *mutators;
   struct gc_event_listener event_listener;
@@ -67,8 +66,8 @@ struct gc_mutator {
   void *freelists[GC_INLINE_FREELIST_COUNT];
   struct gc_heap *heap;
   struct gc_mutator_roots *roots;
-  struct gc_mutator *next; // with heap lock
+  struct gc_mutator *next; // with global bdw lock
-  struct gc_mutator **prev; // with heap lock
+  struct gc_mutator **prev; // with global bdw lock
   void *event_listener_data;
 };
 
@@ -253,6 +252,12 @@ void gc_ephemeron_init(struct gc_mutator *mut, struct gc_ephemeron *ephemeron,
   }
 }
 
+struct gc_ref gc_ephemeron_swap_value(struct gc_mutator *mut,
+                                      struct gc_ephemeron *e,
+                                      struct gc_ref ref) {
+  return gc_ephemeron_swap_value_internal(e, ref);
+}
+
 int gc_visit_ephemeron_key(struct gc_edge edge, struct gc_heap *heap) {
   // Pretend the key is traced, to avoid adding this ephemeron to the
   // global table.
@@ -419,13 +424,13 @@ static inline struct gc_mutator *add_mutator(struct gc_heap *heap) {
   ret->event_listener_data =
     heap->event_listener.mutator_added(heap->event_listener_data);
 
-  pthread_mutex_lock(&heap->lock);
+  GC_alloc_lock();
   ret->next = heap->mutators;
   ret->prev = &heap->mutators;
   if (ret->next)
     ret->next->prev = &ret->next;
   heap->mutators = ret;
-  pthread_mutex_unlock(&heap->lock);
+  GC_alloc_unlock();
 
   return ret;
 }
@@ -607,7 +612,6 @@ int gc_init(const struct gc_options *options, struct gc_stack_addr stack_base,
   }
 
   *heap = GC_generic_malloc(sizeof(struct gc_heap), heap_gc_kind);
-  pthread_mutex_init(&(*heap)->lock, NULL);
 
   (*heap)->event_listener = event_listener;
   (*heap)->event_listener_data = event_listener_data;
@@ -636,12 +640,13 @@ struct gc_mutator* gc_init_for_thread(struct gc_stack_addr stack_base,
   return add_mutator(heap);
 }
 void gc_finish_for_thread(struct gc_mutator *mut) {
-  pthread_mutex_lock(&mut->heap->lock);
+  GC_alloc_lock();
   MUTATOR_EVENT(mut, mutator_removed);
   *mut->prev = mut->next;
   if (mut->next)
     mut->next->prev = mut->prev;
-  pthread_mutex_unlock(&mut->heap->lock);
+  memset(mut, 0, sizeof(*mut));
+  GC_alloc_unlock();
 
   GC_unregister_my_thread();
 }

libguile/whippet/src/gc-ephemeron-internal.h

@@ -47,6 +47,10 @@ GC_INTERNAL void
 gc_sweep_pending_ephemerons(struct gc_pending_ephemerons *state,
                             size_t shard, size_t nshards);
 
+GC_INTERNAL struct gc_ref
+gc_ephemeron_swap_value_internal(struct gc_ephemeron *ephemeron,
+                                 struct gc_ref value);
+
 GC_INTERNAL void gc_ephemeron_init_internal(struct gc_heap *heap,
                                             struct gc_ephemeron *ephemeron,
                                             struct gc_ref key,

libguile/whippet/src/gc-ephemeron.c

@@ -165,19 +165,22 @@
 // Concurrent operations on ephemeron lists
 ////////////////////////////////////////////////////////////////////////
 
+static int
+ephemeron_list_try_push(struct gc_ephemeron **loc,
+                        struct gc_ephemeron *head,
+                        struct gc_ephemeron **tail,
+                        struct gc_ephemeron** (*get_next)(struct gc_ephemeron*)) {
+  *get_next(head) = *tail;
+  return atomic_compare_exchange_weak(loc, tail, head);
+}
+
 static void
 ephemeron_list_push(struct gc_ephemeron **loc,
                     struct gc_ephemeron *head,
                     struct gc_ephemeron** (*get_next)(struct gc_ephemeron*)) {
   struct gc_ephemeron *tail = atomic_load_explicit(loc, memory_order_acquire);
-  while (1) {
+  while (!ephemeron_list_try_push(loc, head, &tail, get_next))
-    // There must be no concurrent readers of HEAD, a precondition that
+    ;
-    // we ensure by only publishing HEAD to LOC at most once per cycle.
-    // Therefore we can use a normal store for the tail pointer.
-    *get_next(head) = tail;
-    if (atomic_compare_exchange_weak(loc, &tail, head))
-      break;
-  }
 }
 
 static struct gc_ephemeron*
@@ -274,6 +277,11 @@ void gc_ephemeron_chain_push(struct gc_ephemeron **loc,
                              struct gc_ephemeron *e) {
   ephemeron_list_push(loc, e, ephemeron_chain);
 }  
+int gc_ephemeron_chain_try_push(struct gc_ephemeron **loc,
+                                struct gc_ephemeron *e,
+                                struct gc_ephemeron **tail) {
+  return ephemeron_list_try_push(loc, e, tail, ephemeron_chain);
+}
 static struct gc_ephemeron* follow_chain(struct gc_ephemeron **loc) {
   return ephemeron_list_follow(loc, ephemeron_chain, ephemeron_is_not_dead);
 }  
@@ -283,6 +291,21 @@ struct gc_ephemeron* gc_ephemeron_chain_head(struct gc_ephemeron **loc) {
 struct gc_ephemeron* gc_ephemeron_chain_next(struct gc_ephemeron *e) {
   return follow_chain(ephemeron_chain(e));
 }
+
+struct gc_ref gc_ephemeron_swap_value_internal(struct gc_ephemeron *e,
+                                               struct gc_ref ref)
+{
+  GC_ASSERT(!gc_ref_is_null(ref));
+
+  uintptr_t prev = atomic_load(&e->value.value);
+  do {
+    if (!prev)
+      break;
+  } while (!atomic_compare_exchange_weak(&e->value.value, &prev, ref.value));
+
+  return gc_ref(prev);
+}
+
 void gc_ephemeron_mark_dead(struct gc_ephemeron *e) {
   atomic_store_explicit(&e->key.value, 0, memory_order_release);
 }
@@ -570,6 +593,9 @@ gc_sweep_pending_ephemerons(struct gc_pending_ephemerons *state,
 void gc_ephemeron_init_internal(struct gc_heap *heap,
                                 struct gc_ephemeron *ephemeron,
                                 struct gc_ref key, struct gc_ref value) {
+  GC_ASSERT(!gc_ref_is_null(key));
+  GC_ASSERT(!gc_ref_is_null(value));
+
   // Caller responsible for any write barrier, though really the
   // assumption is that the ephemeron is younger than the key and the
   // value.

libguile/whippet/src/mmc.c

@@ -1007,6 +1007,14 @@ gc_ephemeron_init(struct gc_mutator *mut, struct gc_ephemeron *ephemeron,
   // key or the value.
 }
 
+struct gc_ref
+gc_ephemeron_swap_value(struct gc_mutator *mut, struct gc_ephemeron *e,
+                        struct gc_ref ref) {
+  gc_write_barrier(mut, gc_ref_from_heap_object(e), gc_ephemeron_size(),
+                   gc_ephemeron_value_edge(e), ref);
+  return gc_ephemeron_swap_value_internal(e, ref);
+}
+
 struct gc_pending_ephemerons *
 gc_heap_pending_ephemerons(struct gc_heap *heap) {
   return heap->pending_ephemerons;

libguile/whippet/src/pcc.c

@@ -1090,6 +1090,14 @@ void gc_ephemeron_init(struct gc_mutator *mut, struct gc_ephemeron *ephemeron,
   gc_ephemeron_init_internal(mutator_heap(mut), ephemeron, key, value);
 }
 
+struct gc_ref
+gc_ephemeron_swap_value(struct gc_mutator *mut, struct gc_ephemeron *e,
+                        struct gc_ref ref) {
+  gc_write_barrier(mut, gc_ref_from_heap_object(e), gc_ephemeron_size(),
+                   gc_ephemeron_value_edge(e), ref);
+  return gc_ephemeron_swap_value_internal(e, ref);
+}
+
 struct gc_pending_ephemerons *gc_heap_pending_ephemerons(struct gc_heap *heap) {
 #if GC_GENERATIONAL
   if (is_minor_collection(heap))

libguile/whippet/src/semi.c

@@ -555,6 +555,12 @@ void gc_ephemeron_init(struct gc_mutator *mut, struct gc_ephemeron *ephemeron,
   gc_ephemeron_init_internal(mutator_heap(mut), ephemeron, key, value);
 }
 
+struct gc_ref gc_ephemeron_swap_value(struct gc_mutator *mut,
+                                      struct gc_ephemeron *e,
+                                      struct gc_ref ref) {
+  return gc_ephemeron_swap_value_internal(e, ref);
+}
+
 struct gc_finalizer* gc_allocate_finalizer(struct gc_mutator *mut) {
   return gc_allocate(mut, gc_finalizer_size(), GC_ALLOCATION_TAGGED);
 }