From 63002037385c9f9995fc7b7f55891ba5004e4952 Mon Sep 17 00:00:00 2001
From: Andy Wingo <wingo@igalia.com>
Date: Tue, 29 Mar 2022 21:58:52 +0200
Subject: [PATCH] Add call_without_gc API

This lets us call pthread_join safely
---
 bdw.h        |  8 ++++++++
 mark-sweep.h | 44 ++++++++++++++++++++++++++++++++++++++++++++
 mt-gcbench.c | 15 ++++++++++++---
 semi.h       |  6 ++++++
 4 files changed, 70 insertions(+), 3 deletions(-)

diff --git a/bdw.h b/bdw.h
index e60802887..0034b0561 100644
--- a/bdw.h
+++ b/bdw.h
@@ -158,6 +158,14 @@ static void finish_gc_for_thread(struct mutator *mut) {
   GC_unregister_my_thread();
 }
 
+static void* call_without_gc(struct mutator *mut, void* (*f)(void*),
+                             void *data) NEVER_INLINE;
+static void* call_without_gc(struct mutator *mut,
+                             void* (*f)(void*),
+                             void *data) {
+  return GC_do_blocking(f, data);
+}
+
 static inline void print_start_gc_stats(struct heap *heap) {
 }
 static inline void print_end_gc_stats(struct heap *heap) {
diff --git a/mark-sweep.h b/mark-sweep.h
index c20ec4b51..a69bd544e 100644
--- a/mark-sweep.h
+++ b/mark-sweep.h
@@ -125,6 +125,7 @@ struct mark_space {
   size_t mem_size;
   long count;
   struct marker marker;
+  struct mutator *deactivated_mutators;
 };
 
 struct heap {
@@ -144,6 +145,7 @@ struct mutator {
   struct heap *heap;
   struct handle *roots;
   struct mutator_mark_buf mark_buf;
+  struct mutator *next;
 };
 
 static inline struct marker* mark_space_marker(struct mark_space *space) {
@@ -352,6 +354,11 @@ static void wait_for_mutators_to_stop(struct mark_space *space) {
     pthread_cond_wait(&space->collector_cond, &space->lock);
 }
 
+static void mark_inactive_mutators(struct mark_space *space) {
+  for (struct mutator *mut = space->deactivated_mutators; mut; mut = mut->next)
+    mark_controlling_mutator_roots(mut);
+}
+
 static void mark_global_roots(struct mark_space *space) {
   for (struct handle *h = space->global_roots; h; h = h->next) {
     struct gcobj *obj = h->v;
@@ -424,6 +431,7 @@ static void collect(struct mark_space *space, struct mutator *mut) {
   request_mutators_to_stop(space);
   mark_controlling_mutator_roots(mut);
   wait_for_mutators_to_stop(space);
+  mark_inactive_mutators(space);
   mark_global_roots(space);
   marker_trace(space);
   marker_release(space);
@@ -834,6 +842,42 @@ static void finish_gc_for_thread(struct mutator *mut) {
   free(mut);
 }
 
+static void deactivate_mutator(struct mark_space *space, struct mutator *mut) {
+  ASSERT(mut->next == NULL);
+  mark_space_lock(space);
+  mut->next = space->deactivated_mutators;
+  space->deactivated_mutators = mut;
+  space->active_mutator_count--;
+  if (!space->active_mutator_count && mutators_are_stopping(space))
+    pthread_cond_signal(&space->collector_cond);
+  mark_space_unlock(space);
+}
+
+static void reactivate_mutator(struct mark_space *space, struct mutator *mut) {
+  mark_space_lock(space);
+  while (mutators_are_stopping(space))
+    pthread_cond_wait(&space->mutator_cond, &space->lock);
+  struct mutator **prev = &space->deactivated_mutators;
+  while (*prev != mut)
+    prev = &(*prev)->next;
+  *prev = mut->next;
+  mut->next = NULL;
+  space->active_mutator_count++;
+  mark_space_unlock(space);
+}
+
+static void* call_without_gc(struct mutator *mut, void* (*f)(void*),
+                             void *data) NEVER_INLINE;
+static void* call_without_gc(struct mutator *mut,
+                             void* (*f)(void*),
+                             void *data) {
+  struct mark_space *space = mutator_mark_space(mut);
+  deactivate_mutator(space, mut);
+  void *ret = f(data);
+  reactivate_mutator(space, mut);
+  return ret;
+}
+
 static inline void print_start_gc_stats(struct heap *heap) {
 }
 
diff --git a/mt-gcbench.c b/mt-gcbench.c
index 28dc92267..b80a0c83c 100644
--- a/mt-gcbench.c
+++ b/mt-gcbench.c
@@ -297,6 +297,14 @@ static void* run_one_test_in_thread(void *arg) {
   return call_with_gc(run_one_test, heap);
 }
 
+struct join_data { int status; pthread_t thread; };
+static void *join_thread(void *data) {
+  struct join_data *join_data = data;
+  void *ret;
+  join_data->status = pthread_join(join_data->thread, &ret);
+  return ret;
+}
+
 int main(int argc, char *argv[]) {
   // Define size of Node without any GC header.
   size_t sizeof_node = 2 * sizeof(Node*) + 2 * sizeof(int);
@@ -350,9 +358,10 @@ int main(int argc, char *argv[]) {
   }
   run_one_test(mut);
   for (size_t i = 1; i < nthreads; i++) {
-    int status = pthread_join(threads[i], NULL);
-    if (status) {
-      errno = status;
+    struct join_data data = { 0, threads[i] };
+    call_without_gc(mut, join_thread, &data);
+    if (data.status) {
+      errno = data.status;
       perror("Failed to join thread");
       return 1;
     }
diff --git a/semi.h b/semi.h
index af58f052d..096c57591 100644
--- a/semi.h
+++ b/semi.h
@@ -204,6 +204,12 @@ static struct mutator* initialize_gc_for_thread(uintptr_t *stack_base,
 static void finish_gc_for_thread(struct mutator *space) {
 }
 
+static void* call_without_gc(struct mutator *mut, void* (*f)(void*),
+                             void *data) {
+  // Can't be threads, then there won't be collection.
+  return f(data);
+}
+
 static inline void print_start_gc_stats(struct heap *heap) {
 }