From 9e8940e59f42de18235037cf5badac68841f608e Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Tue, 16 Aug 2022 11:53:32 +0200 Subject: [PATCH] Get handles out of collectors --- bdw.h | 6 ++++++ conservative-roots.h | 14 ++++++++++++ gc-api.h | 6 ++++++ gc-embedder-api.h | 9 ++++++++ mt-gcbench.c | 41 ++++++++++++++++++----------------- precise-roots.h | 12 +++++++++-- quads.c | 28 +++++++++++++++--------- semi.h | 14 +++++++++--- simple-gc-embedder.h | 25 ++++++++++++++++++++++ whippet.h | 51 ++++++++++++++++++++++++-------------------- 10 files changed, 149 insertions(+), 57 deletions(-) diff --git a/bdw.h b/bdw.h index b01155d4e..7731034ab 100644 --- a/bdw.h +++ b/bdw.h @@ -231,6 +231,12 @@ static void* gc_call_without_gc(struct mutator *mut, return GC_do_blocking(f, data); } +static void gc_mutator_set_roots(struct mutator *mut, + struct gc_mutator_roots *roots) { +} +static void gc_heap_set_roots(struct heap *heap, struct gc_heap_roots *roots) { +} + static void gc_print_stats(struct heap *heap) { printf("Completed %ld collections\n", (long)GC_get_gc_no()); printf("Heap size is %ld\n", (long)GC_get_heap_size()); diff --git a/conservative-roots.h b/conservative-roots.h index f5b1a5708..23fc38de7 100644 --- a/conservative-roots.h +++ b/conservative-roots.h @@ -5,3 +5,17 @@ struct handle { void *unused; }; #define HANDLE_SET(h,val) do { h.v = val; } while (0) #define PUSH_HANDLE(cx, h) do { (void) &h; } while (0) #define POP_HANDLE(cx) do { } while (0) + +static inline void visit_thread_roots(void *thread_roots, + void (*trace_edge)(struct gc_edge edge, + void *trace_data), + void *trace_data) { + abort(); +} + +static inline void visit_roots(struct handle *roots, + void (*trace_edge)(struct gc_edge edge, + void *trace_data), + void *trace_data) { + GC_ASSERT(!roots); +} diff --git a/gc-api.h b/gc-api.h index 86bde3aa4..da513f415 100644 --- a/gc-api.h +++ b/gc-api.h @@ -38,6 +38,12 @@ GC_API_ int gc_option_from_string(const char *str); GC_API_ int gc_init(int argc, struct gc_option argv[], struct heap **heap, struct mutator **mutator); +struct gc_mutator_roots; +struct gc_heap_roots; +GC_API_ void gc_mutator_set_roots(struct mutator *mut, + struct gc_mutator_roots *roots); +GC_API_ void gc_heap_set_roots(struct heap *heap, struct gc_heap_roots *roots); + GC_API_ struct mutator* gc_init_for_thread(uintptr_t *stack_base, struct heap *heap); GC_API_ void gc_finish_for_thread(struct mutator *mut); diff --git a/gc-embedder-api.h b/gc-embedder-api.h index f80ffe995..b74bd0486 100644 --- a/gc-embedder-api.h +++ b/gc-embedder-api.h @@ -13,6 +13,14 @@ GC_EMBEDDER_API inline void gc_trace_object(void *object, void *trace_data), void *trace_data, size_t *size) GC_ALWAYS_INLINE; +GC_EMBEDDER_API inline void gc_trace_mutator_roots(struct gc_mutator_roots *roots, + void (*trace_edge)(struct gc_edge edge, + void *trace_data), + void *trace_data); +GC_EMBEDDER_API inline void gc_trace_heap_roots(struct gc_heap_roots *roots, + void (*trace_edge)(struct gc_edge edge, + void *trace_data), + void *trace_data); GC_EMBEDDER_API inline uintptr_t gc_object_forwarded_nonatomic(void *object); GC_EMBEDDER_API inline void gc_object_forward_nonatomic(void *object, uintptr_t new_addr); @@ -25,4 +33,5 @@ GC_EMBEDDER_API inline void gc_atomic_forward_commit(struct gc_atomic_forward *, uintptr_t new_addr); GC_EMBEDDER_API inline uintptr_t gc_atomic_forward_address(struct gc_atomic_forward *); + #endif // GC_EMBEDDER_API_H diff --git a/mt-gcbench.c b/mt-gcbench.c index 00f1d7e1a..418851ede 100644 --- a/mt-gcbench.c +++ b/mt-gcbench.c @@ -185,6 +185,7 @@ static size_t power_law(size_t *counter) { struct thread { struct mutator *mut; + struct gc_mutator_roots roots; size_t counter; }; @@ -209,13 +210,13 @@ static void populate(struct thread *t, int depth, Node *node) { return; NodeHandle self = { node }; - PUSH_HANDLE(mut, self); + PUSH_HANDLE(t, self); allocate_garbage(t); NodeHandle l = { allocate_node(mut) }; - PUSH_HANDLE(mut, l); + PUSH_HANDLE(t, l); allocate_garbage(t); NodeHandle r = { allocate_node(mut) }; - PUSH_HANDLE(mut, r); + PUSH_HANDLE(t, r); set_field(HANDLE_REF(self), &HANDLE_REF(self)->left, HANDLE_REF(l)); set_field(HANDLE_REF(self), &HANDLE_REF(self)->right, HANDLE_REF(r)); @@ -225,9 +226,9 @@ static void populate(struct thread *t, int depth, Node *node) { populate(t, depth-1, HANDLE_REF(self)->left); populate(t, depth-1, HANDLE_REF(self)->right); - POP_HANDLE(mut); - POP_HANDLE(mut); - POP_HANDLE(mut); + POP_HANDLE(t); + POP_HANDLE(t); + POP_HANDLE(t); } // Build tree bottom-up @@ -237,9 +238,9 @@ static Node* make_tree(struct thread *t, int depth) { return allocate_node(mut); NodeHandle left = { make_tree(t, depth-1) }; - PUSH_HANDLE(mut, left); + PUSH_HANDLE(t, left); NodeHandle right = { make_tree(t, depth-1) }; - PUSH_HANDLE(mut, right); + PUSH_HANDLE(t, right); allocate_garbage(t); Node *result = allocate_node(mut); @@ -248,8 +249,8 @@ static Node* make_tree(struct thread *t, int depth) { // i is 0 because the memory is zeroed. result->j = depth; - POP_HANDLE(mut); - POP_HANDLE(mut); + POP_HANDLE(t); + POP_HANDLE(t); return result; } @@ -274,7 +275,7 @@ static void time_construction(struct thread *t, int depth) { struct mutator *mut = t->mut; int num_iters = compute_num_iters(depth); NodeHandle temp_tree = { NULL }; - PUSH_HANDLE(mut, temp_tree); + PUSH_HANDLE(t, temp_tree); printf("Creating %d trees of depth %d\n", num_iters, depth); @@ -301,7 +302,7 @@ static void time_construction(struct thread *t, int depth) { elapsed_millis(start)); } - POP_HANDLE(mut); + POP_HANDLE(t); } static void* call_with_stack_base(void* (*)(uintptr_t*, void*), void*) GC_NEVER_INLINE; @@ -337,11 +338,12 @@ static void* run_one_test(struct mutator *mut) { NodeHandle long_lived_tree = { NULL }; NodeHandle temp_tree = { NULL }; DoubleArrayHandle array = { NULL }; - struct thread t = { mut, 0 }; + struct thread t = { mut, }; + gc_mutator_set_roots(mut, &t.roots); - PUSH_HANDLE(mut, long_lived_tree); - PUSH_HANDLE(mut, temp_tree); - PUSH_HANDLE(mut, array); + PUSH_HANDLE(&t, long_lived_tree); + PUSH_HANDLE(&t, temp_tree); + PUSH_HANDLE(&t, array); // Create a long lived object printf(" Creating a long-lived binary tree of depth %d\n", @@ -368,9 +370,10 @@ static void* run_one_test(struct mutator *mut) { || HANDLE_REF(array)->values[1000] != 1.0/1000) fprintf(stderr, "Failed\n"); - POP_HANDLE(mut); - POP_HANDLE(mut); - POP_HANDLE(mut); + POP_HANDLE(&t); + POP_HANDLE(&t); + POP_HANDLE(&t); + gc_mutator_set_roots(mut, NULL); return NULL; } diff --git a/precise-roots.h b/precise-roots.h index 0465083b9..2eedb60ed 100644 --- a/precise-roots.h +++ b/precise-roots.h @@ -6,8 +6,8 @@ struct handle { #define HANDLE_TO(T) union { T* v; struct handle handle; } #define HANDLE_REF(h) h.v #define HANDLE_SET(h,val) do { h.v = val; } while (0) -#define PUSH_HANDLE(cx, h) push_handle(&cx->roots, &h.handle) -#define POP_HANDLE(cx) pop_handle(&cx->roots) +#define PUSH_HANDLE(cx, h) push_handle(&(cx)->roots.roots, &h.handle) +#define POP_HANDLE(cx) pop_handle(&(cx)->roots.roots) static inline void push_handle(struct handle **roots, struct handle *handle) { handle->next = *roots; @@ -17,3 +17,11 @@ static inline void push_handle(struct handle **roots, struct handle *handle) { static inline void pop_handle(struct handle **roots) { *roots = (*roots)->next; } + +static inline void visit_roots(struct handle *roots, + void (*trace_edge)(struct gc_edge edge, + void *trace_data), + void *trace_data) { + for (struct handle *h = roots; h; h = h->next) + trace_edge(gc_edge(&h->v), trace_data); +} diff --git a/quads.c b/quads.c index 0b26c5476..d7438a486 100644 --- a/quads.c +++ b/quads.c @@ -38,23 +38,29 @@ static unsigned long current_time(void) return t.tv_sec * 1000 * 1000 + t.tv_usec; } +struct thread { + struct mutator *mut; + struct gc_mutator_roots roots; + size_t counter; +}; + // Build tree bottom-up -static Quad* make_tree(struct mutator *mut, int depth) { +static Quad* make_tree(struct thread *t, int depth) { if (depth<=0) { - return allocate_quad(mut); + return allocate_quad(t->mut); } else { QuadHandle kids[4] = { { NULL }, }; for (size_t i = 0; i < 4; i++) { - HANDLE_SET(kids[i], make_tree(mut, depth-1)); - PUSH_HANDLE(mut, kids[i]); + HANDLE_SET(kids[i], make_tree(t, depth-1)); + PUSH_HANDLE(t, kids[i]); } - Quad *result = allocate_quad(mut); + Quad *result = allocate_quad(t->mut); for (size_t i = 0; i < 4; i++) result->kids[i] = HANDLE_REF(kids[i]); for (size_t i = 0; i < 4; i++) - POP_HANDLE(mut); + POP_HANDLE(t); return result; } @@ -144,15 +150,17 @@ int main(int argc, char *argv[]) { heap_size); return 1; } + struct thread t = { mut, }; + gc_mutator_set_roots(mut, &t.roots); QuadHandle quad = { NULL }; - PUSH_HANDLE(mut, quad); + PUSH_HANDLE(&t, quad); printf("Making quad tree of depth %zu (%zu nodes). Total size %.3fGB.\n", depth, nquads, (nquads * sizeof(Quad)) / 1e9); unsigned long start = current_time(); - HANDLE_SET(quad, make_tree(mut, depth)); + HANDLE_SET(quad, make_tree(&t, depth)); print_elapsed("construction", start); validate_tree(HANDLE_REF(quad), depth); @@ -165,7 +173,7 @@ int main(int argc, char *argv[]) { size_t garbage_depth = 3; start = current_time(); for (size_t i = garbage_step/(tree_size(garbage_depth)*4*sizeof(Quad*)); i; i--) - make_tree(mut, garbage_depth); + make_tree(&t, garbage_depth); print_elapsed("allocating garbage", start); start = current_time(); @@ -176,7 +184,7 @@ int main(int argc, char *argv[]) { gc_print_stats(heap); - POP_HANDLE(mut); + POP_HANDLE(&t); return 0; } diff --git a/semi.h b/semi.h index 224819f2c..def7a607f 100644 --- a/semi.h +++ b/semi.h @@ -27,7 +27,7 @@ struct heap { // One mutator per space, can just store the heap in the mutator. struct mutator { struct heap heap; - struct handle *roots; + struct gc_mutator_roots *roots; }; @@ -152,8 +152,8 @@ static void collect(struct mutator *mut) { large_object_space_start_gc(large, 0); flip(semi); uintptr_t grey = semi->hp; - for (struct handle *h = mut->roots; h; h = h->next) - visit(gc_edge(&h->v), heap); + if (mut->roots) + gc_trace_mutator_roots(mut->roots, visit, heap); // fprintf(stderr, "pushed %zd bytes in roots\n", space->hp - grey); while(grey < semi->hp) grey = scan(heap, grey); @@ -328,6 +328,14 @@ static int gc_init(int argc, struct gc_option argv[], return 1; } +static void gc_mutator_set_roots(struct mutator *mut, + struct gc_mutator_roots *roots) { + mut->roots = roots; +} +static void gc_heap_set_roots(struct heap *heap, struct gc_heap_roots *roots) { + abort(); +} + static struct mutator* gc_init_for_thread(uintptr_t *stack_base, struct heap *heap) { fprintf(stderr, diff --git a/simple-gc-embedder.h b/simple-gc-embedder.h index a198a47ae..5a8adf717 100644 --- a/simple-gc-embedder.h +++ b/simple-gc-embedder.h @@ -23,6 +23,31 @@ static inline void gc_trace_object(void *object, } } +struct handle; +struct gc_heap_roots { struct handle *roots; }; +struct gc_mutator_roots { struct handle *roots; }; + +static inline void visit_roots(struct handle *roots, + void (*trace_edge)(struct gc_edge edge, + void *trace_data), + void *trace_data); + +static inline void gc_trace_mutator_roots(struct gc_mutator_roots *roots, + void (*trace_edge)(struct gc_edge edge, + void *trace_data), + void *trace_data) { + if (roots) + visit_roots(roots->roots, trace_edge, trace_data); +} + +static inline void gc_trace_heap_roots(struct gc_heap_roots *roots, + void (*trace_edge)(struct gc_edge edge, + void *trace_data), + void *trace_data) { + if (roots) + visit_roots(roots->roots, trace_edge, trace_data); +} + static inline uintptr_t gc_object_forwarded_nonatomic(void *object) { uintptr_t tag = *tag_word(object); return (tag & gcobj_not_forwarded_bit) ? 0 : tag; diff --git a/whippet.h b/whippet.h index 6ac499635..4851c874d 100644 --- a/whippet.h +++ b/whippet.h @@ -322,7 +322,7 @@ struct heap { int allow_pinning; size_t active_mutator_count; size_t mutator_count; - struct handle *global_roots; + struct gc_heap_roots *roots; struct mutator *mutator_trace_list; long count; long minor_count; @@ -348,7 +348,7 @@ struct mutator { uintptr_t sweep; uintptr_t block; struct heap *heap; - struct handle *roots; + struct gc_mutator_roots *roots; struct mutator_mark_buf mark_buf; // Three uses for this in-object linked-list pointer: // - inactive (blocked in syscall) mutators @@ -905,29 +905,39 @@ static int mutator_should_mark_while_stopping(struct mutator *mut) { return heap_should_mark_while_stopping(mutator_heap(mut)); } +static void gc_mutator_set_roots(struct mutator *mut, + struct gc_mutator_roots *roots) { + mut->roots = roots; +} +static void gc_heap_set_roots(struct heap *heap, struct gc_heap_roots *roots) { + heap->roots = roots; +} + +static void trace_and_enqueue_locally(struct gc_edge edge, void *data) { + struct mutator *mut = data; + if (trace_edge(mutator_heap(mut), edge)) + mutator_mark_buf_push(&mut->mark_buf, + gc_ref_heap_object(gc_edge_ref(edge))); +} + +static void trace_and_enqueue_globally(struct gc_edge edge, void *data) { + struct heap *heap = data; + if (trace_edge(heap, edge)) + tracer_enqueue_root(&heap->tracer, + gc_ref_heap_object(gc_edge_ref(edge))); +} + // Mark the roots of a mutator that is stopping for GC. We can't // enqueue them directly, so we send them to the controller in a buffer. static void mark_stopping_mutator_roots(struct mutator *mut) { GC_ASSERT(mutator_should_mark_while_stopping(mut)); - struct heap *heap = mutator_heap(mut); - struct mutator_mark_buf *local_roots = &mut->mark_buf; - for (struct handle *h = mut->roots; h; h = h->next) { - struct gc_edge root = gc_edge(&h->v); - if (trace_edge(heap, root)) - mutator_mark_buf_push(local_roots, - gc_ref_heap_object(gc_edge_ref(root))); - } + gc_trace_mutator_roots(mut->roots, trace_and_enqueue_locally, mut); } // Precondition: the caller holds the heap lock. static void mark_mutator_roots_with_lock(struct mutator *mut) { - struct heap *heap = mutator_heap(mut); - for (struct handle *h = mut->roots; h; h = h->next) { - struct gc_edge root = gc_edge(&h->v); - if (trace_edge(heap, root)) - tracer_enqueue_root(&heap->tracer, - gc_ref_heap_object(gc_edge_ref(root))); - } + gc_trace_mutator_roots(mut->roots, trace_and_enqueue_globally, + mutator_heap(mut)); } static void trace_mutator_roots_with_lock(struct mutator *mut) { @@ -976,12 +986,7 @@ static void trace_mutator_roots_after_stop(struct heap *heap) { } static void trace_global_roots(struct heap *heap) { - for (struct handle *h = heap->global_roots; h; h = h->next) { - struct gc_edge edge = gc_edge(&h->v); - if (trace_edge(heap, edge)) - tracer_enqueue_root(&heap->tracer, - gc_ref_heap_object(gc_edge_ref(edge))); - } + gc_trace_heap_roots(heap->roots, trace_and_enqueue_globally, heap); } static inline int