From 52166fe286fb365d0000e87cd79cc38916f7714a Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Sat, 4 Jun 2022 21:54:49 +0200 Subject: [PATCH] Add gc_edge data structure Less casting in user programs, and it's a step on the way to evacuation in whippet. --- gc-types.h | 26 ++++++++++++++++++++++++++ gc.h | 2 ++ heap-objects.h | 3 ++- mt-gcbench.c | 8 ++++---- parallel-tracer.h | 13 ++++++------- quads.c | 4 ++-- semi.h | 20 ++++++++++---------- serial-tracer.h | 14 +++++++------- whippet.h | 39 ++++++++++++++++++++++----------------- 9 files changed, 81 insertions(+), 48 deletions(-) create mode 100644 gc-types.h diff --git a/gc-types.h b/gc-types.h new file mode 100644 index 000000000..4779cbd2c --- /dev/null +++ b/gc-types.h @@ -0,0 +1,26 @@ +#ifndef GC_TYPES_H_ +#define GC_TYPES_H_ + +struct gc_edge { + union { + void *addr; + void **loc; + }; +}; + +static inline struct gc_edge gc_edge(void* addr) { + struct gc_edge edge; + edge.addr = addr; + return edge; +} +static inline struct gc_edge object_field(void* addr) { + return gc_edge(addr); +} +static inline void* dereference_edge(struct gc_edge edge) { + return *edge.loc; +} +static inline void update_edge(struct gc_edge edge, void *value) { + *edge.loc = value; +} + +#endif // GC_TYPES_H_ diff --git a/gc.h b/gc.h index 5d6268d9f..2f6240122 100644 --- a/gc.h +++ b/gc.h @@ -1,6 +1,8 @@ #ifndef GC_H_ #define GC_H_ +#include "gc-types.h" + #if defined(GC_BDW) #include "bdw.h" #elif defined(GC_SEMI) diff --git a/heap-objects.h b/heap-objects.h index db78e7b66..44e282cc1 100644 --- a/heap-objects.h +++ b/heap-objects.h @@ -2,6 +2,7 @@ #define HEAP_OBJECTS_H #include "inline.h" +#include "gc-types.h" #define DECLARE_NODE_TYPE(name, Name, NAME) \ struct Name; \ @@ -18,7 +19,7 @@ enum alloc_kind { #define DEFINE_METHODS(name, Name, NAME) \ static inline size_t name##_size(Name *obj) ALWAYS_INLINE; \ static inline void visit_##name##_fields(Name *obj,\ - void (*visit)(void **loc, void *visit_data), \ + void (*visit)(struct gc_edge edge, void *visit_data), \ void *visit_data) ALWAYS_INLINE; FOR_EACH_HEAP_OBJECT_KIND(DEFINE_METHODS) #undef DEFINE_METHODS diff --git a/mt-gcbench.c b/mt-gcbench.c index 80c802988..0dd9ef6f4 100644 --- a/mt-gcbench.c +++ b/mt-gcbench.c @@ -77,14 +77,14 @@ static inline size_t double_array_size(DoubleArray *array) { } static inline void visit_node_fields(Node *node, - void (*visit)(void **loc, void *visit_data), + void (*visit)(struct gc_edge edge, void *visit_data), void *visit_data) { - visit((void**)&node->left, visit_data); - visit((void**)&node->right, visit_data); + visit(object_field(&node->left), visit_data); + visit(object_field(&node->right), visit_data); } static inline void visit_double_array_fields(DoubleArray *obj, - void (*visit)(void **loc, void *visit_data), + void (*visit)(struct gc_edge edge, void *visit_data), void *visit_data) { } diff --git a/parallel-tracer.h b/parallel-tracer.h index f96e93754..297e2dde8 100644 --- a/parallel-tracer.h +++ b/parallel-tracer.h @@ -452,10 +452,10 @@ static void tracer_release(struct heap *heap) { } struct gcobj; -static inline void tracer_visit(void **loc, void *trace_data) ALWAYS_INLINE; +static inline void tracer_visit(struct gc_edge edge, void *trace_data) ALWAYS_INLINE; static inline void trace_one(struct gcobj *obj, void *trace_data) ALWAYS_INLINE; -static inline int trace_object(struct heap *heap, - struct gcobj *obj) ALWAYS_INLINE; +static inline int trace_edge(struct heap *heap, + struct gc_edge edge) ALWAYS_INLINE; static inline void tracer_share(struct local_tracer *trace) { @@ -465,13 +465,12 @@ tracer_share(struct local_tracer *trace) { } static inline void -tracer_visit(void **loc, void *trace_data) { +tracer_visit(struct gc_edge edge, void *trace_data) { struct local_tracer *trace = trace_data; - struct gcobj *obj = *loc; - if (obj && trace_object(trace->heap, obj)) { + if (trace_edge(trace->heap, edge)) { if (local_trace_queue_full(&trace->local)) tracer_share(trace); - local_trace_queue_push(&trace->local, obj); + local_trace_queue_push(&trace->local, dereference_edge(edge)); } } diff --git a/quads.c b/quads.c index 0f7e01857..0ba9ea3f4 100644 --- a/quads.c +++ b/quads.c @@ -15,10 +15,10 @@ static inline size_t quad_size(Quad *obj) { } static inline void visit_quad_fields(Quad *quad, - void (*visit)(void **loc, void *visit_data), + void (*visit)(struct gc_edge edge, void *visit_data), void *visit_data) { for (size_t i = 0; i < 4; i++) - visit((void**)&quad->kids[i], visit_data); + visit(object_field(&quad->kids[i]), visit_data); } typedef HANDLE_TO(Quad) QuadHandle; diff --git a/semi.h b/semi.h index 3b58376e5..6ed67de8b 100644 --- a/semi.h +++ b/semi.h @@ -57,7 +57,7 @@ static inline void clear_memory(uintptr_t addr, size_t size) { static void collect(struct mutator *mut) NEVER_INLINE; static void collect_for_alloc(struct mutator *mut, size_t bytes) NEVER_INLINE; -static void visit(void **loc, void *visit_data); +static void visit(struct gc_edge edge, void *visit_data); static int semi_space_steal_pages(struct semi_space *space, size_t npages) { size_t stolen_pages = space->stolen_pages + npages; @@ -143,13 +143,13 @@ static void* forward(struct semi_space *space, void *obj) { } static void visit_semi_space(struct heap *heap, struct semi_space *space, - void **loc, void *obj) { - *loc = forward(space, obj); + struct gc_edge edge, void *obj) { + update_edge(edge, forward(space, obj)); } static void visit_large_object_space(struct heap *heap, struct large_object_space *space, - void **loc, void *obj) { + void *obj) { if (large_object_space_copy(space, (uintptr_t)obj)) scan(heap, (uintptr_t)obj); } @@ -158,15 +158,15 @@ static int semi_space_contains(struct semi_space *space, void *obj) { return (((uintptr_t)obj) - space->base) < space->size; } -static void visit(void **loc, void *visit_data) { +static void visit(struct gc_edge edge, void *visit_data) { struct heap *heap = visit_data; - void *obj = *loc; + void *obj = dereference_edge(edge); if (obj == NULL) return; - if (semi_space_contains(heap_semi_space(heap), obj)) - visit_semi_space(heap, heap_semi_space(heap), loc, obj); + else if (semi_space_contains(heap_semi_space(heap), obj)) + visit_semi_space(heap, heap_semi_space(heap), edge, obj); else if (large_object_space_contains(heap_large_object_space(heap), obj)) - visit_large_object_space(heap, heap_large_object_space(heap), loc, obj); + visit_large_object_space(heap, heap_large_object_space(heap), obj); else abort(); } @@ -180,7 +180,7 @@ static void collect(struct mutator *mut) { flip(semi); uintptr_t grey = semi->hp; for (struct handle *h = mut->roots; h; h = h->next) - visit(&h->v, heap); + visit(gc_edge(&h->v), heap); // fprintf(stderr, "pushed %zd bytes in roots\n", space->hp - grey); while(grey < semi->hp) grey = scan(heap, grey); diff --git a/serial-tracer.h b/serial-tracer.h index 7bea9e63e..6b861a471 100644 --- a/serial-tracer.h +++ b/serial-tracer.h @@ -6,6 +6,7 @@ #include "assert.h" #include "debug.h" +#include "gc-types.h" struct gcobj; @@ -137,10 +138,10 @@ static void tracer_release(struct heap *heap) { } struct gcobj; -static inline void tracer_visit(void **loc, void *trace_data) ALWAYS_INLINE; +static inline void tracer_visit(struct gc_edge edge, void *trace_data) ALWAYS_INLINE; static inline void trace_one(struct gcobj *obj, void *trace_data) ALWAYS_INLINE; -static inline int trace_object(struct heap *heap, - struct gcobj *obj) ALWAYS_INLINE; +static inline int trace_edge(struct heap *heap, + struct gc_edge edge) ALWAYS_INLINE; static inline void tracer_enqueue_root(struct tracer *tracer, struct gcobj *obj) { @@ -152,11 +153,10 @@ tracer_enqueue_roots(struct tracer *tracer, struct gcobj **objs, trace_queue_push_many(&tracer->queue, objs, count); } static inline void -tracer_visit(void **loc, void *trace_data) { +tracer_visit(struct gc_edge edge, void *trace_data) { struct heap *heap = trace_data; - struct gcobj *obj = *loc; - if (obj && trace_object(heap, obj)) - tracer_enqueue_root(heap_tracer(heap), obj); + if (trace_edge(heap, edge)) + tracer_enqueue_root(heap_tracer(heap), dereference_edge(edge)); } static inline void tracer_trace(struct heap *heap) { diff --git a/whippet.h b/whippet.h index 3c1fb070b..f124437fc 100644 --- a/whippet.h +++ b/whippet.h @@ -350,8 +350,9 @@ static inline uint8_t* mark_byte(struct mark_space *space, struct gcobj *obj) { return object_metadata_byte(obj); } -static inline int mark_space_trace_object(struct mark_space *space, - struct gcobj *obj) { +static inline int mark_space_mark_object(struct mark_space *space, + struct gc_edge edge) { + struct gcobj *obj = dereference_edge(edge); uint8_t *loc = object_metadata_byte(obj); uint8_t byte = *loc; if (byte & space->marked_mask) @@ -368,16 +369,20 @@ static inline int mark_space_contains(struct mark_space *space, return addr - space->low_addr < space->extent; } -static inline int large_object_space_trace_object(struct large_object_space *space, - struct gcobj *obj) { +static inline int large_object_space_mark_object(struct large_object_space *space, + struct gcobj *obj) { return large_object_space_copy(space, (uintptr_t)obj); } -static inline int trace_object(struct heap *heap, struct gcobj *obj) { - if (LIKELY(mark_space_contains(heap_mark_space(heap), obj))) - return mark_space_trace_object(heap_mark_space(heap), obj); +static inline int trace_edge(struct heap *heap, struct gc_edge edge) { + struct gcobj *obj = dereference_edge(edge); + if (!obj) + return 0; + else if (LIKELY(mark_space_contains(heap_mark_space(heap), obj))) + return mark_space_mark_object(heap_mark_space(heap), edge); else if (large_object_space_contains(heap_large_object_space(heap), obj)) - return large_object_space_trace_object(heap_large_object_space(heap), obj); + return large_object_space_mark_object(heap_large_object_space(heap), + obj); else abort(); } @@ -584,9 +589,9 @@ static void mark_stopping_mutator_roots(struct mutator *mut) { struct heap *heap = mutator_heap(mut); struct mutator_mark_buf *local_roots = &mut->mark_buf; for (struct handle *h = mut->roots; h; h = h->next) { - struct gcobj *root = h->v; - if (root && trace_object(heap, root)) - mutator_mark_buf_push(local_roots, root); + struct gc_edge root = gc_edge(&h->v); + if (trace_edge(heap, root)) + mutator_mark_buf_push(local_roots, dereference_edge(root)); } // Post to global linked-list of thread roots. @@ -602,9 +607,9 @@ static void mark_stopping_mutator_roots(struct mutator *mut) { static void mark_controlling_mutator_roots(struct mutator *mut) { struct heap *heap = mutator_heap(mut); for (struct handle *h = mut->roots; h; h = h->next) { - struct gcobj *root = h->v; - if (root && trace_object(heap, root)) - tracer_enqueue_root(&heap->tracer, root); + struct gc_edge root = gc_edge(&h->v); + if (trace_edge(heap, root)) + tracer_enqueue_root(&heap->tracer, dereference_edge(root)); } } @@ -630,9 +635,9 @@ static void mark_inactive_mutators(struct heap *heap) { static void mark_global_roots(struct heap *heap) { for (struct handle *h = heap->global_roots; h; h = h->next) { - struct gcobj *obj = h->v; - if (obj && trace_object(heap, obj)) - tracer_enqueue_root(&heap->tracer, obj); + struct gc_edge edge = gc_edge(&h->v); + if (trace_edge(heap, edge)) + tracer_enqueue_root(&heap->tracer, dereference_edge(edge)); } struct mutator_mark_buf *roots = atomic_load(&heap->mutator_roots);