From f57a1b8a55fd003f6bc71ddca57f164fb0806537 Mon Sep 17 00:00:00 2001 From: Andy Wingo Date: Fri, 11 Mar 2022 10:17:05 +0100 Subject: [PATCH] Refactor to separate gcbench from gc --- Makefile | 2 +- bdw.h | 11 +++------ gc.h | 17 +++++++++++++ gcbench-types.h | 30 +++++++++++++++++++++++ GCBench.c => gcbench.c | 44 ++++++++++++++------------------- inline.h | 7 ++++++ mark-sweep.h | 38 ++++++++++++----------------- parallel-marker.h | 14 ++++++----- semi.h | 55 ++++++++++++++++++------------------------ serial-marker.h | 15 ++++++------ 10 files changed, 132 insertions(+), 101 deletions(-) create mode 100644 gc.h create mode 100644 gcbench-types.h rename GCBench.c => gcbench.c (90%) create mode 100644 inline.h diff --git a/Makefile b/Makefile index 04a23ed6c..2846749e7 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -TESTS=GCBench # MT_GCBench MT_GCBench2 +TESTS=gcbench # MT_GCBench MT_GCBench2 COLLECTORS=bdw semi mark-sweep parallel-mark-sweep CC=gcc diff --git a/bdw.h b/bdw.h index 7a5538d40..51adea161 100644 --- a/bdw.h +++ b/bdw.h @@ -16,19 +16,16 @@ struct context {}; -enum alloc_kind { NODE, DOUBLE_ARRAY }; - -typedef void (*field_visitor)(struct context *, void **ref); - #define GC_HEADER /**/ static inline void* allocate(struct context *cx, enum alloc_kind kind, size_t size) { - // memset to 0 by the collector. switch (kind) { - case NODE: + case ALLOC_KIND_NODE: + // cleared to 0 by the collector. return GC_malloc(size); - case DOUBLE_ARRAY: + case ALLOC_KIND_DOUBLE_ARRAY: + // warning: not cleared! return GC_malloc_atomic(size); } abort(); diff --git a/gc.h b/gc.h new file mode 100644 index 000000000..2c0c59de0 --- /dev/null +++ b/gc.h @@ -0,0 +1,17 @@ +#ifndef GC_H_ +#define GC_H_ + +#if defined(GC_BDW) +#include "bdw.h" +#elif defined(GC_SEMI) +#include "semi.h" +#elif defined(GC_MARK_SWEEP) +#include "mark-sweep.h" +#elif defined(GC_PARALLEL_MARK_SWEEP) +#define GC_PARALLEL_MARK 1 +#include "mark-sweep.h" +#else +#error unknown gc +#endif + +#endif // GC_H_ diff --git a/gcbench-types.h b/gcbench-types.h new file mode 100644 index 000000000..20cef8be4 --- /dev/null +++ b/gcbench-types.h @@ -0,0 +1,30 @@ +#ifndef GCBENCH_TYPES_H +#define GCBENCH_TYPES_H + +#include "inline.h" + +#define FOR_EACH_HEAP_OBJECT_KIND(M) \ + M(node, Node, NODE) \ + M(double_array, DoubleArray, DOUBLE_ARRAY) + +#define DECLARE_NODE_TYPE(name, Name, NAME) \ + struct Name; \ + typedef struct Name Name; +FOR_EACH_HEAP_OBJECT_KIND(DECLARE_NODE_TYPE) +#undef DECLARE_NODE_TYPE + +#define DEFINE_ENUM(name, Name, NAME) ALLOC_KIND_##NAME, +enum alloc_kind { + FOR_EACH_HEAP_OBJECT_KIND(DEFINE_ENUM) +}; +#undef DEFINE_ENUM + +#define DEFINE_METHODS(name, Name, NAME) \ + static inline size_t name##_size(Name *obj) ALWAYS_INLINE; \ + static inline void visit_##name##_fields(Name *obj,\ + void (*visit)(void **loc, void *visit_data), \ + void *visit_data) ALWAYS_INLINE; +FOR_EACH_HEAP_OBJECT_KIND(DEFINE_METHODS) +#undef DEFINE_METHODS + +#endif // GCBENCH_TYPES_H diff --git a/GCBench.c b/gcbench.c similarity index 90% rename from GCBench.c rename to gcbench.c index f229de866..6d8acd48a 100644 --- a/GCBench.c +++ b/gcbench.c @@ -43,19 +43,8 @@ #include #include "assert.h" - -#if defined(GC_BDW) -#include "bdw.h" -#elif defined(GC_SEMI) -#include "semi.h" -#elif defined(GC_MARK_SWEEP) -#include "mark-sweep.h" -#elif defined(GC_PARALLEL_MARK_SWEEP) -#define GC_PARALLEL_MARK 1 -#include "mark-sweep.h" -#else -#error unknown gc -#endif +#include "gcbench-types.h" +#include "gc.h" static const int kStretchTreeDepth = 18; // about 16Mb static const int kLongLivedTreeDepth = 16; // about 4Mb @@ -76,21 +65,23 @@ typedef struct DoubleArray { double values[0]; } DoubleArray; -static inline size_t node_size(void *obj) { +static inline size_t node_size(Node *obj) { return sizeof(Node); } -static inline size_t double_array_size(void *obj) { - DoubleArray *array = obj; +static inline size_t double_array_size(DoubleArray *array) { return sizeof(*array) + array->length * sizeof(double); } -static inline void visit_node_fields(struct context *cx, void *obj, - field_visitor visit) { - Node *node = obj; - visit(cx, (void**)&node->left); - visit(cx, (void**)&node->right); +static inline void +visit_node_fields(Node *node, + void (*visit)(void **loc, void *visit_data), + void *visit_data) { + visit((void**)&node->left, visit_data); + visit((void**)&node->right, visit_data); } -static inline void visit_double_array_fields(struct context *cx, void *obj, - field_visitor visit) { +static inline void +visit_double_array_fields(DoubleArray *obj, + void (*visit)(void **loc, void *visit_data), + void *visit_data) { } typedef HANDLE_TO(Node) NodeHandle; @@ -98,13 +89,14 @@ typedef HANDLE_TO(DoubleArray) DoubleArrayHandle; static Node* allocate_node(struct context *cx) { // memset to 0 by the collector. - return allocate(cx, NODE, sizeof (Node)); + return allocate(cx, ALLOC_KIND_NODE, sizeof (Node)); } static struct DoubleArray* allocate_double_array(struct context *cx, size_t size) { - // note, not memset to 0 by the collector. - DoubleArray *ret = allocate(cx, DOUBLE_ARRAY, sizeof (double) * size); + // note, we might allow the collector to leave this data uninitialized. + DoubleArray *ret = allocate(cx, ALLOC_KIND_DOUBLE_ARRAY, + sizeof(DoubleArray) + sizeof (double) * size); ret->length = size; return ret; } diff --git a/inline.h b/inline.h new file mode 100644 index 000000000..4e44690f5 --- /dev/null +++ b/inline.h @@ -0,0 +1,7 @@ +#ifndef INLINE_H +#define INLINE_H + +#define ALWAYS_INLINE __attribute__((always_inline)) +#define NEVER_INLINE __attribute__((noinline)) + +#endif // INLINE_H diff --git a/mark-sweep.h b/mark-sweep.h index f6b000fc2..9c586da63 100644 --- a/mark-sweep.h +++ b/mark-sweep.h @@ -6,6 +6,7 @@ #include "assert.h" #include "debug.h" +#include "inline.h" #include "precise-roots.h" #ifdef GC_PARALLEL_MARK #include "parallel-marker.h" @@ -162,20 +163,11 @@ get_small_object_freelist(struct context *cx, enum small_object_size kind) { #define GC_HEADER uintptr_t _gc_header -enum alloc_kind { NODE, DOUBLE_ARRAY }; - -typedef void (*field_visitor)(struct context *, void **ref); - -static inline size_t node_size(void *obj) __attribute__((always_inline)); -static inline size_t double_array_size(void *obj) __attribute__((always_inline)); -static inline void visit_node_fields(struct context *cx, void *obj, field_visitor visit) __attribute__((always_inline)); -static inline void visit_double_array_fields(struct context *cx, void *obj, field_visitor visit) __attribute__((always_inline)); - static inline void clear_memory(uintptr_t addr, size_t size) { memset((char*)addr, 0, size); } -static void collect(struct context *cx) __attribute__((noinline)); +static void collect(struct context *cx) NEVER_INLINE; static inline uint8_t* mark_byte(struct context *cx, struct gcobj *obj) { uintptr_t granule = (((uintptr_t) obj) - cx->heap_base) / GRANULE_SIZE; @@ -193,12 +185,12 @@ static inline int mark_object(struct context *cx, struct gcobj *obj) { static void process(struct context *cx, struct gcobj *obj) { switch (tag_live_alloc_kind(obj->tag)) { - case NODE: - visit_node_fields(cx, obj, marker_visit); - break; - case DOUBLE_ARRAY: - visit_double_array_fields(cx, obj, marker_visit); - break; +#define SCAN_OBJECT(name, Name, NAME) \ + case ALLOC_KIND_##NAME: \ + visit_##name##_fields((Name*)obj, marker_visit, cx); \ + break; + FOR_EACH_HEAP_OBJECT_KIND(SCAN_OBJECT) +#undef SCAN_OBJECT default: abort (); } @@ -215,7 +207,7 @@ static void collect(struct context *cx) { DEBUG("start collect #%ld:\n", cx->count); marker_prepare(cx); for (struct handle *h = cx->roots; h; h = h->next) - marker_visit_root(cx, &h->v); + marker_visit_root(&h->v, cx); marker_trace(cx, process); marker_release(cx); DEBUG("done marking\n"); @@ -301,12 +293,12 @@ static size_t live_object_granules(struct gcobj *obj) { return 1; size_t bytes; switch (tag_live_alloc_kind (obj->tag)) { - case NODE: - bytes = node_size(obj); - break; - case DOUBLE_ARRAY: - bytes = double_array_size(obj); - break; +#define COMPUTE_SIZE(name, Name, NAME) \ + case ALLOC_KIND_##NAME: \ + bytes = name##_size((Name*)obj); \ + break; + FOR_EACH_HEAP_OBJECT_KIND(COMPUTE_SIZE) +#undef COMPUTE_SIZE default: abort (); } diff --git a/parallel-marker.h b/parallel-marker.h index 8bfac725a..e8da3e567 100644 --- a/parallel-marker.h +++ b/parallel-marker.h @@ -7,6 +7,7 @@ #include "assert.h" #include "debug.h" +#include "inline.h" // The Chase-Lev work-stealing deque, as initially described in "Dynamic // Circular Work-Stealing Deque" (Chase and Lev, SPAA'05) @@ -236,22 +237,23 @@ static void marker_release(struct context *cx) { } struct gcobj; -static inline void marker_visit(struct context *cx, void **loc) __attribute__((always_inline)); +static inline void marker_visit(void **loc, void *mark_data) ALWAYS_INLINE; static inline void marker_trace(struct context *cx, void (*)(struct context *, struct gcobj *)) - __attribute__((always_inline)); + ALWAYS_INLINE; static inline int mark_object(struct context *cx, - struct gcobj *obj) __attribute__((always_inline)); + struct gcobj *obj) ALWAYS_INLINE; static inline void -marker_visit(struct context *cx, void **loc) { +marker_visit(void **loc, void *mark_data) { + struct context *cx = mark_data; struct gcobj *obj = *loc; if (obj && mark_object(cx, obj)) mark_deque_push(&context_marker(cx)->deque, (uintptr_t)obj); } static inline void -marker_visit_root(struct context *cx, void **loc) { - marker_visit(cx, loc); +marker_visit_root(void **loc, struct context *cx) { + marker_visit(loc, cx); } static inline void marker_trace(struct context *cx, diff --git a/semi.h b/semi.h index 37b9f4ef4..5fdec4a07 100644 --- a/semi.h +++ b/semi.h @@ -23,22 +23,13 @@ static uintptr_t align_up(uintptr_t addr, size_t align) { #define GC_HEADER uintptr_t _gc_header -enum alloc_kind { NODE, DOUBLE_ARRAY }; - -typedef void (*field_visitor)(struct context *, void **ref); - -static inline size_t node_size(void *obj) __attribute__((always_inline)); -static inline size_t double_array_size(void *obj) __attribute__((always_inline)); -static inline void visit_node_fields(struct context *cx, void *obj, field_visitor visit) __attribute__((always_inline)); -static inline void visit_double_array_fields(struct context *cx, void *obj, field_visitor visit) __attribute__((always_inline)); - static inline void clear_memory(uintptr_t addr, size_t size) { memset((char*)addr, 0, size); } -static void collect(struct context *cx, size_t bytes) __attribute__((noinline)); +static void collect(struct context *cx, size_t bytes) NEVER_INLINE; -static void process(struct context *cx, void **loc); +static void visit(void **loc, void *visit_data); static void flip(struct context *cx) { uintptr_t split = cx->base + (cx->size >> 1); @@ -55,12 +46,12 @@ static void flip(struct context *cx) { static void* copy(struct context *cx, uintptr_t kind, void *obj) { size_t size; switch (kind) { - case NODE: - size = node_size(obj); - break; - case DOUBLE_ARRAY: - size = double_array_size(obj); - break; +#define COMPUTE_SIZE(name, Name, NAME) \ + case ALLOC_KIND_##NAME: \ + size = name##_size(obj); \ + break; + FOR_EACH_HEAP_OBJECT_KIND(COMPUTE_SIZE) +#undef COMPUTE_SIZE default: abort (); } @@ -75,14 +66,12 @@ static uintptr_t scan(struct context *cx, uintptr_t grey) { void *obj = (void*)grey; uintptr_t kind = *(uintptr_t*) obj; switch (kind) { - case NODE: - visit_node_fields(cx, obj, process); - return grey + align_up (node_size(obj), ALIGNMENT); - break; - case DOUBLE_ARRAY: - visit_double_array_fields(cx, obj, process); - return grey + align_up (double_array_size(obj), ALIGNMENT); - break; +#define SCAN_OBJECT(name, Name, NAME) \ + case ALLOC_KIND_##NAME: \ + visit_##name##_fields((Name*)obj, visit, cx); \ + return grey + align_up(name##_size((Name*)obj), ALIGNMENT); + FOR_EACH_HEAP_OBJECT_KIND(SCAN_OBJECT) +#undef SCAN_OBJECT default: abort (); } @@ -91,15 +80,18 @@ static uintptr_t scan(struct context *cx, uintptr_t grey) { static void* forward(struct context *cx, void *obj) { uintptr_t header_word = *(uintptr_t*)obj; switch (header_word) { - case NODE: - case DOUBLE_ARRAY: +#define CASE_ALLOC_KIND(name, Name, NAME) \ + case ALLOC_KIND_##NAME: + FOR_EACH_HEAP_OBJECT_KIND(CASE_ALLOC_KIND) +#undef CASE_ALLOC_KIND return copy(cx, header_word, obj); default: return (void*)header_word; } } -static void process(struct context *cx, void **loc) { +static void visit(void **loc, void *visit_data) { + struct context *cx = visit_data; void *obj = *loc; if (obj != NULL) *loc = forward(cx, obj); @@ -109,7 +101,7 @@ static void collect(struct context *cx, size_t bytes) { flip(cx); uintptr_t grey = cx->hp; for (struct handle *h = cx->roots; h; h = h->next) - process(cx, &h->v); + visit(&h->v, cx); // fprintf(stderr, "pushed %zd bytes in roots\n", cx->hp - grey); while(grey < cx->hp) grey = scan(cx, grey); @@ -134,8 +126,9 @@ static inline void* allocate(struct context *cx, enum alloc_kind kind, void *ret = (void *)addr; uintptr_t *header_word = ret; *header_word = kind; - if (kind == NODE) - clear_memory(addr + sizeof(uintptr_t), size - sizeof(uintptr_t)); + // FIXME: Allow allocator to avoid initializing pointerless memory? + // if (kind == NODE) + clear_memory(addr + sizeof(uintptr_t), size - sizeof(uintptr_t)); return ret; } } diff --git a/serial-marker.h b/serial-marker.h index 755064372..8f7dec01d 100644 --- a/serial-marker.h +++ b/serial-marker.h @@ -49,7 +49,7 @@ mark_queue_put(struct mark_queue *q, size_t idx, uintptr_t x) { q->buf[idx & (q->size - 1)] = x; } -static int mark_queue_grow(struct mark_queue *q) __attribute__((noinline)); +static int mark_queue_grow(struct mark_queue *q) NEVER_INLINE; static int mark_queue_grow(struct mark_queue *q) { @@ -125,22 +125,23 @@ static void marker_release(struct context *cx) { } struct gcobj; -static inline void marker_visit(struct context *cx, void **loc) __attribute__((always_inline)); +static inline void marker_visit(void **loc, void *mark_data) ALWAYS_INLINE; static inline void marker_trace(struct context *cx, void (*)(struct context *, struct gcobj *)) - __attribute__((always_inline)); + ALWAYS_INLINE; static inline int mark_object(struct context *cx, - struct gcobj *obj) __attribute__((always_inline)); + struct gcobj *obj) ALWAYS_INLINE; static inline void -marker_visit(struct context *cx, void **loc) { +marker_visit(void **loc, void *mark_data) { + struct context *cx = mark_data; struct gcobj *obj = *loc; if (obj && mark_object(cx, obj)) mark_queue_push(&context_marker(cx)->queue, obj); } static inline void -marker_visit_root(struct context *cx, void **loc) { - marker_visit(cx, loc); +marker_visit_root(void **loc, struct context *cx) { + marker_visit(loc, cx); } static inline void marker_trace(struct context *cx,