diff --git a/Makefile b/Makefile index 94d25606d..6f5652ad5 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -TESTS=quads mt-gcbench # MT_GCBench MT_GCBench2 +TESTS=quads mt-gcbench ephemerons # MT_GCBench MT_GCBench2 COLLECTORS= \ bdw \ semi \ @@ -19,8 +19,16 @@ COLLECTORS= \ stack-conservative-parallel-generational-whippet \ heap-conservative-parallel-generational-whippet +DEFAULT_BUILD:=opt + +BUILD_CFLAGS_opt=-O2 -g -DNDEBUG +BUILD_CFLAGS_optdebug=-Og -g -DGC_DEBUG=1 +BUILD_CFLAGS_debug=-O0 -g -DGC_DEBUG=1 + +BUILD_CFLAGS=$(BUILD_CFLAGS_$(or $(BUILD),$(DEFAULT_BUILD))) + CC=gcc -CFLAGS=-Wall -O2 -g -flto -fno-strict-aliasing -fvisibility=hidden -Wno-unused -DNDEBUG +CFLAGS=-Wall -flto -fno-strict-aliasing -fvisibility=hidden -Wno-unused $(BUILD_CFLAGS) INCLUDES=-I. LDFLAGS=-lpthread -flto COMPILE=$(CC) $(CFLAGS) $(INCLUDES) @@ -36,92 +44,106 @@ gc-platform.o: gc-platform.h gc-platform-$(PLATFORM).c gc-visibility.h gc-stack.o: gc-stack.c $(COMPILE) -o $@ -c $< +gc-ephemeron-%.o: gc-ephemeron.c gc-ephemeron.h gc-ephemeron-internal.h %-embedder.h + $(COMPILE) -include $*-embedder.h -o $@ -c $< + bdw-%-gc.o: bdw.c %-embedder.h %.c $(COMPILE) -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 `pkg-config --cflags bdw-gc` -include $*-embedder.h -o $@ -c bdw.c bdw-%.o: bdw.c %.c $(COMPILE) -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 -include bdw-attrs.h -o $@ -c $*.c -bdw-%: bdw-%.o bdw-%-gc.o gc-stack.o gc-platform.o +bdw-%: bdw-%.o bdw-%-gc.o gc-stack.o gc-platform.o gc-ephemeron-%.o $(CC) $(LDFLAGS) `pkg-config --libs bdw-gc` -o $@ $^ semi-%-gc.o: semi.c %-embedder.h large-object-space.h assert.h debug.h %.c $(COMPILE) -DGC_PRECISE_ROOTS=1 -include $*-embedder.h -o $@ -c semi.c semi-%.o: semi.c %.c $(COMPILE) -DGC_PRECISE_ROOTS=1 -include semi-attrs.h -o $@ -c $*.c +semi-%: semi-%.o semi-%-gc.o gc-stack.o gc-platform.o gc-ephemeron-%.o + $(CC) $(LDFLAGS) -o $@ $^ whippet-%-gc.o: whippet.c %-embedder.h large-object-space.h serial-tracer.h assert.h debug.h heap-objects.h %.c $(COMPILE) -DGC_PRECISE_ROOTS=1 -include $*-embedder.h -o $@ -c whippet.c whippet-%.o: whippet.c %.c $(COMPILE) -DGC_PRECISE_ROOTS=1 -include whippet-attrs.h -o $@ -c $*.c +whippet-%: whippet-%.o whippet-%-gc.o gc-stack.o gc-platform.o gc-ephemeron-%.o + $(CC) $(LDFLAGS) -o $@ $^ stack-conservative-whippet-%-gc.o: whippet.c %-embedder.h large-object-space.h serial-tracer.h assert.h debug.h heap-objects.h %.c $(COMPILE) -DGC_CONSERVATIVE_ROOTS=1 -include $*-embedder.h -o $@ -c whippet.c stack-conservative-whippet-%.o: whippet.c %.c $(COMPILE) -DGC_CONSERVATIVE_ROOTS=1 -include whippet-attrs.h -o $@ -c $*.c +stack-conservative-whippet-%: stack-conservative-whippet-%.o stack-conservative-whippet-%-gc.o gc-stack.o gc-platform.o gc-ephemeron-%.o + $(CC) $(LDFLAGS) -o $@ $^ heap-conservative-whippet-%-gc.o: whippet.c %-embedder.h large-object-space.h serial-tracer.h assert.h debug.h heap-objects.h %.c $(COMPILE) -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 -include $*-embedder.h -o $@ -c whippet.c heap-conservative-whippet-%.o: whippet.c %.c $(COMPILE) -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 -include whippet-attrs.h -o $@ -c $*.c +heap-conservative-whippet-%: heap-conservative-whippet-%.o heap-conservative-whippet-%-gc.o gc-stack.o gc-platform.o gc-ephemeron-%.o + $(CC) $(LDFLAGS) -o $@ $^ parallel-whippet-%-gc.o: whippet.c %-embedder.h large-object-space.h parallel-tracer.h assert.h debug.h heap-objects.h %.c $(COMPILE) -DGC_PARALLEL=1 -DGC_PRECISE_ROOTS=1 -include $*-embedder.h -o $@ -c whippet.c parallel-whippet-%.o: whippet.c %.c $(COMPILE) -DGC_PARALLEL=1 -DGC_PRECISE_ROOTS=1 -include whippet-attrs.h -o $@ -c $*.c +parallel-whippet-%: parallel-whippet-%.o parallel-whippet-%-gc.o gc-stack.o gc-platform.o gc-ephemeron-%.o + $(CC) $(LDFLAGS) -o $@ $^ stack-conservative-parallel-whippet-%-gc.o: whippet.c %-embedder.h large-object-space.h serial-tracer.h assert.h debug.h heap-objects.h %.c $(COMPILE) -DGC_PARALLEL=1 -DGC_CONSERVATIVE_ROOTS=1 -include $*-embedder.h -o $@ -c whippet.c stack-conservative-parallel-whippet-%.o: whippet.c %.c $(COMPILE) -DGC_PARALLEL=1 -DGC_CONSERVATIVE_ROOTS=1 -include whippet-attrs.h -o $@ -c $*.c +stack-conservative-parallel-whippet-%: stack-conservative-parallel-whippet-%.o stack-conservative-parallel-whippet-%-gc.o gc-stack.o gc-platform.o gc-ephemeron-%.o + $(CC) $(LDFLAGS) -o $@ $^ heap-conservative-parallel-whippet-%-gc.o: whippet.c %-embedder.h large-object-space.h serial-tracer.h assert.h debug.h heap-objects.h %.c $(COMPILE) -DGC_PARALLEL=1 -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 -include $*-embedder.h -o $@ -c whippet.c heap-conservative-parallel-whippet-%.o: whippet.c %.c $(COMPILE) -DGC_PARALLEL=1 -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 -DGC_FULLY_CONSERVATIVE=1 -include whippet-attrs.h -o $@ -c $*.c +heap-conservative-parallel-whippet-%: heap-conservative-parallel-whippet-%.o heap-conservative-parallel-whippet-%-gc.o gc-stack.o gc-platform.o gc-ephemeron-%.o + $(CC) $(LDFLAGS) -o $@ $^ generational-whippet-%-gc.o: whippet.c %-embedder.h large-object-space.h serial-tracer.h assert.h debug.h heap-objects.h %.c $(COMPILE) -DGC_GENERATIONAL=1 -DGC_PRECISE_ROOTS=1 -include $*-embedder.h -o $@ -c whippet.c generational-whippet-%.o: whippet.c %.c $(COMPILE) -DGC_GENERATIONAL=1 -DGC_PRECISE_ROOTS=1 -include whippet-attrs.h -o $@ -c $*.c +generational-whippet-%: generational-whippet-%.o generational-whippet-%-gc.o gc-stack.o gc-platform.o gc-ephemeron-%.o + $(CC) $(LDFLAGS) -o $@ $^ stack-conservative-generational-whippet-%-gc.o: whippet.c %-embedder.h large-object-space.h serial-tracer.h assert.h debug.h heap-objects.h %.c $(COMPILE) -DGC_GENERATIONAL=1 -DGC_CONSERVATIVE_ROOTS=1 -include $*-embedder.h -o $@ -c whippet.c stack-conservative-generational-whippet-%.o: whippet.c %.c $(COMPILE) -DGC_GENERATIONAL=1 -DGC_CONSERVATIVE_ROOTS=1 -include whippet-attrs.h -o $@ -c $*.c +stack-conservative-generational-whippet-%: stack-conservative-generational-whippet-%.o stack-conservative-generational-whippet-%-gc.o gc-stack.o gc-platform.o gc-ephemeron-%.o + $(CC) $(LDFLAGS) -o $@ $^ heap-conservative-generational-whippet-%-gc.o: whippet.c %-embedder.h large-object-space.h serial-tracer.h assert.h debug.h heap-objects.h %.c $(COMPILE) -DGC_GENERATIONAL=1 -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 -include $*-embedder.h -o $@ -c whippet.c heap-conservative-generational-whippet-%.o: whippet.c %.c $(COMPILE) -DGC_GENERATIONAL=1 -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 -include whippet-attrs.h -o $@ -c $*.c +heap-conservative-generational-whippet-%: heap-conservative-generational-whippet-%.o heap-conservative-generational-whippet-%-gc.o gc-stack.o gc-platform.o gc-ephemeron-%.o + $(CC) $(LDFLAGS) -o $@ $^ parallel-generational-whippet-%-gc.o: whippet.c %-embedder.h large-object-space.h parallel-tracer.h assert.h debug.h heap-objects.h %.c $(COMPILE) -DGC_PARALLEL=1 -DGC_GENERATIONAL=1 -DGC_PRECISE_ROOTS=1 -include $*-embedder.h -o $@ -c whippet.c parallel-generational-whippet-%.o: whippet.c %.c $(COMPILE) -DGC_PARALLEL=1 -DGC_GENERATIONAL=1 -DGC_PRECISE_ROOTS=1 -include whippet-attrs.h -o $@ -c $*.c +parallel-generational-whippet-%: parallel-generational-whippet-%.o parallel-generational-whippet-%-gc.o gc-stack.o gc-platform.o gc-ephemeron-%.o + $(CC) $(LDFLAGS) -o $@ $^ stack-conservative-parallel-generational-whippet-%-gc.o: whippet.c %-embedder.h large-object-space.h parallel-tracer.h assert.h debug.h heap-objects.h %.c $(COMPILE) -DGC_PARALLEL=1 -DGC_GENERATIONAL=1 -DGC_CONSERVATIVE_ROOTS=1 -include $*-embedder.h -o $@ -c whippet.c stack-conservative-parallel-generational-whippet-%.o: whippet.c %.c $(COMPILE) -DGC_PARALLEL=1 -DGC_GENERATIONAL=1 -DGC_CONSERVATIVE_ROOTS=1 -include whippet-attrs.h -o $@ -c $*.c +stack-conservative-parallel-generational-whippet-%: stack-conservative-parallel-generational-whippet-%.o stack-conservative-parallel-generational-whippet-%-gc.o gc-stack.o gc-platform.o gc-ephemeron-%.o + $(CC) $(LDFLAGS) -o $@ $^ heap-conservative-parallel-generational-whippet-%-gc.o: whippet.c %-embedder.h large-object-space.h parallel-tracer.h assert.h debug.h heap-objects.h %.c $(COMPILE) -DGC_PARALLEL=1 -DGC_GENERATIONAL=1 -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 -include $*-embedder.h -o $@ -c whippet.c heap-conservative-parallel-generational-whippet-%.o: whippet.c %.c $(COMPILE) -DGC_PARALLEL=1 -DGC_GENERATIONAL=1 -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 -include whippet-attrs.h -o $@ -c $*.c - -%: %.o %-gc.o gc-platform.o gc-stack.o - $(CC) $(LDFLAGS) $($*_LDFLAGS) -o $@ $^ - -check: $(addprefix test-$(TARGET),$(TARGETS)) - -test-%: $(ALL_TESTS) - @echo "Running unit tests..." - @set -e; for test in $?; do \ - echo "Testing: $$test"; \ - ./$$test; \ - done - @echo "Success." - -.PHONY: check +heap-conservative-parallel-generational-whippet-%: heap-conservative-parallel-generational-whippet-%.o heap-conservative-parallel-generational-whippet-%-gc.o gc-stack.o gc-platform.o gc-ephemeron-%.o + $(CC) $(LDFLAGS) -o $@ $^ .PRECIOUS: $(ALL_TESTS) diff --git a/bdw.c b/bdw.c index d4aac0a56..eb05c1b44 100644 --- a/bdw.c +++ b/bdw.c @@ -3,8 +3,11 @@ #include #include -#define GC_API_ #include "gc-api.h" +#include "gc-ephemeron.h" + +#define GC_IMPL 1 +#include "gc-internal.h" #include "bdw-attrs.h" @@ -34,6 +37,7 @@ #include #include /* GC_generic_malloc_many */ +#include /* GC_generic_malloc */ #define GC_INLINE_GRANULE_WORDS 2 #define GC_INLINE_GRANULE_BYTES (sizeof(void *) * GC_INLINE_GRANULE_WORDS) @@ -120,6 +124,85 @@ void gc_collect(struct gc_mutator *mut) { GC_gcollect(); } +// In BDW-GC, we can't hook into the mark phase to call +// gc_trace_ephemerons_for_object, so the advertised ephemeron strategy +// doesn't really work. The primitives that we have are mark functions, +// which run during GC and can't allocate; finalizers, which run after +// GC and can allocate but can't add to the connectivity graph; and +// disappearing links, which are cleared at the end of marking, in the +// stop-the-world phase. It does not appear to be possible to implement +// ephemerons using these primitives. Instead fall back to weak-key +// tables. + +static int ephemeron_gc_kind; + +struct gc_ref gc_allocate_ephemeron(struct gc_mutator *mut) { + void *ret = GC_generic_malloc(gc_ephemeron_size(), ephemeron_gc_kind); + return gc_ref_from_heap_object(ret); +} + +unsigned gc_heap_ephemeron_trace_epoch(struct gc_heap *heap) { + return 0; +} + +void gc_ephemeron_init(struct gc_mutator *mut, struct gc_ephemeron *ephemeron, + struct gc_ref key, struct gc_ref value) { + gc_ephemeron_init_internal(mut->heap, ephemeron, key, value); + if (GC_base((void*)gc_ref_value(key))) { + struct gc_ref *loc = gc_edge_loc(gc_ephemeron_key_edge(ephemeron)); + GC_register_disappearing_link((void**)loc); + } +} + +struct ephemeron_mark_state { + struct GC_ms_entry *mark_stack_ptr; + struct GC_ms_entry *mark_stack_limit; +}; + +int gc_visit_ephemeron_key(struct gc_edge edge, struct gc_heap *heap) { + // Pretend the key is traced, to avoid adding this ephemeron to the + // global table. + return 1; +} +static void trace_ephemeron_edge(struct gc_edge edge, struct gc_heap *heap, + void *visit_data) { + struct ephemeron_mark_state *state = visit_data; + uintptr_t addr = gc_ref_value(gc_edge_ref(edge)); + state->mark_stack_ptr = GC_MARK_AND_PUSH ((void *) addr, + state->mark_stack_ptr, + state->mark_stack_limit, + NULL); +} + +static struct GC_ms_entry * +mark_ephemeron(GC_word *addr, struct GC_ms_entry *mark_stack_ptr, + struct GC_ms_entry *mark_stack_limit, GC_word env) { + + struct ephemeron_mark_state state = { + mark_stack_ptr, + mark_stack_limit, + }; + + struct gc_ephemeron *ephemeron = (struct gc_ephemeron*) addr; + + // If this ephemeron is on a freelist, its first word will be a + // freelist link and everything else will be NULL. + if (!gc_ref_value(gc_edge_ref(gc_ephemeron_value_edge(ephemeron)))) { + trace_ephemeron_edge(gc_edge(addr), NULL, &state); + return state.mark_stack_ptr; + } + + if (!gc_ref_value(gc_edge_ref(gc_ephemeron_key_edge(ephemeron)))) { + // If the key died in a previous collection, the disappearing link + // will have been cleared. Mark the ephemeron as dead. + gc_ephemeron_mark_dead(ephemeron); + } + + gc_trace_ephemeron(ephemeron, trace_ephemeron_edge, NULL, &state); + + return state.mark_stack_ptr; +} + static inline struct gc_mutator *add_mutator(struct gc_heap *heap) { struct gc_mutator *ret = GC_malloc(sizeof(struct gc_mutator)); ret->heap = heap; @@ -224,6 +307,15 @@ int gc_init(int argc, struct gc_option argv[], *heap = GC_malloc(sizeof(struct gc_heap)); pthread_mutex_init(&(*heap)->lock, NULL); *mutator = add_mutator(*heap); + + { + GC_word descriptor = GC_MAKE_PROC(GC_new_proc(mark_ephemeron), 0); + int add_size_to_descriptor = 0; + int clear_memory = 1; + ephemeron_gc_kind = GC_new_kind(GC_new_free_list(), descriptor, + add_size_to_descriptor, clear_memory); + } + return 1; } diff --git a/ephemerons-embedder.h b/ephemerons-embedder.h new file mode 100644 index 000000000..5b17178cd --- /dev/null +++ b/ephemerons-embedder.h @@ -0,0 +1,54 @@ +#ifndef EPHEMERONS_EMBEDDER_H +#define EPHEMERONS_EMBEDDER_H + +#include + +#include "ephemerons-types.h" +#include "gc-ephemeron.h" + +struct gc_heap; + +#define DEFINE_METHODS(name, Name, NAME) \ + static inline size_t name##_size(Name *obj) GC_ALWAYS_INLINE; \ + static inline void visit_##name##_fields(Name *obj,\ + void (*visit)(struct gc_edge edge, \ + struct gc_heap *heap, \ + void *visit_data), \ + struct gc_heap *heap, \ + void *visit_data) GC_ALWAYS_INLINE; +FOR_EACH_HEAP_OBJECT_KIND(DEFINE_METHODS) +#undef DEFINE_METHODS + +static inline size_t small_object_size(SmallObject *obj) { return sizeof(*obj); } +static inline size_t ephemeron_size(Ephemeron *obj) { return gc_ephemeron_size(); } +static inline size_t box_size(Box *obj) { return sizeof(*obj); } + +static inline void +visit_small_object_fields(SmallObject *obj, + void (*visit)(struct gc_edge edge, struct gc_heap *heap, + void *visit_data), + struct gc_heap *heap, + void *visit_data) {} + +static inline void +visit_ephemeron_fields(Ephemeron *ephemeron, + void (*visit)(struct gc_edge edge, struct gc_heap *heap, + void *visit_data), + + struct gc_heap *heap, + void *visit_data) { + gc_trace_ephemeron((struct gc_ephemeron*)ephemeron, visit, heap, visit_data); +} + +static inline void +visit_box_fields(Box *box, + void (*visit)(struct gc_edge edge, struct gc_heap *heap, + void *visit_data), + struct gc_heap *heap, + void *visit_data) { + visit(gc_edge(&box->obj), heap, visit_data); +} + +#include "simple-gc-embedder.h" + +#endif // EPHEMERONS_EMBEDDER_H diff --git a/ephemerons-types.h b/ephemerons-types.h new file mode 100644 index 000000000..d2a4b9a5b --- /dev/null +++ b/ephemerons-types.h @@ -0,0 +1,21 @@ +#ifndef EPHEMERONS_TYPES_H +#define EPHEMERONS_TYPES_H + +#define FOR_EACH_HEAP_OBJECT_KIND(M) \ + M(box, Box, BOX) \ + M(ephemeron, Ephemeron, EPHEMERON) \ + M(small_object, SmallObject, SMALL_OBJECT) + +#include "heap-objects.h" +#include "simple-tagging-scheme.h" + +struct SmallObject { + struct gc_header header; +}; + +struct Box { + struct gc_header header; + void *obj; +}; + +#endif // EPHEMERONS_TYPES_H diff --git a/ephemerons.c b/ephemerons.c new file mode 100644 index 000000000..84fc308f2 --- /dev/null +++ b/ephemerons.c @@ -0,0 +1,270 @@ +#include +#include +#include +#include +#include +#include + +#include "assert.h" +#include "gc-api.h" +#include "gc-ephemeron.h" +#include "simple-roots-api.h" +#include "ephemerons-types.h" +#include "simple-allocator.h" + +typedef HANDLE_TO(SmallObject) SmallObjectHandle; +typedef HANDLE_TO(struct gc_ephemeron) EphemeronHandle; +typedef HANDLE_TO(Box) BoxHandle; + +static SmallObject* allocate_small_object(struct gc_mutator *mut) { + return gc_allocate_with_kind(mut, ALLOC_KIND_SMALL_OBJECT, sizeof(SmallObject)); +} + +static Box* allocate_box(struct gc_mutator *mut) { + return gc_allocate_with_kind(mut, ALLOC_KIND_BOX, sizeof(Box)); +} + +static struct gc_ephemeron* allocate_ephemeron(struct gc_mutator *mut) { + struct gc_ref ret = gc_allocate_ephemeron(mut); + *tag_word(ret) = tag_live(ALLOC_KIND_EPHEMERON); + return gc_ref_heap_object(ret); +} + +/* Get the current time in microseconds */ +static unsigned long current_time(void) +{ + struct timeval t; + if (gettimeofday(&t, NULL) == -1) + return 0; + return t.tv_sec * 1000 * 1000 + t.tv_usec; +} + +struct thread { + struct gc_mutator *mut; + struct gc_mutator_roots roots; +}; + +static void print_elapsed(const char *what, unsigned long start) { + unsigned long end = current_time(); + unsigned long msec = (end - start) / 1000; + unsigned long usec = (end - start) % 1000; + printf("Completed %s in %lu.%.3lu msec\n", what, msec, usec); +} + +struct call_with_gc_data { + void* (*f)(struct thread *); + struct gc_heap *heap; +}; +static void* call_with_gc_inner(struct gc_stack_addr *addr, void *arg) { + struct call_with_gc_data *data = arg; + struct gc_mutator *mut = gc_init_for_thread(addr, data->heap); + struct thread t = { mut, }; + gc_mutator_set_roots(mut, &t.roots); + void *ret = data->f(&t); + gc_finish_for_thread(mut); + return ret; +} +static void* call_with_gc(void* (*f)(struct thread *), + struct gc_heap *heap) { + struct call_with_gc_data data = { f, heap }; + return gc_call_with_stack_addr(call_with_gc_inner, &data); +} + +#define CHECK(x) \ + do { \ + if (!(x)) { \ + fprintf(stderr, "%s:%d: check failed: %s\n", __FILE__, __LINE__, #x); \ + exit(1); \ + } \ + } while (0) + +#define CHECK_EQ(x, y) CHECK((x) == (y)) +#define CHECK_NE(x, y) CHECK((x) != (y)) +#define CHECK_NULL(x) CHECK_EQ(x, NULL) +#define CHECK_NOT_NULL(x) CHECK_NE(x, NULL) + +static size_t ephemeron_chain_length(struct gc_ephemeron **loc, + SmallObject *key) { + struct gc_ephemeron *head = gc_ephemeron_chain_head(loc); + size_t len = 0; + while (head) { + CHECK_EQ(key, (SmallObject*)gc_ref_value(gc_ephemeron_key(head))); + Box *value = gc_ref_heap_object(gc_ephemeron_value(head)); + CHECK_NOT_NULL(value); + key = value->obj; + CHECK_NOT_NULL(key); + head = gc_ephemeron_chain_next(head); + len++; + } + return len; +} + +static double heap_size; +static double heap_multiplier; +static size_t nthreads; + +static void cause_gc(struct gc_mutator *mut) { + gc_collect(mut); +} + +static void make_ephemeron_chain(struct thread *t, EphemeronHandle *head, + SmallObjectHandle *head_key, size_t length) { + BoxHandle tail_box = { NULL }; + PUSH_HANDLE(t, tail_box); + + CHECK_NULL(HANDLE_REF(*head_key)); + HANDLE_SET(*head_key, allocate_small_object(t->mut)); + + for (size_t i = 0; i < length; i++) { + HANDLE_SET(tail_box, allocate_box(t->mut)); + HANDLE_REF(tail_box)->obj = HANDLE_REF(*head_key); + HANDLE_SET(*head_key, allocate_small_object(t->mut)); + struct gc_ephemeron *ephemeron = allocate_ephemeron(t->mut); + gc_ephemeron_init(t->mut, ephemeron, + gc_ref_from_heap_object(HANDLE_REF(*head_key)), + gc_ref_from_heap_object(HANDLE_REF(tail_box))); + gc_ephemeron_chain_push(HANDLE_LOC(*head), ephemeron); + } + + POP_HANDLE(t); +} + +static void* run_one_test(struct thread *t) { + size_t unit_size = gc_ephemeron_size() + sizeof(Box); + size_t list_length = heap_size / nthreads / heap_multiplier / unit_size; + + printf("Allocating ephemeron list %zu nodes long. Total size %.3fGB.\n", + list_length, list_length * unit_size / 1e9); + + unsigned long thread_start = current_time(); + + SmallObjectHandle head_key = { NULL }; + EphemeronHandle head = { NULL }; + + PUSH_HANDLE(t, head_key); + PUSH_HANDLE(t, head); + + make_ephemeron_chain(t, &head, &head_key, list_length); + + size_t measured_length = ephemeron_chain_length(HANDLE_LOC(head), + HANDLE_REF(head_key)); + CHECK_EQ(measured_length, list_length); + + cause_gc(t->mut); + measured_length = ephemeron_chain_length(HANDLE_LOC(head), + HANDLE_REF(head_key)); + CHECK_EQ(measured_length, list_length); + + if (!GC_CONSERVATIVE_ROOTS) { + HANDLE_SET(head_key, NULL); + cause_gc(t->mut); + measured_length = ephemeron_chain_length(HANDLE_LOC(head), + HANDLE_REF(head_key)); + CHECK_EQ(measured_length, 0); + } + + // swap head_key for a key halfway in, cause gc + // check length is expected half-length; warn, or error if precise + // clear and return + + print_elapsed("thread", thread_start); + + POP_HANDLE(t); + POP_HANDLE(t); + + return NULL; +} + +static void* run_one_test_in_thread(void *arg) { + struct gc_heap *heap = arg; + return call_with_gc(run_one_test, heap); +} + +struct join_data { int status; pthread_t thread; }; +static void *join_thread(void *data) { + struct join_data *join_data = data; + void *ret; + join_data->status = pthread_join(join_data->thread, &ret); + return ret; +} + +#define MAX_THREAD_COUNT 256 + +int main(int argc, char *argv[]) { + if (argc != 5) { + fprintf(stderr, "usage: %s HEAP_SIZE MULTIPLIER NTHREADS PARALLELISM\n", argv[0]); + return 1; + } + + heap_size = atof(argv[1]); + heap_multiplier = atof(argv[2]); + nthreads = atol(argv[3]); + size_t parallelism = atol(argv[4]); + + if (heap_size < 8192) { + fprintf(stderr, + "Heap size should probably be at least 8192, right? '%s'\n", + argv[1]); + return 1; + } + if (!(1.0 < heap_multiplier && heap_multiplier < 100)) { + fprintf(stderr, "Failed to parse heap multiplier '%s'\n", argv[2]); + return 1; + } + if (nthreads < 1 || nthreads > MAX_THREAD_COUNT) { + fprintf(stderr, "Expected integer between 1 and %d for thread count, got '%s'\n", + (int)MAX_THREAD_COUNT, argv[2]); + return 1; + } + if (parallelism < 1 || parallelism > MAX_THREAD_COUNT) { + fprintf(stderr, "Expected integer between 1 and %d for parallelism, got '%s'\n", + (int)MAX_THREAD_COUNT, argv[3]); + return 1; + } + + printf("Allocating heap of %.3fGB (%.2f multiplier of live data).\n", + heap_size / 1e9, heap_multiplier); + + struct gc_option options[] = { { GC_OPTION_FIXED_HEAP_SIZE, (size_t) heap_size }, + { GC_OPTION_PARALLELISM, parallelism } }; + struct gc_heap *heap; + struct gc_mutator *mut; + if (!gc_init(sizeof options / sizeof options[0], options, NULL, &heap, + &mut)) { + fprintf(stderr, "Failed to initialize GC with heap size %zu bytes\n", + (size_t)heap_size); + return 1; + } + struct thread main_thread = { mut, }; + gc_mutator_set_roots(mut, &main_thread.roots); + + unsigned long test_start = current_time(); + + pthread_t threads[MAX_THREAD_COUNT]; + // Run one of the threads in the main thread. + for (size_t i = 1; i < nthreads; i++) { + int status = pthread_create(&threads[i], NULL, run_one_test_in_thread, heap); + if (status) { + errno = status; + perror("Failed to create thread"); + return 1; + } + } + run_one_test(&main_thread); + for (size_t i = 1; i < nthreads; i++) { + struct join_data data = { 0, threads[i] }; + gc_call_without_gc(mut, join_thread, &data); + if (data.status) { + errno = data.status; + perror("Failed to join thread"); + return 1; + } + } + + print_elapsed("test", test_start); + + gc_print_stats(heap); + + return 0; +} + diff --git a/gc-api.h b/gc-api.h index bec889cf4..4ffee3fc7 100644 --- a/gc-api.h +++ b/gc-api.h @@ -26,11 +26,6 @@ struct gc_option { double value; }; -// FIXME: Conflict with bdw-gc GC_API. Switch prefix? -#ifndef GC_API_ -#define GC_API_ GC_INTERNAL -#endif - GC_API_ int gc_option_from_string(const char *str); struct gc_stack_addr; diff --git a/gc-edge.h b/gc-edge.h index cfd769c59..72d7b3e5b 100644 --- a/gc-edge.h +++ b/gc-edge.h @@ -13,6 +13,9 @@ static inline struct gc_edge gc_edge(void* addr) { static inline struct gc_ref gc_edge_ref(struct gc_edge edge) { return *edge.dst; } +static inline struct gc_ref* gc_edge_loc(struct gc_edge edge) { + return edge.dst; +} static inline void gc_edge_update(struct gc_edge edge, struct gc_ref ref) { *edge.dst = ref; } diff --git a/gc-embedder-api.h b/gc-embedder-api.h index e0a6b3b5a..8ae45ef61 100644 --- a/gc-embedder-api.h +++ b/gc-embedder-api.h @@ -1,8 +1,11 @@ #ifndef GC_EMBEDDER_API_H #define GC_EMBEDDER_API_H +#include + #include "gc-config.h" #include "gc-edge.h" +#include "gc-inline.h" #include "gc-forwarding.h" #ifndef GC_EMBEDDER_API @@ -13,6 +16,7 @@ struct gc_mutator_roots; struct gc_heap_roots; struct gc_atomic_forward; struct gc_heap; +struct gc_ephemeron; GC_EMBEDDER_API inline int gc_is_valid_conservative_ref_displacement(uintptr_t displacement); diff --git a/gc-ephemeron-internal.h b/gc-ephemeron-internal.h new file mode 100644 index 000000000..8894bbd8f --- /dev/null +++ b/gc-ephemeron-internal.h @@ -0,0 +1,51 @@ +#ifndef GC_EPHEMERON_INTERNAL_H +#define GC_EPHEMERON_INTERNAL_H + +#ifndef GC_IMPL +#error internal header file, not part of API +#endif + +#include "gc-ephemeron.h" + +struct gc_pending_ephemerons; + +// API implemented by collector, for use by ephemerons: +GC_INTERNAL int gc_visit_ephemeron_key(struct gc_edge edge, + struct gc_heap *heap); +GC_INTERNAL struct gc_pending_ephemerons* +gc_heap_pending_ephemerons(struct gc_heap *heap); +GC_INTERNAL unsigned gc_heap_ephemeron_trace_epoch(struct gc_heap *heap); + +// API implemented by ephemerons, for use by collector: +GC_INTERNAL struct gc_edge gc_ephemeron_key_edge(struct gc_ephemeron *eph); +GC_INTERNAL struct gc_edge gc_ephemeron_value_edge(struct gc_ephemeron *eph); + +GC_INTERNAL struct gc_pending_ephemerons* +gc_prepare_pending_ephemerons(struct gc_pending_ephemerons *state, + size_t target_size, double slop); + +GC_INTERNAL void +gc_resolve_pending_ephemerons(struct gc_ref obj, struct gc_heap *heap); + +GC_INTERNAL void +gc_scan_pending_ephemerons(struct gc_pending_ephemerons *state, + struct gc_heap *heap, size_t shard, + size_t nshards); + +GC_INTERNAL int +gc_pop_resolved_ephemerons(struct gc_heap *heap, + void (*visit)(struct gc_edge edge, + struct gc_heap *heap, + void *visit_data), + void *trace_data); + +GC_INTERNAL void +gc_sweep_pending_ephemerons(struct gc_pending_ephemerons *state, + size_t shard, size_t nshards); + +GC_INTERNAL void gc_ephemeron_init_internal(struct gc_heap *heap, + struct gc_ephemeron *ephemeron, + struct gc_ref key, + struct gc_ref value); + +#endif // GC_EPHEMERON_INTERNAL_H diff --git a/gc-ephemeron.c b/gc-ephemeron.c new file mode 100644 index 000000000..a13c4bb98 --- /dev/null +++ b/gc-ephemeron.c @@ -0,0 +1,582 @@ +#include +#include +#include + +#define GC_IMPL 1 + +#include "address-hash.h" +#include "debug.h" +#include "gc-embedder-api.h" +#include "gc-ephemeron-internal.h" + +// # Overview +// +// An ephemeron is a conjunction consisting of the ephemeron object +// itself, a "key" object, and a "value" object. If the ephemeron and +// the key are live, then the value is kept live and can be looked up +// given the ephemeron object. +// +// Sometimes we write this as E×K⇒V, indicating that you need both E and +// K to get V. We'll use this notation in these comments sometimes. +// +// The key and the value of an ephemeron are never modified, except +// possibly via forwarding during GC. +// +// If the key of an ephemeron ever becomes unreachable, the ephemeron +// object will be marked as dead by the collector, and neither key nor +// value will be accessible. Users can also explicitly mark an +// ephemeron as dead. +// +// Users can build collections of ephemerons by chaining them together. +// If an ephemeron ever becomes dead, the ephemeron will be removed from +// the chain by the garbage collector. +// +// # Tracing algorithm +// +// Tracing ephemerons is somewhat complicated. Tracing the live objects +// in a heap is usually a parallelizable fan-out kind of operation, +// requiring minimal synchronization between tracing worker threads. +// However with ephemerons, each worker thread may need to check if +// there is a pending ephemeron E for an object K, marking the +// associated V for later traversal by the tracer. Doing this without +// introducing excessive global serialization points is the motivation +// for the complications that follow. +// +// From the viewpoint of the garbage collector, an ephemeron E×K⇒V has 4 +// possible states: +// +// - Traced: An E that was already fully traced as of a given GC epoch. +// +// - Claimed: GC discovers E for the first time in a GC epoch +// +// - Pending: K's liveness is unknown +// +// - Resolved: K is live; V needs tracing +// +// The ephemeron state is kept in an atomic variable. The pending and +// resolved states also have associated atomic list link fields as well; +// it doesn't appear possible to coalesce them into a single field +// without introducing serialization. Finally, there is a bit to +// indicate whether a "traced" ephemeron is live or dead, and a field to +// indicate the epoch at which it was last traced. +// +// Here is a diagram of the state transitions: +// +// ,----->Traced<-----. +// , | | . +// , v / . +// | Claimed | +// | ,-----/ \---. | +// | v v | +// Pending--------->Resolved +// +// Ephemerons are born in the traced state, for the current GC epoch. +// +// When the tracer sees an ephemeron E in the traced state it checks the +// epoch. If the epoch is up to date, E stays in the traced state and +// we are done. +// +// Otherwise, E transitions from traced to claimed. The thread that +// claims E is then responsible for resetting E's pending and resolved +// links, updating E's epoch, and tracing E's user-controlled chain +// link. +// +// If the claiming thread sees that E was already marked dead by a +// previous GC, or explicitly by the user, the ephemeron then +// transitions from back to traced, ready for the next epoch. +// +// If the claiming thread sees K to already be known to be live, then E +// is added to the global resolved set and E's state becomes resolved. +// +// Otherwise the claiming thread publishes K⇒E to the global pending +// ephemeron table, via the pending link, and E transitions to pending. +// +// A pending ephemeron is a link in a buckets-of-chains concurrent hash +// table. If its K is ever determined to be live, it becomes resolved, +// and is added to a global set of resolved ephemerons. At the end of +// GC, any ephemerons still pending are marked dead, transitioning their +// states to traced. +// +// Note that the claiming thread -- the one that publishes K⇒E to the +// global pending ephemeron table -- needs to re-check that K is still +// untraced after adding K⇒E to the pending table, and move to resolved +// if so. +// +// A resolved ephemeron needs its V to be traced. Incidentally its K +// also needs tracing, to relocate any forwarding pointer. The thread +// that pops an ephemeron from the resolved set is responsible for +// tracing and for moving E's state to traced. +// +// # Concurrency +// +// All operations on ephemerons are wait-free. Sometimes only one +// thread can make progress (for example for an ephemeron in the claimed +// state), but no thread will be stalled waiting on other threads to +// proceed. +// +// There is one interesting (from a concurrency point of view) data +// structure used by the implementation of ephemerons, the singly-linked +// list. Actually there are three of these; one is used as a stack and +// the other two is used as sets. +// +// The resolved set is implemented via a global `struct gc_ephemeron +// *resolved` variable. Resolving an ephemeron does an atomic push to +// this stack, via compare-and-swap (CAS); popping from the stack (also +// via CAS) yields an ephemeron for tracing. Ephemerons are added to +// the resolved set at most once per GC cycle, and the resolved set is +// empty outside of GC. +// +// The operations that are supported on atomic stacks are: +// +// push(LOC, E, OFFSET) -> void +// +// The user-visible chain link and the link for the pending ephemeron +// table are used to build atomic sets. In these you can add an +// ephemeron to the beginning of the list, traverse the list link by +// link to the end (indicated by NULL), and remove any list item. +// Removing a list node proceeds in two phases: one, you mark the node +// for removal, by changing the ephemeron's state; then, possibly on a +// subsequent traversal, any predecessor may forward its link past +// removed nodes. Because node values never change and nodes only go +// from live to dead, the live list tail can always be reached by any +// node, even from dead nodes. +// +// The operations that are supported on these atomic lists: +// +// push(LOC, E, OFFSET) -> void +// pop(LOC, OFFSET) -> ephemeron or null +// follow(LOC, OFFSET, STATE_OFFSET, LIVE_STATE) -> ephemeron or null +// +// These operations are all wait-free. The "push" operation is shared +// between stack and set use cases. "pop" is for stack-like use cases. +// The "follow" operation traverses a list, opportunistically eliding +// nodes that have been marked dead, atomically updating the location +// storing the next item. +// +// There are also accessors on ephemerons to their fields: +// +// key(E) -> value or null +// value(E) -> value or null +// +// These operations retrieve the key and value, respectively, provided +// that the ephemeron is not marked dead. + +//////////////////////////////////////////////////////////////////////// +// Concurrent operations on ephemeron lists +//////////////////////////////////////////////////////////////////////// + +static void +ephemeron_list_push(struct gc_ephemeron **loc, + struct gc_ephemeron *head, + struct gc_ephemeron** (*get_next)(struct gc_ephemeron*)) { + struct gc_ephemeron *tail = atomic_load_explicit(loc, memory_order_acquire); + while (1) { + // There must be no concurrent readers of HEAD, a precondition that + // we ensure by only publishing HEAD to LOC at most once per cycle. + // Therefore we can use a normal store for the tail pointer. + *get_next(head) = tail; + if (atomic_compare_exchange_weak(loc, &tail, head)) + break; + } +} + +static struct gc_ephemeron* +ephemeron_list_pop(struct gc_ephemeron **loc, + struct gc_ephemeron** (*get_next)(struct gc_ephemeron*)) { + struct gc_ephemeron *head = atomic_load_explicit(loc, memory_order_acquire); + while (head) { + // Precondition: the result of get_next on an ephemeron is never + // updated concurrently; OK to load non-atomically. + struct gc_ephemeron *tail = *get_next(head); + if (atomic_compare_exchange_weak(loc, &head, tail)) + break; + } + return head; +} + +static struct gc_ephemeron* +ephemeron_list_follow(struct gc_ephemeron **loc, + struct gc_ephemeron** (*get_next)(struct gc_ephemeron*), + int (*is_live)(struct gc_ephemeron*)) { + struct gc_ephemeron *head = atomic_load_explicit(loc, memory_order_acquire); + + while (1) { + struct gc_ephemeron *new_head = head; + + // Skip past any dead nodes. + while (new_head && !is_live(new_head)) + new_head = atomic_load_explicit(get_next(new_head), memory_order_acquire); + + if (// If we didn't have to advance past any dead nodes, no need to + // update LOC. + (head == new_head) + // Otherwise if we succeed in updating LOC, we're done. + || atomic_compare_exchange_strong(loc, &head, new_head) + // Someone else managed to advance LOC; that's fine too. + || (head == new_head)) + return new_head; + + // Otherwise we lost a race; loop and retry. + } +} + +//////////////////////////////////////////////////////////////////////// +// The ephemeron object type +//////////////////////////////////////////////////////////////////////// + +#ifndef GC_EMBEDDER_EPHEMERON_HEADER +#error Embedder should define GC_EMBEDDER_EPHEMERON_HEADER +#endif + +enum { + EPHEMERON_STATE_TRACED, + EPHEMERON_STATE_CLAIMED, + EPHEMERON_STATE_PENDING, + EPHEMERON_STATE_RESOLVED, +}; + +struct gc_ephemeron { + GC_EMBEDDER_EPHEMERON_HEADER + uint8_t state; + uint8_t is_dead; + unsigned epoch; + struct gc_ephemeron *chain; + struct gc_ephemeron *pending; + struct gc_ephemeron *resolved; + struct gc_ref key; + struct gc_ref value; +}; + +size_t gc_ephemeron_size(void) { return sizeof(struct gc_ephemeron); } + +struct gc_edge gc_ephemeron_key_edge(struct gc_ephemeron *e) { + return gc_edge(&e->key); +} +struct gc_edge gc_ephemeron_value_edge(struct gc_ephemeron *e) { + return gc_edge(&e->value); +} + +//////////////////////////////////////////////////////////////////////// +// Operations on the user-controlled chain field +//////////////////////////////////////////////////////////////////////// + +static struct gc_ephemeron** ephemeron_chain(struct gc_ephemeron *e) { + return &e->chain; +} +static int ephemeron_is_dead(struct gc_ephemeron *e) { + return atomic_load_explicit(&e->is_dead, memory_order_acquire); +} +static int ephemeron_is_not_dead(struct gc_ephemeron *e) { + return !ephemeron_is_dead(e); +} + +void gc_ephemeron_chain_push(struct gc_ephemeron **loc, + struct gc_ephemeron *e) { + ephemeron_list_push(loc, e, ephemeron_chain); +} +static struct gc_ephemeron* follow_chain(struct gc_ephemeron **loc) { + return ephemeron_list_follow(loc, ephemeron_chain, ephemeron_is_not_dead); +} +struct gc_ephemeron* gc_ephemeron_chain_head(struct gc_ephemeron **loc) { + return follow_chain(loc); +} +struct gc_ephemeron* gc_ephemeron_chain_next(struct gc_ephemeron *e) { + return follow_chain(ephemeron_chain(e)); +} +void gc_ephemeron_mark_dead(struct gc_ephemeron *e) { + atomic_store_explicit(&e->is_dead, 1, memory_order_release); +} + +//////////////////////////////////////////////////////////////////////// +// Operations on the GC-managed pending link +//////////////////////////////////////////////////////////////////////// + +static struct gc_ephemeron** ephemeron_pending(struct gc_ephemeron *e) { + return &e->pending; +} +static uint8_t ephemeron_state(struct gc_ephemeron *e) { + return atomic_load_explicit(&e->state, memory_order_acquire); +} +static int ephemeron_is_pending(struct gc_ephemeron *e) { + return ephemeron_state(e) == EPHEMERON_STATE_PENDING; +} + +static void push_pending(struct gc_ephemeron **loc, struct gc_ephemeron *e) { + ephemeron_list_push(loc, e, ephemeron_pending); +} +static struct gc_ephemeron* follow_pending(struct gc_ephemeron **loc) { + return ephemeron_list_follow(loc, ephemeron_pending, ephemeron_is_pending); +} + +//////////////////////////////////////////////////////////////////////// +// Operations on the GC-managed resolved link +//////////////////////////////////////////////////////////////////////// + +static struct gc_ephemeron** ephemeron_resolved(struct gc_ephemeron *e) { + return &e->resolved; +} +static void push_resolved(struct gc_ephemeron **loc, struct gc_ephemeron *e) { + ephemeron_list_push(loc, e, ephemeron_resolved); +} +static struct gc_ephemeron* pop_resolved(struct gc_ephemeron **loc) { + return ephemeron_list_pop(loc, ephemeron_resolved); +} + +//////////////////////////////////////////////////////////////////////// +// Access to the association +//////////////////////////////////////////////////////////////////////// + +struct gc_ref gc_ephemeron_key(struct gc_ephemeron *e) { + return ephemeron_is_dead(e) ? gc_ref_null() : e->key; +} + +struct gc_ref gc_ephemeron_value(struct gc_ephemeron *e) { + return ephemeron_is_dead(e) ? gc_ref_null() : e->value; +} + +//////////////////////////////////////////////////////////////////////// +// Tracing ephemerons +//////////////////////////////////////////////////////////////////////// + +struct gc_pending_ephemerons { + struct gc_ephemeron* resolved; + size_t nbuckets; + double scale; + struct gc_ephemeron* buckets[0]; +}; + +static const size_t MIN_PENDING_EPHEMERONS_SIZE = 32; + +static size_t pending_ephemerons_byte_size(size_t nbuckets) { + return sizeof(struct gc_pending_ephemerons) + + sizeof(struct gc_ephemeron*) * nbuckets; +} + +static struct gc_pending_ephemerons* +gc_make_pending_ephemerons(size_t byte_size) { + size_t nbuckets = byte_size / sizeof(struct gc_ephemeron*); + if (nbuckets < MIN_PENDING_EPHEMERONS_SIZE) + nbuckets = MIN_PENDING_EPHEMERONS_SIZE; + + struct gc_pending_ephemerons *ret = + malloc(pending_ephemerons_byte_size(nbuckets)); + if (!ret) + return NULL; + + ret->resolved = NULL; + ret->nbuckets = nbuckets; + ret->scale = nbuckets / pow(2.0, sizeof(uintptr_t) * 8); + for (size_t i = 0; i < nbuckets; i++) + ret->buckets[i] = NULL; + + return ret; +} + +struct gc_pending_ephemerons* +gc_prepare_pending_ephemerons(struct gc_pending_ephemerons *state, + size_t target_byte_size, double slop) { + size_t existing = + state ? pending_ephemerons_byte_size(state->nbuckets) : 0; + slop += 1.0; + if (existing * slop > target_byte_size && existing < target_byte_size * slop) + return state; + + struct gc_pending_ephemerons *new_state = + gc_make_pending_ephemerons(target_byte_size); + + if (!new_state) + return state; + + free(state); + return new_state; +} + +static struct gc_ephemeron** +pending_ephemeron_bucket(struct gc_pending_ephemerons *state, + struct gc_ref ref) { + uintptr_t hash = hash_address(gc_ref_value(ref)); + size_t idx = hash * state->scale; + GC_ASSERT(idx < state->nbuckets); + return &state->buckets[idx]; +} + +static void +add_pending_ephemeron(struct gc_pending_ephemerons *state, + struct gc_ephemeron *e) { + struct gc_ephemeron **bucket = pending_ephemeron_bucket(state, e->key); + atomic_store_explicit(&e->state, EPHEMERON_STATE_PENDING, + memory_order_release); + push_pending(bucket, e); +} + +static void maybe_resolve_ephemeron(struct gc_pending_ephemerons *state, + struct gc_ephemeron *e) { + uint8_t expected = EPHEMERON_STATE_PENDING; + if (atomic_compare_exchange_strong(&e->state, &expected, + EPHEMERON_STATE_RESOLVED)) + push_resolved(&state->resolved, e); +} + +// Precondition: OBJ has already been copied to tospace, but OBJ is a +// fromspace ref. +void gc_resolve_pending_ephemerons(struct gc_ref obj, struct gc_heap *heap) { + struct gc_pending_ephemerons *state = gc_heap_pending_ephemerons(heap); + struct gc_ephemeron **bucket = pending_ephemeron_bucket(state, obj); + for (struct gc_ephemeron *link = follow_pending(bucket); + link; + link = follow_pending(&link->pending)) { + if (gc_ref_value(obj) == gc_ref_value(link->key)) { + gc_visit_ephemeron_key(gc_ephemeron_key_edge(link), heap); + // PENDING -> RESOLVED, if it was pending. + maybe_resolve_ephemeron(state, link); + } + } +} + +void gc_trace_ephemeron(struct gc_ephemeron *e, + void (*visit)(struct gc_edge edge, struct gc_heap *heap, + void *visit_data), + struct gc_heap *heap, + void *trace_data) { + unsigned epoch = gc_heap_ephemeron_trace_epoch(heap); + uint8_t expected = EPHEMERON_STATE_TRACED; + // TRACED[_] -> CLAIMED[_]. + if (!atomic_compare_exchange_strong(&e->state, &expected, + EPHEMERON_STATE_CLAIMED)) + return; + + + if (e->epoch == epoch) { + // CLAIMED[epoch] -> TRACED[epoch]. + atomic_store_explicit(&e->state, EPHEMERON_STATE_TRACED, + memory_order_release); + return; + } + + // CLAIMED[!epoch] -> CLAIMED[epoch]. + e->epoch = epoch; + e->pending = NULL; + e->resolved = NULL; + + // Trace chain successors, eliding any intermediate dead links. Note + // that there is a race between trace-time evacuation of the next link + // in the chain and any mutation of that link pointer by the mutator + // (which can only be to advance the chain forward past dead links). + // Collectors using this API have to eliminate this race, for example + // by not evacuating while the mutator is running. + follow_chain(&e->chain); + visit(gc_edge(&e->chain), heap, trace_data); + + // Similarly there is a race between the mutator marking an ephemeron + // as dead and here; the consequence would be that we treat an + // ephemeron as live when it's not, but only for this cycle. No big + // deal. + if (atomic_load_explicit(&e->is_dead, memory_order_acquire)) { + // CLAIMED[epoch] -> TRACED[epoch]. + atomic_store_explicit(&e->state, EPHEMERON_STATE_TRACED, + memory_order_release); + return; + } + + // If K is live, trace V and we are done. + if (gc_visit_ephemeron_key(gc_ephemeron_key_edge(e), heap)) { + visit(gc_ephemeron_value_edge(e), heap, trace_data); + // CLAIMED[epoch] -> TRACED[epoch]. + atomic_store_explicit(&e->state, EPHEMERON_STATE_TRACED, + memory_order_release); + return; + } + + // Otherwise K is not yet traced, so we don't know if it is live. + // Publish the ephemeron to a global table. + struct gc_pending_ephemerons *state = gc_heap_pending_ephemerons(heap); + // CLAIMED[epoch] -> PENDING. + add_pending_ephemeron(state, e); + + // Given an ephemeron E×K⇒V, there is a race between marking K and E. + // One thread could go to mark E and see that K is unmarked, so we get + // here. Meanwhile another thread could go to mark K and not see E in + // the global table yet. Therefore after publishing E, we have to + // check the mark on K again. + if (gc_visit_ephemeron_key(gc_ephemeron_key_edge(e), heap)) + // K visited by another thread while we published E; PENDING -> + // RESOLVED, if still PENDING. + maybe_resolve_ephemeron(state, e); +} + +void +gc_scan_pending_ephemerons(struct gc_pending_ephemerons *state, + struct gc_heap *heap, size_t shard, + size_t nshards) { + GC_ASSERT(shard < nshards); + size_t start = state->nbuckets * 1.0 * shard / nshards; + size_t end = state->nbuckets * 1.0 * (shard + 1) / nshards; + for (size_t idx = start; idx < end; idx++) { + for (struct gc_ephemeron *e = follow_pending(&state->buckets[idx]); + e; + e = follow_pending(&e->pending)) { + if (gc_visit_ephemeron_key(gc_ephemeron_key_edge(e), heap)) + // PENDING -> RESOLVED, if PENDING. + maybe_resolve_ephemeron(state, e); + } + } +} + +int +gc_pop_resolved_ephemerons(struct gc_heap *heap, + void (*visit)(struct gc_edge edge, + struct gc_heap *heap, + void *visit_data), + void *trace_data) { + struct gc_pending_ephemerons *state = gc_heap_pending_ephemerons(heap); + struct gc_ephemeron *resolved = atomic_exchange(&state->resolved, NULL); + if (!resolved) + return 0; + for (; resolved; resolved = resolved->resolved) { + visit(gc_ephemeron_value_edge(resolved), heap, trace_data); + // RESOLVED -> TRACED. + atomic_store_explicit(&resolved->state, EPHEMERON_STATE_TRACED, + memory_order_release); + } + return 1; +} + +void +gc_sweep_pending_ephemerons(struct gc_pending_ephemerons *state, + size_t shard, size_t nshards) { + GC_ASSERT(shard < nshards); + size_t start = state->nbuckets * 1.0 * shard / nshards; + size_t end = state->nbuckets * 1.0 * (shard + 1) / nshards; + for (size_t idx = start; idx < end; idx++) { + struct gc_ephemeron **bucket = &state->buckets[idx]; + for (struct gc_ephemeron *e = follow_pending(bucket); + e; + e = follow_pending(&e->pending)) { + // PENDING -> TRACED, but dead. + atomic_store_explicit(&e->is_dead, 1, memory_order_release); + atomic_store_explicit(&e->state, EPHEMERON_STATE_TRACED, + memory_order_release); + } + atomic_store_explicit(bucket, NULL, memory_order_release); + } +} + +//////////////////////////////////////////////////////////////////////// +// Allocation & initialization +//////////////////////////////////////////////////////////////////////// + +void gc_ephemeron_init_internal(struct gc_heap *heap, + struct gc_ephemeron *ephemeron, + struct gc_ref key, struct gc_ref value) { + // Caller responsible for any write barrier, though really the + // assumption is that the ephemeron is younger than the key and the + // value. + ephemeron->state = EPHEMERON_STATE_TRACED; + ephemeron->is_dead = 0; + ephemeron->epoch = gc_heap_ephemeron_trace_epoch(heap) - 1; + ephemeron->chain = NULL; + ephemeron->pending = NULL; + ephemeron->resolved = NULL; + ephemeron->key = key; + ephemeron->value = value; +} diff --git a/gc-ephemeron.h b/gc-ephemeron.h new file mode 100644 index 000000000..d5159dff3 --- /dev/null +++ b/gc-ephemeron.h @@ -0,0 +1,42 @@ +#ifndef GC_EPHEMERON_H_ +#define GC_EPHEMERON_H_ + +#include "gc-edge.h" +#include "gc-ref.h" +#include "gc-visibility.h" + +// Ephemerons establish an association between a "key" object and a +// "value" object. If the ephemeron and the key are live, then the +// value is live, and can be retrieved from the ephemeron. Ephemerons +// can be chained together, which allows them to function as links in a +// buckets-and-chains hash table. +// +// This file defines the user-facing API for ephemerons. + +struct gc_heap; +struct gc_mutator; +struct gc_ephemeron; + +GC_API_ size_t gc_ephemeron_size(void); +GC_API_ struct gc_ref gc_allocate_ephemeron(struct gc_mutator *mut); +GC_API_ void gc_ephemeron_init(struct gc_mutator *mut, + struct gc_ephemeron *ephemeron, + struct gc_ref key, struct gc_ref value); + +GC_API_ struct gc_ref gc_ephemeron_key(struct gc_ephemeron *ephemeron); +GC_API_ struct gc_ref gc_ephemeron_value(struct gc_ephemeron *ephemeron); + +GC_API_ struct gc_ephemeron* gc_ephemeron_chain_head(struct gc_ephemeron **loc); +GC_API_ void gc_ephemeron_chain_push(struct gc_ephemeron **loc, + struct gc_ephemeron *ephemeron); +GC_API_ struct gc_ephemeron* gc_ephemeron_chain_next(struct gc_ephemeron *ephemeron); +GC_API_ void gc_ephemeron_mark_dead(struct gc_ephemeron *ephemeron); + +GC_API_ void gc_trace_ephemeron(struct gc_ephemeron *ephemeron, + void (*visit)(struct gc_edge edge, + struct gc_heap *heap, + void *visit_data), + struct gc_heap *heap, + void *trace_data); + +#endif // GC_EPHEMERON_H_ diff --git a/gc-internal.h b/gc-internal.h new file mode 100644 index 000000000..f74336dc9 --- /dev/null +++ b/gc-internal.h @@ -0,0 +1,10 @@ +#ifndef GC_INTERNAL_H +#define GC_INTERNAL_H + +#ifndef GC_IMPL +#error internal header file, not part of API +#endif + +#include "gc-ephemeron-internal.h" + +#endif // GC_INTERNAL_H diff --git a/gc-visibility.h b/gc-visibility.h index 7360915a0..b7e1995df 100644 --- a/gc-visibility.h +++ b/gc-visibility.h @@ -4,4 +4,9 @@ #define GC_INTERNAL __attribute__((visibility("hidden"))) #define GC_PUBLIC __attribute__((visibility("default"))) +// FIXME: Conflict with bdw-gc GC_API. Switch prefix? +#ifndef GC_API_ +#define GC_API_ GC_INTERNAL +#endif + #endif // GC_VISIBILITY_H diff --git a/large-object-space.h b/large-object-space.h index ddd1bfcde..de41dea60 100644 --- a/large-object-space.h +++ b/large-object-space.h @@ -92,6 +92,16 @@ done: return copied; } +static int large_object_space_is_copied(struct large_object_space *space, + struct gc_ref ref) { + int copied = 0; + uintptr_t addr = gc_ref_value(ref); + pthread_mutex_lock(&space->lock); + copied = address_set_contains(&space->from_space, addr); + pthread_mutex_unlock(&space->lock); + return copied; +} + static int large_object_space_mark_object(struct large_object_space *space, struct gc_ref ref) { return large_object_space_copy(space, ref); diff --git a/semi.c b/semi.c index d9e1110cf..e7c2b59cd 100644 --- a/semi.c +++ b/semi.c @@ -5,9 +5,11 @@ #include #include -#define GC_API_ #include "gc-api.h" +#define GC_IMPL 1 +#include "gc-internal.h" + #include "semi-attrs.h" #include "large-object-space.h" @@ -24,11 +26,16 @@ struct semi_space { size_t stolen_pages; uintptr_t base; size_t size; - long count; }; struct gc_heap { struct semi_space semi_space; struct large_object_space large_object_space; + struct gc_pending_ephemerons *pending_ephemerons; + double pending_ephemerons_size_factor; + double pending_ephemerons_size_slop; + size_t size; + long count; + int check_pending_ephemerons; }; // One mutator per space, can just store the heap in the mutator. struct gc_mutator { @@ -96,16 +103,20 @@ static void flip(struct semi_space *space) { space->from_space = space->to_space; space->to_space = space->hp; space->limit = space->hp + space->size / 2; - space->count++; } -static struct gc_ref copy(struct semi_space *space, struct gc_ref ref) { +static struct gc_ref copy(struct gc_heap *heap, struct semi_space *space, + struct gc_ref ref) { size_t size; gc_trace_object(ref, NULL, NULL, NULL, &size); struct gc_ref new_ref = gc_ref(space->hp); memcpy(gc_ref_heap_object(new_ref), gc_ref_heap_object(ref), size); gc_object_forward_nonatomic(ref, new_ref); space->hp += align_up(size, GC_ALIGNMENT); + + if (GC_UNLIKELY(heap->check_pending_ephemerons)) + gc_resolve_pending_ephemerons(ref, heap); + return new_ref; } @@ -115,21 +126,26 @@ static uintptr_t scan(struct gc_heap *heap, struct gc_ref grey) { return gc_ref_value(grey) + align_up(size, GC_ALIGNMENT); } -static struct gc_ref forward(struct semi_space *space, struct gc_ref obj) { +static struct gc_ref forward(struct gc_heap *heap, struct semi_space *space, + struct gc_ref obj) { uintptr_t forwarded = gc_object_forwarded_nonatomic(obj); - return forwarded ? gc_ref(forwarded) : copy(space, obj); + return forwarded ? gc_ref(forwarded) : copy(heap, space, obj); } static void visit_semi_space(struct gc_heap *heap, struct semi_space *space, struct gc_edge edge, struct gc_ref ref) { - gc_edge_update(edge, forward(space, ref)); + gc_edge_update(edge, forward(heap, space, ref)); } static void visit_large_object_space(struct gc_heap *heap, struct large_object_space *space, struct gc_ref ref) { - if (large_object_space_copy(space, ref)) + if (large_object_space_copy(space, ref)) { + if (GC_UNLIKELY(heap->check_pending_ephemerons)) + gc_resolve_pending_ephemerons(ref, heap); + gc_trace_object(ref, trace, heap, NULL, NULL); + } } static int semi_space_contains(struct semi_space *space, struct gc_ref ref) { @@ -149,6 +165,26 @@ static void visit(struct gc_edge edge, struct gc_heap *heap) { GC_CRASH(); } +struct gc_pending_ephemerons * +gc_heap_pending_ephemerons(struct gc_heap *heap) { + return heap->pending_ephemerons; +} + +int gc_visit_ephemeron_key(struct gc_edge edge, struct gc_heap *heap) { + struct gc_ref ref = gc_edge_ref(edge); + GC_ASSERT(gc_ref_is_heap_object(ref)); + if (semi_space_contains(heap_semi_space(heap), ref)) { + uintptr_t forwarded = gc_object_forwarded_nonatomic(ref); + if (!forwarded) + return 0; + gc_edge_update(edge, gc_ref(forwarded)); + return 1; + } else if (large_object_space_contains(heap_large_object_space(heap), ref)) { + return large_object_space_is_copied(heap_large_object_space(heap), ref); + } + GC_CRASH(); +} + static void trace(struct gc_edge edge, struct gc_heap *heap, void *visit_data) { return visit(edge, heap); } @@ -160,14 +196,22 @@ static void collect(struct gc_mutator *mut) { // fprintf(stderr, "start collect #%ld:\n", space->count); large_object_space_start_gc(large, 0); flip(semi); + heap->count++; + heap->check_pending_ephemerons = 0; uintptr_t grey = semi->hp; if (mut->roots) gc_trace_mutator_roots(mut->roots, trace, heap, NULL); // fprintf(stderr, "pushed %zd bytes in roots\n", space->hp - grey); while(grey < semi->hp) grey = scan(heap, gc_ref(grey)); + gc_scan_pending_ephemerons(heap->pending_ephemerons, heap, 0, 1); + heap->check_pending_ephemerons = 1; + while (gc_pop_resolved_ephemerons(heap, trace, NULL)) + while(grey < semi->hp) + grey = scan(heap, gc_ref(grey)); large_object_space_finish_gc(large, 0); semi_space_set_stolen_pages(semi, large->live_pages_at_last_collection); + gc_sweep_pending_ephemerons(heap->pending_ephemerons, 0, 1); // fprintf(stderr, "%zd bytes copied\n", (space->size>>1)-(space->limit-space->hp)); } @@ -229,6 +273,15 @@ void* gc_allocate_pointerless(struct gc_mutator *mut, size_t size) { return gc_allocate(mut, size); } +struct gc_ref gc_allocate_ephemeron(struct gc_mutator *mut) { + return gc_ref_from_heap_object(gc_allocate(mut, gc_ephemeron_size())); +} + +void gc_ephemeron_init(struct gc_mutator *mut, struct gc_ephemeron *ephemeron, + struct gc_ref key, struct gc_ref value) { + gc_ephemeron_init_internal(mutator_heap(mut), ephemeron, key, value); +} + static int initialize_semi_space(struct semi_space *space, size_t size) { // Allocate even numbers of pages. size_t page_size = getpagesize(); @@ -246,7 +299,6 @@ static int initialize_semi_space(struct semi_space *space, size_t size) { space->page_size = page_size; space->stolen_pages = 0; space->size = size; - space->count = 0; return 1; } @@ -315,6 +367,29 @@ static int parse_options(int argc, struct gc_option argv[], return 1; } +static int heap_prepare_pending_ephemerons(struct gc_heap *heap) { + struct gc_pending_ephemerons *cur = heap->pending_ephemerons; + size_t target = heap->size * heap->pending_ephemerons_size_factor; + double slop = heap->pending_ephemerons_size_slop; + + heap->pending_ephemerons = gc_prepare_pending_ephemerons(cur, target, slop); + + return !!heap->pending_ephemerons; +} + +unsigned gc_heap_ephemeron_trace_epoch(struct gc_heap *heap) { + return heap->count; +} + +static int heap_init(struct gc_heap *heap, size_t size) { + heap->pending_ephemerons_size_factor = 0.01; + heap->pending_ephemerons_size_slop = 0.5; + heap->count = 0; + heap->size = size; + + return heap_prepare_pending_ephemerons(heap); +} + int gc_init(int argc, struct gc_option argv[], struct gc_stack_addr *stack_base, struct gc_heap **heap, struct gc_mutator **mut) { @@ -331,6 +406,9 @@ int gc_init(int argc, struct gc_option argv[], if (!*mut) GC_CRASH(); *heap = mutator_heap(*mut); + if (!heap_init(*heap, options.fixed_heap_size)) + return 0; + struct semi_space *space = mutator_semi_space(*mut); if (!initialize_semi_space(space, options.fixed_heap_size)) return 0; @@ -367,7 +445,6 @@ void* gc_call_without_gc(struct gc_mutator *mut, void* (*f)(void*), } void gc_print_stats(struct gc_heap *heap) { - struct semi_space *space = heap_semi_space(heap); - printf("Completed %ld collections\n", space->count); - printf("Heap size is %zd\n", space->size); + printf("Completed %ld collections\n", heap->count); + printf("Heap size is %zd\n", heap->size); } diff --git a/simple-gc-embedder.h b/simple-gc-embedder.h index 758e56462..70fd5c7a8 100644 --- a/simple-gc-embedder.h +++ b/simple-gc-embedder.h @@ -5,6 +5,8 @@ #include "gc-config.h" #include "gc-embedder-api.h" +#define GC_EMBEDDER_EPHEMERON_HEADER struct gc_header header; + static inline int gc_is_valid_conservative_ref_displacement(uintptr_t displacement) { #if GC_CONSERVATIVE_ROOTS || GC_CONSERVATIVE_TRACE diff --git a/simple-roots-api.h b/simple-roots-api.h index 1cdfc15e0..d94397adf 100644 --- a/simple-roots-api.h +++ b/simple-roots-api.h @@ -5,8 +5,9 @@ #include "simple-roots-types.h" #define HANDLE_TO(T) union { T* v; struct handle handle; } -#define HANDLE_REF(h) h.v -#define HANDLE_SET(h,val) do { h.v = val; } while (0) +#define HANDLE_LOC(h) &(h).v +#define HANDLE_REF(h) (h).v +#define HANDLE_SET(h,val) do { (h).v = val; } while (0) #define PUSH_HANDLE(cx, h) push_handle(&(cx)->roots.roots, &h.handle) #define POP_HANDLE(cx) pop_handle(&(cx)->roots.roots) diff --git a/whippet.c b/whippet.c index ee18445de..9f81948a7 100644 --- a/whippet.c +++ b/whippet.c @@ -7,10 +7,10 @@ #include #include -#define GC_API_ #include "gc-api.h" #define GC_IMPL 1 +#include "gc-internal.h" #include "debug.h" #include "gc-align.h" @@ -77,9 +77,9 @@ enum metadata_byte { METADATA_BYTE_MARK_1 = 4, METADATA_BYTE_MARK_2 = 8, METADATA_BYTE_END = 16, - METADATA_BYTE_UNUSED_1 = 32, - METADATA_BYTE_UNUSED_2 = 64, - METADATA_BYTE_UNUSED_3 = 128 + METADATA_BYTE_EPHEMERON = 32, + METADATA_BYTE_PINNED = 64, + METADATA_BYTE_UNUSED_1 = 128 }; static uint8_t rotate_dead_survivor_marked(uint8_t mask) { @@ -307,6 +307,8 @@ struct gc_heap { size_t size; int collecting; int mark_while_stopping; + int check_pending_ephemerons; + struct gc_pending_ephemerons *pending_ephemerons; enum gc_kind gc_kind; int multithreaded; size_t active_mutator_count; @@ -323,6 +325,8 @@ struct gc_heap { double minor_gc_yield_threshold; double major_gc_yield_threshold; double minimum_major_gc_yield_threshold; + double pending_ephemerons_size_factor; + double pending_ephemerons_size_slop; }; struct gc_mutator_mark_buf { @@ -649,8 +653,8 @@ static inline int mark_space_contains(struct mark_space *space, return mark_space_contains_address(space, gc_ref_value(ref)); } -static inline int trace_edge(struct gc_heap *heap, struct gc_edge edge) { - struct gc_ref ref = gc_edge_ref(edge); +static inline int do_trace(struct gc_heap *heap, struct gc_edge edge, + struct gc_ref ref) { if (!gc_ref_is_heap_object(ref)) return 0; if (GC_LIKELY(mark_space_contains(heap_mark_space(heap), ref))) { @@ -666,6 +670,63 @@ static inline int trace_edge(struct gc_heap *heap, struct gc_edge edge) { GC_CRASH(); } +static inline int trace_edge(struct gc_heap *heap, struct gc_edge edge) { + struct gc_ref ref = gc_edge_ref(edge); + int is_new = do_trace(heap, edge, ref); + + if (GC_UNLIKELY(atomic_load_explicit(&heap->check_pending_ephemerons, + memory_order_relaxed))) + gc_resolve_pending_ephemerons(ref, heap); + + return is_new; +} + +int gc_visit_ephemeron_key(struct gc_edge edge, struct gc_heap *heap) { + struct gc_ref ref = gc_edge_ref(edge); + if (!gc_ref_is_heap_object(ref)) + return 0; + if (GC_LIKELY(mark_space_contains(heap_mark_space(heap), ref))) { + struct mark_space *space = heap_mark_space(heap); + uint8_t *metadata = metadata_byte_for_object(ref); + uint8_t byte = *metadata; + if (byte & space->marked_mask) + return 1; + + if (!space->evacuating) + return 0; + if (!block_summary_has_flag(block_summary_for_addr(gc_ref_value(ref)), + BLOCK_EVACUATE)) + return 0; + + struct gc_atomic_forward fwd = gc_atomic_forward_begin(ref); + switch (fwd.state) { + case GC_FORWARDING_STATE_NOT_FORWARDED: + return 0; + case GC_FORWARDING_STATE_BUSY: + // Someone else claimed this object first. Spin until new address + // known, or evacuation aborts. + for (size_t spin_count = 0;; spin_count++) { + if (gc_atomic_forward_retry_busy(&fwd)) + break; + yield_for_spin(spin_count); + } + if (fwd.state == GC_FORWARDING_STATE_ABORTED) + // Remote evacuation aborted; remote will mark and enqueue. + return 1; + ASSERT(fwd.state == GC_FORWARDING_STATE_FORWARDED); + // Fall through. + case GC_FORWARDING_STATE_FORWARDED: + gc_edge_update(edge, gc_ref(gc_atomic_forward_address(&fwd))); + return 1; + default: + GC_CRASH(); + } + } else if (large_object_space_contains(heap_large_object_space(heap), ref)) { + return large_object_space_is_copied(heap_large_object_space(heap), ref); + } + GC_CRASH(); +} + static inline struct gc_ref mark_space_mark_conservative_ref(struct mark_space *space, struct gc_conservative_ref ref, int possibly_interior) { @@ -732,9 +793,9 @@ static inline struct gc_ref mark_space_mark_conservative_ref(struct mark_space * return gc_ref(addr); } -static inline struct gc_ref trace_conservative_ref(struct gc_heap *heap, - struct gc_conservative_ref ref, - int possibly_interior) { +static inline struct gc_ref do_trace_conservative_ref(struct gc_heap *heap, + struct gc_conservative_ref ref, + int possibly_interior) { if (!gc_conservative_ref_might_be_a_heap_object(ref, possibly_interior)) return gc_ref_null(); @@ -746,6 +807,19 @@ static inline struct gc_ref trace_conservative_ref(struct gc_heap *heap, ref, possibly_interior); } +static inline struct gc_ref trace_conservative_ref(struct gc_heap *heap, + struct gc_conservative_ref ref, + int possibly_interior) { + struct gc_ref ret = do_trace_conservative_ref(heap, ref, possibly_interior); + + if (gc_ref_is_heap_object(ret) && + GC_UNLIKELY(atomic_load_explicit(&heap->check_pending_ephemerons, + memory_order_relaxed))) + gc_resolve_pending_ephemerons(ret, heap); + + return ret; +} + static inline size_t mark_space_object_size(struct mark_space *space, struct gc_ref ref) { uint8_t *loc = metadata_byte_for_object(ref); @@ -1091,18 +1165,37 @@ static inline void tracer_trace_conservative_ref(struct gc_conservative_ref ref, tracer_enqueue(resolved, heap, data); } +static inline void trace_one_conservatively(struct gc_ref ref, + struct gc_heap *heap, + void *mark_data) { + size_t bytes; + if (GC_LIKELY(mark_space_contains(heap_mark_space(heap), ref))) { + // Generally speaking we trace conservatively and don't allow much + // in the way of incremental precise marking on a + // conservative-by-default heap. But, we make an exception for + // ephemerons. + uint8_t meta = *metadata_byte_for_addr(gc_ref_value(ref)); + if (GC_UNLIKELY(meta & METADATA_BYTE_EPHEMERON)) { + gc_trace_ephemeron(gc_ref_heap_object(ref), tracer_visit, heap, + mark_data); + return; + } + bytes = mark_space_object_size(heap_mark_space(heap), ref); + } else { + bytes = large_object_space_object_size(heap_large_object_space(heap), ref); + } + trace_conservative_edges(gc_ref_value(ref), + gc_ref_value(ref) + bytes, + tracer_trace_conservative_ref, heap, + mark_data); +} + static inline void trace_one(struct gc_ref ref, struct gc_heap *heap, void *mark_data) { - if (gc_has_conservative_intraheap_edges()) { - size_t bytes = GC_LIKELY(mark_space_contains(heap_mark_space(heap), ref)) - ? mark_space_object_size(heap_mark_space(heap), ref) - : large_object_space_object_size(heap_large_object_space(heap), ref); - trace_conservative_edges(gc_ref_value(ref), - gc_ref_value(ref) + bytes, - tracer_trace_conservative_ref, heap, mark_data); - } else { + if (gc_has_conservative_intraheap_edges()) + trace_one_conservatively(ref, heap, mark_data); + else gc_trace_object(ref, tracer_visit, heap, mark_data, NULL); - } } static void @@ -1672,6 +1765,26 @@ static void mark_space_finish_gc(struct mark_space *space, release_evacuation_target_blocks(space); } +static void resolve_ephemerons_lazily(struct gc_heap *heap) { + atomic_store_explicit(&heap->check_pending_ephemerons, 0, + memory_order_release); +} + +static void resolve_ephemerons_eagerly(struct gc_heap *heap) { + atomic_store_explicit(&heap->check_pending_ephemerons, 1, + memory_order_release); + gc_scan_pending_ephemerons(heap->pending_ephemerons, heap, 0, 1); +} + +static int enqueue_resolved_ephemerons(struct gc_heap *heap) { + return gc_pop_resolved_ephemerons(heap, trace_and_enqueue_globally, + NULL); +} + +static void sweep_ephemerons(struct gc_heap *heap) { + return gc_sweep_pending_ephemerons(heap->pending_ephemerons, 0, 1); +} + static void collect(struct gc_mutator *mut) { struct gc_heap *heap = mutator_heap(mut); struct mark_space *space = heap_mark_space(heap); @@ -1684,6 +1797,7 @@ static void collect(struct gc_mutator *mut) { enum gc_kind gc_kind = determine_collection_kind(heap); update_mark_patterns(space, !(gc_kind & GC_KIND_FLAG_MINOR)); large_object_space_start_gc(lospace, gc_kind & GC_KIND_FLAG_MINOR); + resolve_ephemerons_lazily(heap); tracer_prepare(heap); request_mutators_to_stop(heap); trace_mutator_roots_with_lock_before_stop(mut); @@ -1697,6 +1811,10 @@ static void collect(struct gc_mutator *mut) { prepare_for_evacuation(heap); trace_roots_after_stop(heap); tracer_trace(heap); + resolve_ephemerons_eagerly(heap); + while (enqueue_resolved_ephemerons(heap)) + tracer_trace(heap); + sweep_ephemerons(heap); tracer_release(heap); mark_space_finish_gc(space, gc_kind); large_object_space_finish_gc(lospace, gc_kind & GC_KIND_FLAG_MINOR); @@ -2054,6 +2172,31 @@ void* gc_allocate_pointerless(struct gc_mutator *mut, size_t size) { return gc_allocate(mut, size); } +struct gc_ref gc_allocate_ephemeron(struct gc_mutator *mut) { + struct gc_ref ret = + gc_ref_from_heap_object(gc_allocate(mut, gc_ephemeron_size())); + if (gc_has_conservative_intraheap_edges()) { + uint8_t *metadata = metadata_byte_for_addr(gc_ref_value(ret)); + *metadata |= METADATA_BYTE_EPHEMERON; + } + return ret; +} + +void gc_ephemeron_init(struct gc_mutator *mut, struct gc_ephemeron *ephemeron, + struct gc_ref key, struct gc_ref value) { + gc_ephemeron_init_internal(mutator_heap(mut), ephemeron, key, value); + // No write barrier: we require that the ephemeron be newer than the + // key or the value. +} + +struct gc_pending_ephemerons *gc_heap_pending_ephemerons(struct gc_heap *heap) { + return heap->pending_ephemerons; +} + +unsigned gc_heap_ephemeron_trace_epoch(struct gc_heap *heap) { + return heap->count; +} + #define FOR_EACH_GC_OPTION(M) \ M(GC_OPTION_FIXED_HEAP_SIZE, "fixed-heap-size") \ M(GC_OPTION_PARALLELISM, "parallelism") @@ -2141,6 +2284,16 @@ static struct slab* allocate_slabs(size_t nslabs) { return (struct slab*) aligned_base; } +static int heap_prepare_pending_ephemerons(struct gc_heap *heap) { + struct gc_pending_ephemerons *cur = heap->pending_ephemerons; + size_t target = heap->size * heap->pending_ephemerons_size_factor; + double slop = heap->pending_ephemerons_size_slop; + + heap->pending_ephemerons = gc_prepare_pending_ephemerons(cur, target, slop); + + return !!heap->pending_ephemerons; +} + static int heap_init(struct gc_heap *heap, struct options *options) { // *heap is already initialized to 0. @@ -2152,6 +2305,8 @@ static int heap_init(struct gc_heap *heap, struct options *options) { if (!tracer_init(heap, options->parallelism)) GC_CRASH(); + heap->pending_ephemerons_size_factor = 0.005; + heap->pending_ephemerons_size_slop = 0.5; heap->fragmentation_low_threshold = 0.05; heap->fragmentation_high_threshold = 0.10; heap->minor_gc_yield_threshold = 0.30; @@ -2159,6 +2314,9 @@ static int heap_init(struct gc_heap *heap, struct options *options) { heap->major_gc_yield_threshold = clamp_major_gc_yield_threshold(heap, heap->minor_gc_yield_threshold); + if (!heap_prepare_pending_ephemerons(heap)) + GC_CRASH(); + return 1; }