mirror/guile
mirror/guile
1
Fork 0
mirror of https://git.savannah.gnu.org/git/guile.git synced 2025-05-20 03:30:27 +02:00
Code Activity

Add ephemeron implementation

Browse source 
This commit adds support for ephemerons to the API and wires it into the
collectors.  It also adds a new test.
This commit is contained in:
Andy Wingo 2022-11-26 22:28:57 +01:00
parent 44f37a373c
commit 78da8d5811
18 changed files with 1455 additions and 56 deletions
Download patch file Download diff file Expand all files Collapse all files

58
Makefile
View file

@ -1,4 +1,4 @@
TESTS=quads mt-gcbench # MT_GCBench MT_GCBench2
TESTS=quads mt-gcbench ephemerons # MT_GCBench MT_GCBench2
COLLECTORS= \
bdw \
semi \
@ -19,8 +19,16 @@ COLLECTORS= \
stack-conservative-parallel-generational-whippet \
heap-conservative-parallel-generational-whippet
DEFAULT_BUILD:=opt
BUILD_CFLAGS_opt=-O2 -g -DNDEBUG
BUILD_CFLAGS_optdebug=-Og -g -DGC_DEBUG=1
BUILD_CFLAGS_debug=-O0 -g -DGC_DEBUG=1
BUILD_CFLAGS=$(BUILD_CFLAGS_$(or $(BUILD),$(DEFAULT_BUILD)))
CC=gcc
CFLAGS=-Wall -O2 -g -flto -fno-strict-aliasing -fvisibility=hidden -Wno-unused -DNDEBUG
CFLAGS=-Wall -flto -fno-strict-aliasing -fvisibility=hidden -Wno-unused $(BUILD_CFLAGS)
INCLUDES=-I.
LDFLAGS=-lpthread -flto
COMPILE=$(CC) $(CFLAGS) $(INCLUDES)
@ -36,92 +44,106 @@ gc-platform.o: gc-platform.h gc-platform-$(PLATFORM).c gc-visibility.h
gc-stack.o: gc-stack.c
$(COMPILE) -o $@ -c $<
gc-ephemeron-%.o: gc-ephemeron.c gc-ephemeron.h gc-ephemeron-internal.h %-embedder.h
$(COMPILE) -include $*-embedder.h -o $@ -c $<
bdw-%-gc.o: bdw.c %-embedder.h %.c
$(COMPILE) -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 `pkg-config --cflags bdw-gc` -include $*-embedder.h -o $@ -c bdw.c
bdw-%.o: bdw.c %.c
$(COMPILE) -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 -include bdw-attrs.h -o $@ -c $*.c
bdw-%: bdw-%.o bdw-%-gc.o gc-stack.o gc-platform.o
bdw-%: bdw-%.o bdw-%-gc.o gc-stack.o gc-platform.o gc-ephemeron-%.o
$(CC) $(LDFLAGS) `pkg-config --libs bdw-gc` -o $@ $^
semi-%-gc.o: semi.c %-embedder.h large-object-space.h assert.h debug.h %.c
$(COMPILE) -DGC_PRECISE_ROOTS=1 -include $*-embedder.h -o $@ -c semi.c
semi-%.o: semi.c %.c
$(COMPILE) -DGC_PRECISE_ROOTS=1 -include semi-attrs.h -o $@ -c $*.c
semi-%: semi-%.o semi-%-gc.o gc-stack.o gc-platform.o gc-ephemeron-%.o
$(CC) $(LDFLAGS) -o $@ $^
whippet-%-gc.o: whippet.c %-embedder.h large-object-space.h serial-tracer.h assert.h debug.h heap-objects.h %.c
$(COMPILE) -DGC_PRECISE_ROOTS=1 -include $*-embedder.h -o $@ -c whippet.c
whippet-%.o: whippet.c %.c
$(COMPILE) -DGC_PRECISE_ROOTS=1 -include whippet-attrs.h -o $@ -c $*.c
whippet-%: whippet-%.o whippet-%-gc.o gc-stack.o gc-platform.o gc-ephemeron-%.o
$(CC) $(LDFLAGS) -o $@ $^
stack-conservative-whippet-%-gc.o: whippet.c %-embedder.h large-object-space.h serial-tracer.h assert.h debug.h heap-objects.h %.c
$(COMPILE) -DGC_CONSERVATIVE_ROOTS=1 -include $*-embedder.h -o $@ -c whippet.c
stack-conservative-whippet-%.o: whippet.c %.c
$(COMPILE) -DGC_CONSERVATIVE_ROOTS=1 -include whippet-attrs.h -o $@ -c $*.c
stack-conservative-whippet-%: stack-conservative-whippet-%.o stack-conservative-whippet-%-gc.o gc-stack.o gc-platform.o gc-ephemeron-%.o
$(CC) $(LDFLAGS) -o $@ $^
heap-conservative-whippet-%-gc.o: whippet.c %-embedder.h large-object-space.h serial-tracer.h assert.h debug.h heap-objects.h %.c
$(COMPILE) -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 -include $*-embedder.h -o $@ -c whippet.c
heap-conservative-whippet-%.o: whippet.c %.c
$(COMPILE) -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 -include whippet-attrs.h -o $@ -c $*.c
heap-conservative-whippet-%: heap-conservative-whippet-%.o heap-conservative-whippet-%-gc.o gc-stack.o gc-platform.o gc-ephemeron-%.o
$(CC) $(LDFLAGS) -o $@ $^
parallel-whippet-%-gc.o: whippet.c %-embedder.h large-object-space.h parallel-tracer.h assert.h debug.h heap-objects.h %.c
$(COMPILE) -DGC_PARALLEL=1 -DGC_PRECISE_ROOTS=1 -include $*-embedder.h -o $@ -c whippet.c
parallel-whippet-%.o: whippet.c %.c
$(COMPILE) -DGC_PARALLEL=1 -DGC_PRECISE_ROOTS=1 -include whippet-attrs.h -o $@ -c $*.c
parallel-whippet-%: parallel-whippet-%.o parallel-whippet-%-gc.o gc-stack.o gc-platform.o gc-ephemeron-%.o
$(CC) $(LDFLAGS) -o $@ $^
stack-conservative-parallel-whippet-%-gc.o: whippet.c %-embedder.h large-object-space.h serial-tracer.h assert.h debug.h heap-objects.h %.c
$(COMPILE) -DGC_PARALLEL=1 -DGC_CONSERVATIVE_ROOTS=1 -include $*-embedder.h -o $@ -c whippet.c
stack-conservative-parallel-whippet-%.o: whippet.c %.c
$(COMPILE) -DGC_PARALLEL=1 -DGC_CONSERVATIVE_ROOTS=1 -include whippet-attrs.h -o $@ -c $*.c
stack-conservative-parallel-whippet-%: stack-conservative-parallel-whippet-%.o stack-conservative-parallel-whippet-%-gc.o gc-stack.o gc-platform.o gc-ephemeron-%.o
$(CC) $(LDFLAGS) -o $@ $^
heap-conservative-parallel-whippet-%-gc.o: whippet.c %-embedder.h large-object-space.h serial-tracer.h assert.h debug.h heap-objects.h %.c
$(COMPILE) -DGC_PARALLEL=1 -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 -include $*-embedder.h -o $@ -c whippet.c
heap-conservative-parallel-whippet-%.o: whippet.c %.c
$(COMPILE) -DGC_PARALLEL=1 -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 -DGC_FULLY_CONSERVATIVE=1 -include whippet-attrs.h -o $@ -c $*.c
heap-conservative-parallel-whippet-%: heap-conservative-parallel-whippet-%.o heap-conservative-parallel-whippet-%-gc.o gc-stack.o gc-platform.o gc-ephemeron-%.o
$(CC) $(LDFLAGS) -o $@ $^
generational-whippet-%-gc.o: whippet.c %-embedder.h large-object-space.h serial-tracer.h assert.h debug.h heap-objects.h %.c
$(COMPILE) -DGC_GENERATIONAL=1 -DGC_PRECISE_ROOTS=1 -include $*-embedder.h -o $@ -c whippet.c
generational-whippet-%.o: whippet.c %.c
$(COMPILE) -DGC_GENERATIONAL=1 -DGC_PRECISE_ROOTS=1 -include whippet-attrs.h -o $@ -c $*.c
generational-whippet-%: generational-whippet-%.o generational-whippet-%-gc.o gc-stack.o gc-platform.o gc-ephemeron-%.o
$(CC) $(LDFLAGS) -o $@ $^
stack-conservative-generational-whippet-%-gc.o: whippet.c %-embedder.h large-object-space.h serial-tracer.h assert.h debug.h heap-objects.h %.c
$(COMPILE) -DGC_GENERATIONAL=1 -DGC_CONSERVATIVE_ROOTS=1 -include $*-embedder.h -o $@ -c whippet.c
stack-conservative-generational-whippet-%.o: whippet.c %.c
$(COMPILE) -DGC_GENERATIONAL=1 -DGC_CONSERVATIVE_ROOTS=1 -include whippet-attrs.h -o $@ -c $*.c
stack-conservative-generational-whippet-%: stack-conservative-generational-whippet-%.o stack-conservative-generational-whippet-%-gc.o gc-stack.o gc-platform.o gc-ephemeron-%.o
$(CC) $(LDFLAGS) -o $@ $^
heap-conservative-generational-whippet-%-gc.o: whippet.c %-embedder.h large-object-space.h serial-tracer.h assert.h debug.h heap-objects.h %.c
$(COMPILE) -DGC_GENERATIONAL=1 -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 -include $*-embedder.h -o $@ -c whippet.c
heap-conservative-generational-whippet-%.o: whippet.c %.c
$(COMPILE) -DGC_GENERATIONAL=1 -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 -include whippet-attrs.h -o $@ -c $*.c
heap-conservative-generational-whippet-%: heap-conservative-generational-whippet-%.o heap-conservative-generational-whippet-%-gc.o gc-stack.o gc-platform.o gc-ephemeron-%.o
$(CC) $(LDFLAGS) -o $@ $^
parallel-generational-whippet-%-gc.o: whippet.c %-embedder.h large-object-space.h parallel-tracer.h assert.h debug.h heap-objects.h %.c
$(COMPILE) -DGC_PARALLEL=1 -DGC_GENERATIONAL=1 -DGC_PRECISE_ROOTS=1 -include $*-embedder.h -o $@ -c whippet.c
parallel-generational-whippet-%.o: whippet.c %.c
$(COMPILE) -DGC_PARALLEL=1 -DGC_GENERATIONAL=1 -DGC_PRECISE_ROOTS=1 -include whippet-attrs.h -o $@ -c $*.c
parallel-generational-whippet-%: parallel-generational-whippet-%.o parallel-generational-whippet-%-gc.o gc-stack.o gc-platform.o gc-ephemeron-%.o
$(CC) $(LDFLAGS) -o $@ $^
stack-conservative-parallel-generational-whippet-%-gc.o: whippet.c %-embedder.h large-object-space.h parallel-tracer.h assert.h debug.h heap-objects.h %.c
$(COMPILE) -DGC_PARALLEL=1 -DGC_GENERATIONAL=1 -DGC_CONSERVATIVE_ROOTS=1 -include $*-embedder.h -o $@ -c whippet.c
stack-conservative-parallel-generational-whippet-%.o: whippet.c %.c
$(COMPILE) -DGC_PARALLEL=1 -DGC_GENERATIONAL=1 -DGC_CONSERVATIVE_ROOTS=1 -include whippet-attrs.h -o $@ -c $*.c
stack-conservative-parallel-generational-whippet-%: stack-conservative-parallel-generational-whippet-%.o stack-conservative-parallel-generational-whippet-%-gc.o gc-stack.o gc-platform.o gc-ephemeron-%.o
$(CC) $(LDFLAGS) -o $@ $^
heap-conservative-parallel-generational-whippet-%-gc.o: whippet.c %-embedder.h large-object-space.h parallel-tracer.h assert.h debug.h heap-objects.h %.c
$(COMPILE) -DGC_PARALLEL=1 -DGC_GENERATIONAL=1 -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 -include $*-embedder.h -o $@ -c whippet.c
heap-conservative-parallel-generational-whippet-%.o: whippet.c %.c
$(COMPILE) -DGC_PARALLEL=1 -DGC_GENERATIONAL=1 -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 -include whippet-attrs.h -o $@ -c $*.c
%: %.o %-gc.o gc-platform.o gc-stack.o
$(CC) $(LDFLAGS) $($*_LDFLAGS) -o $@ $^
check: $(addprefix test-$(TARGET),$(TARGETS))
test-%: $(ALL_TESTS)
@echo "Running unit tests..."
@set -e; for test in $?; do \
echo "Testing: $$test"; \
./$$test; \
done
@echo "Success."
.PHONY: check
heap-conservative-parallel-generational-whippet-%: heap-conservative-parallel-generational-whippet-%.o heap-conservative-parallel-generational-whippet-%-gc.o gc-stack.o gc-platform.o gc-ephemeron-%.o
$(CC) $(LDFLAGS) -o $@ $^
.PRECIOUS: $(ALL_TESTS)

94
bdw.c
View file

@ -3,8 +3,11 @@
#include <stdlib.h>
#include <string.h>
#define GC_API_
#include "gc-api.h"
#include "gc-ephemeron.h"
#define GC_IMPL 1
#include "gc-internal.h"
#include "bdw-attrs.h"
@ -34,6 +37,7 @@
#include <gc/gc.h>
#include <gc/gc_inline.h> /* GC_generic_malloc_many */
#include <gc/gc_mark.h> /* GC_generic_malloc */
#define GC_INLINE_GRANULE_WORDS 2
#define GC_INLINE_GRANULE_BYTES (sizeof(void *) * GC_INLINE_GRANULE_WORDS)
@ -120,6 +124,85 @@ void gc_collect(struct gc_mutator *mut) {
GC_gcollect();
}
// In BDW-GC, we can't hook into the mark phase to call
// gc_trace_ephemerons_for_object, so the advertised ephemeron strategy
// doesn't really work. The primitives that we have are mark functions,
// which run during GC and can't allocate; finalizers, which run after
// GC and can allocate but can't add to the connectivity graph; and
// disappearing links, which are cleared at the end of marking, in the
// stop-the-world phase. It does not appear to be possible to implement
// ephemerons using these primitives. Instead fall back to weak-key
// tables.
static int ephemeron_gc_kind;
struct gc_ref gc_allocate_ephemeron(struct gc_mutator *mut) {
void *ret = GC_generic_malloc(gc_ephemeron_size(), ephemeron_gc_kind);
return gc_ref_from_heap_object(ret);
}
unsigned gc_heap_ephemeron_trace_epoch(struct gc_heap *heap) {
return 0;
}
void gc_ephemeron_init(struct gc_mutator *mut, struct gc_ephemeron *ephemeron,
struct gc_ref key, struct gc_ref value) {
gc_ephemeron_init_internal(mut->heap, ephemeron, key, value);
if (GC_base((void*)gc_ref_value(key))) {
struct gc_ref *loc = gc_edge_loc(gc_ephemeron_key_edge(ephemeron));
GC_register_disappearing_link((void**)loc);
}
}
struct ephemeron_mark_state {
struct GC_ms_entry *mark_stack_ptr;
struct GC_ms_entry *mark_stack_limit;
};
int gc_visit_ephemeron_key(struct gc_edge edge, struct gc_heap *heap) {
// Pretend the key is traced, to avoid adding this ephemeron to the
// global table.
return 1;
}
static void trace_ephemeron_edge(struct gc_edge edge, struct gc_heap *heap,
void *visit_data) {
struct ephemeron_mark_state *state = visit_data;
uintptr_t addr = gc_ref_value(gc_edge_ref(edge));
state->mark_stack_ptr = GC_MARK_AND_PUSH ((void *) addr,
state->mark_stack_ptr,
state->mark_stack_limit,
NULL);
}
static struct GC_ms_entry *
mark_ephemeron(GC_word *addr, struct GC_ms_entry *mark_stack_ptr,
struct GC_ms_entry *mark_stack_limit, GC_word env) {
struct ephemeron_mark_state state = {
mark_stack_ptr,
mark_stack_limit,
};
struct gc_ephemeron *ephemeron = (struct gc_ephemeron*) addr;
// If this ephemeron is on a freelist, its first word will be a
// freelist link and everything else will be NULL.
if (!gc_ref_value(gc_edge_ref(gc_ephemeron_value_edge(ephemeron)))) {
trace_ephemeron_edge(gc_edge(addr), NULL, &state);
return state.mark_stack_ptr;
}
if (!gc_ref_value(gc_edge_ref(gc_ephemeron_key_edge(ephemeron)))) {
// If the key died in a previous collection, the disappearing link
// will have been cleared. Mark the ephemeron as dead.
gc_ephemeron_mark_dead(ephemeron);
}
gc_trace_ephemeron(ephemeron, trace_ephemeron_edge, NULL, &state);
return state.mark_stack_ptr;
}
static inline struct gc_mutator *add_mutator(struct gc_heap *heap) {
struct gc_mutator *ret = GC_malloc(sizeof(struct gc_mutator));
ret->heap = heap;
@ -224,6 +307,15 @@ int gc_init(int argc, struct gc_option argv[],
*heap = GC_malloc(sizeof(struct gc_heap));
pthread_mutex_init(&(*heap)->lock, NULL);
*mutator = add_mutator(*heap);
{
GC_word descriptor = GC_MAKE_PROC(GC_new_proc(mark_ephemeron), 0);
int add_size_to_descriptor = 0;
int clear_memory = 1;
ephemeron_gc_kind = GC_new_kind(GC_new_free_list(), descriptor,
add_size_to_descriptor, clear_memory);
}
return 1;
}

54
ephemerons-embedder.h Normal file
View file

@ -0,0 +1,54 @@
#ifndef EPHEMERONS_EMBEDDER_H
#define EPHEMERONS_EMBEDDER_H
#include <stddef.h>
#include "ephemerons-types.h"
#include "gc-ephemeron.h"
struct gc_heap;
#define DEFINE_METHODS(name, Name, NAME) \
static inline size_t name##_size(Name *obj) GC_ALWAYS_INLINE; \
static inline void visit_##name##_fields(Name *obj,\
void (*visit)(struct gc_edge edge, \
struct gc_heap *heap, \
void *visit_data), \
struct gc_heap *heap, \
void *visit_data) GC_ALWAYS_INLINE;
FOR_EACH_HEAP_OBJECT_KIND(DEFINE_METHODS)
#undef DEFINE_METHODS
static inline size_t small_object_size(SmallObject *obj) { return sizeof(*obj); }
static inline size_t ephemeron_size(Ephemeron *obj) { return gc_ephemeron_size(); }
static inline size_t box_size(Box *obj) { return sizeof(*obj); }
static inline void
visit_small_object_fields(SmallObject *obj,
void (*visit)(struct gc_edge edge, struct gc_heap *heap,
void *visit_data),
struct gc_heap *heap,
void *visit_data) {}
static inline void
visit_ephemeron_fields(Ephemeron *ephemeron,
void (*visit)(struct gc_edge edge, struct gc_heap *heap,
void *visit_data),
struct gc_heap *heap,
void *visit_data) {
gc_trace_ephemeron((struct gc_ephemeron*)ephemeron, visit, heap, visit_data);
}
static inline void
visit_box_fields(Box *box,
void (*visit)(struct gc_edge edge, struct gc_heap *heap,
void *visit_data),
struct gc_heap *heap,
void *visit_data) {
visit(gc_edge(&box->obj), heap, visit_data);
}
#include "simple-gc-embedder.h"
#endif // EPHEMERONS_EMBEDDER_H

21
ephemerons-types.h Normal file
View file

@ -0,0 +1,21 @@
#ifndef EPHEMERONS_TYPES_H
#define EPHEMERONS_TYPES_H
#define FOR_EACH_HEAP_OBJECT_KIND(M) \
M(box, Box, BOX) \
M(ephemeron, Ephemeron, EPHEMERON) \
M(small_object, SmallObject, SMALL_OBJECT)
#include "heap-objects.h"
#include "simple-tagging-scheme.h"
struct SmallObject {
struct gc_header header;
};
struct Box {
struct gc_header header;
void *obj;
};
#endif // EPHEMERONS_TYPES_H

270
ephemerons.c Normal file
View file

@ -0,0 +1,270 @@
#include <errno.h>
#include <pthread.h>
#include <stdio.h>
#include <stddef.h>
#include <stdlib.h>
#include <sys/time.h>
#include "assert.h"
#include "gc-api.h"
#include "gc-ephemeron.h"
#include "simple-roots-api.h"
#include "ephemerons-types.h"
#include "simple-allocator.h"
typedef HANDLE_TO(SmallObject) SmallObjectHandle;
typedef HANDLE_TO(struct gc_ephemeron) EphemeronHandle;
typedef HANDLE_TO(Box) BoxHandle;
static SmallObject* allocate_small_object(struct gc_mutator *mut) {
return gc_allocate_with_kind(mut, ALLOC_KIND_SMALL_OBJECT, sizeof(SmallObject));
}
static Box* allocate_box(struct gc_mutator *mut) {
return gc_allocate_with_kind(mut, ALLOC_KIND_BOX, sizeof(Box));
}
static struct gc_ephemeron* allocate_ephemeron(struct gc_mutator *mut) {
struct gc_ref ret = gc_allocate_ephemeron(mut);
*tag_word(ret) = tag_live(ALLOC_KIND_EPHEMERON);
return gc_ref_heap_object(ret);
}
/* Get the current time in microseconds */
static unsigned long current_time(void)
{
struct timeval t;
if (gettimeofday(&t, NULL) == -1)
return 0;
return t.tv_sec * 1000 * 1000 + t.tv_usec;
}
struct thread {
struct gc_mutator *mut;
struct gc_mutator_roots roots;
};
static void print_elapsed(const char *what, unsigned long start) {
unsigned long end = current_time();
unsigned long msec = (end - start) / 1000;
unsigned long usec = (end - start) % 1000;
printf("Completed %s in %lu.%.3lu msec\n", what, msec, usec);
}
struct call_with_gc_data {
void* (*f)(struct thread *);
struct gc_heap *heap;
};
static void* call_with_gc_inner(struct gc_stack_addr *addr, void *arg) {
struct call_with_gc_data *data = arg;
struct gc_mutator *mut = gc_init_for_thread(addr, data->heap);
struct thread t = { mut, };
gc_mutator_set_roots(mut, &t.roots);
void *ret = data->f(&t);
gc_finish_for_thread(mut);
return ret;
}
static void* call_with_gc(void* (*f)(struct thread *),
struct gc_heap *heap) {
struct call_with_gc_data data = { f, heap };
return gc_call_with_stack_addr(call_with_gc_inner, &data);
}
#define CHECK(x) \
do { \
if (!(x)) { \
fprintf(stderr, "%s:%d: check failed: %s\n", __FILE__, __LINE__, #x); \
exit(1); \
} \
} while (0)
#define CHECK_EQ(x, y) CHECK((x) == (y))
#define CHECK_NE(x, y) CHECK((x) != (y))
#define CHECK_NULL(x) CHECK_EQ(x, NULL)
#define CHECK_NOT_NULL(x) CHECK_NE(x, NULL)
static size_t ephemeron_chain_length(struct gc_ephemeron **loc,
SmallObject *key) {
struct gc_ephemeron *head = gc_ephemeron_chain_head(loc);
size_t len = 0;
while (head) {
CHECK_EQ(key, (SmallObject*)gc_ref_value(gc_ephemeron_key(head)));
Box *value = gc_ref_heap_object(gc_ephemeron_value(head));
CHECK_NOT_NULL(value);
key = value->obj;
CHECK_NOT_NULL(key);
head = gc_ephemeron_chain_next(head);
len++;
}
return len;
}
static double heap_size;
static double heap_multiplier;
static size_t nthreads;
static void cause_gc(struct gc_mutator *mut) {
gc_collect(mut);
}
static void make_ephemeron_chain(struct thread *t, EphemeronHandle *head,
SmallObjectHandle *head_key, size_t length) {
BoxHandle tail_box = { NULL };
PUSH_HANDLE(t, tail_box);
CHECK_NULL(HANDLE_REF(*head_key));
HANDLE_SET(*head_key, allocate_small_object(t->mut));
for (size_t i = 0; i < length; i++) {
HANDLE_SET(tail_box, allocate_box(t->mut));
HANDLE_REF(tail_box)->obj = HANDLE_REF(*head_key);
HANDLE_SET(*head_key, allocate_small_object(t->mut));
struct gc_ephemeron *ephemeron = allocate_ephemeron(t->mut);
gc_ephemeron_init(t->mut, ephemeron,
gc_ref_from_heap_object(HANDLE_REF(*head_key)),
gc_ref_from_heap_object(HANDLE_REF(tail_box)));
gc_ephemeron_chain_push(HANDLE_LOC(*head), ephemeron);
}
POP_HANDLE(t);
}
static void* run_one_test(struct thread *t) {
size_t unit_size = gc_ephemeron_size() + sizeof(Box);
size_t list_length = heap_size / nthreads / heap_multiplier / unit_size;
printf("Allocating ephemeron list %zu nodes long. Total size %.3fGB.\n",
list_length, list_length * unit_size / 1e9);
unsigned long thread_start = current_time();
SmallObjectHandle head_key = { NULL };
EphemeronHandle head = { NULL };
PUSH_HANDLE(t, head_key);
PUSH_HANDLE(t, head);
make_ephemeron_chain(t, &head, &head_key, list_length);
size_t measured_length = ephemeron_chain_length(HANDLE_LOC(head),
HANDLE_REF(head_key));
CHECK_EQ(measured_length, list_length);
cause_gc(t->mut);
measured_length = ephemeron_chain_length(HANDLE_LOC(head),
HANDLE_REF(head_key));
CHECK_EQ(measured_length, list_length);
if (!GC_CONSERVATIVE_ROOTS) {
HANDLE_SET(head_key, NULL);
cause_gc(t->mut);
measured_length = ephemeron_chain_length(HANDLE_LOC(head),
HANDLE_REF(head_key));
CHECK_EQ(measured_length, 0);
}
// swap head_key for a key halfway in, cause gc
// check length is expected half-length; warn, or error if precise
// clear and return
print_elapsed("thread", thread_start);
POP_HANDLE(t);
POP_HANDLE(t);
return NULL;
}
static void* run_one_test_in_thread(void *arg) {
struct gc_heap *heap = arg;
return call_with_gc(run_one_test, heap);
}
struct join_data { int status; pthread_t thread; };
static void *join_thread(void *data) {
struct join_data *join_data = data;
void *ret;
join_data->status = pthread_join(join_data->thread, &ret);
return ret;
}
#define MAX_THREAD_COUNT 256
int main(int argc, char *argv[]) {
if (argc != 5) {
fprintf(stderr, "usage: %s HEAP_SIZE MULTIPLIER NTHREADS PARALLELISM\n", argv[0]);
return 1;
}
heap_size = atof(argv[1]);
heap_multiplier = atof(argv[2]);
nthreads = atol(argv[3]);
size_t parallelism = atol(argv[4]);
if (heap_size < 8192) {
fprintf(stderr,
"Heap size should probably be at least 8192, right? '%s'\n",
argv[1]);
return 1;
}
if (!(1.0 < heap_multiplier && heap_multiplier < 100)) {
fprintf(stderr, "Failed to parse heap multiplier '%s'\n", argv[2]);
return 1;
}
if (nthreads < 1 || nthreads > MAX_THREAD_COUNT) {
fprintf(stderr, "Expected integer between 1 and %d for thread count, got '%s'\n",
(int)MAX_THREAD_COUNT, argv[2]);
return 1;
}
if (parallelism < 1 || parallelism > MAX_THREAD_COUNT) {
fprintf(stderr, "Expected integer between 1 and %d for parallelism, got '%s'\n",
(int)MAX_THREAD_COUNT, argv[3]);
return 1;
}
printf("Allocating heap of %.3fGB (%.2f multiplier of live data).\n",
heap_size / 1e9, heap_multiplier);
struct gc_option options[] = { { GC_OPTION_FIXED_HEAP_SIZE, (size_t) heap_size },
{ GC_OPTION_PARALLELISM, parallelism } };
struct gc_heap *heap;
struct gc_mutator *mut;
if (!gc_init(sizeof options / sizeof options[0], options, NULL, &heap,
&mut)) {
fprintf(stderr, "Failed to initialize GC with heap size %zu bytes\n",
(size_t)heap_size);
return 1;
}
struct thread main_thread = { mut, };
gc_mutator_set_roots(mut, &main_thread.roots);
unsigned long test_start = current_time();
pthread_t threads[MAX_THREAD_COUNT];
// Run one of the threads in the main thread.
for (size_t i = 1; i < nthreads; i++) {
int status = pthread_create(&threads[i], NULL, run_one_test_in_thread, heap);
if (status) {
errno = status;
perror("Failed to create thread");
return 1;
}
}
run_one_test(&main_thread);
for (size_t i = 1; i < nthreads; i++) {
struct join_data data = { 0, threads[i] };
gc_call_without_gc(mut, join_thread, &data);
if (data.status) {
errno = data.status;
perror("Failed to join thread");
return 1;
}
}
print_elapsed("test", test_start);
gc_print_stats(heap);
return 0;
}

5
gc-api.h
View file

@ -26,11 +26,6 @@ struct gc_option {
double value;
};
// FIXME: Conflict with bdw-gc GC_API. Switch prefix?
#ifndef GC_API_
#define GC_API_ GC_INTERNAL
#endif
GC_API_ int gc_option_from_string(const char *str);
struct gc_stack_addr;

3
gc-edge.h
View file

@ -13,6 +13,9 @@ static inline struct gc_edge gc_edge(void* addr) {
static inline struct gc_ref gc_edge_ref(struct gc_edge edge) {
return *edge.dst;
}
static inline struct gc_ref* gc_edge_loc(struct gc_edge edge) {
return edge.dst;
}
static inline void gc_edge_update(struct gc_edge edge, struct gc_ref ref) {
*edge.dst = ref;
}

4
gc-embedder-api.h
View file

@ -1,8 +1,11 @@
#ifndef GC_EMBEDDER_API_H
#define GC_EMBEDDER_API_H
#include <stddef.h>
#include "gc-config.h"
#include "gc-edge.h"
#include "gc-inline.h"
#include "gc-forwarding.h"
#ifndef GC_EMBEDDER_API
@ -13,6 +16,7 @@ struct gc_mutator_roots;
struct gc_heap_roots;
struct gc_atomic_forward;
struct gc_heap;
struct gc_ephemeron;
GC_EMBEDDER_API inline int gc_is_valid_conservative_ref_displacement(uintptr_t displacement);

51
gc-ephemeron-internal.h Normal file
View file

@ -0,0 +1,51 @@
#ifndef GC_EPHEMERON_INTERNAL_H
#define GC_EPHEMERON_INTERNAL_H
#ifndef GC_IMPL
#error internal header file, not part of API
#endif
#include "gc-ephemeron.h"
struct gc_pending_ephemerons;
// API implemented by collector, for use by ephemerons:
GC_INTERNAL int gc_visit_ephemeron_key(struct gc_edge edge,
struct gc_heap *heap);
GC_INTERNAL struct gc_pending_ephemerons*
gc_heap_pending_ephemerons(struct gc_heap *heap);
GC_INTERNAL unsigned gc_heap_ephemeron_trace_epoch(struct gc_heap *heap);
// API implemented by ephemerons, for use by collector:
GC_INTERNAL struct gc_edge gc_ephemeron_key_edge(struct gc_ephemeron *eph);
GC_INTERNAL struct gc_edge gc_ephemeron_value_edge(struct gc_ephemeron *eph);
GC_INTERNAL struct gc_pending_ephemerons*
gc_prepare_pending_ephemerons(struct gc_pending_ephemerons *state,
size_t target_size, double slop);
GC_INTERNAL void
gc_resolve_pending_ephemerons(struct gc_ref obj, struct gc_heap *heap);
GC_INTERNAL void
gc_scan_pending_ephemerons(struct gc_pending_ephemerons *state,
struct gc_heap *heap, size_t shard,
size_t nshards);
GC_INTERNAL int
gc_pop_resolved_ephemerons(struct gc_heap *heap,
void (*visit)(struct gc_edge edge,
struct gc_heap *heap,
void *visit_data),
void *trace_data);
GC_INTERNAL void
gc_sweep_pending_ephemerons(struct gc_pending_ephemerons *state,
size_t shard, size_t nshards);
GC_INTERNAL void gc_ephemeron_init_internal(struct gc_heap *heap,
struct gc_ephemeron *ephemeron,
struct gc_ref key,
struct gc_ref value);
#endif // GC_EPHEMERON_INTERNAL_H

582
gc-ephemeron.c Normal file
View file

@ -0,0 +1,582 @@
#include <math.h>
#include <stdatomic.h>
#include <stdlib.h>
#define GC_IMPL 1
#include "address-hash.h"
#include "debug.h"
#include "gc-embedder-api.h"
#include "gc-ephemeron-internal.h"
// # Overview
//
// An ephemeron is a conjunction consisting of the ephemeron object
// itself, a "key" object, and a "value" object. If the ephemeron and
// the key are live, then the value is kept live and can be looked up
// given the ephemeron object.
//
// Sometimes we write this as E×K⇒V, indicating that you need both E and
// K to get V. We'll use this notation in these comments sometimes.
//
// The key and the value of an ephemeron are never modified, except
// possibly via forwarding during GC.
//
// If the key of an ephemeron ever becomes unreachable, the ephemeron
// object will be marked as dead by the collector, and neither key nor
// value will be accessible. Users can also explicitly mark an
// ephemeron as dead.
//
// Users can build collections of ephemerons by chaining them together.
// If an ephemeron ever becomes dead, the ephemeron will be removed from
// the chain by the garbage collector.
//
// # Tracing algorithm
//
// Tracing ephemerons is somewhat complicated. Tracing the live objects
// in a heap is usually a parallelizable fan-out kind of operation,
// requiring minimal synchronization between tracing worker threads.
// However with ephemerons, each worker thread may need to check if
// there is a pending ephemeron E for an object K, marking the
// associated V for later traversal by the tracer. Doing this without
// introducing excessive global serialization points is the motivation
// for the complications that follow.
//
// From the viewpoint of the garbage collector, an ephemeron E×K⇒V has 4
// possible states:
//
// - Traced: An E that was already fully traced as of a given GC epoch.
//
// - Claimed: GC discovers E for the first time in a GC epoch
//
// - Pending: K's liveness is unknown
//
// - Resolved: K is live; V needs tracing
//
// The ephemeron state is kept in an atomic variable. The pending and
// resolved states also have associated atomic list link fields as well;
// it doesn't appear possible to coalesce them into a single field
// without introducing serialization. Finally, there is a bit to
// indicate whether a "traced" ephemeron is live or dead, and a field to
// indicate the epoch at which it was last traced.
//
// Here is a diagram of the state transitions:
//
// ,----->Traced<-----.
// , | | .
// , v / .
// | Claimed |
// | ,-----/ \---. |
// | v v |
// Pending--------->Resolved
//
// Ephemerons are born in the traced state, for the current GC epoch.
//
// When the tracer sees an ephemeron E in the traced state it checks the
// epoch. If the epoch is up to date, E stays in the traced state and
// we are done.
//
// Otherwise, E transitions from traced to claimed. The thread that
// claims E is then responsible for resetting E's pending and resolved
// links, updating E's epoch, and tracing E's user-controlled chain
// link.
//
// If the claiming thread sees that E was already marked dead by a
// previous GC, or explicitly by the user, the ephemeron then
// transitions from back to traced, ready for the next epoch.
//
// If the claiming thread sees K to already be known to be live, then E
// is added to the global resolved set and E's state becomes resolved.
//
// Otherwise the claiming thread publishes K⇒E to the global pending
// ephemeron table, via the pending link, and E transitions to pending.
//
// A pending ephemeron is a link in a buckets-of-chains concurrent hash
// table. If its K is ever determined to be live, it becomes resolved,
// and is added to a global set of resolved ephemerons. At the end of
// GC, any ephemerons still pending are marked dead, transitioning their
// states to traced.
//
// Note that the claiming thread -- the one that publishes K⇒E to the
// global pending ephemeron table -- needs to re-check that K is still
// untraced after adding K⇒E to the pending table, and move to resolved
// if so.
//
// A resolved ephemeron needs its V to be traced. Incidentally its K
// also needs tracing, to relocate any forwarding pointer. The thread
// that pops an ephemeron from the resolved set is responsible for
// tracing and for moving E's state to traced.
//
// # Concurrency
//
// All operations on ephemerons are wait-free. Sometimes only one
// thread can make progress (for example for an ephemeron in the claimed
// state), but no thread will be stalled waiting on other threads to
// proceed.
//
// There is one interesting (from a concurrency point of view) data
// structure used by the implementation of ephemerons, the singly-linked
// list. Actually there are three of these; one is used as a stack and
// the other two is used as sets.
//
// The resolved set is implemented via a global `struct gc_ephemeron
// *resolved` variable. Resolving an ephemeron does an atomic push to
// this stack, via compare-and-swap (CAS); popping from the stack (also
// via CAS) yields an ephemeron for tracing. Ephemerons are added to
// the resolved set at most once per GC cycle, and the resolved set is
// empty outside of GC.
//
// The operations that are supported on atomic stacks are:
//
// push(LOC, E, OFFSET) -> void
//
// The user-visible chain link and the link for the pending ephemeron
// table are used to build atomic sets. In these you can add an
// ephemeron to the beginning of the list, traverse the list link by
// link to the end (indicated by NULL), and remove any list item.
// Removing a list node proceeds in two phases: one, you mark the node
// for removal, by changing the ephemeron's state; then, possibly on a
// subsequent traversal, any predecessor may forward its link past
// removed nodes. Because node values never change and nodes only go
// from live to dead, the live list tail can always be reached by any
// node, even from dead nodes.
//
// The operations that are supported on these atomic lists:
//
// push(LOC, E, OFFSET) -> void
// pop(LOC, OFFSET) -> ephemeron or null
// follow(LOC, OFFSET, STATE_OFFSET, LIVE_STATE) -> ephemeron or null
//
// These operations are all wait-free. The "push" operation is shared
// between stack and set use cases. "pop" is for stack-like use cases.
// The "follow" operation traverses a list, opportunistically eliding
// nodes that have been marked dead, atomically updating the location
// storing the next item.
//
// There are also accessors on ephemerons to their fields:
//
// key(E) -> value or null
// value(E) -> value or null
//
// These operations retrieve the key and value, respectively, provided
// that the ephemeron is not marked dead.
////////////////////////////////////////////////////////////////////////
// Concurrent operations on ephemeron lists
////////////////////////////////////////////////////////////////////////
static void
ephemeron_list_push(struct gc_ephemeron **loc,
struct gc_ephemeron *head,
struct gc_ephemeron** (*get_next)(struct gc_ephemeron*)) {
struct gc_ephemeron *tail = atomic_load_explicit(loc, memory_order_acquire);
while (1) {
// There must be no concurrent readers of HEAD, a precondition that
// we ensure by only publishing HEAD to LOC at most once per cycle.
// Therefore we can use a normal store for the tail pointer.
*get_next(head) = tail;
if (atomic_compare_exchange_weak(loc, &tail, head))
break;
}
}
static struct gc_ephemeron*
ephemeron_list_pop(struct gc_ephemeron **loc,
struct gc_ephemeron** (*get_next)(struct gc_ephemeron*)) {
struct gc_ephemeron *head = atomic_load_explicit(loc, memory_order_acquire);
while (head) {
// Precondition: the result of get_next on an ephemeron is never
// updated concurrently; OK to load non-atomically.
struct gc_ephemeron *tail = *get_next(head);
if (atomic_compare_exchange_weak(loc, &head, tail))
break;
}
return head;
}
static struct gc_ephemeron*
ephemeron_list_follow(struct gc_ephemeron **loc,
struct gc_ephemeron** (*get_next)(struct gc_ephemeron*),
int (*is_live)(struct gc_ephemeron*)) {
struct gc_ephemeron *head = atomic_load_explicit(loc, memory_order_acquire);
while (1) {
struct gc_ephemeron *new_head = head;
// Skip past any dead nodes.
while (new_head && !is_live(new_head))
new_head = atomic_load_explicit(get_next(new_head), memory_order_acquire);
if (// If we didn't have to advance past any dead nodes, no need to
// update LOC.
(head == new_head)
// Otherwise if we succeed in updating LOC, we're done.
|| atomic_compare_exchange_strong(loc, &head, new_head)
// Someone else managed to advance LOC; that's fine too.
|| (head == new_head))
return new_head;
// Otherwise we lost a race; loop and retry.
}
}
////////////////////////////////////////////////////////////////////////
// The ephemeron object type
////////////////////////////////////////////////////////////////////////
#ifndef GC_EMBEDDER_EPHEMERON_HEADER
#error Embedder should define GC_EMBEDDER_EPHEMERON_HEADER
#endif
enum {
EPHEMERON_STATE_TRACED,
EPHEMERON_STATE_CLAIMED,
EPHEMERON_STATE_PENDING,
EPHEMERON_STATE_RESOLVED,
};
struct gc_ephemeron {
GC_EMBEDDER_EPHEMERON_HEADER
uint8_t state;
uint8_t is_dead;
unsigned epoch;
struct gc_ephemeron *chain;
struct gc_ephemeron *pending;
struct gc_ephemeron *resolved;
struct gc_ref key;
struct gc_ref value;
};
size_t gc_ephemeron_size(void) { return sizeof(struct gc_ephemeron); }
struct gc_edge gc_ephemeron_key_edge(struct gc_ephemeron *e) {
return gc_edge(&e->key);
}
struct gc_edge gc_ephemeron_value_edge(struct gc_ephemeron *e) {
return gc_edge(&e->value);
}
////////////////////////////////////////////////////////////////////////
// Operations on the user-controlled chain field
////////////////////////////////////////////////////////////////////////
static struct gc_ephemeron** ephemeron_chain(struct gc_ephemeron *e) {
return &e->chain;
}
static int ephemeron_is_dead(struct gc_ephemeron *e) {
return atomic_load_explicit(&e->is_dead, memory_order_acquire);
}
static int ephemeron_is_not_dead(struct gc_ephemeron *e) {
return !ephemeron_is_dead(e);
}
void gc_ephemeron_chain_push(struct gc_ephemeron **loc,
struct gc_ephemeron *e) {
ephemeron_list_push(loc, e, ephemeron_chain);
}
static struct gc_ephemeron* follow_chain(struct gc_ephemeron **loc) {
return ephemeron_list_follow(loc, ephemeron_chain, ephemeron_is_not_dead);
}
struct gc_ephemeron* gc_ephemeron_chain_head(struct gc_ephemeron **loc) {
return follow_chain(loc);
}
struct gc_ephemeron* gc_ephemeron_chain_next(struct gc_ephemeron *e) {
return follow_chain(ephemeron_chain(e));
}
void gc_ephemeron_mark_dead(struct gc_ephemeron *e) {
atomic_store_explicit(&e->is_dead, 1, memory_order_release);
}
////////////////////////////////////////////////////////////////////////
// Operations on the GC-managed pending link
////////////////////////////////////////////////////////////////////////
static struct gc_ephemeron** ephemeron_pending(struct gc_ephemeron *e) {
return &e->pending;
}
static uint8_t ephemeron_state(struct gc_ephemeron *e) {
return atomic_load_explicit(&e->state, memory_order_acquire);
}
static int ephemeron_is_pending(struct gc_ephemeron *e) {
return ephemeron_state(e) == EPHEMERON_STATE_PENDING;
}
static void push_pending(struct gc_ephemeron **loc, struct gc_ephemeron *e) {
ephemeron_list_push(loc, e, ephemeron_pending);
}
static struct gc_ephemeron* follow_pending(struct gc_ephemeron **loc) {
return ephemeron_list_follow(loc, ephemeron_pending, ephemeron_is_pending);
}
////////////////////////////////////////////////////////////////////////
// Operations on the GC-managed resolved link
////////////////////////////////////////////////////////////////////////
static struct gc_ephemeron** ephemeron_resolved(struct gc_ephemeron *e) {
return &e->resolved;
}
static void push_resolved(struct gc_ephemeron **loc, struct gc_ephemeron *e) {
ephemeron_list_push(loc, e, ephemeron_resolved);
}
static struct gc_ephemeron* pop_resolved(struct gc_ephemeron **loc) {
return ephemeron_list_pop(loc, ephemeron_resolved);
}
////////////////////////////////////////////////////////////////////////
// Access to the association
////////////////////////////////////////////////////////////////////////
struct gc_ref gc_ephemeron_key(struct gc_ephemeron *e) {
return ephemeron_is_dead(e) ? gc_ref_null() : e->key;
}
struct gc_ref gc_ephemeron_value(struct gc_ephemeron *e) {
return ephemeron_is_dead(e) ? gc_ref_null() : e->value;
}
////////////////////////////////////////////////////////////////////////
// Tracing ephemerons
////////////////////////////////////////////////////////////////////////
struct gc_pending_ephemerons {
struct gc_ephemeron* resolved;
size_t nbuckets;
double scale;
struct gc_ephemeron* buckets[0];
};
static const size_t MIN_PENDING_EPHEMERONS_SIZE = 32;
static size_t pending_ephemerons_byte_size(size_t nbuckets) {
return sizeof(struct gc_pending_ephemerons) +
sizeof(struct gc_ephemeron*) * nbuckets;
}
static struct gc_pending_ephemerons*
gc_make_pending_ephemerons(size_t byte_size) {
size_t nbuckets = byte_size / sizeof(struct gc_ephemeron*);
if (nbuckets < MIN_PENDING_EPHEMERONS_SIZE)
nbuckets = MIN_PENDING_EPHEMERONS_SIZE;
struct gc_pending_ephemerons *ret =
malloc(pending_ephemerons_byte_size(nbuckets));
if (!ret)
return NULL;
ret->resolved = NULL;
ret->nbuckets = nbuckets;
ret->scale = nbuckets / pow(2.0, sizeof(uintptr_t) * 8);
for (size_t i = 0; i < nbuckets; i++)
ret->buckets[i] = NULL;
return ret;
}
struct gc_pending_ephemerons*
gc_prepare_pending_ephemerons(struct gc_pending_ephemerons *state,
size_t target_byte_size, double slop) {
size_t existing =
state ? pending_ephemerons_byte_size(state->nbuckets) : 0;
slop += 1.0;
if (existing * slop > target_byte_size && existing < target_byte_size * slop)
return state;
struct gc_pending_ephemerons *new_state =
gc_make_pending_ephemerons(target_byte_size);
if (!new_state)
return state;
free(state);
return new_state;
}
static struct gc_ephemeron**
pending_ephemeron_bucket(struct gc_pending_ephemerons *state,
struct gc_ref ref) {
uintptr_t hash = hash_address(gc_ref_value(ref));
size_t idx = hash * state->scale;
GC_ASSERT(idx < state->nbuckets);
return &state->buckets[idx];
}
static void
add_pending_ephemeron(struct gc_pending_ephemerons *state,
struct gc_ephemeron *e) {
struct gc_ephemeron **bucket = pending_ephemeron_bucket(state, e->key);
atomic_store_explicit(&e->state, EPHEMERON_STATE_PENDING,
memory_order_release);
push_pending(bucket, e);
}
static void maybe_resolve_ephemeron(struct gc_pending_ephemerons *state,
struct gc_ephemeron *e) {
uint8_t expected = EPHEMERON_STATE_PENDING;
if (atomic_compare_exchange_strong(&e->state, &expected,
EPHEMERON_STATE_RESOLVED))
push_resolved(&state->resolved, e);
}
// Precondition: OBJ has already been copied to tospace, but OBJ is a
// fromspace ref.
void gc_resolve_pending_ephemerons(struct gc_ref obj, struct gc_heap *heap) {
struct gc_pending_ephemerons *state = gc_heap_pending_ephemerons(heap);
struct gc_ephemeron **bucket = pending_ephemeron_bucket(state, obj);
for (struct gc_ephemeron *link = follow_pending(bucket);
link;
link = follow_pending(&link->pending)) {
if (gc_ref_value(obj) == gc_ref_value(link->key)) {
gc_visit_ephemeron_key(gc_ephemeron_key_edge(link), heap);
// PENDING -> RESOLVED, if it was pending.
maybe_resolve_ephemeron(state, link);
}
}
}
void gc_trace_ephemeron(struct gc_ephemeron *e,
void (*visit)(struct gc_edge edge, struct gc_heap *heap,
void *visit_data),
struct gc_heap *heap,
void *trace_data) {
unsigned epoch = gc_heap_ephemeron_trace_epoch(heap);
uint8_t expected = EPHEMERON_STATE_TRACED;
// TRACED[_] -> CLAIMED[_].
if (!atomic_compare_exchange_strong(&e->state, &expected,
EPHEMERON_STATE_CLAIMED))
return;
if (e->epoch == epoch) {
// CLAIMED[epoch] -> TRACED[epoch].
atomic_store_explicit(&e->state, EPHEMERON_STATE_TRACED,
memory_order_release);
return;
}
// CLAIMED[!epoch] -> CLAIMED[epoch].
e->epoch = epoch;
e->pending = NULL;
e->resolved = NULL;
// Trace chain successors, eliding any intermediate dead links. Note
// that there is a race between trace-time evacuation of the next link
// in the chain and any mutation of that link pointer by the mutator
// (which can only be to advance the chain forward past dead links).
// Collectors using this API have to eliminate this race, for example
// by not evacuating while the mutator is running.
follow_chain(&e->chain);
visit(gc_edge(&e->chain), heap, trace_data);
// Similarly there is a race between the mutator marking an ephemeron
// as dead and here; the consequence would be that we treat an
// ephemeron as live when it's not, but only for this cycle. No big
// deal.
if (atomic_load_explicit(&e->is_dead, memory_order_acquire)) {
// CLAIMED[epoch] -> TRACED[epoch].
atomic_store_explicit(&e->state, EPHEMERON_STATE_TRACED,
memory_order_release);
return;
}
// If K is live, trace V and we are done.
if (gc_visit_ephemeron_key(gc_ephemeron_key_edge(e), heap)) {
visit(gc_ephemeron_value_edge(e), heap, trace_data);
// CLAIMED[epoch] -> TRACED[epoch].
atomic_store_explicit(&e->state, EPHEMERON_STATE_TRACED,
memory_order_release);
return;
}
// Otherwise K is not yet traced, so we don't know if it is live.
// Publish the ephemeron to a global table.
struct gc_pending_ephemerons *state = gc_heap_pending_ephemerons(heap);
// CLAIMED[epoch] -> PENDING.
add_pending_ephemeron(state, e);
// Given an ephemeron E×K⇒V, there is a race between marking K and E.
// One thread could go to mark E and see that K is unmarked, so we get
// here. Meanwhile another thread could go to mark K and not see E in
// the global table yet. Therefore after publishing E, we have to
// check the mark on K again.
if (gc_visit_ephemeron_key(gc_ephemeron_key_edge(e), heap))
// K visited by another thread while we published E; PENDING ->
// RESOLVED, if still PENDING.
maybe_resolve_ephemeron(state, e);
}
void
gc_scan_pending_ephemerons(struct gc_pending_ephemerons *state,
struct gc_heap *heap, size_t shard,
size_t nshards) {
GC_ASSERT(shard < nshards);
size_t start = state->nbuckets * 1.0 * shard / nshards;
size_t end = state->nbuckets * 1.0 * (shard + 1) / nshards;
for (size_t idx = start; idx < end; idx++) {
for (struct gc_ephemeron *e = follow_pending(&state->buckets[idx]);
e;
e = follow_pending(&e->pending)) {
if (gc_visit_ephemeron_key(gc_ephemeron_key_edge(e), heap))
// PENDING -> RESOLVED, if PENDING.
maybe_resolve_ephemeron(state, e);
}
}
}
int
gc_pop_resolved_ephemerons(struct gc_heap *heap,
void (*visit)(struct gc_edge edge,
struct gc_heap *heap,
void *visit_data),
void *trace_data) {
struct gc_pending_ephemerons *state = gc_heap_pending_ephemerons(heap);
struct gc_ephemeron *resolved = atomic_exchange(&state->resolved, NULL);
if (!resolved)
return 0;
for (; resolved; resolved = resolved->resolved) {
visit(gc_ephemeron_value_edge(resolved), heap, trace_data);
// RESOLVED -> TRACED.
atomic_store_explicit(&resolved->state, EPHEMERON_STATE_TRACED,
memory_order_release);
}
return 1;
}
void
gc_sweep_pending_ephemerons(struct gc_pending_ephemerons *state,
size_t shard, size_t nshards) {
GC_ASSERT(shard < nshards);
size_t start = state->nbuckets * 1.0 * shard / nshards;
size_t end = state->nbuckets * 1.0 * (shard + 1) / nshards;
for (size_t idx = start; idx < end; idx++) {
struct gc_ephemeron **bucket = &state->buckets[idx];
for (struct gc_ephemeron *e = follow_pending(bucket);
e;
e = follow_pending(&e->pending)) {
// PENDING -> TRACED, but dead.
atomic_store_explicit(&e->is_dead, 1, memory_order_release);
atomic_store_explicit(&e->state, EPHEMERON_STATE_TRACED,
memory_order_release);
}
atomic_store_explicit(bucket, NULL, memory_order_release);
}
}
////////////////////////////////////////////////////////////////////////
// Allocation & initialization
////////////////////////////////////////////////////////////////////////
void gc_ephemeron_init_internal(struct gc_heap *heap,
struct gc_ephemeron *ephemeron,
struct gc_ref key, struct gc_ref value) {
// Caller responsible for any write barrier, though really the
// assumption is that the ephemeron is younger than the key and the
// value.
ephemeron->state = EPHEMERON_STATE_TRACED;
ephemeron->is_dead = 0;
ephemeron->epoch = gc_heap_ephemeron_trace_epoch(heap) - 1;
ephemeron->chain = NULL;
ephemeron->pending = NULL;
ephemeron->resolved = NULL;
ephemeron->key = key;
ephemeron->value = value;
}

42
gc-ephemeron.h Normal file
View file

@ -0,0 +1,42 @@
#ifndef GC_EPHEMERON_H_
#define GC_EPHEMERON_H_
#include "gc-edge.h"
#include "gc-ref.h"
#include "gc-visibility.h"
// Ephemerons establish an association between a "key" object and a
// "value" object. If the ephemeron and the key are live, then the
// value is live, and can be retrieved from the ephemeron. Ephemerons
// can be chained together, which allows them to function as links in a
// buckets-and-chains hash table.
//
// This file defines the user-facing API for ephemerons.
struct gc_heap;
struct gc_mutator;
struct gc_ephemeron;
GC_API_ size_t gc_ephemeron_size(void);
GC_API_ struct gc_ref gc_allocate_ephemeron(struct gc_mutator *mut);
GC_API_ void gc_ephemeron_init(struct gc_mutator *mut,
struct gc_ephemeron *ephemeron,
struct gc_ref key, struct gc_ref value);
GC_API_ struct gc_ref gc_ephemeron_key(struct gc_ephemeron *ephemeron);
GC_API_ struct gc_ref gc_ephemeron_value(struct gc_ephemeron *ephemeron);
GC_API_ struct gc_ephemeron* gc_ephemeron_chain_head(struct gc_ephemeron **loc);
GC_API_ void gc_ephemeron_chain_push(struct gc_ephemeron **loc,
struct gc_ephemeron *ephemeron);
GC_API_ struct gc_ephemeron* gc_ephemeron_chain_next(struct gc_ephemeron *ephemeron);
GC_API_ void gc_ephemeron_mark_dead(struct gc_ephemeron *ephemeron);
GC_API_ void gc_trace_ephemeron(struct gc_ephemeron *ephemeron,
void (*visit)(struct gc_edge edge,
struct gc_heap *heap,
void *visit_data),
struct gc_heap *heap,
void *trace_data);
#endif // GC_EPHEMERON_H_

10
gc-internal.h Normal file
View file

@ -0,0 +1,10 @@
#ifndef GC_INTERNAL_H
#define GC_INTERNAL_H
#ifndef GC_IMPL
#error internal header file, not part of API
#endif
#include "gc-ephemeron-internal.h"
#endif // GC_INTERNAL_H

5
gc-visibility.h
View file

@ -4,4 +4,9 @@
#define GC_INTERNAL __attribute__((visibility("hidden")))
#define GC_PUBLIC __attribute__((visibility("default")))
// FIXME: Conflict with bdw-gc GC_API. Switch prefix?
#ifndef GC_API_
#define GC_API_ GC_INTERNAL
#endif
#endif // GC_VISIBILITY_H

10
large-object-space.h
View file

@ -92,6 +92,16 @@ done:
return copied;
}
static int large_object_space_is_copied(struct large_object_space *space,
struct gc_ref ref) {
int copied = 0;
uintptr_t addr = gc_ref_value(ref);
pthread_mutex_lock(&space->lock);
copied = address_set_contains(&space->from_space, addr);
pthread_mutex_unlock(&space->lock);
return copied;
}
static int large_object_space_mark_object(struct large_object_space *space,
struct gc_ref ref) {
return large_object_space_copy(space, ref);

101
semi.c
View file

@ -5,9 +5,11 @@
#include <sys/mman.h>
#include <unistd.h>
#define GC_API_
#include "gc-api.h"
#define GC_IMPL 1
#include "gc-internal.h"
#include "semi-attrs.h"
#include "large-object-space.h"
@ -24,11 +26,16 @@ struct semi_space {
size_t stolen_pages;
uintptr_t base;
size_t size;
long count;
};
struct gc_heap {
struct semi_space semi_space;
struct large_object_space large_object_space;
struct gc_pending_ephemerons *pending_ephemerons;
double pending_ephemerons_size_factor;
double pending_ephemerons_size_slop;
size_t size;
long count;
int check_pending_ephemerons;
};
// One mutator per space, can just store the heap in the mutator.
struct gc_mutator {
@ -96,16 +103,20 @@ static void flip(struct semi_space *space) {
space->from_space = space->to_space;
space->to_space = space->hp;
space->limit = space->hp + space->size / 2;
space->count++;
}
static struct gc_ref copy(struct semi_space *space, struct gc_ref ref) {
static struct gc_ref copy(struct gc_heap *heap, struct semi_space *space,
struct gc_ref ref) {
size_t size;
gc_trace_object(ref, NULL, NULL, NULL, &size);
struct gc_ref new_ref = gc_ref(space->hp);
memcpy(gc_ref_heap_object(new_ref), gc_ref_heap_object(ref), size);
gc_object_forward_nonatomic(ref, new_ref);
space->hp += align_up(size, GC_ALIGNMENT);
if (GC_UNLIKELY(heap->check_pending_ephemerons))
gc_resolve_pending_ephemerons(ref, heap);
return new_ref;
}
@ -115,21 +126,26 @@ static uintptr_t scan(struct gc_heap *heap, struct gc_ref grey) {
return gc_ref_value(grey) + align_up(size, GC_ALIGNMENT);
}
static struct gc_ref forward(struct semi_space *space, struct gc_ref obj) {
static struct gc_ref forward(struct gc_heap *heap, struct semi_space *space,
struct gc_ref obj) {
uintptr_t forwarded = gc_object_forwarded_nonatomic(obj);
return forwarded ? gc_ref(forwarded) : copy(space, obj);
return forwarded ? gc_ref(forwarded) : copy(heap, space, obj);
}
static void visit_semi_space(struct gc_heap *heap, struct semi_space *space,
struct gc_edge edge, struct gc_ref ref) {
gc_edge_update(edge, forward(space, ref));
gc_edge_update(edge, forward(heap, space, ref));
}
static void visit_large_object_space(struct gc_heap *heap,
struct large_object_space *space,
struct gc_ref ref) {
if (large_object_space_copy(space, ref))
if (large_object_space_copy(space, ref)) {
if (GC_UNLIKELY(heap->check_pending_ephemerons))
gc_resolve_pending_ephemerons(ref, heap);
gc_trace_object(ref, trace, heap, NULL, NULL);
}
}
static int semi_space_contains(struct semi_space *space, struct gc_ref ref) {
@ -149,6 +165,26 @@ static void visit(struct gc_edge edge, struct gc_heap *heap) {
GC_CRASH();
}
struct gc_pending_ephemerons *
gc_heap_pending_ephemerons(struct gc_heap *heap) {
return heap->pending_ephemerons;
}
int gc_visit_ephemeron_key(struct gc_edge edge, struct gc_heap *heap) {
struct gc_ref ref = gc_edge_ref(edge);
GC_ASSERT(gc_ref_is_heap_object(ref));
if (semi_space_contains(heap_semi_space(heap), ref)) {
uintptr_t forwarded = gc_object_forwarded_nonatomic(ref);
if (!forwarded)
return 0;
gc_edge_update(edge, gc_ref(forwarded));
return 1;
} else if (large_object_space_contains(heap_large_object_space(heap), ref)) {
return large_object_space_is_copied(heap_large_object_space(heap), ref);
}
GC_CRASH();
}
static void trace(struct gc_edge edge, struct gc_heap *heap, void *visit_data) {
return visit(edge, heap);
}
@ -160,14 +196,22 @@ static void collect(struct gc_mutator *mut) {
// fprintf(stderr, "start collect #%ld:\n", space->count);
large_object_space_start_gc(large, 0);
flip(semi);
heap->count++;
heap->check_pending_ephemerons = 0;
uintptr_t grey = semi->hp;
if (mut->roots)
gc_trace_mutator_roots(mut->roots, trace, heap, NULL);
// fprintf(stderr, "pushed %zd bytes in roots\n", space->hp - grey);
while(grey < semi->hp)
grey = scan(heap, gc_ref(grey));
gc_scan_pending_ephemerons(heap->pending_ephemerons, heap, 0, 1);
heap->check_pending_ephemerons = 1;
while (gc_pop_resolved_ephemerons(heap, trace, NULL))
while(grey < semi->hp)
grey = scan(heap, gc_ref(grey));
large_object_space_finish_gc(large, 0);
semi_space_set_stolen_pages(semi, large->live_pages_at_last_collection);
gc_sweep_pending_ephemerons(heap->pending_ephemerons, 0, 1);
// fprintf(stderr, "%zd bytes copied\n", (space->size>>1)-(space->limit-space->hp));
}
@ -229,6 +273,15 @@ void* gc_allocate_pointerless(struct gc_mutator *mut, size_t size) {
return gc_allocate(mut, size);
}
struct gc_ref gc_allocate_ephemeron(struct gc_mutator *mut) {
return gc_ref_from_heap_object(gc_allocate(mut, gc_ephemeron_size()));
}
void gc_ephemeron_init(struct gc_mutator *mut, struct gc_ephemeron *ephemeron,
struct gc_ref key, struct gc_ref value) {
gc_ephemeron_init_internal(mutator_heap(mut), ephemeron, key, value);
}
static int initialize_semi_space(struct semi_space *space, size_t size) {
// Allocate even numbers of pages.
size_t page_size = getpagesize();
@ -246,7 +299,6 @@ static int initialize_semi_space(struct semi_space *space, size_t size) {
space->page_size = page_size;
space->stolen_pages = 0;
space->size = size;
space->count = 0;
return 1;
}
@ -315,6 +367,29 @@ static int parse_options(int argc, struct gc_option argv[],
return 1;
}
static int heap_prepare_pending_ephemerons(struct gc_heap *heap) {
struct gc_pending_ephemerons *cur = heap->pending_ephemerons;
size_t target = heap->size * heap->pending_ephemerons_size_factor;
double slop = heap->pending_ephemerons_size_slop;
heap->pending_ephemerons = gc_prepare_pending_ephemerons(cur, target, slop);
return !!heap->pending_ephemerons;
}
unsigned gc_heap_ephemeron_trace_epoch(struct gc_heap *heap) {
return heap->count;
}
static int heap_init(struct gc_heap *heap, size_t size) {
heap->pending_ephemerons_size_factor = 0.01;
heap->pending_ephemerons_size_slop = 0.5;
heap->count = 0;
heap->size = size;
return heap_prepare_pending_ephemerons(heap);
}
int gc_init(int argc, struct gc_option argv[],
struct gc_stack_addr *stack_base, struct gc_heap **heap,
struct gc_mutator **mut) {
@ -331,6 +406,9 @@ int gc_init(int argc, struct gc_option argv[],
if (!*mut) GC_CRASH();
*heap = mutator_heap(*mut);
if (!heap_init(*heap, options.fixed_heap_size))
return 0;
struct semi_space *space = mutator_semi_space(*mut);
if (!initialize_semi_space(space, options.fixed_heap_size))
return 0;
@ -367,7 +445,6 @@ void* gc_call_without_gc(struct gc_mutator *mut, void* (*f)(void*),
}
void gc_print_stats(struct gc_heap *heap) {
struct semi_space *space = heap_semi_space(heap);
printf("Completed %ld collections\n", space->count);
printf("Heap size is %zd\n", space->size);
printf("Completed %ld collections\n", heap->count);
printf("Heap size is %zd\n", heap->size);
}

2
simple-gc-embedder.h
View file

@ -5,6 +5,8 @@
#include "gc-config.h"
#include "gc-embedder-api.h"
#define GC_EMBEDDER_EPHEMERON_HEADER struct gc_header header;
static inline int
gc_is_valid_conservative_ref_displacement(uintptr_t displacement) {
#if GC_CONSERVATIVE_ROOTS || GC_CONSERVATIVE_TRACE

5
simple-roots-api.h
View file

@ -5,8 +5,9 @@
#include "simple-roots-types.h"
#define HANDLE_TO(T) union { T* v; struct handle handle; }
#define HANDLE_REF(h) h.v
#define HANDLE_SET(h,val) do { h.v = val; } while (0)
#define HANDLE_LOC(h) &(h).v
#define HANDLE_REF(h) (h).v
#define HANDLE_SET(h,val) do { (h).v = val; } while (0)
#define PUSH_HANDLE(cx, h) push_handle(&(cx)->roots.roots, &h.handle)
#define POP_HANDLE(cx) pop_handle(&(cx)->roots.roots)

194
whippet.c
View file

@ -7,10 +7,10 @@
#include <string.h>
#include <unistd.h>
#define GC_API_
#include "gc-api.h"
#define GC_IMPL 1
#include "gc-internal.h"
#include "debug.h"
#include "gc-align.h"
@ -77,9 +77,9 @@ enum metadata_byte {
METADATA_BYTE_MARK_1 = 4,
METADATA_BYTE_MARK_2 = 8,
METADATA_BYTE_END = 16,
METADATA_BYTE_UNUSED_1 = 32,
METADATA_BYTE_UNUSED_2 = 64,
METADATA_BYTE_UNUSED_3 = 128
METADATA_BYTE_EPHEMERON = 32,
METADATA_BYTE_PINNED = 64,
METADATA_BYTE_UNUSED_1 = 128
};
static uint8_t rotate_dead_survivor_marked(uint8_t mask) {
@ -307,6 +307,8 @@ struct gc_heap {
size_t size;
int collecting;
int mark_while_stopping;
int check_pending_ephemerons;
struct gc_pending_ephemerons *pending_ephemerons;
enum gc_kind gc_kind;
int multithreaded;
size_t active_mutator_count;
@ -323,6 +325,8 @@ struct gc_heap {
double minor_gc_yield_threshold;
double major_gc_yield_threshold;
double minimum_major_gc_yield_threshold;
double pending_ephemerons_size_factor;
double pending_ephemerons_size_slop;
};
struct gc_mutator_mark_buf {
@ -649,8 +653,8 @@ static inline int mark_space_contains(struct mark_space *space,
return mark_space_contains_address(space, gc_ref_value(ref));
}
static inline int trace_edge(struct gc_heap *heap, struct gc_edge edge) {
struct gc_ref ref = gc_edge_ref(edge);
static inline int do_trace(struct gc_heap *heap, struct gc_edge edge,
struct gc_ref ref) {
if (!gc_ref_is_heap_object(ref))
return 0;
if (GC_LIKELY(mark_space_contains(heap_mark_space(heap), ref))) {
@ -666,6 +670,63 @@ static inline int trace_edge(struct gc_heap *heap, struct gc_edge edge) {
GC_CRASH();
}
static inline int trace_edge(struct gc_heap *heap, struct gc_edge edge) {
struct gc_ref ref = gc_edge_ref(edge);
int is_new = do_trace(heap, edge, ref);
if (GC_UNLIKELY(atomic_load_explicit(&heap->check_pending_ephemerons,
memory_order_relaxed)))
gc_resolve_pending_ephemerons(ref, heap);
return is_new;
}
int gc_visit_ephemeron_key(struct gc_edge edge, struct gc_heap *heap) {
struct gc_ref ref = gc_edge_ref(edge);
if (!gc_ref_is_heap_object(ref))
return 0;
if (GC_LIKELY(mark_space_contains(heap_mark_space(heap), ref))) {
struct mark_space *space = heap_mark_space(heap);
uint8_t *metadata = metadata_byte_for_object(ref);
uint8_t byte = *metadata;
if (byte & space->marked_mask)
return 1;
if (!space->evacuating)
return 0;
if (!block_summary_has_flag(block_summary_for_addr(gc_ref_value(ref)),
BLOCK_EVACUATE))
return 0;
struct gc_atomic_forward fwd = gc_atomic_forward_begin(ref);
switch (fwd.state) {
case GC_FORWARDING_STATE_NOT_FORWARDED:
return 0;
case GC_FORWARDING_STATE_BUSY:
// Someone else claimed this object first. Spin until new address
// known, or evacuation aborts.
for (size_t spin_count = 0;; spin_count++) {
if (gc_atomic_forward_retry_busy(&fwd))
break;
yield_for_spin(spin_count);
}
if (fwd.state == GC_FORWARDING_STATE_ABORTED)
// Remote evacuation aborted; remote will mark and enqueue.
return 1;
ASSERT(fwd.state == GC_FORWARDING_STATE_FORWARDED);
// Fall through.
case GC_FORWARDING_STATE_FORWARDED:
gc_edge_update(edge, gc_ref(gc_atomic_forward_address(&fwd)));
return 1;
default:
GC_CRASH();
}
} else if (large_object_space_contains(heap_large_object_space(heap), ref)) {
return large_object_space_is_copied(heap_large_object_space(heap), ref);
}
GC_CRASH();
}
static inline struct gc_ref mark_space_mark_conservative_ref(struct mark_space *space,
struct gc_conservative_ref ref,
int possibly_interior) {
@ -732,9 +793,9 @@ static inline struct gc_ref mark_space_mark_conservative_ref(struct mark_space *
return gc_ref(addr);
}
static inline struct gc_ref trace_conservative_ref(struct gc_heap *heap,
struct gc_conservative_ref ref,
int possibly_interior) {
static inline struct gc_ref do_trace_conservative_ref(struct gc_heap *heap,
struct gc_conservative_ref ref,
int possibly_interior) {
if (!gc_conservative_ref_might_be_a_heap_object(ref, possibly_interior))
return gc_ref_null();
@ -746,6 +807,19 @@ static inline struct gc_ref trace_conservative_ref(struct gc_heap *heap,
ref, possibly_interior);
}
static inline struct gc_ref trace_conservative_ref(struct gc_heap *heap,
struct gc_conservative_ref ref,
int possibly_interior) {
struct gc_ref ret = do_trace_conservative_ref(heap, ref, possibly_interior);
if (gc_ref_is_heap_object(ret) &&
GC_UNLIKELY(atomic_load_explicit(&heap->check_pending_ephemerons,
memory_order_relaxed)))
gc_resolve_pending_ephemerons(ret, heap);
return ret;
}
static inline size_t mark_space_object_size(struct mark_space *space,
struct gc_ref ref) {
uint8_t *loc = metadata_byte_for_object(ref);
@ -1091,18 +1165,37 @@ static inline void tracer_trace_conservative_ref(struct gc_conservative_ref ref,
tracer_enqueue(resolved, heap, data);
}
static inline void trace_one_conservatively(struct gc_ref ref,
struct gc_heap *heap,
void *mark_data) {
size_t bytes;
if (GC_LIKELY(mark_space_contains(heap_mark_space(heap), ref))) {
// Generally speaking we trace conservatively and don't allow much
// in the way of incremental precise marking on a
// conservative-by-default heap. But, we make an exception for
// ephemerons.
uint8_t meta = *metadata_byte_for_addr(gc_ref_value(ref));
if (GC_UNLIKELY(meta & METADATA_BYTE_EPHEMERON)) {
gc_trace_ephemeron(gc_ref_heap_object(ref), tracer_visit, heap,
mark_data);
return;
}
bytes = mark_space_object_size(heap_mark_space(heap), ref);
} else {
bytes = large_object_space_object_size(heap_large_object_space(heap), ref);
}
trace_conservative_edges(gc_ref_value(ref),
gc_ref_value(ref) + bytes,
tracer_trace_conservative_ref, heap,
mark_data);
}
static inline void trace_one(struct gc_ref ref, struct gc_heap *heap,
void *mark_data) {
if (gc_has_conservative_intraheap_edges()) {
size_t bytes = GC_LIKELY(mark_space_contains(heap_mark_space(heap), ref))
? mark_space_object_size(heap_mark_space(heap), ref)
: large_object_space_object_size(heap_large_object_space(heap), ref);
trace_conservative_edges(gc_ref_value(ref),
gc_ref_value(ref) + bytes,
tracer_trace_conservative_ref, heap, mark_data);
} else {
if (gc_has_conservative_intraheap_edges())
trace_one_conservatively(ref, heap, mark_data);
else
gc_trace_object(ref, tracer_visit, heap, mark_data, NULL);
}
}
static void
@ -1672,6 +1765,26 @@ static void mark_space_finish_gc(struct mark_space *space,
release_evacuation_target_blocks(space);
}
static void resolve_ephemerons_lazily(struct gc_heap *heap) {
atomic_store_explicit(&heap->check_pending_ephemerons, 0,
memory_order_release);
}
static void resolve_ephemerons_eagerly(struct gc_heap *heap) {
atomic_store_explicit(&heap->check_pending_ephemerons, 1,
memory_order_release);
gc_scan_pending_ephemerons(heap->pending_ephemerons, heap, 0, 1);
}
static int enqueue_resolved_ephemerons(struct gc_heap *heap) {
return gc_pop_resolved_ephemerons(heap, trace_and_enqueue_globally,
NULL);
}
static void sweep_ephemerons(struct gc_heap *heap) {
return gc_sweep_pending_ephemerons(heap->pending_ephemerons, 0, 1);
}
static void collect(struct gc_mutator *mut) {
struct gc_heap *heap = mutator_heap(mut);
struct mark_space *space = heap_mark_space(heap);
@ -1684,6 +1797,7 @@ static void collect(struct gc_mutator *mut) {
enum gc_kind gc_kind = determine_collection_kind(heap);
update_mark_patterns(space, !(gc_kind & GC_KIND_FLAG_MINOR));
large_object_space_start_gc(lospace, gc_kind & GC_KIND_FLAG_MINOR);
resolve_ephemerons_lazily(heap);
tracer_prepare(heap);
request_mutators_to_stop(heap);
trace_mutator_roots_with_lock_before_stop(mut);
@ -1697,6 +1811,10 @@ static void collect(struct gc_mutator *mut) {
prepare_for_evacuation(heap);
trace_roots_after_stop(heap);
tracer_trace(heap);
resolve_ephemerons_eagerly(heap);
while (enqueue_resolved_ephemerons(heap))
tracer_trace(heap);
sweep_ephemerons(heap);
tracer_release(heap);
mark_space_finish_gc(space, gc_kind);
large_object_space_finish_gc(lospace, gc_kind & GC_KIND_FLAG_MINOR);
@ -2054,6 +2172,31 @@ void* gc_allocate_pointerless(struct gc_mutator *mut, size_t size) {
return gc_allocate(mut, size);
}
struct gc_ref gc_allocate_ephemeron(struct gc_mutator *mut) {
struct gc_ref ret =
gc_ref_from_heap_object(gc_allocate(mut, gc_ephemeron_size()));
if (gc_has_conservative_intraheap_edges()) {
uint8_t *metadata = metadata_byte_for_addr(gc_ref_value(ret));
*metadata |= METADATA_BYTE_EPHEMERON;
}
return ret;
}
void gc_ephemeron_init(struct gc_mutator *mut, struct gc_ephemeron *ephemeron,
struct gc_ref key, struct gc_ref value) {
gc_ephemeron_init_internal(mutator_heap(mut), ephemeron, key, value);
// No write barrier: we require that the ephemeron be newer than the
// key or the value.
}
struct gc_pending_ephemerons *gc_heap_pending_ephemerons(struct gc_heap *heap) {
return heap->pending_ephemerons;
}
unsigned gc_heap_ephemeron_trace_epoch(struct gc_heap *heap) {
return heap->count;
}
#define FOR_EACH_GC_OPTION(M) \
M(GC_OPTION_FIXED_HEAP_SIZE, "fixed-heap-size") \
M(GC_OPTION_PARALLELISM, "parallelism")
@ -2141,6 +2284,16 @@ static struct slab* allocate_slabs(size_t nslabs) {
return (struct slab*) aligned_base;
}
static int heap_prepare_pending_ephemerons(struct gc_heap *heap) {
struct gc_pending_ephemerons *cur = heap->pending_ephemerons;
size_t target = heap->size * heap->pending_ephemerons_size_factor;
double slop = heap->pending_ephemerons_size_slop;
heap->pending_ephemerons = gc_prepare_pending_ephemerons(cur, target, slop);
return !!heap->pending_ephemerons;
}
static int heap_init(struct gc_heap *heap, struct options *options) {
// *heap is already initialized to 0.
@ -2152,6 +2305,8 @@ static int heap_init(struct gc_heap *heap, struct options *options) {
if (!tracer_init(heap, options->parallelism))
GC_CRASH();
heap->pending_ephemerons_size_factor = 0.005;
heap->pending_ephemerons_size_slop = 0.5;
heap->fragmentation_low_threshold = 0.05;
heap->fragmentation_high_threshold = 0.10;
heap->minor_gc_yield_threshold = 0.30;
@ -2159,6 +2314,9 @@ static int heap_init(struct gc_heap *heap, struct options *options) {
heap->major_gc_yield_threshold =
clamp_major_gc_yield_threshold(heap, heap->minor_gc_yield_threshold);
if (!heap_prepare_pending_ephemerons(heap))
GC_CRASH();
return 1;
}