diff --git a/bdw.c b/bdw.c index caf53b69f..c8fdc5a80 100644 --- a/bdw.c +++ b/bdw.c @@ -20,8 +20,6 @@ #error bdw-gc is a conservative collector #endif -#include "conservative-roots-embedder.h" - // When pthreads are used, let `libgc' know about it and redirect // allocation calls such as `GC_MALLOC ()' to (contention-free, faster) // thread-local allocation. diff --git a/conservative-roots-api.h b/conservative-roots-api.h deleted file mode 100644 index 1619cf640..000000000 --- a/conservative-roots-api.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef CONSERVATIVE_ROOTS_API_H -#define CONSERVATIVE_ROOTS_API_H - -#include "conservative-roots-types.h" - -#define HANDLE_TO(T) union { T* v; struct handle handle; } -#define HANDLE_REF(h) h.v -#define HANDLE_SET(h,val) do { h.v = val; } while (0) -#define PUSH_HANDLE(cx, h) do { (void) &h; } while (0) -#define POP_HANDLE(cx) do { } while (0) - -#endif // CONSERVATIVE_ROOTS_API_H diff --git a/conservative-roots-embedder.h b/conservative-roots-embedder.h deleted file mode 100644 index 4d2c4fa7b..000000000 --- a/conservative-roots-embedder.h +++ /dev/null @@ -1,54 +0,0 @@ -#ifndef CONSERVATIVE_ROOTS_EMBEDDER_H -#define CONSERVATIVE_ROOTS_EMBEDDER_H - -#include "gc-config.h" -#include "gc-embedder-api.h" - -static inline int gc_has_mutator_conservative_roots(void) { - return 1; -} -static inline int gc_mutator_conservative_roots_may_be_interior(void) { - return 1; -} -static inline int gc_has_global_conservative_roots(void) { - return 1; -} -static inline int gc_has_conservative_intraheap_edges(void) { - return GC_CONSERVATIVE_TRACE; -} - -static inline int -gc_is_valid_conservative_ref_displacement(uintptr_t displacement) { - // Here is where you would allow tagged heap object references. - return displacement == 0; -} -static inline int -gc_conservative_ref_might_be_a_heap_object(struct gc_conservative_ref ref, - int possibly_interior) { - // Assume that the minimum page size is 4096, and that the first page - // will contain no heap objects. - if (gc_conservative_ref_value(ref) < 4096) - return 0; - if (possibly_interior) - return 1; - return gc_is_valid_conservative_ref_displacement - (gc_conservative_ref_value(ref) & (sizeof(uintptr_t) - 1)); -} - -static inline void gc_trace_mutator_roots(struct gc_mutator_roots *roots, - void (*trace_edge)(struct gc_edge edge, - struct gc_heap *heap, - void *trace_data), - struct gc_heap *heap, - void *trace_data) { -} - -static inline void gc_trace_heap_roots(struct gc_heap_roots *roots, - void (*trace_edge)(struct gc_edge edge, - struct gc_heap *heap, - void *trace_data), - struct gc_heap *heap, - void *trace_data) { -} - -#endif // CONSERVATIVE_ROOTS_EMBEDDER_H diff --git a/conservative-roots-types.h b/conservative-roots-types.h deleted file mode 100644 index 4744d746e..000000000 --- a/conservative-roots-types.h +++ /dev/null @@ -1,8 +0,0 @@ -#ifndef CONSERVATIVE_ROOTS_TYPES_H -#define CONSERVATIVE_ROOTS_TYPES_H - -struct handle { void *unused; }; -struct gc_heap_roots { void *unused; }; -struct gc_mutator_roots { void *unused; }; - -#endif // CONSERVATIVE_ROOTS_TYPES_H diff --git a/gc-embedder-api.h b/gc-embedder-api.h index 3b3682a71..e0a6b3b5a 100644 --- a/gc-embedder-api.h +++ b/gc-embedder-api.h @@ -1,7 +1,7 @@ #ifndef GC_EMBEDDER_API_H #define GC_EMBEDDER_API_H -#include "gc-conservative-ref.h" +#include "gc-config.h" #include "gc-edge.h" #include "gc-forwarding.h" @@ -14,14 +14,7 @@ struct gc_heap_roots; struct gc_atomic_forward; struct gc_heap; -GC_EMBEDDER_API inline int gc_has_mutator_conservative_roots(void); -GC_EMBEDDER_API inline int gc_has_global_conservative_roots(void); -GC_EMBEDDER_API inline int gc_has_conservative_intraheap_edges(void); -GC_EMBEDDER_API inline int gc_mutator_conservative_roots_may_be_interior(void); - GC_EMBEDDER_API inline int gc_is_valid_conservative_ref_displacement(uintptr_t displacement); -GC_EMBEDDER_API inline int gc_conservative_ref_might_be_a_heap_object(struct gc_conservative_ref, - int possibly_interior); GC_EMBEDDER_API inline void gc_trace_object(struct gc_ref ref, void (*visit)(struct gc_edge edge, diff --git a/gc-trace.h b/gc-trace.h new file mode 100644 index 000000000..b9e4691e8 --- /dev/null +++ b/gc-trace.h @@ -0,0 +1,44 @@ +#ifndef GC_TRACE_H +#define GC_TRACE_H + +#ifndef GC_IMPL +#error internal header file, not part of API +#endif + +#include "gc-config.h" +#include "gc-assert.h" +#include "gc-conservative-ref.h" +#include "gc-embedder-api.h" + +static inline int gc_has_mutator_conservative_roots(void) { + return GC_CONSERVATIVE_ROOTS; +} +static inline int gc_mutator_conservative_roots_may_be_interior(void) { + return 1; +} +static inline int gc_has_global_conservative_roots(void) { + return GC_CONSERVATIVE_ROOTS; +} +static inline int gc_has_conservative_intraheap_edges(void) { + return GC_CONSERVATIVE_TRACE; +} + +static inline int gc_has_conservative_roots(void) { + return gc_has_mutator_conservative_roots() || + gc_has_global_conservative_roots(); +} + +static inline int +gc_conservative_ref_might_be_a_heap_object(struct gc_conservative_ref ref, + int possibly_interior) { + // Assume that the minimum page size is 4096, and that the first page + // will contain no heap objects. + if (gc_conservative_ref_value(ref) < 4096) + return 0; + if (possibly_interior) + return 1; + return gc_is_valid_conservative_ref_displacement + (gc_conservative_ref_value(ref) & (sizeof(uintptr_t) - 1)); +} + +#endif // GC_TRACE_H diff --git a/mt-gcbench.c b/mt-gcbench.c index 744a7e66b..ac0eb1ff9 100644 --- a/mt-gcbench.c +++ b/mt-gcbench.c @@ -47,13 +47,7 @@ #include "assert.h" #include "gc-api.h" #include "mt-gcbench-types.h" -#if GC_PRECISE_ROOTS -#include "precise-roots-api.h" -#endif -#if GC_CONSERVATIVE_ROOTS -#include "conservative-roots-api.h" -#endif -#include "mt-gcbench-types.h" +#include "simple-roots-api.h" #include "simple-allocator.h" #define MAX_THREAD_COUNT 256 diff --git a/precise-roots-embedder.h b/precise-roots-embedder.h deleted file mode 100644 index 94192cb51..000000000 --- a/precise-roots-embedder.h +++ /dev/null @@ -1,61 +0,0 @@ -#ifndef PRECISE_ROOTS_EMBEDDER_H -#define PRECISE_ROOTS_EMBEDDER_H - -#include "gc-edge.h" -#include "gc-embedder-api.h" -#include "precise-roots-types.h" - -static inline int gc_has_mutator_conservative_roots(void) { - return 0; -} -static inline int gc_mutator_conservative_roots_may_be_interior(void) { - return 0; -} -static inline int gc_has_global_conservative_roots(void) { - return 0; -} -static inline int gc_has_conservative_intraheap_edges(void) { - return 0; -} - -static inline int -gc_is_valid_conservative_ref_displacement(uintptr_t displacement) { - GC_CRASH(); -} -static inline int -gc_conservative_ref_might_be_a_heap_object(struct gc_conservative_ref ref, - int possibly_interior) { - GC_CRASH(); -} - -static inline void visit_roots(struct handle *roots, - void (*trace_edge)(struct gc_edge edge, - struct gc_heap *heap, - void *trace_data), - struct gc_heap *heap, - void *trace_data) { - for (struct handle *h = roots; h; h = h->next) - trace_edge(gc_edge(&h->v), heap, trace_data); -} - -static inline void gc_trace_mutator_roots(struct gc_mutator_roots *roots, - void (*trace_edge)(struct gc_edge edge, - struct gc_heap *heap, - void *trace_data), - struct gc_heap *heap, - void *trace_data) { - if (roots) - visit_roots(roots->roots, trace_edge, heap, trace_data); -} - -static inline void gc_trace_heap_roots(struct gc_heap_roots *roots, - void (*trace_edge)(struct gc_edge edge, - struct gc_heap *heap, - void *trace_data), - struct gc_heap *heap, - void *trace_data) { - if (roots) - visit_roots(roots->roots, trace_edge, heap, trace_data); -} - -#endif // PRECISE_ROOTS_EMBEDDER_H diff --git a/quads.c b/quads.c index b7d1bccc3..27923f1f7 100644 --- a/quads.c +++ b/quads.c @@ -5,12 +5,7 @@ #include "assert.h" #include "gc-api.h" -#if GC_PRECISE_ROOTS -#include "precise-roots-api.h" -#endif -#if GC_CONSERVATIVE_ROOTS -#include "conservative-roots-api.h" -#endif +#include "simple-roots-api.h" #include "quads-types.h" #include "simple-allocator.h" diff --git a/semi.c b/semi.c index 2a3b19f23..4c505b60e 100644 --- a/semi.c +++ b/semi.c @@ -11,10 +11,6 @@ #include "semi-attrs.h" #include "large-object-space.h" -#if GC_PRECISE_ROOTS -#include "precise-roots-embedder.h" -#endif - #if GC_CONSERVATIVE_ROOTS #error semi is a precise collector #endif diff --git a/simple-gc-embedder.h b/simple-gc-embedder.h index b97d1d7f0..758e56462 100644 --- a/simple-gc-embedder.h +++ b/simple-gc-embedder.h @@ -1,8 +1,21 @@ #include #include "simple-tagging-scheme.h" +#include "simple-roots-types.h" +#include "gc-config.h" #include "gc-embedder-api.h" +static inline int +gc_is_valid_conservative_ref_displacement(uintptr_t displacement) { +#if GC_CONSERVATIVE_ROOTS || GC_CONSERVATIVE_TRACE + // Here is where you would allow tagged heap object references. + return displacement == 0; +#else + // Shouldn't get here. + GC_CRASH(); +#endif +} + static inline void gc_trace_object(struct gc_ref ref, void (*trace_edge)(struct gc_edge edge, struct gc_heap *heap, @@ -10,6 +23,10 @@ static inline void gc_trace_object(struct gc_ref ref, struct gc_heap *heap, void *trace_data, size_t *size) { +#if GC_CONSERVATIVE_TRACE + // Shouldn't get here. + GC_CRASH(); +#else switch (tag_live_alloc_kind(*tag_word(ref))) { #define SCAN_OBJECT(name, Name, NAME) \ case ALLOC_KIND_##NAME: \ @@ -24,15 +41,38 @@ static inline void gc_trace_object(struct gc_ref ref, default: GC_CRASH(); } +#endif } -#if GC_PRECISE_ROOTS -#include "precise-roots-embedder.h" -#endif +static inline void visit_roots(struct handle *roots, + void (*trace_edge)(struct gc_edge edge, + struct gc_heap *heap, + void *trace_data), + struct gc_heap *heap, + void *trace_data) { + for (struct handle *h = roots; h; h = h->next) + trace_edge(gc_edge(&h->v), heap, trace_data); +} -#if GC_CONSERVATIVE_ROOTS -#include "conservative-roots-embedder.h" -#endif +static inline void gc_trace_mutator_roots(struct gc_mutator_roots *roots, + void (*trace_edge)(struct gc_edge edge, + struct gc_heap *heap, + void *trace_data), + struct gc_heap *heap, + void *trace_data) { + if (roots) + visit_roots(roots->roots, trace_edge, heap, trace_data); +} + +static inline void gc_trace_heap_roots(struct gc_heap_roots *roots, + void (*trace_edge)(struct gc_edge edge, + struct gc_heap *heap, + void *trace_data), + struct gc_heap *heap, + void *trace_data) { + if (roots) + visit_roots(roots->roots, trace_edge, heap, trace_data); +} static inline uintptr_t gc_object_forwarded_nonatomic(struct gc_ref ref) { uintptr_t tag = *tag_word(ref); @@ -108,9 +148,3 @@ gc_atomic_forward_address(struct gc_atomic_forward *fwd) { GC_ASSERT(fwd->state == GC_FORWARDING_STATE_FORWARDED); return fwd->data; } - -static inline uintptr_t -gc_conservative_ref_heap_address(struct gc_conservative_ref ref) { - // The specific spaces are responsible for checking alignment. - return gc_conservative_ref_value(ref); -} diff --git a/precise-roots-api.h b/simple-roots-api.h similarity index 60% rename from precise-roots-api.h rename to simple-roots-api.h index ced560d15..1cdfc15e0 100644 --- a/precise-roots-api.h +++ b/simple-roots-api.h @@ -1,7 +1,8 @@ -#ifndef PRECISE_ROOTS_API_H -#define PRECISE_ROOTS_API_H +#ifndef SIMPLE_ROOTS_API_H +#define SIMPLE_ROOTS_API_H -#include "precise-roots-types.h" +#include "gc-config.h" +#include "simple-roots-types.h" #define HANDLE_TO(T) union { T* v; struct handle handle; } #define HANDLE_REF(h) h.v @@ -10,12 +11,15 @@ #define POP_HANDLE(cx) pop_handle(&(cx)->roots.roots) static inline void push_handle(struct handle **roots, struct handle *handle) { - handle->next = *roots; - *roots = handle; + if (GC_PRECISE_ROOTS) { + handle->next = *roots; + *roots = handle; + } } static inline void pop_handle(struct handle **roots) { - *roots = (*roots)->next; + if (GC_PRECISE_ROOTS) + *roots = (*roots)->next; } -#endif // PRECISE_ROOTS_API_H +#endif // SIMPLE_ROOTS_API_H diff --git a/precise-roots-types.h b/simple-roots-types.h similarity index 63% rename from precise-roots-types.h rename to simple-roots-types.h index d2dc96491..6d47fa788 100644 --- a/precise-roots-types.h +++ b/simple-roots-types.h @@ -1,5 +1,5 @@ -#ifndef PRECISE_ROOTS_TYPES_H -#define PRECISE_ROOTS_TYPES_H +#ifndef SIMPLE_ROOTS_TYPES_H +#define SIMPLE_ROOTS_TYPES_H struct handle { void *v; @@ -14,4 +14,4 @@ struct gc_mutator_roots { struct handle *roots; }; -#endif // PRECISE_ROOTS_TYPES_H +#endif // SIMPLE_ROOTS_TYPES_H diff --git a/whippet.c b/whippet.c index 7ae78671b..90f2b470c 100644 --- a/whippet.c +++ b/whippet.c @@ -17,6 +17,7 @@ #include "gc-inline.h" #include "gc-platform.h" #include "gc-stack.h" +#include "gc-trace.h" #include "large-object-space.h" #if GC_PARALLEL #include "parallel-tracer.h" @@ -26,14 +27,6 @@ #include "spin.h" #include "whippet-attrs.h" -#if GC_PRECISE_ROOTS -#include "precise-roots-embedder.h" -#endif - -#if GC_CONSERVATIVE_ROOTS -#include "conservative-roots-embedder.h" -#endif - #define GRANULE_SIZE 16 #define GRANULE_SIZE_LOG_2 4 #define MEDIUM_OBJECT_THRESHOLD 256 @@ -1455,11 +1448,6 @@ static double clamp_major_gc_yield_threshold(struct gc_heap *heap, return threshold; } -static inline int has_conservative_roots(void) { - return gc_has_mutator_conservative_roots() || - gc_has_global_conservative_roots(); -} - static enum gc_kind determine_collection_kind(struct gc_heap *heap) { struct mark_space *mark_space = heap_mark_space(heap); enum gc_kind previous_gc_kind = atomic_load(&heap->gc_kind); @@ -1489,7 +1477,7 @@ static enum gc_kind determine_collection_kind(struct gc_heap *heap) { // blocks, try to avoid any pinning caused by the ragged-stop // marking. Of course if the mutator has conservative roots we will // have pinning anyway and might as well allow ragged stops. - mark_while_stopping = has_conservative_roots(); + mark_while_stopping = gc_has_conservative_roots(); } else if (previous_gc_kind == GC_KIND_MAJOR_EVACUATING && fragmentation >= heap->fragmentation_low_threshold) { DEBUG("continuing evacuation due to fragmentation %.2f%% > %.2f%%\n",