diff --git a/conservative-roots-embedder.h b/conservative-roots-embedder.h index 2ac2d2b78..15447a2c4 100644 --- a/conservative-roots-embedder.h +++ b/conservative-roots-embedder.h @@ -16,16 +16,34 @@ static inline int gc_has_conservative_intraheap_edges(void) { return 0; } -static inline void gc_trace_precise_mutator_roots(struct gc_mutator_roots *roots, - void (*trace_edge)(struct gc_edge edge, - void *trace_data), - void *trace_data) { +static inline int +gc_is_valid_conservative_ref_displacement(uintptr_t displacement) { + // Here is where you would allow tagged heap object references. + return displacement == 0; +} +static inline int +gc_conservative_ref_might_be_a_heap_object(struct gc_conservative_ref ref, + int possibly_interior) { + // Assume that the minimum page size is 4096, and that the first page + // will contain no heap objects. + if (gc_conservative_ref_value(ref) < 4096) + return 0; + if (possibly_interior) + return 1; + return gc_is_valid_conservative_ref_displacement + (gc_conservative_ref_value(ref) & (sizeof(uintptr_t) - 1)); } -static inline void gc_trace_precise_heap_roots(struct gc_heap_roots *roots, - void (*trace_edge)(struct gc_edge edge, - void *trace_data), - void *trace_data) { +static inline void gc_trace_mutator_roots(struct gc_mutator_roots *roots, + void (*trace_edge)(struct gc_edge edge, + void *trace_data), + void *trace_data) { +} + +static inline void gc_trace_heap_roots(struct gc_heap_roots *roots, + void (*trace_edge)(struct gc_edge edge, + void *trace_data), + void *trace_data) { } #endif // CONSERVATIVE_ROOTS_EMBEDDER_H diff --git a/gc-conservative-ref.h b/gc-conservative-ref.h new file mode 100644 index 000000000..a2b260384 --- /dev/null +++ b/gc-conservative-ref.h @@ -0,0 +1,17 @@ +#ifndef GC_CONSERVATIVE_REF_H +#define GC_CONSERVATIVE_REF_H + +#include + +struct gc_conservative_ref { + uintptr_t value; +}; + +static inline struct gc_conservative_ref gc_conservative_ref(uintptr_t value) { + return (struct gc_conservative_ref){value}; +} +static inline uintptr_t gc_conservative_ref_value(struct gc_conservative_ref ref) { + return ref.value; +} + +#endif // GC_CONSERVATIVE_REF_H diff --git a/gc-embedder-api.h b/gc-embedder-api.h index b24483245..2d74ed0a4 100644 --- a/gc-embedder-api.h +++ b/gc-embedder-api.h @@ -1,6 +1,7 @@ #ifndef GC_EMBEDDER_API_H #define GC_EMBEDDER_API_H +#include "gc-conservative-ref.h" #include "gc-edge.h" #include "gc-forwarding.h" @@ -17,19 +18,24 @@ GC_EMBEDDER_API inline int gc_has_global_conservative_roots(void); GC_EMBEDDER_API inline int gc_has_conservative_intraheap_edges(void); GC_EMBEDDER_API inline int gc_mutator_conservative_roots_may_be_interior(void); +GC_EMBEDDER_API inline int gc_is_valid_conservative_ref_displacement(uintptr_t displacement); +GC_EMBEDDER_API inline int gc_conservative_ref_might_be_a_heap_object(struct gc_conservative_ref, + int possibly_interior); + GC_EMBEDDER_API inline void gc_trace_object(struct gc_ref ref, void (*trace_edge)(struct gc_edge edge, void *trace_data), void *trace_data, size_t *size) GC_ALWAYS_INLINE; -GC_EMBEDDER_API inline void gc_trace_precise_mutator_roots(struct gc_mutator_roots *roots, + +GC_EMBEDDER_API inline void gc_trace_mutator_roots(struct gc_mutator_roots *roots, void (*trace_edge)(struct gc_edge edge, void *trace_data), void *trace_data); -GC_EMBEDDER_API inline void gc_trace_precise_heap_roots(struct gc_heap_roots *roots, - void (*trace_edge)(struct gc_edge edge, - void *trace_data), - void *trace_data); +GC_EMBEDDER_API inline void gc_trace_heap_roots(struct gc_heap_roots *roots, + void (*trace_edge)(struct gc_edge edge, + void *trace_data), + void *trace_data); GC_EMBEDDER_API inline uintptr_t gc_object_forwarded_nonatomic(struct gc_ref ref); GC_EMBEDDER_API inline void gc_object_forward_nonatomic(struct gc_ref ref, diff --git a/large-object-space.h b/large-object-space.h index 01bc4cfc7..6bb7a5af7 100644 --- a/large-object-space.h +++ b/large-object-space.h @@ -10,6 +10,7 @@ #include #include "gc-ref.h" +#include "gc-conservative-ref.h" #include "address-map.h" #include "address-set.h" @@ -90,6 +91,11 @@ done: return copied; } +static int large_object_space_mark_object(struct large_object_space *space, + struct gc_ref ref) { + return large_object_space_copy(space, ref); +} + static void large_object_space_reclaim_one(uintptr_t addr, void *data) { struct large_object_space *space = data; size_t npages = address_map_lookup(&space->object_pages, addr, 0); @@ -145,6 +151,38 @@ static void large_object_space_finish_gc(struct large_object_space *space, pthread_mutex_unlock(&space->lock); } +static inline struct gc_ref +large_object_space_mark_conservative_ref(struct large_object_space *space, + struct gc_conservative_ref ref, + int possibly_interior) { + uintptr_t addr = gc_conservative_ref_value(ref); + + if (possibly_interior) { + // FIXME: This only allows interior pointers within the first page. + // BDW-GC doesn't have all-interior-pointers on for intraheap edges + // or edges originating in static data but by default does allow + // them from stack edges; probably we should too. + addr &= ~(space->page_size - 1); + } else { + // Addr not aligned on page boundary? Not a large object. + uintptr_t displacement = addr & (space->page_size - 1); + if (!gc_is_valid_conservative_ref_displacement(displacement)) + return gc_ref_null(); + addr -= displacement; + } + + pthread_mutex_lock(&space->lock); + // ptr might be in fromspace or tospace. Just check the object_pages table, which + // contains both, as well as object_pages for free blocks. + int found = address_map_contains(&space->object_pages, addr); + pthread_mutex_unlock(&space->lock); + + if (found && large_object_space_copy(space, gc_ref(addr))) + return gc_ref(addr); + + return gc_ref_null(); +} + static inline int large_object_space_contains(struct large_object_space *space, struct gc_ref ref) { pthread_mutex_lock(&space->lock); diff --git a/precise-roots-embedder.h b/precise-roots-embedder.h index cf649e14d..bde6be36e 100644 --- a/precise-roots-embedder.h +++ b/precise-roots-embedder.h @@ -18,6 +18,16 @@ static inline int gc_has_conservative_intraheap_edges(void) { return 0; } +static inline int +gc_is_valid_conservative_ref_displacement(uintptr_t displacement) { + GC_CRASH(); +} +static inline int +gc_conservative_ref_might_be_a_heap_object(struct gc_conservative_ref ref, + int possibly_interior) { + GC_CRASH(); +} + static inline void visit_roots(struct handle *roots, void (*trace_edge)(struct gc_edge edge, void *trace_data), @@ -26,18 +36,18 @@ static inline void visit_roots(struct handle *roots, trace_edge(gc_edge(&h->v), trace_data); } -static inline void gc_trace_precise_mutator_roots(struct gc_mutator_roots *roots, - void (*trace_edge)(struct gc_edge edge, - void *trace_data), - void *trace_data) { +static inline void gc_trace_mutator_roots(struct gc_mutator_roots *roots, + void (*trace_edge)(struct gc_edge edge, + void *trace_data), + void *trace_data) { if (roots) visit_roots(roots->roots, trace_edge, trace_data); } -static inline void gc_trace_precise_heap_roots(struct gc_heap_roots *roots, - void (*trace_edge)(struct gc_edge edge, - void *trace_data), - void *trace_data) { +static inline void gc_trace_heap_roots(struct gc_heap_roots *roots, + void (*trace_edge)(struct gc_edge edge, + void *trace_data), + void *trace_data) { if (roots) visit_roots(roots->roots, trace_edge, trace_data); } diff --git a/semi.c b/semi.c index e771f7f3d..3ad765416 100644 --- a/semi.c +++ b/semi.c @@ -161,7 +161,7 @@ static void collect(struct gc_mutator *mut) { flip(semi); uintptr_t grey = semi->hp; if (mut->roots) - gc_trace_precise_mutator_roots(mut->roots, visit, heap); + gc_trace_mutator_roots(mut->roots, visit, heap); // fprintf(stderr, "pushed %zd bytes in roots\n", space->hp - grey); while(grey < semi->hp) grey = scan(heap, gc_ref(grey)); diff --git a/simple-gc-embedder.h b/simple-gc-embedder.h index 71255256d..457d9b09e 100644 --- a/simple-gc-embedder.h +++ b/simple-gc-embedder.h @@ -104,3 +104,9 @@ gc_atomic_forward_address(struct gc_atomic_forward *fwd) { GC_ASSERT(fwd->state == GC_FORWARDING_STATE_FORWARDED); return fwd->data; } + +static inline uintptr_t +gc_conservative_ref_heap_address(struct gc_conservative_ref ref) { + // The specific spaces are responsible for checking alignment. + return gc_conservative_ref_value(ref); +} diff --git a/whippet.c b/whippet.c index ad1e02260..687d6e307 100644 --- a/whippet.c +++ b/whippet.c @@ -15,6 +15,7 @@ #include "debug.h" #include "gc-align.h" #include "gc-inline.h" +#include "gc-platform.h" #include "gc-stack.h" #include "large-object-space.h" #if GC_PARALLEL @@ -597,15 +598,19 @@ static inline int mark_space_evacuate_or_mark_object(struct mark_space *space, return 1; } -static inline int mark_space_contains(struct mark_space *space, - struct gc_ref ref) { - uintptr_t addr = gc_ref_value(ref); +static inline int mark_space_contains_address(struct mark_space *space, + uintptr_t addr) { return addr - space->low_addr < space->extent; } -static inline int large_object_space_mark_object(struct large_object_space *space, - struct gc_ref ref) { - return large_object_space_copy(space, ref); +static inline int mark_space_contains_conservative_ref(struct mark_space *space, + struct gc_conservative_ref ref) { + return mark_space_contains_address(space, gc_conservative_ref_value(ref)); +} + +static inline int mark_space_contains(struct mark_space *space, + struct gc_ref ref) { + return mark_space_contains_address(space, gc_ref_value(ref)); } static inline int trace_edge(struct gc_heap *heap, struct gc_edge edge) { @@ -625,18 +630,84 @@ static inline int trace_edge(struct gc_heap *heap, struct gc_edge edge) { GC_CRASH(); } -static inline int trace_ref(struct gc_heap *heap, struct gc_ref ref) { - if (!gc_ref_is_heap_object(ref)) - return 0; - if (GC_LIKELY(mark_space_contains(heap_mark_space(heap), ref))) { - GC_ASSERT(!heap_mark_space(heap)->evacuating); - return mark_space_mark_object(heap_mark_space(heap), ref); +static inline struct gc_ref mark_space_mark_conservative_ref(struct mark_space *space, + struct gc_conservative_ref ref, + int possibly_interior) { + uintptr_t addr = gc_conservative_ref_value(ref); + + if (possibly_interior) { + addr = align_down(addr, GRANULE_SIZE); + } else { + // Addr not an aligned granule? Not an object. + uintptr_t displacement = addr & (GRANULE_SIZE - 1); + if (!gc_is_valid_conservative_ref_displacement(displacement)) + return gc_ref_null(); + addr -= displacement; } - else if (large_object_space_contains(heap_large_object_space(heap), ref)) - return large_object_space_mark_object(heap_large_object_space(heap), - ref); + + // Addr in meta block? Not an object. + if ((addr & (SLAB_SIZE - 1)) < META_BLOCKS_PER_SLAB * BLOCK_SIZE) + return gc_ref_null(); + + // Addr in block that has been paged out? Not an object. + struct block_summary *summary = block_summary_for_addr(addr); + if (block_summary_has_flag(summary, BLOCK_UNAVAILABLE)) + return gc_ref_null(); + + uint8_t *loc = metadata_byte_for_addr(addr); + uint8_t byte = atomic_load_explicit(loc, memory_order_relaxed); + + // Already marked object? Nothing to do. + if (byte & space->marked_mask) + return gc_ref_null(); + + // Addr is the not start of an unmarked object? Search backwards if + // we have interior pointers, otherwise not an object. + uint8_t object_start_mask = space->live_mask | METADATA_BYTE_YOUNG; + if (!(byte & object_start_mask)) { + if (!possibly_interior) + return gc_ref_null(); + + uintptr_t block_base = align_down(addr, BLOCK_SIZE); + uint8_t *loc_base = metadata_byte_for_addr(block_base); + do { + // Searched past block? Not an object. + if (loc-- == loc_base) + return gc_ref_null(); + + byte = atomic_load_explicit(loc, memory_order_relaxed); + + // Ran into the end of some other allocation? Not an object, then. + if (byte & METADATA_BYTE_END) + return gc_ref_null(); + + // Continue until we find object start. + } while (!(byte & object_start_mask)); + + // Found object start, and object is unmarked; adjust addr. + addr = block_base + (loc - loc_base) * GRANULE_SIZE; + } + + uint8_t mask = METADATA_BYTE_YOUNG | METADATA_BYTE_MARK_0 + | METADATA_BYTE_MARK_1 | METADATA_BYTE_MARK_2; + atomic_store_explicit(loc, (byte & ~mask) | space->marked_mask, + memory_order_relaxed); + + return gc_ref(addr); +} + +static inline struct gc_ref trace_conservative_ref(struct gc_heap *heap, + struct gc_conservative_ref ref, + int possibly_interior) { + if (!gc_conservative_ref_might_be_a_heap_object(ref, possibly_interior)) + return gc_ref_null(); + + if (GC_LIKELY(mark_space_contains_conservative_ref(heap_mark_space(heap), ref))) + return mark_space_mark_conservative_ref(heap_mark_space(heap), ref, + possibly_interior); else - GC_CRASH(); + return large_object_space_mark_conservative_ref(heap_large_object_space(heap), + ref, possibly_interior); } static inline void trace_one(struct gc_ref ref, void *mark_data) { @@ -894,10 +965,24 @@ static void trace_and_enqueue_locally(struct gc_edge edge, void *data) { mutator_mark_buf_push(&mut->mark_buf, gc_edge_ref(edge)); } -static void trace_ref_and_enqueue_locally(struct gc_ref ref, void *data) { +static inline void do_trace_conservative_ref_and_enqueue_locally(struct gc_conservative_ref ref, + void *data, + int possibly_interior) { struct gc_mutator *mut = data; - if (trace_ref(mutator_heap(mut), ref)) - mutator_mark_buf_push(&mut->mark_buf, ref); + struct gc_ref object = trace_conservative_ref(mutator_heap(mut), ref, + possibly_interior); + if (gc_ref_is_heap_object(object)) + mutator_mark_buf_push(&mut->mark_buf, object); +} + +static void trace_possibly_interior_conservative_ref_and_enqueue_locally + (struct gc_conservative_ref ref, void *data) { + return do_trace_conservative_ref_and_enqueue_locally(ref, data, 1); +} + +static void trace_conservative_ref_and_enqueue_locally + (struct gc_conservative_ref ref, void *data) { + return do_trace_conservative_ref_and_enqueue_locally(ref, data, 0); } static void trace_and_enqueue_globally(struct gc_edge edge, void *data) { @@ -906,40 +991,105 @@ static void trace_and_enqueue_globally(struct gc_edge edge, void *data) { tracer_enqueue_root(&heap->tracer, gc_edge_ref(edge)); } -static void trace_ref_and_enqueue_globally(struct gc_ref ref, void *data) { +static inline void do_trace_conservative_ref_and_enqueue_globally(struct gc_conservative_ref ref, + void *data, + int possibly_interior) { struct gc_heap *heap = data; - if (trace_ref(heap, ref)) - tracer_enqueue_root(&heap->tracer, ref); + struct gc_ref object = trace_conservative_ref(heap, ref, possibly_interior); + if (gc_ref_is_heap_object(object)) + tracer_enqueue_root(&heap->tracer, object); +} + +static void trace_possibly_interior_conservative_ref_and_enqueue_globally(struct gc_conservative_ref ref, + void *data) { + return do_trace_conservative_ref_and_enqueue_globally(ref, data, 1); +} + +static void trace_conservative_ref_and_enqueue_globally(struct gc_conservative_ref ref, + void *data) { + return do_trace_conservative_ref_and_enqueue_globally(ref, data, 0); +} + +static inline struct gc_conservative_ref +load_conservative_ref(uintptr_t addr) { + GC_ASSERT((addr & (sizeof(uintptr_t) - 1)) == 0); + uintptr_t val; + memcpy(&val, (char*)addr, sizeof(uintptr_t)); + return gc_conservative_ref(val); +} + +static inline void +trace_conservative_edges(uintptr_t low, + uintptr_t high, + void (*trace)(struct gc_conservative_ref, void *), + void *data) { + GC_ASSERT(low == align_down(low, sizeof(uintptr_t))); + GC_ASSERT(high == align_down(high, sizeof(uintptr_t))); + for (uintptr_t addr = low; addr < high; addr += sizeof(uintptr_t)) + trace(load_conservative_ref(addr), data); +} + +static void +mark_and_globally_enqueue_mutator_conservative_roots(uintptr_t low, + uintptr_t high, + void *data) { + trace_conservative_edges(low, high, + gc_mutator_conservative_roots_may_be_interior() + ? trace_possibly_interior_conservative_ref_and_enqueue_globally + : trace_conservative_ref_and_enqueue_globally, + data); +} + +static void +mark_and_globally_enqueue_heap_conservative_roots(uintptr_t low, + uintptr_t high, + void *data) { + trace_conservative_edges(low, high, + trace_conservative_ref_and_enqueue_globally, + data); +} + +static void +mark_and_locally_enqueue_mutator_conservative_roots(uintptr_t low, + uintptr_t high, + void *data) { + trace_conservative_edges(low, high, + gc_mutator_conservative_roots_may_be_interior() + ? trace_possibly_interior_conservative_ref_and_enqueue_locally + : trace_conservative_ref_and_enqueue_locally, + data); +} + +static inline void +trace_mutator_conservative_roots(struct gc_mutator *mut, + void (*trace_range)(uintptr_t low, + uintptr_t high, + void *data), + void *data) { + if (gc_has_mutator_conservative_roots()) + gc_stack_visit(&mut->stack, trace_range, data); } // Mark the roots of a mutator that is stopping for GC. We can't // enqueue them directly, so we send them to the controller in a buffer. static void trace_stopping_mutator_roots(struct gc_mutator *mut) { GC_ASSERT(mutator_should_mark_while_stopping(mut)); - /* trace_mutator_conservative_roots(mut, - mark_and_locally_enqueue_conservative_roots, + mark_and_locally_enqueue_mutator_conservative_roots, mut); - */ - gc_trace_precise_mutator_roots(mut->roots, trace_and_enqueue_locally, mut); -} - -static void trace_precise_mutator_roots_with_lock(struct gc_mutator *mut) { - gc_trace_precise_mutator_roots(mut->roots, trace_and_enqueue_globally, - mutator_heap(mut)); + gc_trace_mutator_roots(mut->roots, trace_and_enqueue_locally, mut); } static void trace_mutator_conservative_roots_with_lock(struct gc_mutator *mut) { - /* trace_mutator_conservative_roots(mut, - mark_and_globally_enqueue_conservative_roots, + mark_and_globally_enqueue_mutator_conservative_roots, mutator_heap(mut)); - */ } static void trace_mutator_roots_with_lock(struct gc_mutator *mut) { trace_mutator_conservative_roots_with_lock(mut); - trace_precise_mutator_roots_with_lock(mut); + gc_trace_mutator_roots(mut->roots, trace_and_enqueue_globally, + mutator_heap(mut)); } static void trace_mutator_roots_with_lock_before_stop(struct gc_mutator *mut) { @@ -965,12 +1115,11 @@ static void finish_sweeping_in_block(struct gc_mutator *mut); static void trace_mutator_conservative_roots_after_stop(struct gc_heap *heap) { int active_mutators_already_marked = heap_should_mark_while_stopping(heap); - if (!active_mutators_already_marked) { + if (!active_mutators_already_marked) for (struct gc_mutator *mut = atomic_load(&heap->mutator_trace_list); mut; mut = mut->next) trace_mutator_conservative_roots_with_lock(mut); - } for (struct gc_mutator *mut = heap->deactivated_mutators; mut; @@ -978,7 +1127,7 @@ static void trace_mutator_conservative_roots_after_stop(struct gc_heap *heap) { trace_mutator_conservative_roots_with_lock(mut); } -static void trace_precise_mutator_roots_after_stop(struct gc_heap *heap) { +static void trace_mutator_roots_after_stop(struct gc_heap *heap) { struct gc_mutator *mut = atomic_load(&heap->mutator_trace_list); int active_mutators_already_marked = heap_should_mark_while_stopping(heap); while (mut) { @@ -988,7 +1137,7 @@ static void trace_precise_mutator_roots_after_stop(struct gc_heap *heap) { tracer_enqueue_roots(&heap->tracer, mut->mark_buf.objects, mut->mark_buf.size); else - trace_precise_mutator_roots_with_lock(mut); + trace_mutator_roots_with_lock(mut); // Also unlink mutator_trace_list chain. struct gc_mutator *next = mut->next; mut->next = NULL; @@ -998,20 +1147,14 @@ static void trace_precise_mutator_roots_after_stop(struct gc_heap *heap) { for (struct gc_mutator *mut = heap->deactivated_mutators; mut; mut = mut->next) { finish_sweeping_in_block(mut); - trace_precise_mutator_roots_with_lock(mut); + trace_mutator_roots_with_lock(mut); } } -static void trace_precise_global_roots(struct gc_heap *heap) { - gc_trace_precise_heap_roots(heap->roots, trace_and_enqueue_globally, heap); -} - static void trace_global_conservative_roots(struct gc_heap *heap) { - /* if (gc_has_global_conservative_roots()) gc_platform_visit_global_conservative_roots - (mark_and_globally_enqueue_conservative_roots, heap); - */ + (mark_and_globally_enqueue_heap_conservative_roots, heap); } static inline uint64_t load_eight_aligned_bytes(uint8_t *mark) { @@ -1456,9 +1599,9 @@ static void trace_pinned_roots_after_stop(struct gc_heap *heap) { trace_conservative_roots_after_stop(heap); } -static void trace_precise_roots_after_stop(struct gc_heap *heap) { - trace_precise_mutator_roots_after_stop(heap); - trace_precise_global_roots(heap); +static void trace_roots_after_stop(struct gc_heap *heap) { + trace_mutator_roots_after_stop(heap); + gc_trace_heap_roots(heap->roots, trace_and_enqueue_globally, heap); trace_generational_roots(heap); } @@ -1494,7 +1637,7 @@ static void collect(struct gc_mutator *mut) { detect_out_of_memory(heap); trace_pinned_roots_after_stop(heap); prepare_for_evacuation(heap); - trace_precise_roots_after_stop(heap); + trace_roots_after_stop(heap); tracer_trace(heap); tracer_release(heap); mark_space_finish_gc(space, gc_kind);