diff --git a/conservative-roots-embedder.h b/conservative-roots-embedder.h index 15447a2c4..c8004f00c 100644 --- a/conservative-roots-embedder.h +++ b/conservative-roots-embedder.h @@ -36,13 +36,17 @@ gc_conservative_ref_might_be_a_heap_object(struct gc_conservative_ref ref, static inline void gc_trace_mutator_roots(struct gc_mutator_roots *roots, void (*trace_edge)(struct gc_edge edge, + struct gc_heap *heap, void *trace_data), + struct gc_heap *heap, void *trace_data) { } static inline void gc_trace_heap_roots(struct gc_heap_roots *roots, void (*trace_edge)(struct gc_edge edge, + struct gc_heap *heap, void *trace_data), + struct gc_heap *heap, void *trace_data) { } diff --git a/gc-embedder-api.h b/gc-embedder-api.h index 2d74ed0a4..3b3682a71 100644 --- a/gc-embedder-api.h +++ b/gc-embedder-api.h @@ -12,6 +12,7 @@ struct gc_mutator_roots; struct gc_heap_roots; struct gc_atomic_forward; +struct gc_heap; GC_EMBEDDER_API inline int gc_has_mutator_conservative_roots(void); GC_EMBEDDER_API inline int gc_has_global_conservative_roots(void); @@ -23,18 +24,24 @@ GC_EMBEDDER_API inline int gc_conservative_ref_might_be_a_heap_object(struct gc_ int possibly_interior); GC_EMBEDDER_API inline void gc_trace_object(struct gc_ref ref, - void (*trace_edge)(struct gc_edge edge, - void *trace_data), + void (*visit)(struct gc_edge edge, + struct gc_heap *heap, + void *visit_data), + struct gc_heap *heap, void *trace_data, size_t *size) GC_ALWAYS_INLINE; GC_EMBEDDER_API inline void gc_trace_mutator_roots(struct gc_mutator_roots *roots, void (*trace_edge)(struct gc_edge edge, + struct gc_heap *heap, void *trace_data), + struct gc_heap *heap, void *trace_data); GC_EMBEDDER_API inline void gc_trace_heap_roots(struct gc_heap_roots *roots, void (*trace_edge)(struct gc_edge edge, + struct gc_heap *heap, void *trace_data), + struct gc_heap *heap, void *trace_data); GC_EMBEDDER_API inline uintptr_t gc_object_forwarded_nonatomic(struct gc_ref ref); diff --git a/gc-platform-gnu-linux.c b/gc-platform-gnu-linux.c index b0b9f0983..66e2a73df 100644 --- a/gc-platform-gnu-linux.c +++ b/gc-platform-gnu-linux.c @@ -11,7 +11,7 @@ #include "debug.h" #include "gc-assert.h" #include "gc-inline.h" -//#include "gc-stack.h" +#include "gc-platform.h" void gc_platform_init(void) { // Nothing to do. @@ -59,7 +59,8 @@ uintptr_t gc_platform_current_thread_stack_base(void) { } struct visit_data { - void (*f)(uintptr_t start, uintptr_t end, void *data); + void (*f)(uintptr_t start, uintptr_t end, struct gc_heap *heap, void *data); + struct gc_heap *heap; void *data; }; @@ -85,7 +86,7 @@ static int visit_roots(struct dl_phdr_info *info, size_t size, void *data) { uintptr_t end = start + p->p_memsz; DEBUG("found roots for '%s': [%p,%p)\n", object_name, (void*)start, (void*)end); - visit_data->f(start, end, visit_data->data); + visit_data->f(start, end, visit_data->heap, visit_data->data); } } @@ -94,8 +95,10 @@ static int visit_roots(struct dl_phdr_info *info, size_t size, void *data) { void gc_platform_visit_global_conservative_roots(void (*f)(uintptr_t start, uintptr_t end, + struct gc_heap*, void *data), + struct gc_heap *heap, void *data) { - struct visit_data visit_data = { f, data }; + struct visit_data visit_data = { f, heap, data }; dl_iterate_phdr(visit_roots, &visit_data); } diff --git a/gc-platform.h b/gc-platform.h index acc0096f9..b22787d19 100644 --- a/gc-platform.h +++ b/gc-platform.h @@ -9,12 +9,16 @@ #include "gc-visibility.h" +struct gc_heap; + GC_INTERNAL void gc_platform_init(void); GC_INTERNAL uintptr_t gc_platform_current_thread_stack_base(void); GC_INTERNAL void gc_platform_visit_global_conservative_roots(void (*f)(uintptr_t start, uintptr_t end, + struct gc_heap *heap, void *data), + struct gc_heap *heap, void *data); #endif // GC_PLATFORM_H diff --git a/gc-stack.c b/gc-stack.c index bc8c9e64f..54c6fdb0c 100644 --- a/gc-stack.c +++ b/gc-stack.c @@ -63,7 +63,9 @@ void* gc_call_with_stack_addr(void* (*f)(struct gc_stack_addr *base, } void gc_stack_visit(struct gc_stack *stack, - void (*visit)(uintptr_t low, uintptr_t high, void *data), + void (*visit)(uintptr_t low, uintptr_t high, + struct gc_heap *heap, void *data), + struct gc_heap *heap, void *data) { { uintptr_t low = (uintptr_t)stack->registers; @@ -71,7 +73,7 @@ void gc_stack_visit(struct gc_stack *stack, uintptr_t high = low + sizeof(jmp_buf); DEBUG("found mutator register roots for %p: [%p,%p)\n", stack, (void*)low, (void*)high); - visit(low, high, data); + visit(low, high, heap, data); } if (0 HOTTER_THAN 1) { @@ -79,12 +81,12 @@ void gc_stack_visit(struct gc_stack *stack, (void*)stack->hot.addr, (void*)stack->cold.addr); visit(align_up(stack->hot.addr, sizeof(uintptr_t)), align_down(stack->cold.addr, sizeof(uintptr_t)), - data); + heap, data); } else { DEBUG("found mutator stack roots for %p: [%p,%p)\n", stack, (void*)stack->cold.addr, (void*)stack->hot.addr); visit(align_up(stack->cold.addr, sizeof(uintptr_t)), align_down(stack->hot.addr, sizeof(uintptr_t)), - data); + heap, data); } } diff --git a/gc-stack.h b/gc-stack.h index fa228b210..15df9df6d 100644 --- a/gc-stack.h +++ b/gc-stack.h @@ -18,12 +18,16 @@ struct gc_stack { jmp_buf registers; }; +struct gc_heap; + GC_INTERNAL void gc_stack_init(struct gc_stack *stack, struct gc_stack_addr *base); GC_INTERNAL void gc_stack_capture_hot(struct gc_stack *stack); GC_INTERNAL void gc_stack_visit(struct gc_stack *stack, void (*visit)(uintptr_t low, uintptr_t high, + struct gc_heap *heap, void *data), + struct gc_heap *heap, void *data); #endif // GC_STACK_H diff --git a/mt-gcbench-embedder.h b/mt-gcbench-embedder.h index 3162a8bf3..1ac42a327 100644 --- a/mt-gcbench-embedder.h +++ b/mt-gcbench-embedder.h @@ -3,10 +3,15 @@ #include "mt-gcbench-types.h" +struct gc_heap; + #define DEFINE_METHODS(name, Name, NAME) \ static inline size_t name##_size(Name *obj) GC_ALWAYS_INLINE; \ static inline void visit_##name##_fields(Name *obj,\ - void (*visit)(struct gc_edge edge, void *visit_data), \ + void (*visit)(struct gc_edge edge, \ + struct gc_heap *heap, \ + void *visit_data), \ + struct gc_heap *heap, \ void *visit_data) GC_ALWAYS_INLINE; FOR_EACH_HEAP_OBJECT_KIND(DEFINE_METHODS) #undef DEFINE_METHODS @@ -22,20 +27,23 @@ static inline size_t hole_size(Hole *hole) { } static inline void visit_node_fields(Node *node, - void (*visit)(struct gc_edge edge, void *visit_data), - void *visit_data) { - visit(gc_edge(&node->left), visit_data); - visit(gc_edge(&node->right), visit_data); + void (*visit)(struct gc_edge edge, struct gc_heap *heap, + void *visit_data), + struct gc_heap *heap, void *visit_data) { + visit(gc_edge(&node->left), heap, visit_data); + visit(gc_edge(&node->right), heap, visit_data); } static inline void visit_double_array_fields(DoubleArray *obj, - void (*visit)(struct gc_edge edge, void *visit_data), - void *visit_data) { + void (*visit)(struct gc_edge edge, + struct gc_heap *heap, void *visit_data), + struct gc_heap *heap, void *visit_data) { } static inline void visit_hole_fields(Hole *obj, - void (*visit)(struct gc_edge edge, void *visit_data), - void *visit_data) { + void (*visit)(struct gc_edge edge, + struct gc_heap *heap, void *visit_data), + struct gc_heap *heap, void *visit_data) { #if GC_PRECISE GC_CRASH(); #endif diff --git a/parallel-tracer.h b/parallel-tracer.h index bcc7910c2..df6cc89ae 100644 --- a/parallel-tracer.h +++ b/parallel-tracer.h @@ -320,7 +320,6 @@ struct tracer { struct local_tracer { struct trace_worker *worker; struct trace_deque *share_deque; - struct gc_heap *heap; struct local_trace_queue local; }; @@ -449,8 +448,10 @@ static void tracer_release(struct gc_heap *heap) { trace_deque_release(&tracer->workers[i].deque); } -static inline void tracer_visit(struct gc_edge edge, void *trace_data) GC_ALWAYS_INLINE; -static inline void trace_one(struct gc_ref ref, void *trace_data) GC_ALWAYS_INLINE; +static inline void tracer_visit(struct gc_edge edge, struct gc_heap *heap, + void *trace_data) GC_ALWAYS_INLINE; +static inline void trace_one(struct gc_ref ref, struct gc_heap *heap, + void *trace_data) GC_ALWAYS_INLINE; static inline int trace_edge(struct gc_heap *heap, struct gc_edge edge) GC_ALWAYS_INLINE; @@ -462,9 +463,9 @@ tracer_share(struct local_tracer *trace) { } static inline void -tracer_visit(struct gc_edge edge, void *trace_data) { - struct local_tracer *trace = trace_data; - if (trace_edge(trace->heap, edge)) { +tracer_visit(struct gc_edge edge, struct gc_heap *heap, void *trace_data) { + if (trace_edge(heap, edge)) { + struct local_tracer *trace = trace_data; if (local_trace_queue_full(&trace->local)) tracer_share(trace); local_trace_queue_push(&trace->local, gc_edge_ref(edge)); @@ -544,8 +545,8 @@ trace_worker_check_termination(struct trace_worker *worker, static struct gc_ref trace_worker_steal(struct local_tracer *trace) { - struct tracer *tracer = heap_tracer(trace->heap); struct trace_worker *worker = trace->worker; + struct tracer *tracer = heap_tracer(worker->heap); // It could be that the worker's local trace queue has simply // overflowed. In that case avoid contention by trying to pop @@ -573,7 +574,7 @@ trace_worker_trace(struct trace_worker *worker) { struct local_tracer trace; trace.worker = worker; trace.share_deque = &worker->deque; - trace.heap = worker->heap; + struct gc_heap *heap = worker->heap; local_trace_queue_init(&trace.local); size_t n = 0; @@ -587,7 +588,7 @@ trace_worker_trace(struct trace_worker *worker) { if (!gc_ref_is_heap_object(ref)) break; } - trace_one(ref, &trace); + trace_one(ref, heap, &trace); n++; } DEBUG("tracer #%zu: done tracing, %zu objects traced\n", worker->id, n); diff --git a/precise-roots-embedder.h b/precise-roots-embedder.h index bde6be36e..94192cb51 100644 --- a/precise-roots-embedder.h +++ b/precise-roots-embedder.h @@ -30,26 +30,32 @@ gc_conservative_ref_might_be_a_heap_object(struct gc_conservative_ref ref, static inline void visit_roots(struct handle *roots, void (*trace_edge)(struct gc_edge edge, + struct gc_heap *heap, void *trace_data), + struct gc_heap *heap, void *trace_data) { for (struct handle *h = roots; h; h = h->next) - trace_edge(gc_edge(&h->v), trace_data); + trace_edge(gc_edge(&h->v), heap, trace_data); } static inline void gc_trace_mutator_roots(struct gc_mutator_roots *roots, void (*trace_edge)(struct gc_edge edge, + struct gc_heap *heap, void *trace_data), + struct gc_heap *heap, void *trace_data) { if (roots) - visit_roots(roots->roots, trace_edge, trace_data); + visit_roots(roots->roots, trace_edge, heap, trace_data); } static inline void gc_trace_heap_roots(struct gc_heap_roots *roots, void (*trace_edge)(struct gc_edge edge, + struct gc_heap *heap, void *trace_data), + struct gc_heap *heap, void *trace_data) { if (roots) - visit_roots(roots->roots, trace_edge, trace_data); + visit_roots(roots->roots, trace_edge, heap, trace_data); } #endif // PRECISE_ROOTS_EMBEDDER_H diff --git a/quads-embedder.h b/quads-embedder.h index 714415dd0..1d9d3f71c 100644 --- a/quads-embedder.h +++ b/quads-embedder.h @@ -5,10 +5,15 @@ #include "quads-types.h" +struct gc_heap; + #define DEFINE_METHODS(name, Name, NAME) \ static inline size_t name##_size(Name *obj) GC_ALWAYS_INLINE; \ static inline void visit_##name##_fields(Name *obj,\ - void (*visit)(struct gc_edge edge, void *visit_data), \ + void (*visit)(struct gc_edge edge, \ + struct gc_heap *heap, \ + void *visit_data), \ + struct gc_heap *heap, \ void *visit_data) GC_ALWAYS_INLINE; FOR_EACH_HEAP_OBJECT_KIND(DEFINE_METHODS) #undef DEFINE_METHODS @@ -19,10 +24,12 @@ static inline size_t quad_size(Quad *obj) { static inline void visit_quad_fields(Quad *quad, - void (*visit)(struct gc_edge edge, void *visit_data), + void (*visit)(struct gc_edge edge, struct gc_heap *heap, + void *visit_data), + struct gc_heap *heap, void *visit_data) { for (size_t i = 0; i < 4; i++) - visit(gc_edge(&quad->kids[i]), visit_data); + visit(gc_edge(&quad->kids[i]), heap, visit_data); } #include "simple-gc-embedder.h" diff --git a/semi.c b/semi.c index 3ad765416..2c1eae600 100644 --- a/semi.c +++ b/semi.c @@ -63,7 +63,7 @@ static uintptr_t align_up(uintptr_t addr, size_t align) { static void collect(struct gc_mutator *mut) GC_NEVER_INLINE; static void collect_for_alloc(struct gc_mutator *mut, size_t bytes) GC_NEVER_INLINE; -static void visit(struct gc_edge edge, void *visit_data); +static void trace(struct gc_edge edge, struct gc_heap *heap, void *visit_data); static int semi_space_steal_pages(struct semi_space *space, size_t npages) { size_t stolen_pages = space->stolen_pages + npages; @@ -103,7 +103,7 @@ static void flip(struct semi_space *space) { static struct gc_ref copy(struct semi_space *space, struct gc_ref ref) { size_t size; - gc_trace_object(ref, NULL, NULL, &size); + gc_trace_object(ref, NULL, NULL, NULL, &size); struct gc_ref new_ref = gc_ref(space->hp); memcpy(gc_ref_heap_object(new_ref), gc_ref_heap_object(ref), size); gc_object_forward_nonatomic(ref, new_ref); @@ -113,7 +113,7 @@ static struct gc_ref copy(struct semi_space *space, struct gc_ref ref) { static uintptr_t scan(struct gc_heap *heap, struct gc_ref grey) { size_t size; - gc_trace_object(grey, visit, heap, &size); + gc_trace_object(grey, trace, heap, NULL, &size); return gc_ref_value(grey) + align_up(size, GC_ALIGNMENT); } @@ -131,7 +131,7 @@ static void visit_large_object_space(struct gc_heap *heap, struct large_object_space *space, struct gc_ref ref) { if (large_object_space_copy(space, ref)) - gc_trace_object(ref, visit, heap, NULL); + gc_trace_object(ref, trace, heap, NULL, NULL); } static int semi_space_contains(struct semi_space *space, struct gc_ref ref) { @@ -139,8 +139,7 @@ static int semi_space_contains(struct semi_space *space, struct gc_ref ref) { return addr - space->base < space->size; } -static void visit(struct gc_edge edge, void *visit_data) { - struct gc_heap *heap = visit_data; +static void visit(struct gc_edge edge, struct gc_heap *heap) { struct gc_ref ref = gc_edge_ref(edge); if (!gc_ref_is_heap_object(ref)) return; @@ -152,6 +151,10 @@ static void visit(struct gc_edge edge, void *visit_data) { GC_CRASH(); } +static void trace(struct gc_edge edge, struct gc_heap *heap, void *visit_data) { + return visit(edge, heap); +} + static void collect(struct gc_mutator *mut) { struct gc_heap *heap = mutator_heap(mut); struct semi_space *semi = heap_semi_space(heap); @@ -161,7 +164,7 @@ static void collect(struct gc_mutator *mut) { flip(semi); uintptr_t grey = semi->hp; if (mut->roots) - gc_trace_mutator_roots(mut->roots, visit, heap); + gc_trace_mutator_roots(mut->roots, trace, heap, NULL); // fprintf(stderr, "pushed %zd bytes in roots\n", space->hp - grey); while(grey < semi->hp) grey = scan(heap, gc_ref(grey)); diff --git a/serial-tracer.h b/serial-tracer.h index b4194c160..7c0bdcca9 100644 --- a/serial-tracer.h +++ b/serial-tracer.h @@ -135,8 +135,10 @@ static void tracer_release(struct gc_heap *heap) { trace_queue_release(&heap_tracer(heap)->queue); } -static inline void tracer_visit(struct gc_edge edge, void *trace_data) GC_ALWAYS_INLINE; -static inline void trace_one(struct gc_ref ref, void *trace_data) GC_ALWAYS_INLINE; +static inline void tracer_visit(struct gc_edge edge, struct gc_heap *heap, + void *trace_data) GC_ALWAYS_INLINE; +static inline void trace_one(struct gc_ref ref, struct gc_heap *heap, + void *trace_data) GC_ALWAYS_INLINE; static inline int trace_edge(struct gc_heap *heap, struct gc_edge edge) GC_ALWAYS_INLINE; @@ -150,8 +152,7 @@ tracer_enqueue_roots(struct tracer *tracer, struct gc_ref *objs, trace_queue_push_many(&tracer->queue, objs, count); } static inline void -tracer_visit(struct gc_edge edge, void *trace_data) { - struct gc_heap *heap = trace_data; +tracer_visit(struct gc_edge edge, struct gc_heap *heap, void *trace_data) { if (trace_edge(heap, edge)) tracer_enqueue_root(heap_tracer(heap), gc_edge_ref(edge)); } @@ -161,7 +162,7 @@ tracer_trace(struct gc_heap *heap) { struct gc_ref obj = trace_queue_pop(&heap_tracer(heap)->queue); if (!gc_ref_is_heap_object(obj)) break; - trace_one(obj, heap); + trace_one(obj, heap, NULL); } while (1); } diff --git a/simple-gc-embedder.h b/simple-gc-embedder.h index 457d9b09e..7b691acfa 100644 --- a/simple-gc-embedder.h +++ b/simple-gc-embedder.h @@ -5,7 +5,9 @@ static inline void gc_trace_object(struct gc_ref ref, void (*trace_edge)(struct gc_edge edge, + struct gc_heap *heap, void *trace_data), + struct gc_heap *heap, void *trace_data, size_t *size) { switch (tag_live_alloc_kind(*tag_word(ref))) { @@ -13,7 +15,7 @@ static inline void gc_trace_object(struct gc_ref ref, case ALLOC_KIND_##NAME: \ if (trace_edge) \ visit_##name##_fields(gc_ref_heap_object(ref), trace_edge, \ - trace_data); \ + heap, trace_data); \ if (size) \ *size = name##_size(gc_ref_heap_object(ref)); \ break; diff --git a/whippet.c b/whippet.c index 687d6e307..5dbfc80ec 100644 --- a/whippet.c +++ b/whippet.c @@ -710,8 +710,9 @@ static inline struct gc_ref trace_conservative_ref(struct gc_heap *heap, ref, possibly_interior); } -static inline void trace_one(struct gc_ref ref, void *mark_data) { - gc_trace_object(ref, tracer_visit, mark_data, NULL); +static inline void trace_one(struct gc_ref ref, struct gc_heap *heap, + void *mark_data) { + gc_trace_object(ref, tracer_visit, heap, mark_data, NULL); } static int heap_has_multiple_mutators(struct gc_heap *heap) { @@ -959,55 +960,60 @@ void gc_heap_set_roots(struct gc_heap *heap, struct gc_heap_roots *roots) { heap->roots = roots; } -static void trace_and_enqueue_locally(struct gc_edge edge, void *data) { +static void trace_and_enqueue_locally(struct gc_edge edge, + struct gc_heap *heap, + void *data) { struct gc_mutator *mut = data; - if (trace_edge(mutator_heap(mut), edge)) + if (trace_edge(heap, edge)) mutator_mark_buf_push(&mut->mark_buf, gc_edge_ref(edge)); } static inline void do_trace_conservative_ref_and_enqueue_locally(struct gc_conservative_ref ref, + struct gc_heap *heap, void *data, int possibly_interior) { struct gc_mutator *mut = data; - struct gc_ref object = trace_conservative_ref(mutator_heap(mut), ref, - possibly_interior); + struct gc_ref object = trace_conservative_ref(heap, ref, possibly_interior); if (gc_ref_is_heap_object(object)) mutator_mark_buf_push(&mut->mark_buf, object); } static void trace_possibly_interior_conservative_ref_and_enqueue_locally - (struct gc_conservative_ref ref, void *data) { - return do_trace_conservative_ref_and_enqueue_locally(ref, data, 1); + (struct gc_conservative_ref ref, struct gc_heap *heap, void *data) { + return do_trace_conservative_ref_and_enqueue_locally(ref, heap, data, 1); } static void trace_conservative_ref_and_enqueue_locally - (struct gc_conservative_ref ref, void *data) { - return do_trace_conservative_ref_and_enqueue_locally(ref, data, 0); + (struct gc_conservative_ref ref, struct gc_heap *heap, void *data) { + return do_trace_conservative_ref_and_enqueue_locally(ref, heap, data, 0); } -static void trace_and_enqueue_globally(struct gc_edge edge, void *data) { - struct gc_heap *heap = data; +static void trace_and_enqueue_globally(struct gc_edge edge, + struct gc_heap *heap, + void *unused) { if (trace_edge(heap, edge)) tracer_enqueue_root(&heap->tracer, gc_edge_ref(edge)); } static inline void do_trace_conservative_ref_and_enqueue_globally(struct gc_conservative_ref ref, + struct gc_heap *heap, void *data, int possibly_interior) { - struct gc_heap *heap = data; struct gc_ref object = trace_conservative_ref(heap, ref, possibly_interior); if (gc_ref_is_heap_object(object)) tracer_enqueue_root(&heap->tracer, object); } static void trace_possibly_interior_conservative_ref_and_enqueue_globally(struct gc_conservative_ref ref, + struct gc_heap *heap, void *data) { - return do_trace_conservative_ref_and_enqueue_globally(ref, data, 1); + return do_trace_conservative_ref_and_enqueue_globally(ref, heap, data, 1); } static void trace_conservative_ref_and_enqueue_globally(struct gc_conservative_ref ref, + struct gc_heap *heap, void *data) { - return do_trace_conservative_ref_and_enqueue_globally(ref, data, 0); + return do_trace_conservative_ref_and_enqueue_globally(ref, heap, data, 0); } static inline struct gc_conservative_ref @@ -1021,75 +1027,84 @@ load_conservative_ref(uintptr_t addr) { static inline void trace_conservative_edges(uintptr_t low, uintptr_t high, - void (*trace)(struct gc_conservative_ref, void *), + void (*trace)(struct gc_conservative_ref, + struct gc_heap *, void *), + struct gc_heap *heap, void *data) { GC_ASSERT(low == align_down(low, sizeof(uintptr_t))); GC_ASSERT(high == align_down(high, sizeof(uintptr_t))); for (uintptr_t addr = low; addr < high; addr += sizeof(uintptr_t)) - trace(load_conservative_ref(addr), data); + trace(load_conservative_ref(addr), heap, data); } static void mark_and_globally_enqueue_mutator_conservative_roots(uintptr_t low, uintptr_t high, + struct gc_heap *heap, void *data) { trace_conservative_edges(low, high, gc_mutator_conservative_roots_may_be_interior() ? trace_possibly_interior_conservative_ref_and_enqueue_globally : trace_conservative_ref_and_enqueue_globally, - data); + heap, data); } static void mark_and_globally_enqueue_heap_conservative_roots(uintptr_t low, uintptr_t high, + struct gc_heap *heap, void *data) { trace_conservative_edges(low, high, trace_conservative_ref_and_enqueue_globally, - data); + heap, data); } static void mark_and_locally_enqueue_mutator_conservative_roots(uintptr_t low, uintptr_t high, + struct gc_heap *heap, void *data) { trace_conservative_edges(low, high, gc_mutator_conservative_roots_may_be_interior() ? trace_possibly_interior_conservative_ref_and_enqueue_locally : trace_conservative_ref_and_enqueue_locally, - data); + heap, data); } static inline void trace_mutator_conservative_roots(struct gc_mutator *mut, void (*trace_range)(uintptr_t low, uintptr_t high, + struct gc_heap *heap, void *data), + struct gc_heap *heap, void *data) { if (gc_has_mutator_conservative_roots()) - gc_stack_visit(&mut->stack, trace_range, data); + gc_stack_visit(&mut->stack, trace_range, heap, data); } // Mark the roots of a mutator that is stopping for GC. We can't // enqueue them directly, so we send them to the controller in a buffer. static void trace_stopping_mutator_roots(struct gc_mutator *mut) { GC_ASSERT(mutator_should_mark_while_stopping(mut)); + struct gc_heap *heap = mutator_heap(mut); trace_mutator_conservative_roots(mut, mark_and_locally_enqueue_mutator_conservative_roots, - mut); - gc_trace_mutator_roots(mut->roots, trace_and_enqueue_locally, mut); + heap, mut); + gc_trace_mutator_roots(mut->roots, trace_and_enqueue_locally, heap, mut); } static void trace_mutator_conservative_roots_with_lock(struct gc_mutator *mut) { trace_mutator_conservative_roots(mut, mark_and_globally_enqueue_mutator_conservative_roots, - mutator_heap(mut)); + mutator_heap(mut), + NULL); } static void trace_mutator_roots_with_lock(struct gc_mutator *mut) { trace_mutator_conservative_roots_with_lock(mut); gc_trace_mutator_roots(mut->roots, trace_and_enqueue_globally, - mutator_heap(mut)); + mutator_heap(mut), NULL); } static void trace_mutator_roots_with_lock_before_stop(struct gc_mutator *mut) { @@ -1154,7 +1169,7 @@ static void trace_mutator_roots_after_stop(struct gc_heap *heap) { static void trace_global_conservative_roots(struct gc_heap *heap) { if (gc_has_global_conservative_roots()) gc_platform_visit_global_conservative_roots - (mark_and_globally_enqueue_heap_conservative_roots, heap); + (mark_and_globally_enqueue_heap_conservative_roots, heap, NULL); } static inline uint64_t load_eight_aligned_bytes(uint8_t *mark) { @@ -1601,7 +1616,7 @@ static void trace_pinned_roots_after_stop(struct gc_heap *heap) { static void trace_roots_after_stop(struct gc_heap *heap) { trace_mutator_roots_after_stop(heap); - gc_trace_heap_roots(heap->roots, trace_and_enqueue_globally, heap); + gc_trace_heap_roots(heap->roots, trace_and_enqueue_globally, heap, NULL); trace_generational_roots(heap); }