mirror/guile
mirror/guile
1
Fork 0
mirror of https://git.savannah.gnu.org/git/guile.git synced 2025-05-09 07:00:23 +02:00
Code Activity

Pass heap to tracer functions

Browse source 
This will allow conservative intra-heap edges.  Hopefully no overhead?
This commit is contained in:
Andy Wingo 2022-10-25 14:25:55 +02:00
parent 5e986e84e9
commit 053dbf0b61
14 changed files with 141 additions and 74 deletions
Download patch file Download diff file Expand all files Collapse all files

4
conservative-roots-embedder.h
View file

@ -36,13 +36,17 @@ gc_conservative_ref_might_be_a_heap_object(struct gc_conservative_ref ref,
static inline void gc_trace_mutator_roots(struct gc_mutator_roots *roots, static inline void gc_trace_mutator_roots(struct gc_mutator_roots *roots,
void (*trace_edge)(struct gc_edge edge, void (*trace_edge)(struct gc_edge edge,
struct gc_heap *heap,
void *trace_data), void *trace_data),
struct gc_heap *heap,
void *trace_data) { void *trace_data) {
} }
static inline void gc_trace_heap_roots(struct gc_heap_roots *roots, static inline void gc_trace_heap_roots(struct gc_heap_roots *roots,
void (*trace_edge)(struct gc_edge edge, void (*trace_edge)(struct gc_edge edge,
struct gc_heap *heap,
void *trace_data), void *trace_data),
struct gc_heap *heap,
void *trace_data) { void *trace_data) {
} }

11
gc-embedder-api.h
View file

@ -12,6 +12,7 @@
struct gc_mutator_roots; struct gc_mutator_roots;
struct gc_heap_roots; struct gc_heap_roots;
struct gc_atomic_forward; struct gc_atomic_forward;
struct gc_heap;
GC_EMBEDDER_API inline int gc_has_mutator_conservative_roots(void); GC_EMBEDDER_API inline int gc_has_mutator_conservative_roots(void);
GC_EMBEDDER_API inline int gc_has_global_conservative_roots(void); GC_EMBEDDER_API inline int gc_has_global_conservative_roots(void);
@ -23,18 +24,24 @@ GC_EMBEDDER_API inline int gc_conservative_ref_might_be_a_heap_object(struct gc_
int possibly_interior); int possibly_interior);
GC_EMBEDDER_API inline void gc_trace_object(struct gc_ref ref, GC_EMBEDDER_API inline void gc_trace_object(struct gc_ref ref,
void (*trace_edge)(struct gc_edge edge, void (*visit)(struct gc_edge edge,
void *trace_data), struct gc_heap *heap,
void *visit_data),
struct gc_heap *heap,
void *trace_data, void *trace_data,
size_t *size) GC_ALWAYS_INLINE; size_t *size) GC_ALWAYS_INLINE;
GC_EMBEDDER_API inline void gc_trace_mutator_roots(struct gc_mutator_roots *roots, GC_EMBEDDER_API inline void gc_trace_mutator_roots(struct gc_mutator_roots *roots,
void (*trace_edge)(struct gc_edge edge, void (*trace_edge)(struct gc_edge edge,
struct gc_heap *heap,
void *trace_data), void *trace_data),
struct gc_heap *heap,
void *trace_data); void *trace_data);
GC_EMBEDDER_API inline void gc_trace_heap_roots(struct gc_heap_roots *roots, GC_EMBEDDER_API inline void gc_trace_heap_roots(struct gc_heap_roots *roots,
void (*trace_edge)(struct gc_edge edge, void (*trace_edge)(struct gc_edge edge,
struct gc_heap *heap,
void *trace_data), void *trace_data),
struct gc_heap *heap,
void *trace_data); void *trace_data);
GC_EMBEDDER_API inline uintptr_t gc_object_forwarded_nonatomic(struct gc_ref ref); GC_EMBEDDER_API inline uintptr_t gc_object_forwarded_nonatomic(struct gc_ref ref);

11
gc-platform-gnu-linux.c
View file

@ -11,7 +11,7 @@
#include "debug.h" #include "debug.h"
#include "gc-assert.h" #include "gc-assert.h"
#include "gc-inline.h" #include "gc-inline.h"
//#include "gc-stack.h" #include "gc-platform.h"
void gc_platform_init(void) { void gc_platform_init(void) {
// Nothing to do. // Nothing to do.
@ -59,7 +59,8 @@ uintptr_t gc_platform_current_thread_stack_base(void) {
} }
struct visit_data { struct visit_data {
void (*f)(uintptr_t start, uintptr_t end, void *data); void (*f)(uintptr_t start, uintptr_t end, struct gc_heap *heap, void *data);
struct gc_heap *heap;
void *data; void *data;
}; };
@ -85,7 +86,7 @@ static int visit_roots(struct dl_phdr_info *info, size_t size, void *data) {
uintptr_t end = start + p->p_memsz; uintptr_t end = start + p->p_memsz;
DEBUG("found roots for '%s': [%p,%p)\n", object_name, DEBUG("found roots for '%s': [%p,%p)\n", object_name,
(void*)start, (void*)end); (void*)start, (void*)end);
visit_data->f(start, end, visit_data->data); visit_data->f(start, end, visit_data->heap, visit_data->data);
} }
} }
@ -94,8 +95,10 @@ static int visit_roots(struct dl_phdr_info *info, size_t size, void *data) {
void gc_platform_visit_global_conservative_roots(void (*f)(uintptr_t start, void gc_platform_visit_global_conservative_roots(void (*f)(uintptr_t start,
uintptr_t end, uintptr_t end,
struct gc_heap*,
void *data), void *data),
struct gc_heap *heap,
void *data) { void *data) {
struct visit_data visit_data = { f, data }; struct visit_data visit_data = { f, heap, data };
dl_iterate_phdr(visit_roots, &visit_data); dl_iterate_phdr(visit_roots, &visit_data);
} }

4
gc-platform.h
View file

@ -9,12 +9,16 @@
#include "gc-visibility.h" #include "gc-visibility.h"
struct gc_heap;
GC_INTERNAL void gc_platform_init(void); GC_INTERNAL void gc_platform_init(void);
GC_INTERNAL uintptr_t gc_platform_current_thread_stack_base(void); GC_INTERNAL uintptr_t gc_platform_current_thread_stack_base(void);
GC_INTERNAL GC_INTERNAL
void gc_platform_visit_global_conservative_roots(void (*f)(uintptr_t start, void gc_platform_visit_global_conservative_roots(void (*f)(uintptr_t start,
uintptr_t end, uintptr_t end,
struct gc_heap *heap,
void *data), void *data),
struct gc_heap *heap,
void *data); void *data);
#endif // GC_PLATFORM_H #endif // GC_PLATFORM_H

10
gc-stack.c
View file

@ -63,7 +63,9 @@ void* gc_call_with_stack_addr(void* (*f)(struct gc_stack_addr *base,
} }
void gc_stack_visit(struct gc_stack *stack, void gc_stack_visit(struct gc_stack *stack,
void (*visit)(uintptr_t low, uintptr_t high, void *data), void (*visit)(uintptr_t low, uintptr_t high,
struct gc_heap *heap, void *data),
struct gc_heap *heap,
void *data) { void *data) {
{ {
uintptr_t low = (uintptr_t)stack->registers; uintptr_t low = (uintptr_t)stack->registers;
@ -71,7 +73,7 @@ void gc_stack_visit(struct gc_stack *stack,
uintptr_t high = low + sizeof(jmp_buf); uintptr_t high = low + sizeof(jmp_buf);
DEBUG("found mutator register roots for %p: [%p,%p)\n", stack, DEBUG("found mutator register roots for %p: [%p,%p)\n", stack,
(void*)low, (void*)high); (void*)low, (void*)high);
visit(low, high, data); visit(low, high, heap, data);
} }
if (0 HOTTER_THAN 1) { if (0 HOTTER_THAN 1) {
@ -79,12 +81,12 @@ void gc_stack_visit(struct gc_stack *stack,
(void*)stack->hot.addr, (void*)stack->cold.addr); (void*)stack->hot.addr, (void*)stack->cold.addr);
visit(align_up(stack->hot.addr, sizeof(uintptr_t)), visit(align_up(stack->hot.addr, sizeof(uintptr_t)),
align_down(stack->cold.addr, sizeof(uintptr_t)), align_down(stack->cold.addr, sizeof(uintptr_t)),
data); heap, data);
} else { } else {
DEBUG("found mutator stack roots for %p: [%p,%p)\n", stack, DEBUG("found mutator stack roots for %p: [%p,%p)\n", stack,
(void*)stack->cold.addr, (void*)stack->hot.addr); (void*)stack->cold.addr, (void*)stack->hot.addr);
visit(align_up(stack->cold.addr, sizeof(uintptr_t)), visit(align_up(stack->cold.addr, sizeof(uintptr_t)),
align_down(stack->hot.addr, sizeof(uintptr_t)), align_down(stack->hot.addr, sizeof(uintptr_t)),
data); heap, data);
} }
} }

4
gc-stack.h
View file

@ -18,12 +18,16 @@ struct gc_stack {
jmp_buf registers; jmp_buf registers;
}; };
struct gc_heap;
GC_INTERNAL void gc_stack_init(struct gc_stack *stack, GC_INTERNAL void gc_stack_init(struct gc_stack *stack,
struct gc_stack_addr *base); struct gc_stack_addr *base);
GC_INTERNAL void gc_stack_capture_hot(struct gc_stack *stack); GC_INTERNAL void gc_stack_capture_hot(struct gc_stack *stack);
GC_INTERNAL void gc_stack_visit(struct gc_stack *stack, GC_INTERNAL void gc_stack_visit(struct gc_stack *stack,
void (*visit)(uintptr_t low, uintptr_t high, void (*visit)(uintptr_t low, uintptr_t high,
struct gc_heap *heap,
void *data), void *data),
struct gc_heap *heap,
void *data); void *data);
#endif // GC_STACK_H #endif // GC_STACK_H

26
mt-gcbench-embedder.h
View file

@ -3,10 +3,15 @@
#include "mt-gcbench-types.h" #include "mt-gcbench-types.h"
struct gc_heap;
#define DEFINE_METHODS(name, Name, NAME) \ #define DEFINE_METHODS(name, Name, NAME) \
static inline size_t name##_size(Name *obj) GC_ALWAYS_INLINE; \ static inline size_t name##_size(Name *obj) GC_ALWAYS_INLINE; \
static inline void visit_##name##_fields(Name *obj,\ static inline void visit_##name##_fields(Name *obj,\
void (*visit)(struct gc_edge edge, void *visit_data), \ void (*visit)(struct gc_edge edge, \
struct gc_heap *heap, \
void *visit_data), \
struct gc_heap *heap, \
void *visit_data) GC_ALWAYS_INLINE; void *visit_data) GC_ALWAYS_INLINE;
FOR_EACH_HEAP_OBJECT_KIND(DEFINE_METHODS) FOR_EACH_HEAP_OBJECT_KIND(DEFINE_METHODS)
#undef DEFINE_METHODS #undef DEFINE_METHODS
@ -22,20 +27,23 @@ static inline size_t hole_size(Hole *hole) {
} }
static inline void static inline void
visit_node_fields(Node *node, visit_node_fields(Node *node,
void (*visit)(struct gc_edge edge, void *visit_data), void (*visit)(struct gc_edge edge, struct gc_heap *heap,
void *visit_data) { void *visit_data),
visit(gc_edge(&node->left), visit_data); struct gc_heap *heap, void *visit_data) {
visit(gc_edge(&node->right), visit_data); visit(gc_edge(&node->left), heap, visit_data);
visit(gc_edge(&node->right), heap, visit_data);
} }
static inline void static inline void
visit_double_array_fields(DoubleArray *obj, visit_double_array_fields(DoubleArray *obj,
void (*visit)(struct gc_edge edge, void *visit_data), void (*visit)(struct gc_edge edge,
void *visit_data) { struct gc_heap *heap, void *visit_data),
struct gc_heap *heap, void *visit_data) {
} }
static inline void static inline void
visit_hole_fields(Hole *obj, visit_hole_fields(Hole *obj,
void (*visit)(struct gc_edge edge, void *visit_data), void (*visit)(struct gc_edge edge,
void *visit_data) { struct gc_heap *heap, void *visit_data),
struct gc_heap *heap, void *visit_data) {
#if GC_PRECISE #if GC_PRECISE
GC_CRASH(); GC_CRASH();
#endif #endif

19
parallel-tracer.h
View file

@ -320,7 +320,6 @@ struct tracer {
struct local_tracer { struct local_tracer {
struct trace_worker *worker; struct trace_worker *worker;
struct trace_deque *share_deque; struct trace_deque *share_deque;
struct gc_heap *heap;
struct local_trace_queue local; struct local_trace_queue local;
}; };
@ -449,8 +448,10 @@ static void tracer_release(struct gc_heap *heap) {
trace_deque_release(&tracer->workers[i].deque); trace_deque_release(&tracer->workers[i].deque);
} }
static inline void tracer_visit(struct gc_edge edge, void *trace_data) GC_ALWAYS_INLINE; static inline void tracer_visit(struct gc_edge edge, struct gc_heap *heap,
static inline void trace_one(struct gc_ref ref, void *trace_data) GC_ALWAYS_INLINE; void *trace_data) GC_ALWAYS_INLINE;
static inline void trace_one(struct gc_ref ref, struct gc_heap *heap,
void *trace_data) GC_ALWAYS_INLINE;
static inline int trace_edge(struct gc_heap *heap, static inline int trace_edge(struct gc_heap *heap,
struct gc_edge edge) GC_ALWAYS_INLINE; struct gc_edge edge) GC_ALWAYS_INLINE;
@ -462,9 +463,9 @@ tracer_share(struct local_tracer *trace) {
} }
static inline void static inline void
tracer_visit(struct gc_edge edge, void *trace_data) { tracer_visit(struct gc_edge edge, struct gc_heap *heap, void *trace_data) {
struct local_tracer *trace = trace_data; if (trace_edge(heap, edge)) {
if (trace_edge(trace->heap, edge)) { struct local_tracer *trace = trace_data;
if (local_trace_queue_full(&trace->local)) if (local_trace_queue_full(&trace->local))
tracer_share(trace); tracer_share(trace);
local_trace_queue_push(&trace->local, gc_edge_ref(edge)); local_trace_queue_push(&trace->local, gc_edge_ref(edge));
@ -544,8 +545,8 @@ trace_worker_check_termination(struct trace_worker *worker,
static struct gc_ref static struct gc_ref
trace_worker_steal(struct local_tracer *trace) { trace_worker_steal(struct local_tracer *trace) {
struct tracer *tracer = heap_tracer(trace->heap);
struct trace_worker *worker = trace->worker; struct trace_worker *worker = trace->worker;
struct tracer *tracer = heap_tracer(worker->heap);
// It could be that the worker's local trace queue has simply // It could be that the worker's local trace queue has simply
// overflowed. In that case avoid contention by trying to pop // overflowed. In that case avoid contention by trying to pop
@ -573,7 +574,7 @@ trace_worker_trace(struct trace_worker *worker) {
struct local_tracer trace; struct local_tracer trace;
trace.worker = worker; trace.worker = worker;
trace.share_deque = &worker->deque; trace.share_deque = &worker->deque;
trace.heap = worker->heap; struct gc_heap *heap = worker->heap;
local_trace_queue_init(&trace.local); local_trace_queue_init(&trace.local);
size_t n = 0; size_t n = 0;
@ -587,7 +588,7 @@ trace_worker_trace(struct trace_worker *worker) {
if (!gc_ref_is_heap_object(ref)) if (!gc_ref_is_heap_object(ref))
break; break;
} }
trace_one(ref, &trace); trace_one(ref, heap, &trace);
n++; n++;
} }
DEBUG("tracer #%zu: done tracing, %zu objects traced\n", worker->id, n); DEBUG("tracer #%zu: done tracing, %zu objects traced\n", worker->id, n);

12
precise-roots-embedder.h
View file

@ -30,26 +30,32 @@ gc_conservative_ref_might_be_a_heap_object(struct gc_conservative_ref ref,
static inline void visit_roots(struct handle *roots, static inline void visit_roots(struct handle *roots,
void (*trace_edge)(struct gc_edge edge, void (*trace_edge)(struct gc_edge edge,
struct gc_heap *heap,
void *trace_data), void *trace_data),
struct gc_heap *heap,
void *trace_data) { void *trace_data) {
for (struct handle *h = roots; h; h = h->next) for (struct handle *h = roots; h; h = h->next)
trace_edge(gc_edge(&h->v), trace_data); trace_edge(gc_edge(&h->v), heap, trace_data);
} }
static inline void gc_trace_mutator_roots(struct gc_mutator_roots *roots, static inline void gc_trace_mutator_roots(struct gc_mutator_roots *roots,
void (*trace_edge)(struct gc_edge edge, void (*trace_edge)(struct gc_edge edge,
struct gc_heap *heap,
void *trace_data), void *trace_data),
struct gc_heap *heap,
void *trace_data) { void *trace_data) {
if (roots) if (roots)
visit_roots(roots->roots, trace_edge, trace_data); visit_roots(roots->roots, trace_edge, heap, trace_data);
} }
static inline void gc_trace_heap_roots(struct gc_heap_roots *roots, static inline void gc_trace_heap_roots(struct gc_heap_roots *roots,
void (*trace_edge)(struct gc_edge edge, void (*trace_edge)(struct gc_edge edge,
struct gc_heap *heap,
void *trace_data), void *trace_data),
struct gc_heap *heap,
void *trace_data) { void *trace_data) {
if (roots) if (roots)
visit_roots(roots->roots, trace_edge, trace_data); visit_roots(roots->roots, trace_edge, heap, trace_data);
} }
#endif // PRECISE_ROOTS_EMBEDDER_H #endif // PRECISE_ROOTS_EMBEDDER_H

13
quads-embedder.h
View file

@ -5,10 +5,15 @@
#include "quads-types.h" #include "quads-types.h"
struct gc_heap;
#define DEFINE_METHODS(name, Name, NAME) \ #define DEFINE_METHODS(name, Name, NAME) \
static inline size_t name##_size(Name *obj) GC_ALWAYS_INLINE; \ static inline size_t name##_size(Name *obj) GC_ALWAYS_INLINE; \
static inline void visit_##name##_fields(Name *obj,\ static inline void visit_##name##_fields(Name *obj,\
void (*visit)(struct gc_edge edge, void *visit_data), \ void (*visit)(struct gc_edge edge, \
struct gc_heap *heap, \
void *visit_data), \
struct gc_heap *heap, \
void *visit_data) GC_ALWAYS_INLINE; void *visit_data) GC_ALWAYS_INLINE;
FOR_EACH_HEAP_OBJECT_KIND(DEFINE_METHODS) FOR_EACH_HEAP_OBJECT_KIND(DEFINE_METHODS)
#undef DEFINE_METHODS #undef DEFINE_METHODS
@ -19,10 +24,12 @@ static inline size_t quad_size(Quad *obj) {
static inline void static inline void
visit_quad_fields(Quad *quad, visit_quad_fields(Quad *quad,
void (*visit)(struct gc_edge edge, void *visit_data), void (*visit)(struct gc_edge edge, struct gc_heap *heap,
void *visit_data),
struct gc_heap *heap,
void *visit_data) { void *visit_data) {
for (size_t i = 0; i < 4; i++) for (size_t i = 0; i < 4; i++)
visit(gc_edge(&quad->kids[i]), visit_data); visit(gc_edge(&quad->kids[i]), heap, visit_data);
} }
#include "simple-gc-embedder.h" #include "simple-gc-embedder.h"

17
semi.c
View file

@ -63,7 +63,7 @@ static uintptr_t align_up(uintptr_t addr, size_t align) {
static void collect(struct gc_mutator *mut) GC_NEVER_INLINE; static void collect(struct gc_mutator *mut) GC_NEVER_INLINE;
static void collect_for_alloc(struct gc_mutator *mut, size_t bytes) GC_NEVER_INLINE; static void collect_for_alloc(struct gc_mutator *mut, size_t bytes) GC_NEVER_INLINE;
static void visit(struct gc_edge edge, void *visit_data); static void trace(struct gc_edge edge, struct gc_heap *heap, void *visit_data);
static int semi_space_steal_pages(struct semi_space *space, size_t npages) { static int semi_space_steal_pages(struct semi_space *space, size_t npages) {
size_t stolen_pages = space->stolen_pages + npages; size_t stolen_pages = space->stolen_pages + npages;
@ -103,7 +103,7 @@ static void flip(struct semi_space *space) {
static struct gc_ref copy(struct semi_space *space, struct gc_ref ref) { static struct gc_ref copy(struct semi_space *space, struct gc_ref ref) {
size_t size; size_t size;
gc_trace_object(ref, NULL, NULL, &size); gc_trace_object(ref, NULL, NULL, NULL, &size);
struct gc_ref new_ref = gc_ref(space->hp); struct gc_ref new_ref = gc_ref(space->hp);
memcpy(gc_ref_heap_object(new_ref), gc_ref_heap_object(ref), size); memcpy(gc_ref_heap_object(new_ref), gc_ref_heap_object(ref), size);
gc_object_forward_nonatomic(ref, new_ref); gc_object_forward_nonatomic(ref, new_ref);
@ -113,7 +113,7 @@ static struct gc_ref copy(struct semi_space *space, struct gc_ref ref) {
static uintptr_t scan(struct gc_heap *heap, struct gc_ref grey) { static uintptr_t scan(struct gc_heap *heap, struct gc_ref grey) {
size_t size; size_t size;
gc_trace_object(grey, visit, heap, &size); gc_trace_object(grey, trace, heap, NULL, &size);
return gc_ref_value(grey) + align_up(size, GC_ALIGNMENT); return gc_ref_value(grey) + align_up(size, GC_ALIGNMENT);
} }
@ -131,7 +131,7 @@ static void visit_large_object_space(struct gc_heap *heap,
struct large_object_space *space, struct large_object_space *space,
struct gc_ref ref) { struct gc_ref ref) {
if (large_object_space_copy(space, ref)) if (large_object_space_copy(space, ref))
gc_trace_object(ref, visit, heap, NULL); gc_trace_object(ref, trace, heap, NULL, NULL);
} }
static int semi_space_contains(struct semi_space *space, struct gc_ref ref) { static int semi_space_contains(struct semi_space *space, struct gc_ref ref) {
@ -139,8 +139,7 @@ static int semi_space_contains(struct semi_space *space, struct gc_ref ref) {
return addr - space->base < space->size; return addr - space->base < space->size;
} }
static void visit(struct gc_edge edge, void *visit_data) { static void visit(struct gc_edge edge, struct gc_heap *heap) {
struct gc_heap *heap = visit_data;
struct gc_ref ref = gc_edge_ref(edge); struct gc_ref ref = gc_edge_ref(edge);
if (!gc_ref_is_heap_object(ref)) if (!gc_ref_is_heap_object(ref))
return; return;
@ -152,6 +151,10 @@ static void visit(struct gc_edge edge, void *visit_data) {
GC_CRASH(); GC_CRASH();
} }
static void trace(struct gc_edge edge, struct gc_heap *heap, void *visit_data) {
return visit(edge, heap);
}
static void collect(struct gc_mutator *mut) { static void collect(struct gc_mutator *mut) {
struct gc_heap *heap = mutator_heap(mut); struct gc_heap *heap = mutator_heap(mut);
struct semi_space *semi = heap_semi_space(heap); struct semi_space *semi = heap_semi_space(heap);
@ -161,7 +164,7 @@ static void collect(struct gc_mutator *mut) {
flip(semi); flip(semi);
uintptr_t grey = semi->hp; uintptr_t grey = semi->hp;
if (mut->roots) if (mut->roots)
gc_trace_mutator_roots(mut->roots, visit, heap); gc_trace_mutator_roots(mut->roots, trace, heap, NULL);
// fprintf(stderr, "pushed %zd bytes in roots\n", space->hp - grey); // fprintf(stderr, "pushed %zd bytes in roots\n", space->hp - grey);
while(grey < semi->hp) while(grey < semi->hp)
grey = scan(heap, gc_ref(grey)); grey = scan(heap, gc_ref(grey));

11
serial-tracer.h
View file

@ -135,8 +135,10 @@ static void tracer_release(struct gc_heap *heap) {
trace_queue_release(&heap_tracer(heap)->queue); trace_queue_release(&heap_tracer(heap)->queue);
} }
static inline void tracer_visit(struct gc_edge edge, void *trace_data) GC_ALWAYS_INLINE; static inline void tracer_visit(struct gc_edge edge, struct gc_heap *heap,
static inline void trace_one(struct gc_ref ref, void *trace_data) GC_ALWAYS_INLINE; void *trace_data) GC_ALWAYS_INLINE;
static inline void trace_one(struct gc_ref ref, struct gc_heap *heap,
void *trace_data) GC_ALWAYS_INLINE;
static inline int trace_edge(struct gc_heap *heap, static inline int trace_edge(struct gc_heap *heap,
struct gc_edge edge) GC_ALWAYS_INLINE; struct gc_edge edge) GC_ALWAYS_INLINE;
@ -150,8 +152,7 @@ tracer_enqueue_roots(struct tracer *tracer, struct gc_ref *objs,
trace_queue_push_many(&tracer->queue, objs, count); trace_queue_push_many(&tracer->queue, objs, count);
} }
static inline void static inline void
tracer_visit(struct gc_edge edge, void *trace_data) { tracer_visit(struct gc_edge edge, struct gc_heap *heap, void *trace_data) {
struct gc_heap *heap = trace_data;
if (trace_edge(heap, edge)) if (trace_edge(heap, edge))
tracer_enqueue_root(heap_tracer(heap), gc_edge_ref(edge)); tracer_enqueue_root(heap_tracer(heap), gc_edge_ref(edge));
} }
@ -161,7 +162,7 @@ tracer_trace(struct gc_heap *heap) {
struct gc_ref obj = trace_queue_pop(&heap_tracer(heap)->queue); struct gc_ref obj = trace_queue_pop(&heap_tracer(heap)->queue);
if (!gc_ref_is_heap_object(obj)) if (!gc_ref_is_heap_object(obj))
break; break;
trace_one(obj, heap); trace_one(obj, heap, NULL);
} while (1); } while (1);
} }

4
simple-gc-embedder.h
View file

@ -5,7 +5,9 @@
static inline void gc_trace_object(struct gc_ref ref, static inline void gc_trace_object(struct gc_ref ref,
void (*trace_edge)(struct gc_edge edge, void (*trace_edge)(struct gc_edge edge,
struct gc_heap *heap,
void *trace_data), void *trace_data),
struct gc_heap *heap,
void *trace_data, void *trace_data,
size_t *size) { size_t *size) {
switch (tag_live_alloc_kind(*tag_word(ref))) { switch (tag_live_alloc_kind(*tag_word(ref))) {
@ -13,7 +15,7 @@ static inline void gc_trace_object(struct gc_ref ref,
case ALLOC_KIND_##NAME: \ case ALLOC_KIND_##NAME: \
if (trace_edge) \ if (trace_edge) \
visit_##name##_fields(gc_ref_heap_object(ref), trace_edge, \ visit_##name##_fields(gc_ref_heap_object(ref), trace_edge, \
trace_data); \ heap, trace_data); \
if (size) \ if (size) \
*size = name##_size(gc_ref_heap_object(ref)); \ *size = name##_size(gc_ref_heap_object(ref)); \
break; break;

69
whippet.c
View file

@ -710,8 +710,9 @@ static inline struct gc_ref trace_conservative_ref(struct gc_heap *heap,
ref, possibly_interior); ref, possibly_interior);
} }
static inline void trace_one(struct gc_ref ref, void *mark_data) { static inline void trace_one(struct gc_ref ref, struct gc_heap *heap,
gc_trace_object(ref, tracer_visit, mark_data, NULL); void *mark_data) {
gc_trace_object(ref, tracer_visit, heap, mark_data, NULL);
} }
static int heap_has_multiple_mutators(struct gc_heap *heap) { static int heap_has_multiple_mutators(struct gc_heap *heap) {
@ -959,55 +960,60 @@ void gc_heap_set_roots(struct gc_heap *heap, struct gc_heap_roots *roots) {
heap->roots = roots; heap->roots = roots;
} }
static void trace_and_enqueue_locally(struct gc_edge edge, void *data) { static void trace_and_enqueue_locally(struct gc_edge edge,
struct gc_heap *heap,
void *data) {
struct gc_mutator *mut = data; struct gc_mutator *mut = data;
if (trace_edge(mutator_heap(mut), edge)) if (trace_edge(heap, edge))
mutator_mark_buf_push(&mut->mark_buf, gc_edge_ref(edge)); mutator_mark_buf_push(&mut->mark_buf, gc_edge_ref(edge));
} }
static inline void do_trace_conservative_ref_and_enqueue_locally(struct gc_conservative_ref ref, static inline void do_trace_conservative_ref_and_enqueue_locally(struct gc_conservative_ref ref,
struct gc_heap *heap,
void *data, void *data,
int possibly_interior) { int possibly_interior) {
struct gc_mutator *mut = data; struct gc_mutator *mut = data;
struct gc_ref object = trace_conservative_ref(mutator_heap(mut), ref, struct gc_ref object = trace_conservative_ref(heap, ref, possibly_interior);
possibly_interior);
if (gc_ref_is_heap_object(object)) if (gc_ref_is_heap_object(object))
mutator_mark_buf_push(&mut->mark_buf, object); mutator_mark_buf_push(&mut->mark_buf, object);
} }
static void trace_possibly_interior_conservative_ref_and_enqueue_locally static void trace_possibly_interior_conservative_ref_and_enqueue_locally
(struct gc_conservative_ref ref, void *data) { (struct gc_conservative_ref ref, struct gc_heap *heap, void *data) {
return do_trace_conservative_ref_and_enqueue_locally(ref, data, 1); return do_trace_conservative_ref_and_enqueue_locally(ref, heap, data, 1);
} }
static void trace_conservative_ref_and_enqueue_locally static void trace_conservative_ref_and_enqueue_locally
(struct gc_conservative_ref ref, void *data) { (struct gc_conservative_ref ref, struct gc_heap *heap, void *data) {
return do_trace_conservative_ref_and_enqueue_locally(ref, data, 0); return do_trace_conservative_ref_and_enqueue_locally(ref, heap, data, 0);
} }
static void trace_and_enqueue_globally(struct gc_edge edge, void *data) { static void trace_and_enqueue_globally(struct gc_edge edge,
struct gc_heap *heap = data; struct gc_heap *heap,
void *unused) {
if (trace_edge(heap, edge)) if (trace_edge(heap, edge))
tracer_enqueue_root(&heap->tracer, gc_edge_ref(edge)); tracer_enqueue_root(&heap->tracer, gc_edge_ref(edge));
} }
static inline void do_trace_conservative_ref_and_enqueue_globally(struct gc_conservative_ref ref, static inline void do_trace_conservative_ref_and_enqueue_globally(struct gc_conservative_ref ref,
struct gc_heap *heap,
void *data, void *data,
int possibly_interior) { int possibly_interior) {
struct gc_heap *heap = data;
struct gc_ref object = trace_conservative_ref(heap, ref, possibly_interior); struct gc_ref object = trace_conservative_ref(heap, ref, possibly_interior);
if (gc_ref_is_heap_object(object)) if (gc_ref_is_heap_object(object))
tracer_enqueue_root(&heap->tracer, object); tracer_enqueue_root(&heap->tracer, object);
} }
static void trace_possibly_interior_conservative_ref_and_enqueue_globally(struct gc_conservative_ref ref, static void trace_possibly_interior_conservative_ref_and_enqueue_globally(struct gc_conservative_ref ref,
struct gc_heap *heap,
void *data) { void *data) {
return do_trace_conservative_ref_and_enqueue_globally(ref, data, 1); return do_trace_conservative_ref_and_enqueue_globally(ref, heap, data, 1);
} }
static void trace_conservative_ref_and_enqueue_globally(struct gc_conservative_ref ref, static void trace_conservative_ref_and_enqueue_globally(struct gc_conservative_ref ref,
struct gc_heap *heap,
void *data) { void *data) {
return do_trace_conservative_ref_and_enqueue_globally(ref, data, 0); return do_trace_conservative_ref_and_enqueue_globally(ref, heap, data, 0);
} }
static inline struct gc_conservative_ref static inline struct gc_conservative_ref
@ -1021,75 +1027,84 @@ load_conservative_ref(uintptr_t addr) {
static inline void static inline void
trace_conservative_edges(uintptr_t low, trace_conservative_edges(uintptr_t low,
uintptr_t high, uintptr_t high,
void (*trace)(struct gc_conservative_ref, void *), void (*trace)(struct gc_conservative_ref,
struct gc_heap *, void *),
struct gc_heap *heap,
void *data) { void *data) {
GC_ASSERT(low == align_down(low, sizeof(uintptr_t))); GC_ASSERT(low == align_down(low, sizeof(uintptr_t)));
GC_ASSERT(high == align_down(high, sizeof(uintptr_t))); GC_ASSERT(high == align_down(high, sizeof(uintptr_t)));
for (uintptr_t addr = low; addr < high; addr += sizeof(uintptr_t)) for (uintptr_t addr = low; addr < high; addr += sizeof(uintptr_t))
trace(load_conservative_ref(addr), data); trace(load_conservative_ref(addr), heap, data);
} }
static void static void
mark_and_globally_enqueue_mutator_conservative_roots(uintptr_t low, mark_and_globally_enqueue_mutator_conservative_roots(uintptr_t low,
uintptr_t high, uintptr_t high,
struct gc_heap *heap,
void *data) { void *data) {
trace_conservative_edges(low, high, trace_conservative_edges(low, high,
gc_mutator_conservative_roots_may_be_interior() gc_mutator_conservative_roots_may_be_interior()
? trace_possibly_interior_conservative_ref_and_enqueue_globally ? trace_possibly_interior_conservative_ref_and_enqueue_globally
: trace_conservative_ref_and_enqueue_globally, : trace_conservative_ref_and_enqueue_globally,
data); heap, data);
} }
static void static void
mark_and_globally_enqueue_heap_conservative_roots(uintptr_t low, mark_and_globally_enqueue_heap_conservative_roots(uintptr_t low,
uintptr_t high, uintptr_t high,
struct gc_heap *heap,
void *data) { void *data) {
trace_conservative_edges(low, high, trace_conservative_edges(low, high,
trace_conservative_ref_and_enqueue_globally, trace_conservative_ref_and_enqueue_globally,
data); heap, data);
} }
static void static void
mark_and_locally_enqueue_mutator_conservative_roots(uintptr_t low, mark_and_locally_enqueue_mutator_conservative_roots(uintptr_t low,
uintptr_t high, uintptr_t high,
struct gc_heap *heap,
void *data) { void *data) {
trace_conservative_edges(low, high, trace_conservative_edges(low, high,
gc_mutator_conservative_roots_may_be_interior() gc_mutator_conservative_roots_may_be_interior()
? trace_possibly_interior_conservative_ref_and_enqueue_locally ? trace_possibly_interior_conservative_ref_and_enqueue_locally
: trace_conservative_ref_and_enqueue_locally, : trace_conservative_ref_and_enqueue_locally,
data); heap, data);
} }
static inline void static inline void
trace_mutator_conservative_roots(struct gc_mutator *mut, trace_mutator_conservative_roots(struct gc_mutator *mut,
void (*trace_range)(uintptr_t low, void (*trace_range)(uintptr_t low,
uintptr_t high, uintptr_t high,
struct gc_heap *heap,
void *data), void *data),
struct gc_heap *heap,
void *data) { void *data) {
if (gc_has_mutator_conservative_roots()) if (gc_has_mutator_conservative_roots())
gc_stack_visit(&mut->stack, trace_range, data); gc_stack_visit(&mut->stack, trace_range, heap, data);
} }
// Mark the roots of a mutator that is stopping for GC. We can't // Mark the roots of a mutator that is stopping for GC. We can't
// enqueue them directly, so we send them to the controller in a buffer. // enqueue them directly, so we send them to the controller in a buffer.
static void trace_stopping_mutator_roots(struct gc_mutator *mut) { static void trace_stopping_mutator_roots(struct gc_mutator *mut) {
GC_ASSERT(mutator_should_mark_while_stopping(mut)); GC_ASSERT(mutator_should_mark_while_stopping(mut));
struct gc_heap *heap = mutator_heap(mut);
trace_mutator_conservative_roots(mut, trace_mutator_conservative_roots(mut,
mark_and_locally_enqueue_mutator_conservative_roots, mark_and_locally_enqueue_mutator_conservative_roots,
mut); heap, mut);
gc_trace_mutator_roots(mut->roots, trace_and_enqueue_locally, mut); gc_trace_mutator_roots(mut->roots, trace_and_enqueue_locally, heap, mut);
} }
static void trace_mutator_conservative_roots_with_lock(struct gc_mutator *mut) { static void trace_mutator_conservative_roots_with_lock(struct gc_mutator *mut) {
trace_mutator_conservative_roots(mut, trace_mutator_conservative_roots(mut,
mark_and_globally_enqueue_mutator_conservative_roots, mark_and_globally_enqueue_mutator_conservative_roots,
mutator_heap(mut)); mutator_heap(mut),
NULL);
} }
static void trace_mutator_roots_with_lock(struct gc_mutator *mut) { static void trace_mutator_roots_with_lock(struct gc_mutator *mut) {
trace_mutator_conservative_roots_with_lock(mut); trace_mutator_conservative_roots_with_lock(mut);
gc_trace_mutator_roots(mut->roots, trace_and_enqueue_globally, gc_trace_mutator_roots(mut->roots, trace_and_enqueue_globally,
mutator_heap(mut)); mutator_heap(mut), NULL);
} }
static void trace_mutator_roots_with_lock_before_stop(struct gc_mutator *mut) { static void trace_mutator_roots_with_lock_before_stop(struct gc_mutator *mut) {
@ -1154,7 +1169,7 @@ static void trace_mutator_roots_after_stop(struct gc_heap *heap) {
static void trace_global_conservative_roots(struct gc_heap *heap) { static void trace_global_conservative_roots(struct gc_heap *heap) {
if (gc_has_global_conservative_roots()) if (gc_has_global_conservative_roots())
gc_platform_visit_global_conservative_roots gc_platform_visit_global_conservative_roots
(mark_and_globally_enqueue_heap_conservative_roots, heap); (mark_and_globally_enqueue_heap_conservative_roots, heap, NULL);
} }
static inline uint64_t load_eight_aligned_bytes(uint8_t *mark) { static inline uint64_t load_eight_aligned_bytes(uint8_t *mark) {
@ -1601,7 +1616,7 @@ static void trace_pinned_roots_after_stop(struct gc_heap *heap) {
static void trace_roots_after_stop(struct gc_heap *heap) { static void trace_roots_after_stop(struct gc_heap *heap) {
trace_mutator_roots_after_stop(heap); trace_mutator_roots_after_stop(heap);
gc_trace_heap_roots(heap->roots, trace_and_enqueue_globally, heap); gc_trace_heap_roots(heap->roots, trace_and_enqueue_globally, heap, NULL);
trace_generational_roots(heap); trace_generational_roots(heap);
} }