mirror/guile
mirror/guile
1
Fork 0
mirror of https://git.savannah.gnu.org/git/guile.git synced 2025-07-07 18:10:21 +02:00
Code Activity

Tighten up typing of trace workers and trace worker data

Browse source
This commit is contained in:
Andy Wingo 2024-07-08 18:44:24 +02:00
parent 5ff78f01c8
commit 247f9432a4
4 changed files with 78 additions and 79 deletions
Download patch file Download diff file Expand all files Collapse all files

90
src/parallel-tracer.h
View file

@ -23,7 +23,7 @@ enum trace_worker_state {
}; };
struct gc_heap; struct gc_heap;
struct trace_worker { struct gc_trace_worker {
struct gc_heap *heap; struct gc_heap *heap;
struct gc_tracer *tracer; struct gc_tracer *tracer;
size_t id; size_t id;
@ -31,7 +31,8 @@ struct trace_worker {
pthread_t thread; pthread_t thread;
enum trace_worker_state state; enum trace_worker_state state;
pthread_mutex_t lock; pthread_mutex_t lock;
struct shared_worklist deque; struct shared_worklist shared;
struct local_worklist local;
struct gc_trace_worker_data *data; struct gc_trace_worker_data *data;
}; };
@ -44,17 +45,11 @@ struct gc_tracer {
long epoch; long epoch;
pthread_mutex_t lock; pthread_mutex_t lock;
pthread_cond_t cond; pthread_cond_t cond;
struct trace_worker workers[TRACE_WORKERS_MAX_COUNT]; struct gc_trace_worker workers[TRACE_WORKERS_MAX_COUNT];
};
struct local_tracer {
struct trace_worker *worker;
struct shared_worklist *share_deque;
struct local_worklist local;
}; };
static int static int
trace_worker_init(struct trace_worker *worker, struct gc_heap *heap, trace_worker_init(struct gc_trace_worker *worker, struct gc_heap *heap,
struct gc_tracer *tracer, size_t id) { struct gc_tracer *tracer, size_t id) {
worker->heap = heap; worker->heap = heap;
worker->tracer = tracer; worker->tracer = tracer;
@ -64,14 +59,15 @@ trace_worker_init(struct trace_worker *worker, struct gc_heap *heap,
worker->state = TRACE_WORKER_STOPPED; worker->state = TRACE_WORKER_STOPPED;
pthread_mutex_init(&worker->lock, NULL); pthread_mutex_init(&worker->lock, NULL);
worker->data = NULL; worker->data = NULL;
return shared_worklist_init(&worker->deque); local_worklist_init(&worker->local);
return shared_worklist_init(&worker->shared);
} }
static void trace_worker_trace(struct trace_worker *worker); static void trace_worker_trace(struct gc_trace_worker *worker);
static void* static void*
trace_worker_thread(void *data) { trace_worker_thread(void *data) {
struct trace_worker *worker = data; struct gc_trace_worker *worker = data;
struct gc_tracer *tracer = worker->tracer; struct gc_tracer *tracer = worker->tracer;
long trace_epoch = 0; long trace_epoch = 0;
@ -88,7 +84,7 @@ trace_worker_thread(void *data) {
} }
static int static int
trace_worker_spawn(struct trace_worker *worker) { trace_worker_spawn(struct gc_trace_worker *worker) {
if (pthread_create(&worker->thread, NULL, trace_worker_thread, worker)) { if (pthread_create(&worker->thread, NULL, trace_worker_thread, worker)) {
perror("spawning tracer thread failed"); perror("spawning tracer thread failed");
return 0; return 0;
@ -129,7 +125,7 @@ static void gc_tracer_prepare(struct gc_tracer *tracer) {
} }
static void gc_tracer_release(struct gc_tracer *tracer) { static void gc_tracer_release(struct gc_tracer *tracer) {
for (size_t i = 0; i < tracer->worker_count; i++) for (size_t i = 0; i < tracer->worker_count; i++)
shared_worklist_release(&tracer->workers[i].deque); shared_worklist_release(&tracer->workers[i].shared);
} }
static inline void static inline void
@ -151,41 +147,40 @@ tracer_maybe_unpark_workers(struct gc_tracer *tracer) {
} }
static inline void static inline void
tracer_share(struct local_tracer *trace) { tracer_share(struct gc_trace_worker *worker) {
DEBUG("tracer #%zu: sharing\n", trace->worker->id); DEBUG("tracer #%zu: sharing\n", worker->id);
size_t to_share = LOCAL_WORKLIST_SHARE_AMOUNT; size_t to_share = LOCAL_WORKLIST_SHARE_AMOUNT;
while (to_share) { while (to_share) {
struct gc_ref *objv; struct gc_ref *objv;
size_t count = local_worklist_pop_many(&trace->local, &objv, to_share); size_t count = local_worklist_pop_many(&worker->local, &objv, to_share);
shared_worklist_push_many(trace->share_deque, objv, count); shared_worklist_push_many(&worker->shared, objv, count);
to_share -= count; to_share -= count;
} }
tracer_maybe_unpark_workers(trace->worker->tracer); tracer_maybe_unpark_workers(worker->tracer);
} }
static inline void static inline void
gc_tracer_enqueue(struct gc_tracer *tracer, struct gc_ref ref, gc_trace_worker_enqueue(struct gc_trace_worker *worker, struct gc_ref ref) {
void *trace_data) { if (local_worklist_full(&worker->local))
struct local_tracer *trace = trace_data; tracer_share(worker);
if (local_worklist_full(&trace->local)) local_worklist_push(&worker->local, ref);
tracer_share(trace);
local_worklist_push(&trace->local, ref);
} }
static struct gc_ref static struct gc_ref
tracer_steal_from_worker(struct gc_tracer *tracer, size_t id) { tracer_steal_from_worker(struct gc_tracer *tracer, size_t id) {
ASSERT(id < tracer->worker_count); ASSERT(id < tracer->worker_count);
return shared_worklist_steal(&tracer->workers[id].deque); return shared_worklist_steal(&tracer->workers[id].shared);
} }
static int static int
tracer_can_steal_from_worker(struct gc_tracer *tracer, size_t id) { tracer_can_steal_from_worker(struct gc_tracer *tracer, size_t id) {
ASSERT(id < tracer->worker_count); ASSERT(id < tracer->worker_count);
return shared_worklist_can_steal(&tracer->workers[id].deque); return shared_worklist_can_steal(&tracer->workers[id].shared);
} }
static struct gc_ref static struct gc_ref
trace_worker_steal_from_any(struct trace_worker *worker, struct gc_tracer *tracer) { trace_worker_steal_from_any(struct gc_trace_worker *worker,
struct gc_tracer *tracer) {
for (size_t i = 0; i < tracer->worker_count; i++) { for (size_t i = 0; i < tracer->worker_count; i++) {
DEBUG("tracer #%zu: stealing from #%zu\n", worker->id, worker->steal_id); DEBUG("tracer #%zu: stealing from #%zu\n", worker->id, worker->steal_id);
struct gc_ref obj = tracer_steal_from_worker(tracer, worker->steal_id); struct gc_ref obj = tracer_steal_from_worker(tracer, worker->steal_id);
@ -201,7 +196,7 @@ trace_worker_steal_from_any(struct trace_worker *worker, struct gc_tracer *trace
} }
static int static int
trace_worker_can_steal_from_any(struct trace_worker *worker, trace_worker_can_steal_from_any(struct gc_trace_worker *worker,
struct gc_tracer *tracer) { struct gc_tracer *tracer) {
DEBUG("tracer #%zu: checking if any worker has tasks\n", worker->id); DEBUG("tracer #%zu: checking if any worker has tasks\n", worker->id);
for (size_t i = 0; i < tracer->worker_count; i++) { for (size_t i = 0; i < tracer->worker_count; i++) {
@ -218,7 +213,7 @@ trace_worker_can_steal_from_any(struct trace_worker *worker,
} }
static int static int
trace_worker_should_continue(struct trace_worker *worker) { trace_worker_should_continue(struct gc_trace_worker *worker) {
// Helper workers should park themselves immediately if they have no work. // Helper workers should park themselves immediately if they have no work.
if (worker->id != 0) if (worker->id != 0)
return 0; return 0;
@ -252,8 +247,7 @@ trace_worker_should_continue(struct trace_worker *worker) {
} }
static struct gc_ref static struct gc_ref
trace_worker_steal(struct local_tracer *trace) { trace_worker_steal(struct gc_trace_worker *worker) {
struct trace_worker *worker = trace->worker;
struct gc_tracer *tracer = worker->tracer; struct gc_tracer *tracer = worker->tracer;
// It could be that the worker's local trace queue has simply // It could be that the worker's local trace queue has simply
@ -261,7 +255,7 @@ trace_worker_steal(struct local_tracer *trace) {
// something from the worker's own queue. // something from the worker's own queue.
{ {
DEBUG("tracer #%zu: trying to pop worker's own deque\n", worker->id); DEBUG("tracer #%zu: trying to pop worker's own deque\n", worker->id);
struct gc_ref obj = shared_worklist_try_pop(&worker->deque); struct gc_ref obj = shared_worklist_try_pop(&worker->shared);
if (gc_ref_is_heap_object(obj)) if (gc_ref_is_heap_object(obj))
return obj; return obj;
} }
@ -277,16 +271,10 @@ trace_worker_steal(struct local_tracer *trace) {
static void static void
trace_with_data(struct gc_tracer *tracer, trace_with_data(struct gc_tracer *tracer,
struct gc_heap *heap, struct gc_heap *heap,
struct gc_trace_worker_data *worker_data, struct gc_trace_worker *worker,
void *data) { struct gc_trace_worker_data *data) {
struct trace_worker *worker = data;
atomic_fetch_add_explicit(&tracer->active_tracers, 1, memory_order_acq_rel); atomic_fetch_add_explicit(&tracer->active_tracers, 1, memory_order_acq_rel);
worker->data = worker_data; worker->data = data;
struct local_tracer trace;
trace.worker = worker;
trace.share_deque = &worker->deque;
local_worklist_init(&trace.local);
size_t n = 0; size_t n = 0;
DEBUG("tracer #%zu: running trace loop\n", worker->id); DEBUG("tracer #%zu: running trace loop\n", worker->id);
@ -294,14 +282,14 @@ trace_with_data(struct gc_tracer *tracer,
do { do {
while (1) { while (1) {
struct gc_ref ref; struct gc_ref ref;
if (!local_worklist_empty(&trace.local)) { if (!local_worklist_empty(&worker->local)) {
ref = local_worklist_pop(&trace.local); ref = local_worklist_pop(&worker->local);
} else { } else {
ref = trace_worker_steal(&trace); ref = trace_worker_steal(worker);
if (!gc_ref_is_heap_object(ref)) if (!gc_ref_is_heap_object(ref))
break; break;
} }
trace_one(ref, heap, &trace); trace_one(ref, heap, worker);
n++; n++;
} }
} while (trace_worker_should_continue(worker)); } while (trace_worker_should_continue(worker));
@ -313,21 +301,21 @@ trace_with_data(struct gc_tracer *tracer,
} }
static void static void
trace_worker_trace(struct trace_worker *worker) { trace_worker_trace(struct gc_trace_worker *worker) {
gc_trace_worker_call_with_data(trace_with_data, worker->tracer, gc_trace_worker_call_with_data(trace_with_data, worker->tracer,
worker->heap, worker); worker->heap, worker);
} }
static inline void static inline void
gc_tracer_enqueue_root(struct gc_tracer *tracer, struct gc_ref ref) { gc_tracer_enqueue_root(struct gc_tracer *tracer, struct gc_ref ref) {
struct shared_worklist *worker0_deque = &tracer->workers[0].deque; struct shared_worklist *worker0_deque = &tracer->workers[0].shared;
shared_worklist_push(worker0_deque, ref); shared_worklist_push(worker0_deque, ref);
} }
static inline void static inline void
gc_tracer_enqueue_roots(struct gc_tracer *tracer, struct gc_ref *objv, gc_tracer_enqueue_roots(struct gc_tracer *tracer, struct gc_ref *objv,
size_t count) { size_t count) {
struct shared_worklist *worker0_deque = &tracer->workers[0].deque; struct shared_worklist *worker0_deque = &tracer->workers[0].shared;
shared_worklist_push_many(worker0_deque, objv, count); shared_worklist_push_many(worker0_deque, objv, count);
} }
@ -337,7 +325,7 @@ gc_tracer_trace(struct gc_tracer *tracer) {
ssize_t parallel_threshold = ssize_t parallel_threshold =
LOCAL_WORKLIST_SIZE - LOCAL_WORKLIST_SHARE_AMOUNT; LOCAL_WORKLIST_SIZE - LOCAL_WORKLIST_SHARE_AMOUNT;
if (shared_worklist_size(&tracer->workers[0].deque) >= parallel_threshold) { if (shared_worklist_size(&tracer->workers[0].shared) >= parallel_threshold) {
DEBUG("waking workers\n"); DEBUG("waking workers\n");
tracer_unpark_all_workers(tracer); tracer_unpark_all_workers(tracer);
} else { } else {

20
src/serial-tracer.h
View file

@ -14,6 +14,11 @@ struct gc_tracer {
struct simple_worklist worklist; struct simple_worklist worklist;
}; };
struct gc_trace_worker {
struct gc_tracer *tracer;
struct gc_trace_worker_data *data;
};
static int static int
gc_tracer_init(struct gc_tracer *tracer, struct gc_heap *heap, gc_tracer_init(struct gc_tracer *tracer, struct gc_heap *heap,
size_t parallelism) { size_t parallelism) {
@ -35,25 +40,26 @@ gc_tracer_enqueue_roots(struct gc_tracer *tracer, struct gc_ref *objs,
simple_worklist_push_many(&tracer->worklist, objs, count); simple_worklist_push_many(&tracer->worklist, objs, count);
} }
static inline void static inline void
gc_tracer_enqueue(struct gc_tracer *tracer, struct gc_ref ref, gc_trace_worker_enqueue(struct gc_trace_worker *worker, struct gc_ref ref) {
void *trace_data) { gc_tracer_enqueue_root(worker->tracer, ref);
gc_tracer_enqueue_root(tracer, ref);
} }
static inline void static inline void
tracer_trace_with_data(struct gc_tracer *tracer, struct gc_heap *heap, tracer_trace_with_data(struct gc_tracer *tracer, struct gc_heap *heap,
struct gc_trace_worker_data *worker_data, struct gc_trace_worker *worker,
void *data) { struct gc_trace_worker_data *data) {
worker->data = data;
do { do {
struct gc_ref obj = simple_worklist_pop(&tracer->worklist); struct gc_ref obj = simple_worklist_pop(&tracer->worklist);
if (!gc_ref_is_heap_object(obj)) if (!gc_ref_is_heap_object(obj))
break; break;
trace_one(obj, heap, NULL); trace_one(obj, heap, worker);
} while (1); } while (1);
} }
static inline void static inline void
gc_tracer_trace(struct gc_tracer *tracer) { gc_tracer_trace(struct gc_tracer *tracer) {
struct gc_trace_worker worker = { tracer };
gc_trace_worker_call_with_data(tracer_trace_with_data, tracer, tracer->heap, gc_trace_worker_call_with_data(tracer_trace_with_data, tracer, tracer->heap,
NULL); &worker);
} }
#endif // SERIAL_TRACER_H #endif // SERIAL_TRACER_H

21
src/tracer.h
View file

@ -6,24 +6,28 @@
struct gc_heap; struct gc_heap;
// Data types to be implemented by tracer.
struct gc_tracer;
struct gc_trace_worker;
// Data types to be implemented by collector.
struct gc_trace_worker_data;
//////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////
/// To be implemented by collector. /// To be implemented by collector.
//////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////
struct gc_tracer;
struct gc_trace_worker_data;
// Visit all fields in an object. // Visit all fields in an object.
static inline void trace_one(struct gc_ref ref, struct gc_heap *heap, static inline void trace_one(struct gc_ref ref, struct gc_heap *heap,
void *trace_data) GC_ALWAYS_INLINE; struct gc_trace_worker *worker) GC_ALWAYS_INLINE;
static void static void
gc_trace_worker_call_with_data(void (*f)(struct gc_tracer *tracer, gc_trace_worker_call_with_data(void (*f)(struct gc_tracer *tracer,
struct gc_heap *heap, struct gc_heap *heap,
struct gc_trace_worker_data *worker_data, struct gc_trace_worker *worker,
void *data), struct gc_trace_worker_data *data),
struct gc_tracer *tracer, struct gc_tracer *tracer,
struct gc_heap *heap, struct gc_heap *heap,
void *data); struct gc_trace_worker *worker);
//////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////
/// To be implemented by tracer. /// To be implemented by tracer.
@ -47,9 +51,8 @@ static inline void gc_tracer_enqueue_roots(struct gc_tracer *tracer,
size_t count); size_t count);
// Given that an object has been shaded grey, enqueue for tracing. // Given that an object has been shaded grey, enqueue for tracing.
static inline void gc_tracer_enqueue(struct gc_tracer *tracer, static inline void gc_trace_worker_enqueue(struct gc_trace_worker *worker,
struct gc_ref ref, struct gc_ref ref) GC_ALWAYS_INLINE;
void *trace_data) GC_ALWAYS_INLINE;
// Run the full trace. // Run the full trace.
static inline void gc_tracer_trace(struct gc_tracer *tracer); static inline void gc_tracer_trace(struct gc_tracer *tracer);

26
src/whippet.c
View file

@ -1097,20 +1097,21 @@ void gc_heap_set_extern_space(struct gc_heap *heap,
static void static void
gc_trace_worker_call_with_data(void (*f)(struct gc_tracer *tracer, gc_trace_worker_call_with_data(void (*f)(struct gc_tracer *tracer,
struct gc_heap *heap, struct gc_heap *heap,
struct gc_trace_worker_data *worker_data, struct gc_trace_worker *worker,
void *data), struct gc_trace_worker_data *data),
struct gc_tracer *tracer, struct gc_tracer *tracer,
struct gc_heap *heap, struct gc_heap *heap,
void *data) { struct gc_trace_worker *worker) {
f(tracer, heap, NULL, data); f(tracer, heap, worker, NULL);
} }
static inline void tracer_visit(struct gc_edge edge, struct gc_heap *heap, static inline void tracer_visit(struct gc_edge edge, struct gc_heap *heap,
void *trace_data) GC_ALWAYS_INLINE; void *trace_data) GC_ALWAYS_INLINE;
static inline void static inline void
tracer_visit(struct gc_edge edge, struct gc_heap *heap, void *trace_data) { tracer_visit(struct gc_edge edge, struct gc_heap *heap, void *trace_data) {
struct gc_trace_worker *worker = trace_data;
if (trace_edge(heap, edge)) if (trace_edge(heap, edge))
gc_tracer_enqueue(&heap->tracer, gc_edge_ref(edge), trace_data); gc_trace_worker_enqueue(worker, gc_edge_ref(edge));
} }
static void trace_and_enqueue_locally(struct gc_edge edge, static void trace_and_enqueue_locally(struct gc_edge edge,
@ -1193,15 +1194,16 @@ trace_conservative_edges(uintptr_t low,
static inline void tracer_trace_conservative_ref(struct gc_conservative_ref ref, static inline void tracer_trace_conservative_ref(struct gc_conservative_ref ref,
struct gc_heap *heap, struct gc_heap *heap,
void *data) { void *data) {
struct gc_trace_worker *worker = data;
int possibly_interior = 0; int possibly_interior = 0;
struct gc_ref resolved = trace_conservative_ref(heap, ref, possibly_interior); struct gc_ref resolved = trace_conservative_ref(heap, ref, possibly_interior);
if (gc_ref_is_heap_object(resolved)) if (gc_ref_is_heap_object(resolved))
gc_tracer_enqueue(&heap->tracer, resolved, data); gc_trace_worker_enqueue(worker, resolved);
} }
static inline void trace_one_conservatively(struct gc_ref ref, static inline void trace_one_conservatively(struct gc_ref ref,
struct gc_heap *heap, struct gc_heap *heap,
void *mark_data) { struct gc_trace_worker *worker) {
size_t bytes; size_t bytes;
if (GC_LIKELY(mark_space_contains(heap_mark_space(heap), ref))) { if (GC_LIKELY(mark_space_contains(heap_mark_space(heap), ref))) {
// Generally speaking we trace conservatively and don't allow much // Generally speaking we trace conservatively and don't allow much
@ -1211,7 +1213,7 @@ static inline void trace_one_conservatively(struct gc_ref ref,
uint8_t meta = *metadata_byte_for_addr(gc_ref_value(ref)); uint8_t meta = *metadata_byte_for_addr(gc_ref_value(ref));
if (GC_UNLIKELY(meta & METADATA_BYTE_EPHEMERON)) { if (GC_UNLIKELY(meta & METADATA_BYTE_EPHEMERON)) {
gc_trace_ephemeron(gc_ref_heap_object(ref), tracer_visit, heap, gc_trace_ephemeron(gc_ref_heap_object(ref), tracer_visit, heap,
mark_data); worker);
return; return;
} }
bytes = mark_space_object_size(heap_mark_space(heap), ref); bytes = mark_space_object_size(heap_mark_space(heap), ref);
@ -1221,15 +1223,15 @@ static inline void trace_one_conservatively(struct gc_ref ref,
trace_conservative_edges(gc_ref_value(ref), trace_conservative_edges(gc_ref_value(ref),
gc_ref_value(ref) + bytes, gc_ref_value(ref) + bytes,
tracer_trace_conservative_ref, heap, tracer_trace_conservative_ref, heap,
mark_data); worker);
} }
static inline void trace_one(struct gc_ref ref, struct gc_heap *heap, static inline void trace_one(struct gc_ref ref, struct gc_heap *heap,
void *mark_data) { struct gc_trace_worker *worker) {
if (gc_has_conservative_intraheap_edges()) if (gc_has_conservative_intraheap_edges())
trace_one_conservatively(ref, heap, mark_data); trace_one_conservatively(ref, heap, worker);
else else
gc_trace_object(ref, tracer_visit, heap, mark_data, NULL); gc_trace_object(ref, tracer_visit, heap, worker, NULL);
} }
static void static void