diff --git a/gc-api.h b/gc-api.h index 4779cbd2c..6c072bc31 100644 --- a/gc-api.h +++ b/gc-api.h @@ -1,26 +1,62 @@ -#ifndef GC_TYPES_H_ -#define GC_TYPES_H_ +#ifndef GC_API_H_ +#define GC_API_H_ + +#include + +#ifndef GC_DEBUG +#define GC_DEBUG 0 +#endif + +#define GC_UNLIKELY(e) __builtin_expect(e, 0) +#define GC_LIKELY(e) __builtin_expect(e, 1) + +#if GC_DEBUG +#define GC_ASSERT(x) do { if (GC_UNLIKELY(!(x))) __builtin_trap(); } while (0) +#else +#define GC_ASSERT(x) do { } while (0) +#endif + +struct gc_ref { + uintptr_t value; +}; + +static inline struct gc_ref gc_ref(uintptr_t value) { + return (struct gc_ref){value}; +} +static inline uintptr_t gc_ref_value(struct gc_ref ref) { + return ref.value; +} + +static inline struct gc_ref gc_ref_null(void) { + return gc_ref(0); +} +static inline int gc_ref_is_heap_object(struct gc_ref ref) { + return ref.value != 0; +} +static inline struct gc_ref gc_ref_from_heap_object_or_null(void *obj) { + return gc_ref((uintptr_t) obj); +} +static inline struct gc_ref gc_ref_from_heap_object(void *obj) { + GC_ASSERT(obj); + return gc_ref_from_heap_object_or_null(obj); +} +static inline void* gc_ref_heap_object(struct gc_ref ref) { + GC_ASSERT(gc_ref_is_heap_object(ref)); + return (void *) gc_ref_value(ref); +} struct gc_edge { - union { - void *addr; - void **loc; - }; + struct gc_ref *dst; }; static inline struct gc_edge gc_edge(void* addr) { - struct gc_edge edge; - edge.addr = addr; - return edge; + return (struct gc_edge){addr}; } -static inline struct gc_edge object_field(void* addr) { - return gc_edge(addr); +static struct gc_ref gc_edge_ref(struct gc_edge edge) { + return *edge.dst; } -static inline void* dereference_edge(struct gc_edge edge) { - return *edge.loc; -} -static inline void update_edge(struct gc_edge edge, void *value) { - *edge.loc = value; +static inline void gc_edge_update(struct gc_edge edge, struct gc_ref ref) { + *edge.dst = ref; } -#endif // GC_TYPES_H_ +#endif // GC_API_H_ diff --git a/mt-gcbench.c b/mt-gcbench.c index 1819655fb..4d1c92255 100644 --- a/mt-gcbench.c +++ b/mt-gcbench.c @@ -88,8 +88,8 @@ static inline void visit_node_fields(Node *node, void (*visit)(struct gc_edge edge, void *visit_data), void *visit_data) { - visit(object_field(&node->left), visit_data); - visit(object_field(&node->right), visit_data); + visit(gc_edge(&node->left), visit_data); + visit(gc_edge(&node->right), visit_data); } static inline void visit_double_array_fields(DoubleArray *obj, diff --git a/parallel-tracer.h b/parallel-tracer.h index 02f641352..92c0a86d4 100644 --- a/parallel-tracer.h +++ b/parallel-tracer.h @@ -471,7 +471,8 @@ tracer_visit(struct gc_edge edge, void *trace_data) { if (trace_edge(trace->heap, edge)) { if (local_trace_queue_full(&trace->local)) tracer_share(trace); - local_trace_queue_push(&trace->local, dereference_edge(edge)); + local_trace_queue_push(&trace->local, + gc_ref_heap_object(gc_edge_ref(edge))); } } diff --git a/quads.c b/quads.c index 9743b88a8..d2cba2bf0 100644 --- a/quads.c +++ b/quads.c @@ -18,7 +18,7 @@ visit_quad_fields(Quad *quad, void (*visit)(struct gc_edge edge, void *visit_data), void *visit_data) { for (size_t i = 0; i < 4; i++) - visit(object_field(&quad->kids[i]), visit_data); + visit(gc_edge(&quad->kids[i]), visit_data); } typedef HANDLE_TO(Quad) QuadHandle; diff --git a/semi.h b/semi.h index e2769fe7f..7cd776398 100644 --- a/semi.h +++ b/semi.h @@ -144,7 +144,7 @@ static void* forward(struct semi_space *space, void *obj) { static void visit_semi_space(struct heap *heap, struct semi_space *space, struct gc_edge edge, void *obj) { - update_edge(edge, forward(space, obj)); + gc_edge_update(edge, gc_ref_from_heap_object(forward(space, obj))); } static void visit_large_object_space(struct heap *heap, @@ -160,10 +160,11 @@ static int semi_space_contains(struct semi_space *space, void *obj) { static void visit(struct gc_edge edge, void *visit_data) { struct heap *heap = visit_data; - void *obj = dereference_edge(edge); - if (obj == NULL) + struct gc_ref ref = gc_edge_ref(edge); + if (!gc_ref_is_heap_object(ref)) return; - else if (semi_space_contains(heap_semi_space(heap), obj)) + void *obj = gc_ref_heap_object(ref); + if (semi_space_contains(heap_semi_space(heap), obj)) visit_semi_space(heap, heap_semi_space(heap), edge, obj); else if (large_object_space_contains(heap_large_object_space(heap), obj)) visit_large_object_space(heap, heap_large_object_space(heap), obj); diff --git a/serial-tracer.h b/serial-tracer.h index 81664ddf3..3376d9608 100644 --- a/serial-tracer.h +++ b/serial-tracer.h @@ -156,7 +156,8 @@ static inline void tracer_visit(struct gc_edge edge, void *trace_data) { struct heap *heap = trace_data; if (trace_edge(heap, edge)) - tracer_enqueue_root(heap_tracer(heap), dereference_edge(edge)); + tracer_enqueue_root(heap_tracer(heap), + gc_ref_heap_object(gc_edge_ref(edge))); } static inline void tracer_trace(struct heap *heap) { diff --git a/whippet.h b/whippet.h index f8232bf44..93936aaae 100644 --- a/whippet.h +++ b/whippet.h @@ -423,8 +423,8 @@ static size_t mark_space_live_object_granules(uint8_t *metadata) { } static inline int mark_space_mark_object(struct mark_space *space, - struct gc_edge edge) { - struct gcobj *obj = dereference_edge(edge); + struct gc_ref ref) { + struct gcobj *obj = gc_ref_heap_object(ref); uint8_t *loc = object_metadata_byte(obj); uint8_t byte = *loc; if (byte & space->marked_mask) @@ -567,8 +567,9 @@ static struct gcobj *evacuation_allocate(struct mark_space *space, } static inline int mark_space_evacuate_or_mark_object(struct mark_space *space, - struct gc_edge edge) { - struct gcobj *obj = dereference_edge(edge); + struct gc_edge edge, + struct gc_ref old_ref) { + struct gcobj *obj = gc_ref_heap_object(old_ref); uint8_t *metadata = object_metadata_byte(obj); uint8_t byte = *metadata; if (byte & space->marked_mask) @@ -588,8 +589,7 @@ static inline int mark_space_evacuate_or_mark_object(struct mark_space *space, // The object has been evacuated already. Update the edge; // whoever forwarded the object will make sure it's eventually // traced. - struct gcobj *forwarded = (struct gcobj*) header_word; - update_edge(edge, forwarded); + gc_edge_update(edge, gc_ref(header_word)); return 0; } // Otherwise try to claim it for evacuation. @@ -613,7 +613,7 @@ static inline int mark_space_evacuate_or_mark_object(struct mark_space *space, object_granules * GRANULE_SIZE - sizeof(header_word)); uint8_t *new_metadata = object_metadata_byte(new_obj); memcpy(new_metadata + 1, metadata + 1, object_granules - 1); - update_edge(edge, new_obj); + gc_edge_update(edge, gc_ref_from_heap_object(new_obj)); obj = new_obj; metadata = new_metadata; // Fall through to set mark bits. @@ -634,10 +634,8 @@ static inline int mark_space_evacuate_or_mark_object(struct mark_space *space, break; yield_for_spin(spin_count); } - if ((header_word & gcobj_not_forwarded_bit) == 0) { - struct gcobj *forwarded = (struct gcobj*) header_word; - update_edge(edge, forwarded); - } + if ((header_word & gcobj_not_forwarded_bit) == 0) + gc_edge_update(edge, gc_ref(header_word)); // Either way, the other party is responsible for adding the // object to the mark queue. return 0; @@ -661,13 +659,15 @@ static inline int large_object_space_mark_object(struct large_object_space *spac } static inline int trace_edge(struct heap *heap, struct gc_edge edge) { - struct gcobj *obj = dereference_edge(edge); - if (!obj) + struct gc_ref ref = gc_edge_ref(edge); + if (!gc_ref_is_heap_object(ref)) return 0; - else if (LIKELY(mark_space_contains(heap_mark_space(heap), obj))) { + struct gcobj *obj = gc_ref_heap_object(ref); + if (LIKELY(mark_space_contains(heap_mark_space(heap), obj))) { if (heap_mark_space(heap)->evacuating) - return mark_space_evacuate_or_mark_object(heap_mark_space(heap), edge); - return mark_space_mark_object(heap_mark_space(heap), edge); + return mark_space_evacuate_or_mark_object(heap_mark_space(heap), edge, + ref); + return mark_space_mark_object(heap_mark_space(heap), ref); } else if (large_object_space_contains(heap_large_object_space(heap), obj)) return large_object_space_mark_object(heap_large_object_space(heap), @@ -955,7 +955,8 @@ static void mark_stopping_mutator_roots(struct mutator *mut) { for (struct handle *h = mut->roots; h; h = h->next) { struct gc_edge root = gc_edge(&h->v); if (trace_edge(heap, root)) - mutator_mark_buf_push(local_roots, dereference_edge(root)); + mutator_mark_buf_push(local_roots, + gc_ref_heap_object(gc_edge_ref(root))); } } @@ -965,7 +966,8 @@ static void mark_mutator_roots_with_lock(struct mutator *mut) { for (struct handle *h = mut->roots; h; h = h->next) { struct gc_edge root = gc_edge(&h->v); if (trace_edge(heap, root)) - tracer_enqueue_root(&heap->tracer, dereference_edge(root)); + tracer_enqueue_root(&heap->tracer, + gc_ref_heap_object(gc_edge_ref(root))); } } @@ -1018,7 +1020,8 @@ static void trace_global_roots(struct heap *heap) { for (struct handle *h = heap->global_roots; h; h = h->next) { struct gc_edge edge = gc_edge(&h->v); if (trace_edge(heap, edge)) - tracer_enqueue_root(&heap->tracer, dereference_edge(edge)); + tracer_enqueue_root(&heap->tracer, + gc_ref_heap_object(gc_edge_ref(edge))); } }