diff --git a/api/gc-ephemeron.h b/api/gc-ephemeron.h index 1d9e59b55..fd5ef8346 100644 --- a/api/gc-ephemeron.h +++ b/api/gc-ephemeron.h @@ -26,9 +26,17 @@ GC_API_ void gc_ephemeron_init(struct gc_mutator *mut, GC_API_ struct gc_ref gc_ephemeron_key(struct gc_ephemeron *ephemeron); GC_API_ struct gc_ref gc_ephemeron_value(struct gc_ephemeron *ephemeron); +GC_API_ struct gc_ref gc_ephemeron_swap_value(struct gc_mutator *mut, + struct gc_ephemeron *ephemeron, + struct gc_ref ref); + GC_API_ struct gc_ephemeron* gc_ephemeron_chain_head(struct gc_ephemeron **loc); + GC_API_ void gc_ephemeron_chain_push(struct gc_ephemeron **loc, struct gc_ephemeron *ephemeron); +GC_API_ int gc_ephemeron_chain_try_push(struct gc_ephemeron **loc, + struct gc_ephemeron *ephemeron, + struct gc_ephemeron **tail); GC_API_ struct gc_ephemeron* gc_ephemeron_chain_next(struct gc_ephemeron *ephemeron); GC_API_ void gc_ephemeron_mark_dead(struct gc_ephemeron *ephemeron); diff --git a/src/bdw.c b/src/bdw.c index 046e0181d..e08f595c8 100644 --- a/src/bdw.c +++ b/src/bdw.c @@ -253,6 +253,12 @@ void gc_ephemeron_init(struct gc_mutator *mut, struct gc_ephemeron *ephemeron, } } +struct gc_ref gc_ephemeron_swap_value(struct gc_mutator *mut, + struct gc_ephemeron *e, + struct gc_ref ref) { + return gc_ephemeron_swap_value_internal(e, ref); +} + int gc_visit_ephemeron_key(struct gc_edge edge, struct gc_heap *heap) { // Pretend the key is traced, to avoid adding this ephemeron to the // global table. diff --git a/src/gc-ephemeron-internal.h b/src/gc-ephemeron-internal.h index 3d34cf188..007ba28a2 100644 --- a/src/gc-ephemeron-internal.h +++ b/src/gc-ephemeron-internal.h @@ -47,6 +47,10 @@ GC_INTERNAL void gc_sweep_pending_ephemerons(struct gc_pending_ephemerons *state, size_t shard, size_t nshards); +GC_INTERNAL struct gc_ref +gc_ephemeron_swap_value_internal(struct gc_ephemeron *ephemeron, + struct gc_ref value); + GC_INTERNAL void gc_ephemeron_init_internal(struct gc_heap *heap, struct gc_ephemeron *ephemeron, struct gc_ref key, diff --git a/src/gc-ephemeron.c b/src/gc-ephemeron.c index 7ef5b53f6..81d2b6e50 100644 --- a/src/gc-ephemeron.c +++ b/src/gc-ephemeron.c @@ -165,19 +165,22 @@ // Concurrent operations on ephemeron lists //////////////////////////////////////////////////////////////////////// +static int +ephemeron_list_try_push(struct gc_ephemeron **loc, + struct gc_ephemeron *head, + struct gc_ephemeron **tail, + struct gc_ephemeron** (*get_next)(struct gc_ephemeron*)) { + *get_next(head) = *tail; + return atomic_compare_exchange_weak(loc, tail, head); +} + static void ephemeron_list_push(struct gc_ephemeron **loc, struct gc_ephemeron *head, struct gc_ephemeron** (*get_next)(struct gc_ephemeron*)) { struct gc_ephemeron *tail = atomic_load_explicit(loc, memory_order_acquire); - while (1) { - // There must be no concurrent readers of HEAD, a precondition that - // we ensure by only publishing HEAD to LOC at most once per cycle. - // Therefore we can use a normal store for the tail pointer. - *get_next(head) = tail; - if (atomic_compare_exchange_weak(loc, &tail, head)) - break; - } + while (!ephemeron_list_try_push(loc, head, &tail, get_next)) + ; } static struct gc_ephemeron* @@ -274,6 +277,11 @@ void gc_ephemeron_chain_push(struct gc_ephemeron **loc, struct gc_ephemeron *e) { ephemeron_list_push(loc, e, ephemeron_chain); } +int gc_ephemeron_chain_try_push(struct gc_ephemeron **loc, + struct gc_ephemeron *e, + struct gc_ephemeron **tail) { + return ephemeron_list_try_push(loc, e, tail, ephemeron_chain); +} static struct gc_ephemeron* follow_chain(struct gc_ephemeron **loc) { return ephemeron_list_follow(loc, ephemeron_chain, ephemeron_is_not_dead); } @@ -283,6 +291,21 @@ struct gc_ephemeron* gc_ephemeron_chain_head(struct gc_ephemeron **loc) { struct gc_ephemeron* gc_ephemeron_chain_next(struct gc_ephemeron *e) { return follow_chain(ephemeron_chain(e)); } + +struct gc_ref gc_ephemeron_swap_value_internal(struct gc_ephemeron *e, + struct gc_ref ref) +{ + GC_ASSERT(!gc_ref_is_null(ref)); + + uintptr_t prev = atomic_load(&e->value.value); + do { + if (!prev) + break; + } while (!atomic_compare_exchange_weak(&e->value.value, &prev, ref.value)); + + return gc_ref(prev); +} + void gc_ephemeron_mark_dead(struct gc_ephemeron *e) { atomic_store_explicit(&e->key.value, 0, memory_order_release); } @@ -570,6 +593,9 @@ gc_sweep_pending_ephemerons(struct gc_pending_ephemerons *state, void gc_ephemeron_init_internal(struct gc_heap *heap, struct gc_ephemeron *ephemeron, struct gc_ref key, struct gc_ref value) { + GC_ASSERT(!gc_ref_is_null(key)); + GC_ASSERT(!gc_ref_is_null(value)); + // Caller responsible for any write barrier, though really the // assumption is that the ephemeron is younger than the key and the // value. diff --git a/src/mmc.c b/src/mmc.c index 6055905be..1ae342f5e 100644 --- a/src/mmc.c +++ b/src/mmc.c @@ -1007,6 +1007,14 @@ gc_ephemeron_init(struct gc_mutator *mut, struct gc_ephemeron *ephemeron, // key or the value. } +struct gc_ref +gc_ephemeron_swap_value(struct gc_mutator *mut, struct gc_ephemeron *e, + struct gc_ref ref) { + gc_write_barrier(mut, gc_ref_from_heap_object(e), gc_ephemeron_size(), + gc_ephemeron_value_edge(e), ref); + return gc_ephemeron_swap_value_internal(e, ref); +} + struct gc_pending_ephemerons * gc_heap_pending_ephemerons(struct gc_heap *heap) { return heap->pending_ephemerons; diff --git a/src/pcc.c b/src/pcc.c index 4419f519f..6b656360c 100644 --- a/src/pcc.c +++ b/src/pcc.c @@ -1090,6 +1090,14 @@ void gc_ephemeron_init(struct gc_mutator *mut, struct gc_ephemeron *ephemeron, gc_ephemeron_init_internal(mutator_heap(mut), ephemeron, key, value); } +struct gc_ref +gc_ephemeron_swap_value(struct gc_mutator *mut, struct gc_ephemeron *e, + struct gc_ref ref) { + gc_write_barrier(mut, gc_ref_from_heap_object(e), gc_ephemeron_size(), + gc_ephemeron_value_edge(e), ref); + return gc_ephemeron_swap_value_internal(e, ref); +} + struct gc_pending_ephemerons *gc_heap_pending_ephemerons(struct gc_heap *heap) { #if GC_GENERATIONAL if (is_minor_collection(heap)) diff --git a/src/semi.c b/src/semi.c index 57bb762c7..9754b3fa2 100644 --- a/src/semi.c +++ b/src/semi.c @@ -555,6 +555,12 @@ void gc_ephemeron_init(struct gc_mutator *mut, struct gc_ephemeron *ephemeron, gc_ephemeron_init_internal(mutator_heap(mut), ephemeron, key, value); } +struct gc_ref gc_ephemeron_swap_value(struct gc_mutator *mut, + struct gc_ephemeron *e, + struct gc_ref ref) { + return gc_ephemeron_swap_value_internal(e, ref); +} + struct gc_finalizer* gc_allocate_finalizer(struct gc_mutator *mut) { return gc_allocate(mut, gc_finalizer_size(), GC_ALLOCATION_TAGGED); }