diff --git a/api/bdw-attrs.h b/api/bdw-attrs.h index 51b4e72f1..05f7e4cb7 100644 --- a/api/bdw-attrs.h +++ b/api/bdw-attrs.h @@ -40,6 +40,13 @@ static inline int gc_allocator_needs_clear(void) { return 0; } +static inline enum gc_old_generation_check_kind gc_old_generation_check_kind(size_t) { + return GC_OLD_GENERATION_CHECK_NONE; +} +static inline uint8_t gc_old_generation_check_alloc_table_bit_pattern(void) { + GC_CRASH(); +} + static inline enum gc_write_barrier_kind gc_write_barrier_kind(size_t) { return GC_WRITE_BARRIER_NONE; } diff --git a/api/gc-api.h b/api/gc-api.h index cb1e4e819..8f5565428 100644 --- a/api/gc-api.h +++ b/api/gc-api.h @@ -179,6 +179,33 @@ static inline void* gc_allocate(struct gc_mutator *mut, size_t size) { // FIXME: remove :P GC_API_ void* gc_allocate_pointerless(struct gc_mutator *mut, size_t bytes); +GC_API_ int gc_object_is_old_generation_slow(struct gc_mutator *mut, + struct gc_ref obj) GC_NEVER_INLINE; + +static inline int gc_object_is_old_generation(struct gc_mutator *mut, + struct gc_ref obj, + size_t obj_size) GC_ALWAYS_INLINE; +static inline int gc_object_is_old_generation(struct gc_mutator *mut, + struct gc_ref obj, + size_t obj_size) { + switch (gc_old_generation_check_kind(obj_size)) { + case GC_OLD_GENERATION_CHECK_ALLOC_TABLE: { + size_t alignment = gc_allocator_alloc_table_alignment(); + GC_ASSERT(alignment); + uintptr_t addr = gc_ref_value(obj); + uintptr_t base = addr & ~(alignment - 1); + size_t granule_size = gc_allocator_small_granule_size(); + uintptr_t granule = (addr & (alignment - 1)) / granule_size; + uint8_t *byte = (uint8_t*)(base + granule); + return (*byte) & gc_old_generation_check_alloc_table_bit_pattern(); + } + case GC_OLD_GENERATION_CHECK_SLOW: + return gc_object_is_old_generation_slow(mut, obj); + default: + GC_CRASH(); + } +} + GC_API_ void gc_write_barrier_slow(struct gc_mutator *mut, struct gc_ref obj, size_t obj_size, struct gc_edge edge, struct gc_ref new_val) GC_NEVER_INLINE; @@ -202,11 +229,14 @@ static inline void gc_write_barrier(struct gc_mutator *mut, struct gc_ref obj, return; } case GC_WRITE_BARRIER_FIELD: { + if (!gc_object_is_old_generation(mut, obj, obj_size)) + return; + size_t field_table_alignment = gc_write_barrier_field_table_alignment(); size_t fields_per_byte = gc_write_barrier_field_fields_per_byte(); uint8_t first_bit_pattern = gc_write_barrier_field_first_bit_pattern(); - uintptr_t addr = (uintptr_t) gc_edge_loc(edge); + uintptr_t addr = gc_ref_value(obj); uintptr_t base = addr & ~(field_table_alignment - 1); uintptr_t field = (addr & (field_table_alignment - 1)) / sizeof(uintptr_t); uintptr_t log_byte = field / fields_per_byte; diff --git a/api/gc-attrs.h b/api/gc-attrs.h index 39faceae2..389cb536e 100644 --- a/api/gc-attrs.h +++ b/api/gc-attrs.h @@ -27,6 +27,16 @@ static inline uint8_t gc_allocator_alloc_table_end_pattern(void) GC_ALWAYS_INLIN static inline int gc_allocator_needs_clear(void) GC_ALWAYS_INLINE; +enum gc_old_generation_check_kind { + GC_OLD_GENERATION_CHECK_NONE, + GC_OLD_GENERATION_CHECK_ALLOC_TABLE, + GC_OLD_GENERATION_CHECK_SLOW +}; + +static inline enum gc_old_generation_check_kind gc_old_generation_check_kind(size_t obj_size) GC_ALWAYS_INLINE; + +static uint8_t gc_old_generation_check_alloc_table_bit_pattern(void) GC_ALWAYS_INLINE; + enum gc_write_barrier_kind { GC_WRITE_BARRIER_NONE, GC_WRITE_BARRIER_CARD, diff --git a/api/mmc-attrs.h b/api/mmc-attrs.h index 6da4b2a68..f527c127b 100644 --- a/api/mmc-attrs.h +++ b/api/mmc-attrs.h @@ -40,6 +40,19 @@ static inline int gc_allocator_needs_clear(void) { return 0; } +static inline enum gc_old_generation_check_kind gc_old_generation_check_kind(size_t obj_size) { + if (GC_GENERATIONAL) { + if (obj_size <= gc_allocator_large_threshold()) + return GC_OLD_GENERATION_CHECK_ALLOC_TABLE; + return GC_OLD_GENERATION_CHECK_SLOW; + } + return GC_OLD_GENERATION_CHECK_NONE; +} +static inline uint8_t gc_old_generation_check_alloc_table_bit_pattern(void) { + // The three mark bits. + return 2 + 4 + 8; +} + static inline enum gc_write_barrier_kind gc_write_barrier_kind(size_t obj_size) { if (GC_GENERATIONAL) { if (obj_size <= gc_allocator_large_threshold()) diff --git a/api/pcc-attrs.h b/api/pcc-attrs.h index 5dd38c42f..c86d79471 100644 --- a/api/pcc-attrs.h +++ b/api/pcc-attrs.h @@ -43,6 +43,13 @@ static inline int gc_allocator_needs_clear(void) { return 0; } +static inline enum gc_old_generation_check_kind gc_old_generation_check_kind(size_t) { + return GC_OLD_GENERATION_CHECK_NONE; +} +static inline uint8_t gc_old_generation_check_alloc_table_bit_pattern(void) { + GC_CRASH(); +} + static inline enum gc_write_barrier_kind gc_write_barrier_kind(size_t obj_size) { return GC_WRITE_BARRIER_NONE; } diff --git a/api/semi-attrs.h b/api/semi-attrs.h index 94e2dc814..997b031ee 100644 --- a/api/semi-attrs.h +++ b/api/semi-attrs.h @@ -42,6 +42,13 @@ static inline uint8_t gc_allocator_alloc_table_end_pattern(void) { GC_CRASH(); } +static inline enum gc_old_generation_check_kind gc_old_generation_check_kind(size_t) { + return GC_OLD_GENERATION_CHECK_NONE; +} +static inline uint8_t gc_old_generation_check_alloc_table_bit_pattern(void) { + GC_CRASH(); +} + static inline enum gc_write_barrier_kind gc_write_barrier_kind(size_t) { return GC_WRITE_BARRIER_NONE; } diff --git a/src/bdw.c b/src/bdw.c index d4cbe4c41..d1478d805 100644 --- a/src/bdw.c +++ b/src/bdw.c @@ -149,6 +149,11 @@ void gc_collect(struct gc_mutator *mut, } } +int gc_object_is_old_generation_slow(struct gc_mutator *mut, + struct gc_ref obj) { + return 0; +} + void gc_write_barrier_slow(struct gc_mutator *mut, struct gc_ref obj, size_t obj_size, struct gc_edge edge, struct gc_ref new_val) { diff --git a/src/large-object-space.h b/src/large-object-space.h index 358826b36..323c3f2f8 100644 --- a/src/large-object-space.h +++ b/src/large-object-space.h @@ -176,8 +176,8 @@ static int large_object_space_is_copied(struct large_object_space *space, return copied; } -static int large_object_space_is_old(struct large_object_space *space, - struct gc_ref ref) { +static int large_object_space_is_survivor(struct large_object_space *space, + struct gc_ref ref) { GC_ASSERT(large_object_space_contains(space, ref)); int old = 0; uintptr_t addr = gc_ref_value(ref); diff --git a/src/mmc.c b/src/mmc.c index e14346bec..ce1571118 100644 --- a/src/mmc.c +++ b/src/mmc.c @@ -879,6 +879,23 @@ gc_pin_object(struct gc_mutator *mut, struct gc_ref ref) { // Otherwise if it's a large or external object, it won't move. } +int gc_object_is_old_generation_slow(struct gc_mutator *mut, + struct gc_ref obj) { + if (!GC_GENERATIONAL) + return 0; + + struct gc_heap *heap = mutator_heap(mut); + struct nofl_space *nofl_space = heap_nofl_space(heap); + if (nofl_space_contains(nofl_space, obj)) + return nofl_space_is_survivor(nofl_space, obj); + + struct large_object_space *lospace = heap_large_object_space(heap); + if (large_object_space_contains(lospace, obj)) + return large_object_space_is_survivor(lospace, obj); + + return 0; +} + void gc_write_barrier_slow(struct gc_mutator *mut, struct gc_ref obj, size_t obj_size, struct gc_edge edge, @@ -887,7 +904,7 @@ gc_write_barrier_slow(struct gc_mutator *mut, struct gc_ref obj, GC_ASSERT(obj_size > gc_allocator_large_threshold()); struct gc_heap *heap = mutator_heap(mut); struct large_object_space *space = heap_large_object_space(heap); - if (!large_object_space_is_old(space, obj)) + if (!large_object_space_is_survivor(space, obj)) return; if (gc_object_set_remembered(obj)) large_object_space_remember_object(space, obj); diff --git a/src/nofl-space.h b/src/nofl-space.h index 93904aee5..e47162853 100644 --- a/src/nofl-space.h +++ b/src/nofl-space.h @@ -1431,6 +1431,15 @@ nofl_space_pin_object(struct nofl_space *space, struct gc_ref ref) { memory_order_acquire)); } +static inline int +nofl_space_is_survivor(struct nofl_space *space, struct gc_ref ref) { + uint8_t *metadata = nofl_metadata_byte_for_object(ref); + uint8_t mask = NOFL_METADATA_BYTE_MARK_0 + | NOFL_METADATA_BYTE_MARK_1 | NOFL_METADATA_BYTE_MARK_2; + uint8_t byte = atomic_load_explicit(metadata, memory_order_relaxed); + return byte & mask; +} + static inline int nofl_space_evacuate(struct nofl_space *space, uint8_t *metadata, uint8_t byte, struct gc_edge edge, diff --git a/src/pcc.c b/src/pcc.c index 10d5f8382..593f43fc6 100644 --- a/src/pcc.c +++ b/src/pcc.c @@ -483,6 +483,11 @@ void gc_pin_object(struct gc_mutator *mut, struct gc_ref ref) { GC_CRASH(); } +int gc_object_is_old_generation_slow(struct gc_mutator *mut, + struct gc_ref obj) { + return 0; +} + void gc_write_barrier_slow(struct gc_mutator *mut, struct gc_ref obj, size_t obj_size, struct gc_edge edge, struct gc_ref new_val) { diff --git a/src/semi.c b/src/semi.c index aab5af233..ca7a31607 100644 --- a/src/semi.c +++ b/src/semi.c @@ -453,6 +453,11 @@ void gc_collect(struct gc_mutator *mut, collect(mut, 0); } +int gc_object_is_old_generation_slow(struct gc_mutator *mut, + struct gc_ref obj) { + return 0; +} + void gc_write_barrier_slow(struct gc_mutator *mut, struct gc_ref obj, size_t obj_size, struct gc_edge edge, struct gc_ref new_val) {