#ifndef GC_API_H_ #define GC_API_H_ #include "gc-config.h" #include "gc-allocation-kind.h" #include "gc-assert.h" #include "gc-attrs.h" #include "gc-collection-kind.h" #include "gc-edge.h" #include "gc-event-listener.h" #include "gc-inline.h" #include "gc-options.h" #include "gc-ref.h" #include "gc-visibility.h" #include #include #include struct gc_heap; struct gc_mutator; struct gc_stack_addr; GC_API_ void* gc_call_with_stack_addr(void* (*f)(struct gc_stack_addr *, void *), void *data) GC_NEVER_INLINE; GC_API_ int gc_init(const struct gc_options *options, struct gc_stack_addr *base, struct gc_heap **heap, struct gc_mutator **mutator, struct gc_event_listener event_listener, void *event_listener_data); GC_API_ uint64_t gc_allocation_counter(struct gc_heap *heap); GC_API_ struct gc_heap* gc_mutator_heap(struct gc_mutator *mut); GC_API_ uintptr_t gc_small_object_nursery_low_address(struct gc_heap *heap); GC_API_ uintptr_t gc_small_object_nursery_high_address(struct gc_heap *heap); struct gc_mutator_roots; GC_API_ void gc_mutator_set_roots(struct gc_mutator *mut, struct gc_mutator_roots *roots); struct gc_heap_roots; GC_API_ void gc_heap_set_roots(struct gc_heap *heap, struct gc_heap_roots *roots); struct gc_extern_space; GC_API_ void gc_heap_set_extern_space(struct gc_heap *heap, struct gc_extern_space *space); GC_API_ struct gc_mutator* gc_init_for_thread(struct gc_stack_addr *base, struct gc_heap *heap); GC_API_ void gc_finish_for_thread(struct gc_mutator *mut); GC_API_ void* gc_call_without_gc(struct gc_mutator *mut, void* (*f)(void*), void *data) GC_NEVER_INLINE; GC_API_ void gc_collect(struct gc_mutator *mut, enum gc_collection_kind requested_kind); static inline void gc_update_alloc_table(struct gc_ref obj, size_t size, enum gc_allocation_kind kind) GC_ALWAYS_INLINE; static inline void gc_update_alloc_table(struct gc_ref obj, size_t size, enum gc_allocation_kind kind) { size_t alignment = gc_allocator_alloc_table_alignment(); if (!alignment) return; uintptr_t addr = gc_ref_value(obj); uintptr_t base = addr & ~(alignment - 1); size_t granule_size = gc_allocator_small_granule_size(); uintptr_t granule = (addr & (alignment - 1)) / granule_size; uint8_t *alloc = (uint8_t*)(base + granule); uint8_t begin_pattern = gc_allocator_alloc_table_begin_pattern(kind); uint8_t end_pattern = gc_allocator_alloc_table_end_pattern(); if (end_pattern) { size_t granules = size / granule_size; if (granules == 1) { alloc[0] = begin_pattern | end_pattern; } else { alloc[0] = begin_pattern; if (granules > 2) memset(alloc + 1, 0, granules - 2); alloc[granules - 1] = end_pattern; } } else { alloc[0] = begin_pattern; } } GC_API_ void* gc_allocate_slow(struct gc_mutator *mut, size_t bytes, enum gc_allocation_kind kind) GC_NEVER_INLINE; static inline void* gc_allocate_small_fast_bump_pointer(struct gc_mutator *mut, size_t size, enum gc_allocation_kind kind) GC_ALWAYS_INLINE; static inline void* gc_allocate_small_fast_bump_pointer(struct gc_mutator *mut, size_t size, enum gc_allocation_kind kind) { GC_ASSERT(size <= gc_allocator_large_threshold()); size_t granule_size = gc_allocator_small_granule_size(); size_t hp_offset = gc_allocator_allocation_pointer_offset(); size_t limit_offset = gc_allocator_allocation_limit_offset(); uintptr_t base_addr = (uintptr_t)mut; uintptr_t *hp_loc = (uintptr_t*)(base_addr + hp_offset); uintptr_t *limit_loc = (uintptr_t*)(base_addr + limit_offset); size = (size + granule_size - 1) & ~(granule_size - 1); uintptr_t hp = *hp_loc; uintptr_t limit = *limit_loc; uintptr_t new_hp = hp + size; if (GC_UNLIKELY (new_hp > limit)) return NULL; *hp_loc = new_hp; gc_update_alloc_table(gc_ref(hp), size, kind); return (void*)hp; } static inline void* gc_allocate_small_fast_freelist(struct gc_mutator *mut, size_t size, enum gc_allocation_kind kind) GC_ALWAYS_INLINE; static inline void* gc_allocate_small_fast_freelist(struct gc_mutator *mut, size_t size, enum gc_allocation_kind kind) { GC_ASSERT(size <= gc_allocator_large_threshold()); size_t freelist_offset = gc_allocator_freelist_offset(size, kind); uintptr_t base_addr = (uintptr_t)mut; void **freelist_loc = (void**)(base_addr + freelist_offset); void *head = *freelist_loc; if (GC_UNLIKELY(!head)) return NULL; *freelist_loc = *(void**)head; gc_update_alloc_table(gc_ref_from_heap_object(head), size, kind); return head; } static inline void* gc_allocate_small_fast(struct gc_mutator *mut, size_t size, enum gc_allocation_kind kind) GC_ALWAYS_INLINE; static inline void* gc_allocate_small_fast(struct gc_mutator *mut, size_t size, enum gc_allocation_kind kind) { GC_ASSERT(size != 0); GC_ASSERT(size <= gc_allocator_large_threshold()); switch (gc_allocator_kind()) { case GC_ALLOCATOR_INLINE_BUMP_POINTER: return gc_allocate_small_fast_bump_pointer(mut, size, kind); case GC_ALLOCATOR_INLINE_FREELIST: return gc_allocate_small_fast_freelist(mut, size, kind); case GC_ALLOCATOR_INLINE_NONE: return NULL; default: GC_CRASH(); } } static inline void* gc_allocate_fast(struct gc_mutator *mut, size_t size, enum gc_allocation_kind kind) GC_ALWAYS_INLINE; static inline void* gc_allocate_fast(struct gc_mutator *mut, size_t size, enum gc_allocation_kind kind) { GC_ASSERT(size != 0); if (size > gc_allocator_large_threshold()) return NULL; return gc_allocate_small_fast(mut, size, kind); } static inline void* gc_allocate(struct gc_mutator *mut, size_t size, enum gc_allocation_kind kind) GC_ALWAYS_INLINE; static inline void* gc_allocate(struct gc_mutator *mut, size_t size, enum gc_allocation_kind kind) { void *ret = gc_allocate_fast(mut, size, kind); if (GC_LIKELY(ret != NULL)) return ret; return gc_allocate_slow(mut, size, kind); } GC_API_ int gc_object_is_old_generation_slow(struct gc_mutator *mut, struct gc_ref obj) GC_NEVER_INLINE; static inline int gc_object_is_old_generation(struct gc_mutator *mut, struct gc_ref obj, size_t obj_size) GC_ALWAYS_INLINE; static inline int gc_object_is_old_generation(struct gc_mutator *mut, struct gc_ref obj, size_t obj_size) { switch (gc_old_generation_check_kind(obj_size)) { case GC_OLD_GENERATION_CHECK_ALLOC_TABLE: { size_t alignment = gc_allocator_alloc_table_alignment(); GC_ASSERT(alignment); uintptr_t addr = gc_ref_value(obj); uintptr_t base = addr & ~(alignment - 1); size_t granule_size = gc_allocator_small_granule_size(); uintptr_t granule = (addr & (alignment - 1)) / granule_size; uint8_t *byte_loc = (uint8_t*)(base + granule); uint8_t byte = atomic_load_explicit(byte_loc, memory_order_relaxed); uint8_t mask = gc_old_generation_check_alloc_table_tag_mask(); uint8_t young = gc_old_generation_check_alloc_table_young_tag(); return (byte & mask) != young; } case GC_OLD_GENERATION_CHECK_SMALL_OBJECT_NURSERY: { struct gc_heap *heap = gc_mutator_heap(mut); // Note that these addresses are fixed and that the embedder might // want to store them somewhere or inline them into the output of // JIT-generated code. They may also be power-of-two aligned. uintptr_t low_addr = gc_small_object_nursery_low_address(heap); uintptr_t high_addr = gc_small_object_nursery_high_address(heap); uintptr_t size = high_addr - low_addr; uintptr_t addr = gc_ref_value(obj); return addr - low_addr >= size; } case GC_OLD_GENERATION_CHECK_SLOW: return gc_object_is_old_generation_slow(mut, obj); default: GC_CRASH(); } } GC_API_ void gc_write_barrier_slow(struct gc_mutator *mut, struct gc_ref obj, size_t obj_size, struct gc_edge edge, struct gc_ref new_val) GC_NEVER_INLINE; static inline int gc_write_barrier_fast(struct gc_mutator *mut, struct gc_ref obj, size_t obj_size, struct gc_edge edge, struct gc_ref new_val) GC_ALWAYS_INLINE; static inline int gc_write_barrier_fast(struct gc_mutator *mut, struct gc_ref obj, size_t obj_size, struct gc_edge edge, struct gc_ref new_val) { switch (gc_write_barrier_kind(obj_size)) { case GC_WRITE_BARRIER_NONE: return 0; case GC_WRITE_BARRIER_FIELD: { if (!gc_object_is_old_generation(mut, obj, obj_size)) return 0; size_t field_table_alignment = gc_write_barrier_field_table_alignment(); size_t fields_per_byte = gc_write_barrier_field_fields_per_byte(); uint8_t first_bit_pattern = gc_write_barrier_field_first_bit_pattern(); ssize_t table_offset = gc_write_barrier_field_table_offset(); uintptr_t addr = gc_edge_address(edge); uintptr_t base = addr & ~(field_table_alignment - 1); uintptr_t field = (addr & (field_table_alignment - 1)) / sizeof(uintptr_t); uintptr_t log_byte = field / fields_per_byte; uint8_t log_bit = first_bit_pattern << (field % fields_per_byte); uint8_t *byte_loc = (uint8_t*)(base + table_offset + log_byte); uint8_t byte = atomic_load_explicit(byte_loc, memory_order_relaxed); return !(byte & log_bit); } case GC_WRITE_BARRIER_SLOW: return 1; default: GC_CRASH(); } } static inline void gc_write_barrier(struct gc_mutator *mut, struct gc_ref obj, size_t obj_size, struct gc_edge edge, struct gc_ref new_val) GC_ALWAYS_INLINE; static inline void gc_write_barrier(struct gc_mutator *mut, struct gc_ref obj, size_t obj_size, struct gc_edge edge, struct gc_ref new_val) { if (GC_UNLIKELY(gc_write_barrier_fast(mut, obj, obj_size, edge, new_val))) gc_write_barrier_slow(mut, obj, obj_size, edge, new_val); } GC_API_ void gc_pin_object(struct gc_mutator *mut, struct gc_ref obj); GC_API_ void gc_safepoint_slow(struct gc_mutator *mut) GC_NEVER_INLINE; GC_API_ int* gc_safepoint_flag_loc(struct gc_mutator *mut); static inline int gc_should_stop_for_safepoint(struct gc_mutator *mut) { switch (gc_cooperative_safepoint_kind()) { case GC_COOPERATIVE_SAFEPOINT_NONE: return 0; case GC_COOPERATIVE_SAFEPOINT_MUTATOR_FLAG: case GC_COOPERATIVE_SAFEPOINT_HEAP_FLAG: { return atomic_load_explicit(gc_safepoint_flag_loc(mut), memory_order_relaxed); } default: GC_CRASH(); } } static inline void gc_safepoint(struct gc_mutator *mut) { if (GC_UNLIKELY(gc_should_stop_for_safepoint(mut))) gc_safepoint_slow(mut); } #endif // GC_API_H_