diff --git a/src/copy-space.h b/src/copy-space.h index d09609dfe..9be2f8d2a 100644 --- a/src/copy-space.h +++ b/src/copy-space.h @@ -3,7 +3,6 @@ #include #include -#include #include "gc-api.h" @@ -18,6 +17,7 @@ #include "gc-attrs.h" #include "gc-inline.h" #include "gc-lock.h" +#include "gc-platform.h" #include "spin.h" // A copy space: a block-structured space that traces via evacuation. @@ -620,27 +620,8 @@ copy_space_allocator_finish(struct copy_space_allocator *alloc, static struct copy_space_slab* copy_space_allocate_slabs(size_t nslabs) { - size_t size = nslabs * COPY_SPACE_SLAB_SIZE; - size_t extent = size + COPY_SPACE_SLAB_SIZE; - - char *mem = mmap(NULL, extent, PROT_READ|PROT_WRITE, - MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); - if (mem == MAP_FAILED) { - perror("mmap failed"); - return NULL; - } - - uintptr_t base = (uintptr_t) mem; - uintptr_t end = base + extent; - uintptr_t aligned_base = align_up(base, COPY_SPACE_SLAB_SIZE); - uintptr_t aligned_end = aligned_base + size; - - if (aligned_base - base) - munmap((void*)base, aligned_base - base); - if (end - aligned_end) - munmap((void*)aligned_end, end - aligned_end); - - return (struct copy_space_slab*) aligned_base; + return gc_platform_acquire_memory(nslabs * COPY_SPACE_SLAB_SIZE, + COPY_SPACE_SLAB_SIZE); } static void @@ -715,8 +696,8 @@ copy_space_page_out_blocks(void *data) { if (!block) break; block->in_core = 0; block->all_zeroes[0] = block->all_zeroes[1] = 1; - madvise(copy_space_block_payload(block), COPY_SPACE_BLOCK_SIZE, - MADV_DONTNEED); + gc_platform_discard_memory(copy_space_block_payload(block), + COPY_SPACE_BLOCK_SIZE); copy_space_block_stack_push(&space->paged_out[age + 1], block, &lock); } gc_lock_release(&lock); diff --git a/src/gc-platform-gnu-linux.c b/src/gc-platform-gnu-linux.c index ebcfa5579..0b267032a 100644 --- a/src/gc-platform-gnu-linux.c +++ b/src/gc-platform-gnu-linux.c @@ -5,12 +5,14 @@ #include #include #include +#include #include #include #define GC_IMPL 1 #include "debug.h" +#include "gc-align.h" #include "gc-assert.h" #include "gc-inline.h" #include "gc-platform.h" @@ -121,3 +123,58 @@ uint64_t gc_platform_monotonic_nanoseconds(void) { uint64_t ns_per_sec = 1000000000; return s * ns_per_sec + ns; } + +size_t gc_platform_page_size(void) { + return getpagesize(); +} + +void* gc_platform_acquire_memory(size_t size, size_t alignment) { + GC_ASSERT_EQ(size, align_down(size, getpagesize())); + GC_ASSERT_EQ(alignment & (alignment - 1), 0); + GC_ASSERT_EQ(alignment, align_down(alignment, getpagesize())); + + size_t extent = size + alignment; + char *mem = mmap(NULL, extent, PROT_READ|PROT_WRITE, + MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); + if (mem == MAP_FAILED) { + perror("mmap failed"); + return NULL; + } + + uintptr_t base = (uintptr_t) mem; + uintptr_t end = base + extent; + uintptr_t aligned_base = alignment ? align_up(base, alignment) : base; + uintptr_t aligned_end = aligned_base + size; + + if (aligned_base - base) + munmap((void*)base, aligned_base - base); + if (end - aligned_end) + munmap((void*)aligned_end, end - aligned_end); + + return (void*) aligned_base; +} + +void gc_platform_release_memory(void *ptr, size_t size) { + GC_ASSERT_EQ((uintptr_t)ptr, align_down((uintptr_t)ptr, getpagesize())); + GC_ASSERT_EQ(size, align_down(size, getpagesize())); + if (munmap(ptr, size) != 0) + perror("failed to unmap memory"); +} + +int gc_platform_populate_memory(void *ptr, size_t size) { + GC_ASSERT_EQ((uintptr_t)ptr, align_down((uintptr_t)ptr, getpagesize())); + GC_ASSERT_EQ(size, align_down(size, getpagesize())); + if (madvise(ptr, size, MADV_WILLNEED) == 0) + return 1; + perror("failed to populate memory"); + return 0; +} + +int gc_platform_discard_memory(void *ptr, size_t size) { + GC_ASSERT_EQ((uintptr_t)ptr, align_down((uintptr_t)ptr, getpagesize())); + GC_ASSERT_EQ(size, align_down(size, getpagesize())); + if (madvise(ptr, size, MADV_DONTNEED) == 0) + return 1; + perror("failed to discard memory"); + return 0; +} diff --git a/src/gc-platform.h b/src/gc-platform.h index 42335ed7a..ddedbb984 100644 --- a/src/gc-platform.h +++ b/src/gc-platform.h @@ -23,4 +23,11 @@ void gc_platform_visit_global_conservative_roots(void (*f)(uintptr_t start, GC_INTERNAL int gc_platform_processor_count(void); GC_INTERNAL uint64_t gc_platform_monotonic_nanoseconds(void); +GC_INTERNAL size_t gc_platform_page_size(void); +GC_INTERNAL void* gc_platform_acquire_memory(size_t size, size_t alignment); +GC_INTERNAL void gc_platform_release_memory(void *base, size_t size); + +GC_INTERNAL int gc_platform_populate_memory(void *addr, size_t size); +GC_INTERNAL int gc_platform_discard_memory(void *addr, size_t size); + #endif // GC_PLATFORM_H diff --git a/src/large-object-space.h b/src/large-object-space.h index 44f4095f2..7aba13d9a 100644 --- a/src/large-object-space.h +++ b/src/large-object-space.h @@ -321,8 +321,7 @@ static void* large_object_space_obtain_and_alloc(struct large_object_space *space, size_t npages) { size_t bytes = npages * space->page_size; - void *ret = mmap(NULL, bytes, PROT_READ|PROT_WRITE, - MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); + void *ret = gc_platform_acquire_memory(bytes, 0); if (ret == MAP_FAILED) return NULL; diff --git a/src/nofl-space.h b/src/nofl-space.h index 4bf99efdf..9e4edf912 100644 --- a/src/nofl-space.h +++ b/src/nofl-space.h @@ -5,7 +5,6 @@ #include #include #include -#include #include "gc-api.h" @@ -19,6 +18,7 @@ #include "gc-attrs.h" #include "gc-inline.h" #include "gc-lock.h" +#include "gc-platform.h" #include "spin.h" #include "swar.h" @@ -1675,27 +1675,7 @@ nofl_space_object_size(struct nofl_space *space, struct gc_ref ref) { static struct nofl_slab* nofl_allocate_slabs(size_t nslabs) { - size_t size = nslabs * NOFL_SLAB_SIZE; - size_t extent = size + NOFL_SLAB_SIZE; - - char *mem = mmap(NULL, extent, PROT_READ|PROT_WRITE, - MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); - if (mem == MAP_FAILED) { - perror("mmap failed"); - return NULL; - } - - uintptr_t base = (uintptr_t) mem; - uintptr_t end = base + extent; - uintptr_t aligned_base = align_up(base, NOFL_SLAB_SIZE); - uintptr_t aligned_end = aligned_base + size; - - if (aligned_base - base) - munmap((void*)base, aligned_base - base); - if (end - aligned_end) - munmap((void*)aligned_end, end - aligned_end); - - return (struct nofl_slab*) aligned_base; + return gc_platform_acquire_memory(nslabs * NOFL_SLAB_SIZE, NOFL_SLAB_SIZE); } static void @@ -1813,7 +1793,7 @@ nofl_space_page_out_blocks(void *data) { if (nofl_block_is_null(block)) break; nofl_block_set_flag(block, NOFL_BLOCK_ZERO | NOFL_BLOCK_PAGED_OUT); - madvise((void*)block.addr, NOFL_BLOCK_SIZE, MADV_DONTNEED); + gc_platform_discard_memory((void*)block.addr, NOFL_BLOCK_SIZE); nofl_block_stack_push(&space->paged_out[age + 1], block, &lock); } gc_lock_release(&lock); diff --git a/src/semi.c b/src/semi.c index 239367999..725a75f30 100644 --- a/src/semi.c +++ b/src/semi.c @@ -2,8 +2,6 @@ #include #include #include -#include -#include #include "gc-api.h" @@ -100,13 +98,13 @@ static void region_trim_by(struct region *region, size_t newly_unavailable) { GC_ASSERT(newly_unavailable <= region->active_size); region->active_size -= newly_unavailable; - madvise((void*)(region->base + region->active_size), newly_unavailable, - MADV_DONTNEED); + gc_platform_discard_memory((void*)(region->base + region->active_size), + newly_unavailable); } static void region_set_active_size(struct region *region, size_t size) { GC_ASSERT(size <= region->mapped_size); - GC_ASSERT(size == align_up(size, getpagesize())); + GC_ASSERT(size == align_up(size, gc_platform_page_size())); if (size < region->active_size) region_trim_by(region, region->active_size - size); else @@ -274,15 +272,12 @@ static int grow_region_if_needed(struct region *region, size_t new_size) { if (new_size <= region->mapped_size) return 1; - void *mem = mmap(NULL, new_size, PROT_READ|PROT_WRITE, - MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); + void *mem = gc_platform_acquire_memory(new_size, 0); DEBUG("new size %zx\n", new_size); - if (mem == MAP_FAILED) { - perror("mmap failed"); + if (!mem) return 0; - } if (region->mapped_size) - munmap((void*)region->base, region->mapped_size); + gc_platform_release_memory((void*)region->base, region->mapped_size); region->base = (uintptr_t)mem; region->active_size = 0; region->mapped_size = new_size; @@ -294,7 +289,7 @@ static void truncate_region(struct region *region, size_t new_size) { size_t bytes = region->mapped_size - new_size; if (bytes) { - munmap((void*)(region->base + new_size), bytes); + gc_platform_release_memory((void*)(region->base + new_size), bytes); region->mapped_size = new_size; if (region->active_size > new_size) region->active_size = new_size; @@ -569,7 +564,7 @@ static int region_init(struct region *region, size_t size) { static int semi_space_init(struct semi_space *space, struct gc_heap *heap) { // Allocate even numbers of pages. - size_t page_size = getpagesize(); + size_t page_size = gc_platform_page_size(); size_t size = align_up(heap->size, page_size * 2); space->page_size = page_size; diff --git a/src/shared-worklist.h b/src/shared-worklist.h index afefb11e2..979c87178 100644 --- a/src/shared-worklist.h +++ b/src/shared-worklist.h @@ -2,13 +2,12 @@ #define SHARED_WORKLIST_H #include -#include -#include #include "assert.h" #include "debug.h" #include "gc-align.h" #include "gc-inline.h" +#include "gc-platform.h" #include "spin.h" // The Chase-Lev work-stealing deque, as initially described in "Dynamic @@ -36,9 +35,8 @@ shared_worklist_buf_init(struct shared_worklist_buf *buf, unsigned log_size) { ASSERT(log_size >= shared_worklist_buf_min_log_size); ASSERT(log_size <= shared_worklist_buf_max_log_size); size_t size = (1 << log_size) * sizeof(uintptr_t); - void *mem = mmap(NULL, size, PROT_READ|PROT_WRITE, - MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); - if (mem == MAP_FAILED) { + void *mem = gc_platform_acquire_memory(size, 0); + if (!mem) { perror("Failed to grow work-stealing dequeue"); DEBUG("Failed to allocate %zu bytes", size); return 0; @@ -63,13 +61,13 @@ static void shared_worklist_buf_release(struct shared_worklist_buf *buf) { size_t byte_size = shared_worklist_buf_byte_size(buf); if (buf->data && byte_size >= shared_worklist_release_byte_threshold) - madvise(buf->data, byte_size, MADV_DONTNEED); + gc_platform_discard_memory(buf->data, byte_size); } static void shared_worklist_buf_destroy(struct shared_worklist_buf *buf) { if (buf->data) { - munmap(buf->data, shared_worklist_buf_byte_size(buf)); + gc_platform_release_memory(buf->data, shared_worklist_buf_byte_size(buf)); buf->data = NULL; buf->log_size = 0; buf->size = 0; diff --git a/src/simple-worklist.h b/src/simple-worklist.h index bae33b470..61f92a31d 100644 --- a/src/simple-worklist.h +++ b/src/simple-worklist.h @@ -1,13 +1,11 @@ #ifndef SIMPLE_WORKLIST_H #define SIMPLE_WORKLIST_H -#include -#include - #include "assert.h" #include "debug.h" #include "gc-inline.h" #include "gc-ref.h" +#include "gc-platform.h" struct simple_worklist { size_t size; @@ -22,9 +20,8 @@ static const size_t simple_worklist_release_byte_threshold = 1 * 1024 * 1024; static struct gc_ref * simple_worklist_alloc(size_t size) { - void *mem = mmap(NULL, size * sizeof(struct gc_ref), PROT_READ|PROT_WRITE, - MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); - if (mem == MAP_FAILED) { + void *mem = gc_platform_acquire_memory(size * sizeof(struct gc_ref), 0); + if (!mem) { perror("Failed to grow trace queue"); DEBUG("Failed to allocate %zu bytes", size); return NULL; @@ -34,7 +31,7 @@ simple_worklist_alloc(size_t size) { static int simple_worklist_init(struct simple_worklist *q) { - q->size = getpagesize() / sizeof(struct gc_ref); + q->size = gc_platform_page_size() / sizeof(struct gc_ref); q->read = 0; q->write = 0; q->buf = simple_worklist_alloc(q->size);