Move mmap / munmap / madvise to gc-platform

src/copy-space.h

@@ -3,7 +3,6 @@
 
 #include <pthread.h>
 #include <stdlib.h>
-#include <sys/mman.h>
 
 #include "gc-api.h"
 
@@ -18,6 +17,7 @@
 #include "gc-attrs.h"
 #include "gc-inline.h"
 #include "gc-lock.h"
+#include "gc-platform.h"
 #include "spin.h"
 
 // A copy space: a block-structured space that traces via evacuation.
@@ -620,27 +620,8 @@ copy_space_allocator_finish(struct copy_space_allocator *alloc,
 
 static struct copy_space_slab*
 copy_space_allocate_slabs(size_t nslabs) {
-  size_t size = nslabs * COPY_SPACE_SLAB_SIZE;
-  size_t extent = size + COPY_SPACE_SLAB_SIZE;
-
-  char *mem = mmap(NULL, extent, PROT_READ|PROT_WRITE,
-                   MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
-  if (mem == MAP_FAILED) {
-    perror("mmap failed");
-    return NULL;
-  }
-
-  uintptr_t base = (uintptr_t) mem;
-  uintptr_t end = base + extent;
-  uintptr_t aligned_base = align_up(base, COPY_SPACE_SLAB_SIZE);
-  uintptr_t aligned_end = aligned_base + size;
-
-  if (aligned_base - base)
-    munmap((void*)base, aligned_base - base);
-  if (end - aligned_end)
-    munmap((void*)aligned_end, end - aligned_end);
-
-  return (struct copy_space_slab*) aligned_base;
+  return gc_platform_acquire_memory(nslabs * COPY_SPACE_SLAB_SIZE,
+                                    COPY_SPACE_SLAB_SIZE);
 }
 
 static void
@@ -715,8 +696,8 @@ copy_space_page_out_blocks(void *data) {
     if (!block) break;
     block->in_core = 0;
     block->all_zeroes[0] = block->all_zeroes[1] = 1;
-    madvise(copy_space_block_payload(block), COPY_SPACE_BLOCK_SIZE,
-            MADV_DONTNEED);
+    gc_platform_discard_memory(copy_space_block_payload(block),
+                               COPY_SPACE_BLOCK_SIZE);
     copy_space_block_stack_push(&space->paged_out[age + 1], block, &lock);
   }
   gc_lock_release(&lock);

src/gc-platform-gnu-linux.c

@@ -5,12 +5,14 @@
 #include <pthread.h>
 #include <sched.h>
 #include <stdio.h>
+#include <sys/mman.h>
 #include <time.h>
 #include <unistd.h>
 
 #define GC_IMPL 1
 
 #include "debug.h"
+#include "gc-align.h"
 #include "gc-assert.h"
 #include "gc-inline.h"
 #include "gc-platform.h"
@@ -121,3 +123,58 @@ uint64_t gc_platform_monotonic_nanoseconds(void) {
   uint64_t ns_per_sec = 1000000000;
   return s * ns_per_sec + ns;
 }
+
+size_t gc_platform_page_size(void) {
+  return getpagesize();
+}
+
+void* gc_platform_acquire_memory(size_t size, size_t alignment) {
+  GC_ASSERT_EQ(size, align_down(size, getpagesize()));
+  GC_ASSERT_EQ(alignment & (alignment - 1), 0);
+  GC_ASSERT_EQ(alignment, align_down(alignment, getpagesize()));
+
+  size_t extent = size + alignment;
+  char *mem = mmap(NULL, extent, PROT_READ|PROT_WRITE,
+                   MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
+  if (mem == MAP_FAILED) {
+    perror("mmap failed");
+    return NULL;
+  }
+
+  uintptr_t base = (uintptr_t) mem;
+  uintptr_t end = base + extent;
+  uintptr_t aligned_base = alignment ? align_up(base, alignment) : base;
+  uintptr_t aligned_end = aligned_base + size;
+
+  if (aligned_base - base)
+    munmap((void*)base, aligned_base - base);
+  if (end - aligned_end)
+    munmap((void*)aligned_end, end - aligned_end);
+
+  return (void*) aligned_base;
+}
+
+void gc_platform_release_memory(void *ptr, size_t size) {
+  GC_ASSERT_EQ((uintptr_t)ptr, align_down((uintptr_t)ptr, getpagesize()));
+  GC_ASSERT_EQ(size, align_down(size, getpagesize()));
+  if (munmap(ptr, size) != 0)
+    perror("failed to unmap memory");
+}
+
+int gc_platform_populate_memory(void *ptr, size_t size) {
+  GC_ASSERT_EQ((uintptr_t)ptr, align_down((uintptr_t)ptr, getpagesize()));
+  GC_ASSERT_EQ(size, align_down(size, getpagesize()));
+  if (madvise(ptr, size, MADV_WILLNEED) == 0)
+    return 1;
+  perror("failed to populate memory");
+  return 0;
+}
+
+int gc_platform_discard_memory(void *ptr, size_t size) {
+  GC_ASSERT_EQ((uintptr_t)ptr, align_down((uintptr_t)ptr, getpagesize()));
+  GC_ASSERT_EQ(size, align_down(size, getpagesize()));
+  if (madvise(ptr, size, MADV_DONTNEED) == 0)
+    return 1;
+  perror("failed to discard memory");
+  return 0;
+}

src/gc-platform.h

@@ -23,4 +23,11 @@ void gc_platform_visit_global_conservative_roots(void (*f)(uintptr_t start,
 GC_INTERNAL int gc_platform_processor_count(void);
 GC_INTERNAL uint64_t gc_platform_monotonic_nanoseconds(void);
 
+GC_INTERNAL size_t gc_platform_page_size(void);
+GC_INTERNAL void* gc_platform_acquire_memory(size_t size, size_t alignment);
+GC_INTERNAL void gc_platform_release_memory(void *base, size_t size);
+
+GC_INTERNAL int gc_platform_populate_memory(void *addr, size_t size);
+GC_INTERNAL int gc_platform_discard_memory(void *addr, size_t size);
+
 #endif // GC_PLATFORM_H

src/large-object-space.h

@@ -321,8 +321,7 @@ static void*
 large_object_space_obtain_and_alloc(struct large_object_space *space,
                                     size_t npages) {
   size_t bytes = npages * space->page_size;
-  void *ret = mmap(NULL, bytes, PROT_READ|PROT_WRITE,
-                   MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
+  void *ret = gc_platform_acquire_memory(bytes, 0);
   if (ret == MAP_FAILED)
     return NULL;
 

src/nofl-space.h

@@ -5,7 +5,6 @@
 #include <stdatomic.h>
 #include <stdint.h>
 #include <string.h>
-#include <sys/mman.h>
 
 #include "gc-api.h"
 
@@ -19,6 +18,7 @@
 #include "gc-attrs.h"
 #include "gc-inline.h"
 #include "gc-lock.h"
+#include "gc-platform.h"
 #include "spin.h"
 #include "swar.h"
 
@@ -1675,27 +1675,7 @@ nofl_space_object_size(struct nofl_space *space, struct gc_ref ref) {
 
 static struct nofl_slab*
 nofl_allocate_slabs(size_t nslabs) {
-  size_t size = nslabs * NOFL_SLAB_SIZE;
-  size_t extent = size + NOFL_SLAB_SIZE;
-
-  char *mem = mmap(NULL, extent, PROT_READ|PROT_WRITE,
-                   MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
-  if (mem == MAP_FAILED) {
-    perror("mmap failed");
-    return NULL;
-  }
-
-  uintptr_t base = (uintptr_t) mem;
-  uintptr_t end = base + extent;
-  uintptr_t aligned_base = align_up(base, NOFL_SLAB_SIZE);
-  uintptr_t aligned_end = aligned_base + size;
-
-  if (aligned_base - base)
-    munmap((void*)base, aligned_base - base);
-  if (end - aligned_end)
-    munmap((void*)aligned_end, end - aligned_end);
-
-  return (struct nofl_slab*) aligned_base;
+  return gc_platform_acquire_memory(nslabs * NOFL_SLAB_SIZE, NOFL_SLAB_SIZE);
 }
 
 static void
@@ -1813,7 +1793,7 @@ nofl_space_page_out_blocks(void *data) {
     if (nofl_block_is_null(block))
       break;
     nofl_block_set_flag(block, NOFL_BLOCK_ZERO | NOFL_BLOCK_PAGED_OUT);
-    madvise((void*)block.addr, NOFL_BLOCK_SIZE, MADV_DONTNEED);
+    gc_platform_discard_memory((void*)block.addr, NOFL_BLOCK_SIZE);
     nofl_block_stack_push(&space->paged_out[age + 1], block, &lock);
   }
   gc_lock_release(&lock);

src/semi.c

@@ -2,8 +2,6 @@
 #include <stdint.h>
 #include <stdio.h>
 #include <string.h>
-#include <sys/mman.h>
-#include <unistd.h>
 
 #include "gc-api.h"
 
@@ -100,13 +98,13 @@ static void region_trim_by(struct region *region, size_t newly_unavailable) {
   GC_ASSERT(newly_unavailable <= region->active_size);
 
   region->active_size -= newly_unavailable;
-  madvise((void*)(region->base + region->active_size), newly_unavailable,
-          MADV_DONTNEED);
+  gc_platform_discard_memory((void*)(region->base + region->active_size),
+                             newly_unavailable);
 }
 
 static void region_set_active_size(struct region *region, size_t size) {
   GC_ASSERT(size <= region->mapped_size);
-  GC_ASSERT(size == align_up(size, getpagesize()));
+  GC_ASSERT(size == align_up(size, gc_platform_page_size()));
   if (size < region->active_size)
     region_trim_by(region, region->active_size - size);
   else
@@ -274,15 +272,12 @@ static int grow_region_if_needed(struct region *region, size_t new_size) {
   if (new_size <= region->mapped_size)
     return 1;
 
-  void *mem = mmap(NULL, new_size, PROT_READ|PROT_WRITE,
-                   MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
+  void *mem = gc_platform_acquire_memory(new_size, 0);
   DEBUG("new size %zx\n", new_size);
-  if (mem == MAP_FAILED) {
-    perror("mmap failed");
+  if (!mem)
     return 0;
-  }
   if (region->mapped_size)
-    munmap((void*)region->base, region->mapped_size);
+    gc_platform_release_memory((void*)region->base, region->mapped_size);
   region->base = (uintptr_t)mem;
   region->active_size = 0;
   region->mapped_size = new_size;
@@ -294,7 +289,7 @@ static void truncate_region(struct region *region, size_t new_size) {
 
   size_t bytes = region->mapped_size - new_size;
   if (bytes) {
-    munmap((void*)(region->base + new_size), bytes);
+    gc_platform_release_memory((void*)(region->base + new_size), bytes);
     region->mapped_size = new_size;
     if (region->active_size > new_size)
       region->active_size = new_size;
@@ -569,7 +564,7 @@ static int region_init(struct region *region, size_t size) {
 
 static int semi_space_init(struct semi_space *space, struct gc_heap *heap) {
   // Allocate even numbers of pages.
-  size_t page_size = getpagesize();
+  size_t page_size = gc_platform_page_size();
   size_t size = align_up(heap->size, page_size * 2);
 
   space->page_size = page_size;

src/shared-worklist.h

@@ -2,13 +2,12 @@
 #define SHARED_WORKLIST_H
 
 #include <stdatomic.h>
-#include <sys/mman.h>
-#include <unistd.h>
 
 #include "assert.h"
 #include "debug.h"
 #include "gc-align.h"
 #include "gc-inline.h"
+#include "gc-platform.h"
 #include "spin.h"
 
 // The Chase-Lev work-stealing deque, as initially described in "Dynamic
@@ -36,9 +35,8 @@ shared_worklist_buf_init(struct shared_worklist_buf *buf, unsigned log_size) {
   ASSERT(log_size >= shared_worklist_buf_min_log_size);
   ASSERT(log_size <= shared_worklist_buf_max_log_size);
   size_t size = (1 << log_size) * sizeof(uintptr_t);
-  void *mem = mmap(NULL, size, PROT_READ|PROT_WRITE,
-                   MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
-  if (mem == MAP_FAILED) {
+  void *mem = gc_platform_acquire_memory(size, 0);
+  if (!mem) {
     perror("Failed to grow work-stealing dequeue");
     DEBUG("Failed to allocate %zu bytes", size);
     return 0;
@@ -63,13 +61,13 @@ static void
 shared_worklist_buf_release(struct shared_worklist_buf *buf) {
   size_t byte_size = shared_worklist_buf_byte_size(buf);
   if (buf->data && byte_size >= shared_worklist_release_byte_threshold)
-    madvise(buf->data, byte_size, MADV_DONTNEED);
+    gc_platform_discard_memory(buf->data, byte_size);
 }
 
 static void
 shared_worklist_buf_destroy(struct shared_worklist_buf *buf) {
   if (buf->data) {
-    munmap(buf->data, shared_worklist_buf_byte_size(buf));
+    gc_platform_release_memory(buf->data, shared_worklist_buf_byte_size(buf));
     buf->data = NULL;
     buf->log_size = 0;
     buf->size = 0;

src/simple-worklist.h

@@ -1,13 +1,11 @@
 #ifndef SIMPLE_WORKLIST_H
 #define SIMPLE_WORKLIST_H
 
-#include <sys/mman.h>
-#include <unistd.h>
-
 #include "assert.h"
 #include "debug.h"
 #include "gc-inline.h"
 #include "gc-ref.h"
+#include "gc-platform.h"
 
 struct simple_worklist {
   size_t size;
@@ -22,9 +20,8 @@ static const size_t simple_worklist_release_byte_threshold = 1 * 1024 * 1024;
 
 static struct gc_ref *
 simple_worklist_alloc(size_t size) {
-  void *mem = mmap(NULL, size * sizeof(struct gc_ref), PROT_READ|PROT_WRITE,
-                   MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
-  if (mem == MAP_FAILED) {
+  void *mem = gc_platform_acquire_memory(size * sizeof(struct gc_ref), 0);
+  if (!mem) {
     perror("Failed to grow trace queue");
     DEBUG("Failed to allocate %zu bytes", size);
     return NULL;
@@ -34,7 +31,7 @@ simple_worklist_alloc(size_t size) {
 
 static int
 simple_worklist_init(struct simple_worklist *q) {
-  q->size = getpagesize() / sizeof(struct gc_ref);
+  q->size = gc_platform_page_size() / sizeof(struct gc_ref);
   q->read = 0;
   q->write = 0;
   q->buf = simple_worklist_alloc(q->size);