Reorganize source tree and document

Makefile

@@ -29,7 +29,7 @@ BUILD_CFLAGS=$(BUILD_CFLAGS_$(or $(BUILD),$(DEFAULT_BUILD)))
 
 CC=gcc
 CFLAGS=-Wall -flto -fno-strict-aliasing -fvisibility=hidden -Wno-unused $(BUILD_CFLAGS)
-INCLUDES=-I.
+INCLUDES=-Iapi
 LDFLAGS=-lpthread -flto
 COMPILE=$(CC) $(CFLAGS) $(INCLUDES)
 PLATFORM=gnu-linux
@@ -38,113 +38,113 @@ ALL_TESTS=$(foreach COLLECTOR,$(COLLECTORS),$(addprefix $(COLLECTOR)-,$(TESTS)))
 
 all: $(ALL_TESTS)
 
-gc-platform.o: gc-platform.h gc-platform-$(PLATFORM).c gc-visibility.h
-	$(COMPILE) -o $@ -c gc-platform-$(PLATFORM).c
+gc-platform.o: src/gc-platform.h src/gc-platform-$(PLATFORM).c api/gc-visibility.h
+	$(COMPILE) -o $@ -c src/gc-platform-$(PLATFORM).c
 
-gc-stack.o: gc-stack.c
+gc-stack.o: src/gc-stack.c
 	$(COMPILE) -o $@ -c $<
 
-gc-options.o: gc-options.c gc-options.h gc-options-internal.h
+gc-options.o: src/gc-options.c api/gc-options.h src/gc-options-internal.h
 	$(COMPILE) -o $@ -c $<
 
-gc-ephemeron-%.o: gc-ephemeron.c gc-ephemeron.h gc-ephemeron-internal.h %-embedder.h
-	$(COMPILE) -include $*-embedder.h -o $@ -c $<
+gc-ephemeron-%.o: src/gc-ephemeron.c api/gc-ephemeron.h src/gc-ephemeron-internal.h benchmarks/%-embedder.h
+	$(COMPILE) -include benchmarks/$*-embedder.h -o $@ -c $<
 
-bdw-%-gc.o: bdw.c %-embedder.h %.c
-	$(COMPILE) -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 `pkg-config --cflags bdw-gc` -include $*-embedder.h -o $@ -c bdw.c
-bdw-%.o: bdw.c %.c
-	$(COMPILE) -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 -include bdw-attrs.h -o $@ -c $*.c
+bdw-%-gc.o: src/bdw.c benchmarks/%-embedder.h benchmarks/%.c
+	$(COMPILE) -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 `pkg-config --cflags bdw-gc` -include benchmarks/$*-embedder.h -o $@ -c src/bdw.c
+bdw-%.o: src/bdw.c benchmarks/%.c
+	$(COMPILE) -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 -include api/bdw-attrs.h -o $@ -c benchmarks/$*.c
 bdw-%: bdw-%.o bdw-%-gc.o gc-stack.o gc-options.o gc-platform.o gc-ephemeron-%.o
 	$(CC) $(LDFLAGS) `pkg-config --libs bdw-gc` -o $@ $^
 
-semi-%-gc.o: semi.c %-embedder.h large-object-space.h assert.h debug.h %.c
-	$(COMPILE) -DGC_PRECISE_ROOTS=1 -include $*-embedder.h -o $@ -c semi.c
-semi-%.o: semi.c %.c
-	$(COMPILE) -DGC_PRECISE_ROOTS=1 -include semi-attrs.h -o $@ -c $*.c
+semi-%-gc.o: src/semi.c benchmarks/%-embedder.h
+	$(COMPILE) -DGC_PRECISE_ROOTS=1 -include benchmarks/$*-embedder.h -o $@ -c src/semi.c
+semi-%.o: src/semi.c benchmarks/%.c
+	$(COMPILE) -DGC_PRECISE_ROOTS=1 -include api/semi-attrs.h -o $@ -c benchmarks/$*.c
 semi-%: semi-%.o semi-%-gc.o gc-stack.o gc-options.o gc-platform.o gc-ephemeron-%.o
 	$(CC) $(LDFLAGS) -o $@ $^
 
-whippet-%-gc.o: whippet.c %-embedder.h large-object-space.h serial-tracer.h assert.h debug.h heap-objects.h %.c
-	$(COMPILE) -DGC_PRECISE_ROOTS=1 -include $*-embedder.h -o $@ -c whippet.c
-whippet-%.o: whippet.c %.c
-	$(COMPILE) -DGC_PRECISE_ROOTS=1 -include whippet-attrs.h -o $@ -c $*.c
+whippet-%-gc.o: src/whippet.c benchmarks/%-embedder.h
+	$(COMPILE) -DGC_PRECISE_ROOTS=1 -include benchmarks/$*-embedder.h -o $@ -c src/whippet.c
+whippet-%.o: src/whippet.c benchmarks/%.c
+	$(COMPILE) -DGC_PRECISE_ROOTS=1 -include api/whippet-attrs.h -o $@ -c benchmarks/$*.c
 whippet-%: whippet-%.o whippet-%-gc.o gc-stack.o gc-options.o gc-platform.o gc-ephemeron-%.o
 	$(CC) $(LDFLAGS) -o $@ $^
 
-stack-conservative-whippet-%-gc.o: whippet.c %-embedder.h large-object-space.h serial-tracer.h assert.h debug.h heap-objects.h %.c
-	$(COMPILE) -DGC_CONSERVATIVE_ROOTS=1 -include $*-embedder.h -o $@ -c whippet.c
-stack-conservative-whippet-%.o: whippet.c %.c
-	$(COMPILE) -DGC_CONSERVATIVE_ROOTS=1 -include whippet-attrs.h -o $@ -c $*.c
+stack-conservative-whippet-%-gc.o: src/whippet.c benchmarks/%-embedder.h
+	$(COMPILE) -DGC_CONSERVATIVE_ROOTS=1 -include benchmarks/$*-embedder.h -o $@ -c src/whippet.c
+stack-conservative-whippet-%.o: src/whippet.c benchmarks/%.c
+	$(COMPILE) -DGC_CONSERVATIVE_ROOTS=1 -include api/whippet-attrs.h -o $@ -c benchmarks/$*.c
 stack-conservative-whippet-%: stack-conservative-whippet-%.o stack-conservative-whippet-%-gc.o gc-stack.o gc-options.o gc-platform.o gc-ephemeron-%.o
 	$(CC) $(LDFLAGS) -o $@ $^
 
-heap-conservative-whippet-%-gc.o: whippet.c %-embedder.h large-object-space.h serial-tracer.h assert.h debug.h heap-objects.h %.c
-	$(COMPILE) -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 -include $*-embedder.h -o $@ -c whippet.c
-heap-conservative-whippet-%.o: whippet.c %.c
-	$(COMPILE) -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 -include whippet-attrs.h -o $@ -c $*.c
+heap-conservative-whippet-%-gc.o: src/whippet.c benchmarks/%-embedder.h
+	$(COMPILE) -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 -include benchmarks/$*-embedder.h -o $@ -c src/whippet.c
+heap-conservative-whippet-%.o: src/whippet.c benchmarks/%.c
+	$(COMPILE) -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 -include api/whippet-attrs.h -o $@ -c benchmarks/$*.c
 heap-conservative-whippet-%: heap-conservative-whippet-%.o heap-conservative-whippet-%-gc.o gc-stack.o gc-options.o gc-platform.o gc-ephemeron-%.o
 	$(CC) $(LDFLAGS) -o $@ $^
 
-parallel-whippet-%-gc.o: whippet.c %-embedder.h large-object-space.h parallel-tracer.h assert.h debug.h heap-objects.h %.c
-	$(COMPILE) -DGC_PARALLEL=1 -DGC_PRECISE_ROOTS=1 -include $*-embedder.h -o $@ -c whippet.c
-parallel-whippet-%.o: whippet.c %.c
-	$(COMPILE) -DGC_PARALLEL=1 -DGC_PRECISE_ROOTS=1 -include whippet-attrs.h -o $@ -c $*.c
+parallel-whippet-%-gc.o: src/whippet.c benchmarks/%-embedder.h
+	$(COMPILE) -DGC_PARALLEL=1 -DGC_PRECISE_ROOTS=1 -include benchmarks/$*-embedder.h -o $@ -c src/whippet.c
+parallel-whippet-%.o: src/whippet.c benchmarks/%.c
+	$(COMPILE) -DGC_PARALLEL=1 -DGC_PRECISE_ROOTS=1 -include api/whippet-attrs.h -o $@ -c benchmarks/$*.c
 parallel-whippet-%: parallel-whippet-%.o parallel-whippet-%-gc.o gc-stack.o gc-options.o gc-platform.o gc-ephemeron-%.o
 	$(CC) $(LDFLAGS) -o $@ $^
 
-stack-conservative-parallel-whippet-%-gc.o: whippet.c %-embedder.h large-object-space.h serial-tracer.h assert.h debug.h heap-objects.h %.c
-	$(COMPILE) -DGC_PARALLEL=1 -DGC_CONSERVATIVE_ROOTS=1 -include $*-embedder.h -o $@ -c whippet.c
-stack-conservative-parallel-whippet-%.o: whippet.c %.c
-	$(COMPILE) -DGC_PARALLEL=1 -DGC_CONSERVATIVE_ROOTS=1 -include whippet-attrs.h -o $@ -c $*.c
+stack-conservative-parallel-whippet-%-gc.o: src/whippet.c benchmarks/%-embedder.h
+	$(COMPILE) -DGC_PARALLEL=1 -DGC_CONSERVATIVE_ROOTS=1 -include benchmarks/$*-embedder.h -o $@ -c src/whippet.c
+stack-conservative-parallel-whippet-%.o: src/whippet.c benchmarks/%.c
+	$(COMPILE) -DGC_PARALLEL=1 -DGC_CONSERVATIVE_ROOTS=1 -include api/whippet-attrs.h -o $@ -c benchmarks/$*.c
 stack-conservative-parallel-whippet-%: stack-conservative-parallel-whippet-%.o stack-conservative-parallel-whippet-%-gc.o gc-stack.o gc-options.o gc-platform.o gc-ephemeron-%.o
 	$(CC) $(LDFLAGS) -o $@ $^
 
-heap-conservative-parallel-whippet-%-gc.o: whippet.c %-embedder.h large-object-space.h serial-tracer.h assert.h debug.h heap-objects.h %.c
-	$(COMPILE) -DGC_PARALLEL=1 -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 -include $*-embedder.h -o $@ -c whippet.c
-heap-conservative-parallel-whippet-%.o: whippet.c %.c
-	$(COMPILE) -DGC_PARALLEL=1 -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 -DGC_FULLY_CONSERVATIVE=1 -include whippet-attrs.h -o $@ -c $*.c
+heap-conservative-parallel-whippet-%-gc.o: src/whippet.c benchmarks/%-embedder.h
+	$(COMPILE) -DGC_PARALLEL=1 -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 -include benchmarks/$*-embedder.h -o $@ -c src/whippet.c
+heap-conservative-parallel-whippet-%.o: src/whippet.c benchmarks/%.c
+	$(COMPILE) -DGC_PARALLEL=1 -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 -DGC_FULLY_CONSERVATIVE=1 -include api/whippet-attrs.h -o $@ -c benchmarks/$*.c
 heap-conservative-parallel-whippet-%: heap-conservative-parallel-whippet-%.o heap-conservative-parallel-whippet-%-gc.o gc-stack.o gc-options.o gc-platform.o gc-ephemeron-%.o
 	$(CC) $(LDFLAGS) -o $@ $^
 
-generational-whippet-%-gc.o: whippet.c %-embedder.h large-object-space.h serial-tracer.h assert.h debug.h heap-objects.h %.c
-	$(COMPILE) -DGC_GENERATIONAL=1 -DGC_PRECISE_ROOTS=1 -include $*-embedder.h -o $@ -c whippet.c
-generational-whippet-%.o: whippet.c %.c
-	$(COMPILE) -DGC_GENERATIONAL=1 -DGC_PRECISE_ROOTS=1 -include whippet-attrs.h -o $@ -c $*.c
+generational-whippet-%-gc.o: src/whippet.c benchmarks/%-embedder.h
+	$(COMPILE) -DGC_GENERATIONAL=1 -DGC_PRECISE_ROOTS=1 -include benchmarks/$*-embedder.h -o $@ -c src/whippet.c
+generational-whippet-%.o: src/whippet.c benchmarks/%.c
+	$(COMPILE) -DGC_GENERATIONAL=1 -DGC_PRECISE_ROOTS=1 -include api/whippet-attrs.h -o $@ -c benchmarks/$*.c
 generational-whippet-%: generational-whippet-%.o generational-whippet-%-gc.o gc-stack.o gc-options.o gc-platform.o gc-ephemeron-%.o
 	$(CC) $(LDFLAGS) -o $@ $^
 
-stack-conservative-generational-whippet-%-gc.o: whippet.c %-embedder.h large-object-space.h serial-tracer.h assert.h debug.h heap-objects.h %.c
-	$(COMPILE) -DGC_GENERATIONAL=1 -DGC_CONSERVATIVE_ROOTS=1 -include $*-embedder.h -o $@ -c whippet.c
-stack-conservative-generational-whippet-%.o: whippet.c %.c
-	$(COMPILE) -DGC_GENERATIONAL=1 -DGC_CONSERVATIVE_ROOTS=1 -include whippet-attrs.h -o $@ -c $*.c
+stack-conservative-generational-whippet-%-gc.o: src/whippet.c benchmarks/%-embedder.h
+	$(COMPILE) -DGC_GENERATIONAL=1 -DGC_CONSERVATIVE_ROOTS=1 -include benchmarks/$*-embedder.h -o $@ -c src/whippet.c
+stack-conservative-generational-whippet-%.o: src/whippet.c benchmarks/%.c
+	$(COMPILE) -DGC_GENERATIONAL=1 -DGC_CONSERVATIVE_ROOTS=1 -include api/whippet-attrs.h -o $@ -c benchmarks/$*.c
 stack-conservative-generational-whippet-%: stack-conservative-generational-whippet-%.o stack-conservative-generational-whippet-%-gc.o gc-stack.o gc-options.o gc-platform.o gc-ephemeron-%.o
 	$(CC) $(LDFLAGS) -o $@ $^
 
-heap-conservative-generational-whippet-%-gc.o: whippet.c %-embedder.h large-object-space.h serial-tracer.h assert.h debug.h heap-objects.h %.c
-	$(COMPILE) -DGC_GENERATIONAL=1 -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 -include $*-embedder.h -o $@ -c whippet.c
-heap-conservative-generational-whippet-%.o: whippet.c %.c
-	$(COMPILE) -DGC_GENERATIONAL=1 -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 -include whippet-attrs.h -o $@ -c $*.c
+heap-conservative-generational-whippet-%-gc.o: src/whippet.c benchmarks/%-embedder.h
+	$(COMPILE) -DGC_GENERATIONAL=1 -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 -include benchmarks/$*-embedder.h -o $@ -c src/whippet.c
+heap-conservative-generational-whippet-%.o: src/whippet.c benchmarks/%.c
+	$(COMPILE) -DGC_GENERATIONAL=1 -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 -include api/whippet-attrs.h -o $@ -c benchmarks/$*.c
 heap-conservative-generational-whippet-%: heap-conservative-generational-whippet-%.o heap-conservative-generational-whippet-%-gc.o gc-stack.o gc-options.o gc-platform.o gc-ephemeron-%.o
 	$(CC) $(LDFLAGS) -o $@ $^
 
-parallel-generational-whippet-%-gc.o: whippet.c %-embedder.h large-object-space.h parallel-tracer.h assert.h debug.h heap-objects.h %.c
-	$(COMPILE) -DGC_PARALLEL=1 -DGC_GENERATIONAL=1 -DGC_PRECISE_ROOTS=1 -include $*-embedder.h -o $@ -c whippet.c
-parallel-generational-whippet-%.o: whippet.c %.c
-	$(COMPILE) -DGC_PARALLEL=1 -DGC_GENERATIONAL=1 -DGC_PRECISE_ROOTS=1 -include whippet-attrs.h -o $@ -c $*.c
+parallel-generational-whippet-%-gc.o: src/whippet.c benchmarks/%-embedder.h
+	$(COMPILE) -DGC_PARALLEL=1 -DGC_GENERATIONAL=1 -DGC_PRECISE_ROOTS=1 -include benchmarks/$*-embedder.h -o $@ -c src/whippet.c
+parallel-generational-whippet-%.o: src/whippet.c benchmarks/%.c
+	$(COMPILE) -DGC_PARALLEL=1 -DGC_GENERATIONAL=1 -DGC_PRECISE_ROOTS=1 -include api/whippet-attrs.h -o $@ -c benchmarks/$*.c
 parallel-generational-whippet-%: parallel-generational-whippet-%.o parallel-generational-whippet-%-gc.o gc-stack.o gc-options.o gc-platform.o gc-ephemeron-%.o
 	$(CC) $(LDFLAGS) -o $@ $^
 
-stack-conservative-parallel-generational-whippet-%-gc.o: whippet.c %-embedder.h large-object-space.h parallel-tracer.h assert.h debug.h heap-objects.h %.c
-	$(COMPILE) -DGC_PARALLEL=1 -DGC_GENERATIONAL=1 -DGC_CONSERVATIVE_ROOTS=1 -include $*-embedder.h -o $@ -c whippet.c
-stack-conservative-parallel-generational-whippet-%.o: whippet.c %.c
-	$(COMPILE) -DGC_PARALLEL=1 -DGC_GENERATIONAL=1 -DGC_CONSERVATIVE_ROOTS=1 -include whippet-attrs.h -o $@ -c $*.c
+stack-conservative-parallel-generational-whippet-%-gc.o: src/whippet.c benchmarks/%-embedder.h
+	$(COMPILE) -DGC_PARALLEL=1 -DGC_GENERATIONAL=1 -DGC_CONSERVATIVE_ROOTS=1 -include benchmarks/$*-embedder.h -o $@ -c src/whippet.c
+stack-conservative-parallel-generational-whippet-%.o: src/whippet.c benchmarks/%.c
+	$(COMPILE) -DGC_PARALLEL=1 -DGC_GENERATIONAL=1 -DGC_CONSERVATIVE_ROOTS=1 -include api/whippet-attrs.h -o $@ -c benchmarks/$*.c
 stack-conservative-parallel-generational-whippet-%: stack-conservative-parallel-generational-whippet-%.o stack-conservative-parallel-generational-whippet-%-gc.o gc-stack.o gc-options.o gc-platform.o gc-ephemeron-%.o
 	$(CC) $(LDFLAGS) -o $@ $^
 
-heap-conservative-parallel-generational-whippet-%-gc.o: whippet.c %-embedder.h large-object-space.h parallel-tracer.h assert.h debug.h heap-objects.h %.c
-	$(COMPILE) -DGC_PARALLEL=1 -DGC_GENERATIONAL=1 -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 -include $*-embedder.h -o $@ -c whippet.c
-heap-conservative-parallel-generational-whippet-%.o: whippet.c %.c
-	$(COMPILE) -DGC_PARALLEL=1 -DGC_GENERATIONAL=1 -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 -include whippet-attrs.h -o $@ -c $*.c
+heap-conservative-parallel-generational-whippet-%-gc.o: src/whippet.c benchmarks/%-embedder.h
+	$(COMPILE) -DGC_PARALLEL=1 -DGC_GENERATIONAL=1 -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 -include benchmarks/$*-embedder.h -o $@ -c src/whippet.c
+heap-conservative-parallel-generational-whippet-%.o: src/whippet.c benchmarks/%.c
+	$(COMPILE) -DGC_PARALLEL=1 -DGC_GENERATIONAL=1 -DGC_CONSERVATIVE_ROOTS=1 -DGC_CONSERVATIVE_TRACE=1 -include api/whippet-attrs.h -o $@ -c benchmarks/$*.c
 heap-conservative-parallel-generational-whippet-%: heap-conservative-parallel-generational-whippet-%.o heap-conservative-parallel-generational-whippet-%-gc.o gc-stack.o gc-options.o gc-platform.o gc-ephemeron-%.o
 	$(CC) $(LDFLAGS) -o $@ $^
 

README.md

@@ -4,139 +4,37 @@ This repository is for development of Whippet, a new garbage collector
 implementation, eventually for use in [Guile
 Scheme](https://gnu.org/s/guile).
 
-## Design
+Whippet is an embed-only C library, designed to be copied into a
+program's source tree.  It exposes an abstract C API for managed memory
+allocation, and provides a number of implementations of that API.
 
-Whippet is mainly a mark-region collector, like
-[Immix](http://users.cecs.anu.edu.au/~steveb/pubs/papers/immix-pldi-2008.pdf).
-See also the lovely detailed [Rust
-implementation](http://users.cecs.anu.edu.au/~steveb/pubs/papers/rust-ismm-2016.pdf).
+One of the implementations is also called "whippet", and is the
+motivation for creating this library.  For a detailed introduction, see
+[Whippet: Towards a new local
+maximum](https://wingolog.org/archives/2023/02/07/whippet-towards-a-new-local-maximum),
+a talk given at FOSDEM 2023.
 
-To a first approximation, Whippet is a whole-heap Immix collector with a
-large object space on the side.  See the Immix paper for full details,
-but basically Immix divides the heap into 32kB blocks, and then divides
-those blocks into 128B lines.  An Immix allocation never spans blocks;
-allocations larger than 8kB go into a separate large object space.
-Mutators request blocks from the global store and allocate into those
-blocks using bump-pointer allocation.  When all blocks are consumed,
-Immix stops the world and traces the object graph, marking objects but
-also the lines that objects are on.  After marking, blocks contain some
-lines with live objects and others that are completely free.  Spans of
-free lines are called holes.  When a mutator gets a recycled block from
-the global block store, it allocates into those holes.  Also, sometimes
-Immix can choose to evacuate rather than mark.  Bump-pointer-into-holes
-allocation is quite compatible with conservative roots, so it's an
-interesting option for Guile, which has a lot of legacy C API users.
+## Documentation
 
-The essential difference of Whippet from Immix stems from a simple
-observation: Immix needs a side table of line mark bytes and also a mark
-bit or bits in each object (or in a side table).  But if instead you
-choose to store mark bytes instead of bits (for concurrency reasons) in
-a side table, with one mark byte per granule (unit of allocation,
-perhaps 16 bytes), then you effectively have a line mark table where the
-granule size is the line size.  You can bump-pointer allocate into holes
-in the mark byte table.
+ * [Design](./doc/design.md): What's the general idea?
+ * [Manual](./doc/manual.md): How do you get your program to use
+   Whippet?  What is the API?
+ * [Guile](./doc/guile.md): Some notes on a potential rebase of Guile on
+   top of Whippet.
 
-You might think this is a bad tradeoff, and perhaps it is: I don't know
-yet.  If your granule size is two pointers, then one mark byte per
-granule is 6.25% overhead on 64-bit, or 12.5% on 32-bit.  Especially on
-32-bit, it's a lot!  On the other hand, instead of the worst case of one
-survivor object wasting a line (or two, in the case of conservative line
-marking), granule-size-is-line-size instead wastes nothing.  Also, you
-don't need GC bits in the object itself, and you can use the mark byte
-array to record the object end, so that finding holes in a block can
-just read the mark table and can avoid looking at object memory.
+## Source repository structure
 
-Other ideas in Whippet:
-
- * Minimize stop-the-world phase via parallel marking and punting all
-   sweeping to mutators
-
- * Enable mutator parallelism via lock-free block acquisition and lazy
-   statistics collation
-
- * Allocate block space using aligned 4 MB slabs, with embedded metadata
-   to allow metadata bytes, slab headers, and block metadata to be
-   located via address arithmetic
-
- * Facilitate conservative collection via mark byte array, oracle for
-   "does this address start an object"
-
- * Enable in-place generational collection via card table with one entry
-   per 256B or so
-
- * Enable concurrent marking by having three mark bit states (dead,
-   survivor, marked) that rotate at each collection, and sweeping a
-   block clears metadata for dead objects; but concurrent marking and
-   associated SATB barrier not yet implemented
-
-## What's there
-
-This repository is a workspace for Whippet implementation.  As such, it
-has files implementing Whippet itself.  It also has some benchmarks to
-use in optimizing Whippet:
-
- - [`mt-gcbench.c`](./mt-gcbench.c): The multi-threaded [GCBench
-   benchmark](https://hboehm.info/gc/gc_bench.html).  An old but
-   standard benchmark that allocates different sizes of binary trees.
-   As parameters it takes a heap multiplier and a number of mutator
-   threads.  We analytically compute the peak amount of live data and
-   then size the GC heap as a multiplier of that size.  It has a peak
-   heap consumption of 10 MB or so per mutator thread: not very large.
-   At a 2x heap multiplier, it causes about 30 collections for the
-   whippet collector, and runs somewhere around 200-400 milliseconds in
-   single-threaded mode, on the machines I have in 2022.  For low thread
-   counts, the GCBench benchmark is small; but then again many Guile
-   processes also are quite short-lived, so perhaps it is useful to
-   ensure that small heaps remain lightweight.
-
-   To stress Whippet's handling of fragmentation, we modified this
-   benchmark to intersperse pseudorandomly-sized holes between tree
-   nodes.
-
- - [`quads.c`](./quads.c): A synthetic benchmark that allocates quad
-   trees.  The mutator begins by allocating one long-lived tree of depth
-   N, and then allocates 13% of the heap in depth-3 trees, 20 times,
-   simulating a fixed working set and otherwise an allocation-heavy
-   workload.  By observing the times to allocate 13% of the heap in
-   garbage we can infer mutator overheads, and also note the variance
-   for the cycles in which GC hits.
-
-The repository has two other collector implementations, to appropriately
-situate Whippet's performance in context:
-
- - `bdw.h`: The external BDW-GC conservative parallel stop-the-world
-   mark-sweep segregated-fits collector with lazy sweeping.
- - `semi.h`: Semispace copying collector.
- - `whippet.h`: The whippet collector.  Two different marking
-   implementations: single-threaded and parallel.  Generational and
-   non-generational variants, also.
-
-## Guile
-
-If the Whippet collector works out, it could replace Guile's garbage
-collector.  Guile currently uses BDW-GC.  Guile has a widely used C API
-and implements part of its run-time in C.  For this reason it may be
-infeasible to require precise enumeration of GC roots -- we may need to
-allow GC roots to be conservatively identified from data sections and
-from stacks.  Such conservative roots would be pinned, but other objects
-can be moved by the collector if it chooses to do so.  We assume that
-object references within a heap object can be precisely identified.
-(However, Guile currently uses BDW-GC in its default configuration,
-which scans for references conservatively even on the heap.)
-
-The existing C API allows direct access to mutable object fields,
-without the mediation of read or write barriers.  Therefore it may be
-impossible to switch to collector strategies that need barriers, such as
-generational or concurrent collectors.  However, we shouldn't write off
-this possibility entirely; an ideal replacement for Guile's GC will
-offer the possibility of migration to other GC designs without imposing
-new requirements on C API users in the initial phase.
-
-In this regard, the Whippet experiment also has the goal of identifying
-a smallish GC abstraction in Guile, so that we might consider evolving
-GC implementation in the future without too much pain.  If we switch
-away from BDW-GC, we should be able to evaluate that it's a win for a
-large majority of use cases.
+ * [api/](./api/): The user-facing API.  Also, the "embedder API"; see
+   the [manual](./doc/manual.md) for more.
+ * [doc/](./doc/): Documentation, such as it is.
+ * [src/](./src/): The actual GC implementation.  The specific
+   implementations of the Whippet API are [`semi.c`](./src/semi.c), a
+   semi-space collector; [`bdw.c`](./src/bdw.c), the third-party
+   [BDW-GC](https://github.com/ivmai/bdwgc) conservative parallel
+   stop-the-world mark-sweep segregated-fits collector with lazy
+   sweeping; and [`whippet.c`](./src/whippet.c), the whippet collector.
+ * [benchmarks/](./benchmarks/): Benchmarks.  A work in progress.
+ * [test/](./test/): A dusty attic of minimal testing.
 
 ## To do
 
@@ -166,14 +64,8 @@ size.  It would be nice if whippet-gc turns out to have this property.
 
 ## License
 
-gcbench.c, MT_GCBench.c, and MT_GCBench2.c are from
-https://hboehm.info/gc/gc_bench/ and have a somewhat unclear license.  I
-have modified GCBench significantly so that I can slot in different GC
-implementations.  The GC implementations themselves are available under
-a MIT-style license, the text of which follows:
-
 ```
-Copyright (c) 2022 Andy Wingo
+Copyright (c) 2022-2023 Andy Wingo
 
 Permission is hereby granted, free of charge, to any person obtaining a
 copy of this software and associated documentation files (the
@@ -194,3 +86,6 @@ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 ```
+
+Note that some benchmarks have other licenses; see
+[`benchmarks/README.md`](./benchmarks/README.md) for more.

benchmarks/README.md

@@ -0,0 +1,35 @@
+# Benchmarks
+
+ - [`mt-gcbench.c`](./mt-gcbench.c): The multi-threaded [GCBench
+   benchmark](https://hboehm.info/gc/gc_bench.html).  An old but
+   standard benchmark that allocates different sizes of binary trees.
+   As parameters it takes a heap multiplier and a number of mutator
+   threads.  We analytically compute the peak amount of live data and
+   then size the GC heap as a multiplier of that size.  It has a peak
+   heap consumption of 10 MB or so per mutator thread: not very large.
+   At a 2x heap multiplier, it causes about 30 collections for the
+   whippet collector, and runs somewhere around 200-400 milliseconds in
+   single-threaded mode, on the machines I have in 2022.  For low thread
+   counts, the GCBench benchmark is small; but then again many Guile
+   processes also are quite short-lived, so perhaps it is useful to
+   ensure that small heaps remain lightweight.
+
+   To stress Whippet's handling of fragmentation, we modified this
+   benchmark to intersperse pseudorandomly-sized holes between tree
+   nodes.
+
+ - [`quads.c`](./quads.c): A synthetic benchmark that allocates quad
+   trees.  The mutator begins by allocating one long-lived tree of depth
+   N, and then allocates 13% of the heap in depth-3 trees, 20 times,
+   simulating a fixed working set and otherwise an allocation-heavy
+   workload.  By observing the times to allocate 13% of the heap in
+   garbage we can infer mutator overheads, and also note the variance
+   for the cycles in which GC hits.
+
+## License
+
+mt-gcbench.c was originally from https://hboehm.info/gc/gc_bench/, which
+has a somewhat unclear license.  I have modified GCBench significantly
+so that I can slot in different GC implementations.  Other files are
+distributed under the Whippet license; see the top-level
+[README.md](../README.md) for more.

doc/design.md

@@ -0,0 +1,64 @@
+# Design
+
+Whippet is mainly a mark-region collector, like
+[Immix](http://users.cecs.anu.edu.au/~steveb/pubs/papers/immix-pldi-2008.pdf).
+See also the lovely detailed [Rust
+implementation](http://users.cecs.anu.edu.au/~steveb/pubs/papers/rust-ismm-2016.pdf).
+
+To a first approximation, Whippet is a whole-heap Immix collector with a
+large object space on the side.  See the Immix paper for full details,
+but basically Immix divides the heap into 32kB blocks, and then divides
+those blocks into 128B lines.  An Immix allocation never spans blocks;
+allocations larger than 8kB go into a separate large object space.
+Mutators request blocks from the global store and allocate into those
+blocks using bump-pointer allocation.  When all blocks are consumed,
+Immix stops the world and traces the object graph, marking objects but
+also the lines that objects are on.  After marking, blocks contain some
+lines with live objects and others that are completely free.  Spans of
+free lines are called holes.  When a mutator gets a recycled block from
+the global block store, it allocates into those holes.  Also, sometimes
+Immix can choose to evacuate rather than mark.  Bump-pointer-into-holes
+allocation is quite compatible with conservative roots, so it's an
+interesting option for Guile, which has a lot of legacy C API users.
+
+The essential difference of Whippet from Immix stems from a simple
+observation: Immix needs a side table of line mark bytes and also a mark
+bit or bits in each object (or in a side table).  But if instead you
+choose to store mark bytes instead of bits (for concurrency reasons) in
+a side table, with one mark byte per granule (unit of allocation,
+perhaps 16 bytes), then you effectively have a line mark table where the
+granule size is the line size.  You can bump-pointer allocate into holes
+in the mark byte table.
+
+You might think this is a bad tradeoff, and perhaps it is: I don't know
+yet.  If your granule size is two pointers, then one mark byte per
+granule is 6.25% overhead on 64-bit, or 12.5% on 32-bit.  Especially on
+32-bit, it's a lot!  On the other hand, instead of the worst case of one
+survivor object wasting a line (or two, in the case of conservative line
+marking), granule-size-is-line-size instead wastes nothing.  Also, you
+don't need GC bits in the object itself, and you can use the mark byte
+array to record the object end, so that finding holes in a block can
+just read the mark table and can avoid looking at object memory.
+
+Other ideas in Whippet:
+
+ * Minimize stop-the-world phase via parallel marking and punting all
+   sweeping to mutators
+
+ * Enable mutator parallelism via lock-free block acquisition and lazy
+   statistics collation
+
+ * Allocate block space using aligned 4 MB slabs, with embedded metadata
+   to allow metadata bytes, slab headers, and block metadata to be
+   located via address arithmetic
+
+ * Facilitate conservative collection via mark byte array, oracle for
+   "does this address start an object"
+
+ * Enable in-place generational collection via card table with one entry
+   per 256B or so
+
+ * Enable concurrent marking by having three mark bit states (dead,
+   survivor, marked) that rotate at each collection, and sweeping a
+   block clears metadata for dead objects; but concurrent marking and
+   associated SATB barrier not yet implemented

doc/guile.md

@@ -0,0 +1,26 @@
+# Whippet and Guile
+
+If the Whippet collector works out, it could replace Guile's garbage
+collector.  Guile currently uses BDW-GC.  Guile has a widely used C API
+and implements part of its run-time in C.  For this reason it may be
+infeasible to require precise enumeration of GC roots -- we may need to
+allow GC roots to be conservatively identified from data sections and
+from stacks.  Such conservative roots would be pinned, but other objects
+can be moved by the collector if it chooses to do so.  We assume that
+object references within a heap object can be precisely identified.
+(However, Guile currently uses BDW-GC in its default configuration,
+which scans for references conservatively even on the heap.)
+
+The existing C API allows direct access to mutable object fields,
+without the mediation of read or write barriers.  Therefore it may be
+impossible to switch to collector strategies that need barriers, such as
+generational or concurrent collectors.  However, we shouldn't write off
+this possibility entirely; an ideal replacement for Guile's GC will
+offer the possibility of migration to other GC designs without imposing
+new requirements on C API users in the initial phase.
+
+In this regard, the Whippet experiment also has the goal of identifying
+a smallish GC abstraction in Guile, so that we might consider evolving
+GC implementation in the future without too much pain.  If we switch
+away from BDW-GC, we should be able to evaluate that it's a win for a
+large majority of use cases.

doc/manual.md

@@ -51,7 +51,7 @@ itself.  This is the *embedder API*, and this document describes what
 Whippet requires from a program.
 
 A program should provide a header file implementing the API in
-[`gc-embedder-api.h`](./gc-embedder-api.h).  This header should only be
+[`gc-embedder-api.h`](../api/gc-embedder-api.h).  This header should only be
 included when compiling Whippet itself; it is not part of the API that
 Whippet exposes to the program.
 
@@ -83,7 +83,7 @@ Most kinds of GC-managed object are defined by the program, but the GC
 itself has support for a specific object kind: ephemerons.  If the
 program allocates ephemerons, it should trace them in the
 `gc_trace_object` function by calling `gc_trace_ephemeron` from
-[`gc-ephemerons.h`](./gc-ephemerons.h).
+[`gc-ephemerons.h`](../api/gc-ephemerons.h).
 
 ### Remembered-set bits
 
@@ -91,7 +91,7 @@ When built to support generational garbage collection, Whippet requires
 that all "large" or potentially large objects have a flag bit reserved
 for use of the garbage collector.  A large object is one whose size
 exceeds the `gc_allocator_large_threshold()` (see
-[`gc-attrs.h`](./gc-attrs.h)), which is a collector-specific value.
+[`gc-attrs.h`](../api/gc-attrs.h)), which is a collector-specific value.
 Currently the only generational collector is the in-place Whippet
 collector, whose large object threshold is 4096 bytes.  The
 `gc_object_set_remembered`, `gc_object_is_remembered_nonatomic`, and
@@ -116,7 +116,7 @@ The atomic API is gnarly.  It is used by parallel collectors, in which
 multiple collector threads can race to evacuate an object.
 
 There is a state machine associated with the `gc_atomic_forward`
-structure from [`gc-forwarding.h`](./gc-forwarding.h); the embedder API
+structure from [`gc-forwarding.h`](../api/gc-forwarding.h); the embedder API
 implements the state changes.  The collector calls
 `gc_atomic_forward_begin` on an object to begin a forwarding attempt,
 and the resulting `gc_atomic_forward` can be in the `NOT_FORWARDED`,
@@ -379,7 +379,7 @@ program?  No, because your program isn't written yet?  Well this section
 is for you: we describe the user-facing API of Whippet, where "user" in
 this case denotes the embedding program.
 
-What is the API, you ask?  It is in [`gc-api.h`](./gc-api.h).
+What is the API, you ask?  It is in [`gc-api.h`](../api/gc-api.h).
 
 ### Heaps and mutators
 
@@ -442,7 +442,7 @@ defined for all collectors:
    processors, with a maximum of 8.
 
 You can set these options via `gc_option_set_int` and so on; see
-[`gc-options.h`](./gc-options.h).  Or, you can parse options from
+[`gc-options.h`](../api/gc-options.h).  Or, you can parse options from
 strings: `heap-size-policy`, `heap-size`, `maximum-heap-size`, and so
 on.  Use `gc_option_from_string` to determine if a string is really an
 option.  Use `gc_option_parse_and_set` to parse a value for an option.
@@ -519,7 +519,7 @@ Whippet supports ephemerons, first-class objects that weakly associate
 keys with values.  If the an ephemeron's key ever becomes unreachable,
 the ephemeron becomes dead and loses its value.
 
-The user-facing API is in [`gc-ephemeron.h`](./gc-ephemeron.h).  To
+The user-facing API is in [`gc-ephemeron.h`](../api/gc-ephemeron.h).  To
 allocate an ephemeron, call `gc_allocate_ephemeron`, then initialize its
 key and value via `gc_ephemeron_init`.  Get the key and value via
 `gc_ephemeron_key` and `gc_ephemeron_value`, respectively.

gc.h

@@ -1,30 +0,0 @@
-#ifndef GC_H_
-#define GC_H_
-
-#include "gc-api.h"
-
-#if defined(GC_BDW)
-#include "bdw.h"
-#elif defined(GC_SEMI)
-#include "semi.h"
-#elif defined(GC_WHIPPET)
-#define GC_PARALLEL_TRACE 0
-#define GC_GENERATIONAL 0
-#include "whippet.h"
-#elif defined(GC_PARALLEL_WHIPPET)
-#define GC_PARALLEL_TRACE 1
-#define GC_GENERATIONAL 0
-#include "whippet.h"
-#elif defined(GC_GENERATIONAL_WHIPPET)
-#define GC_PARALLEL_TRACE 0
-#define GC_GENERATIONAL 1
-#include "whippet.h"
-#elif defined(GC_PARALLEL_GENERATIONAL_WHIPPET)
-#define GC_PARALLEL_TRACE 1
-#define GC_GENERATIONAL 1
-#include "whippet.h"
-#else
-#error unknown gc
-#endif
-
-#endif // GC_H_