runtime: preserve malloc allocations until free

C malloc storage has explicit lifetime: it must remain allocated until
free even when no GC-visible pointer references it. Treating it as an
ordinary NoPtrs allocation breaks bare-metal C object graphs, while
conservatively scanning arbitrary C bytes creates false Go roots.

Add allocManual/freeManual so collectors can represent pointer-free,
explicitly managed storage. Block GC keeps these objects permanently
marked and releases their blocks on free; Boehm uses atomic uncollectable
allocations; leaking and custom collectors provide equivalent behavior.
Wasm keeps its allocation map only for validation and sizes, and WASIp2
realloc now copies min(oldSize, newSize).

Bump the Boehm library cache version because enabling atomic
uncollectable allocations changes its compiled flags and exported API.

Also handle zero-size and overflowing allocations, serialize allocation
registries, reject Go finalizers on manual storage, and add CGo regressions
for C pointer graphs, hidden until-free allocations, repeated free/reuse,
and allocation edge cases.

Author: jakebailey

CHANGELOG.md

@@ -84,6 +84,7 @@
   - gc: correct the old size calculation in block realloc
   - gc: correct the leaking allocator bounds and overflow checks
   - gc: move objHeader to the end of the block header
+  - cgo: keep malloc allocations alive until free and stop the program for invalid or repeated free calls
   - fix the leaking GC build with the cores scheduler
   - rp2040: fix -gc=leaking and -gc=none
   - rp2: handle the RP2350 shared FIFO IRQ for GC (#5482)

builder/bdwgc.go

@@ -30,10 +30,11 @@ var BoehmGC = Library{
 			// Use a minimal environment.
 			"-DNO_MSGBOX_ON_ERROR", // don't call MessageBoxA on Windows
 			"-DDONT_USE_ATEXIT",
-			"-DNO_GETENV",          // smaller binary, more predictable configuration
-			"-DNO_CLOCK",           // don't use system clock
-			"-DNO_DEBUGGING",       // reduce code size
-			"-DGC_NO_FINALIZATION", // finalization is not used at the moment
+			"-DNO_GETENV",               // smaller binary, more predictable configuration
+			"-DNO_CLOCK",                // don't use system clock
+			"-DNO_DEBUGGING",            // reduce code size
+			"-DGC_NO_FINALIZATION",      // finalization is not used at the moment
+			"-DGC_ATOMIC_UNCOLLECTABLE", // pointer-free storage retained until GC_free
 
 			// Special flag to work around the lack of __data_start in ld.lld.
 			// TODO: try to fix this in LLVM/lld directly so we don't have to

builder/testdata/binary-size.txt

@@ -1,4 +1,4 @@
 target      package               code rodata data  bss
-hifive1b    examples/echo         4405    323    0 2268
-microbit    examples/serial       2922    382    8 2264
-wioterminal examples/pininterrupt 8251   1717  148 7496
+hifive1b    examples/echo         4533    323    0 2268
+microbit    examples/serial       3002    382    8 2264
+wioterminal examples/pininterrupt 8331   1717  148 7496

compileopts/config.go

@@ -24,7 +24,7 @@ import (
 // library path in advance in several places).
 var libVersions = map[string]int{
 	"musl":         3,
-	"bdwgc":        2,
+	"bdwgc":        3,
 	"picolibc":     2,
 	"wasmbuiltins": 1,
 }

main_test.go

@@ -114,7 +114,13 @@ func TestBuild(t *testing.T) {
 		// This makes it possible to run one specific test (instead of all),
 		// which is especially useful to quickly check whether some changes
 		// affect a particular target architecture.
-		runPlatTests(optionsFromTarget(*testTarget, sema), tests, t)
+		options := optionsFromTarget(*testTarget, sema)
+		runPlatTests(options, tests, t)
+		if *testTarget == "wasip1" {
+			t.Run("cgo-realloc", func(t *testing.T) {
+				runTest("cgo-realloc/", options, t, nil, nil)
+			})
+		}
 		return
 	}
 
@@ -223,7 +229,11 @@ func TestBuild(t *testing.T) {
 		})
 		t.Run("WASIp1", func(t *testing.T) {
 			t.Parallel()
-			runPlatTests(optionsFromTarget("wasip1", sema), tests, t)
+			options := optionsFromTarget("wasip1", sema)
+			runPlatTests(options, tests, t)
+			t.Run("cgo-realloc", func(t *testing.T) {
+				runTest("cgo-realloc/", options, t, nil, nil)
+			})
 
 			// Test with -gc=boehm.
 			t.Run("gc.go-boehm", func(t *testing.T) {

src/runtime/arch_tinygowasm_malloc.go

@@ -3,29 +3,29 @@
 package runtime
 
 import (
-	"internal/gclayout"
+	"internal/task"
 	"unsafe"
 )
 
 // The below functions override the default allocator of wasi-libc. This ensures
 // code linked from other languages can allocate memory without colliding with
 // our GC allocations.
 
-// Map of allocations, where the key is the allocated pointer and the value is
-// the size of the allocation.
-// TODO: make this a map[unsafe.Pointer]uintptr, since that results in slightly
-// smaller binaries. But for that to work, unsafe.Pointer needs to be seen as a
-// binary key (which it is not at the moment).
-// See https://github.com/tinygo-org/tinygo/pull/4898 for details.
-var allocs = make(map[*byte]uintptr)
+// Map of allocations, where the key is the allocation address and the value is
+// its size. Integer keys intentionally do not act as GC roots: manual
+// allocations are retained by the allocator until free.
+var allocs = make(map[uintptr]uintptr)
+var allocsLock task.PMutex
 
 //export malloc
 func libc_malloc(size uintptr) unsafe.Pointer {
 	if size == 0 {
 		return nil
 	}
-	ptr := alloc(size, gclayout.NoPtrs.AsPtr())
-	allocs[(*byte)(ptr)] = size
+	ptr := allocManual(size)
+	allocsLock.Lock()
+	allocs[uintptr(ptr)] = size
+	allocsLock.Unlock()
 	return ptr
 }
 
@@ -34,16 +34,22 @@ func libc_free(ptr unsafe.Pointer) {
 	if ptr == nil {
 		return
 	}
-	if _, ok := allocs[(*byte)(ptr)]; ok {
-		delete(allocs, (*byte)(ptr))
+	allocsLock.Lock()
+	if _, ok := allocs[uintptr(ptr)]; ok {
+		delete(allocs, uintptr(ptr))
+		allocsLock.Unlock()
+		freeManual(ptr)
 	} else {
+		allocsLock.Unlock()
 		runtimeFatal("free: invalid pointer")
 	}
 }
 
 //export calloc
 func libc_calloc(nmemb, size uintptr) unsafe.Pointer {
-	// No difference between calloc and malloc.
+	if size != 0 && nmemb > ^uintptr(0)/size {
+		return nil
+	}
 	return libc_malloc(nmemb * size)
 }
 
@@ -54,22 +60,37 @@ func libc_realloc(oldPtr unsafe.Pointer, size uintptr) unsafe.Pointer {
 		return nil
 	}
 
+	var oldSize uintptr
+	if oldPtr != nil {
+		allocsLock.Lock()
+		var ok bool
+		oldSize, ok = allocs[uintptr(oldPtr)]
+		allocsLock.Unlock()
+		if !ok {
+			runtimeFatal("realloc: invalid pointer")
+		}
+	}
+
 	// It's hard to optimize this to expand the current buffer with our GC, but
 	// it is theoretically possible. For now, just always allocate fresh.
 	// TODO: we could skip this if the new allocation is smaller than the old.
-	ptr := alloc(size, gclayout.NoPtrs.AsPtr())
+	ptr := allocManual(size)
 
+	allocsLock.Lock()
 	if oldPtr != nil {
-		if oldSize, ok := allocs[(*byte)(oldPtr)]; ok {
-			oldBuf := unsafe.Slice((*byte)(oldPtr), oldSize)
-			newBuf := unsafe.Slice((*byte)(ptr), size)
-			copy(newBuf, oldBuf)
-			delete(allocs, (*byte)(oldPtr))
-		} else {
+		if currentSize, ok := allocs[uintptr(oldPtr)]; !ok || currentSize != oldSize {
+			allocsLock.Unlock()
 			runtimeFatal("realloc: invalid pointer")
 		}
+		oldBuf := unsafe.Slice((*byte)(oldPtr), oldSize)
+		newBuf := unsafe.Slice((*byte)(ptr), size)
+		copy(newBuf, oldBuf)
+		delete(allocs, uintptr(oldPtr))
+	}
+	allocs[uintptr(ptr)] = size
+	allocsLock.Unlock()
+	if oldPtr != nil {
+		freeManual(oldPtr)
 	}
-
-	allocs[(*byte)(ptr)] = size
 	return ptr
 }

src/runtime/baremetal.go

@@ -3,7 +3,6 @@
 package runtime
 
 import (
-	"internal/gclayout"
 	"sync/atomic"
 	"unsafe"
 )
@@ -12,18 +11,20 @@ import (
 func libc_malloc(size uintptr) unsafe.Pointer {
 	// Note: this zeroes the returned buffer which is not necessary.
 	// The same goes for bytealg.MakeNoZero.
-	return alloc(size, gclayout.NoPtrs.AsPtr())
+	return allocManual(size)
 }
 
 //export calloc
 func libc_calloc(nmemb, size uintptr) unsafe.Pointer {
-	// No difference between calloc and malloc.
+	if size != 0 && nmemb > ^uintptr(0)/size {
+		return nil
+	}
 	return libc_malloc(nmemb * size)
 }
 
 //export free
 func libc_free(ptr unsafe.Pointer) {
-	free(ptr)
+	freeManual(ptr)
 }
 
 //export runtime_putchar

src/runtime/gc_blocks.go

@@ -205,8 +205,9 @@ func (b gcBlock) free() {
 
 // objHeader is a structure appended to every heap object to hold metadata.
 type objHeader struct {
-	// next is the next object to scan after this.
-	next *objHeader
+	// next links the GC scan list. Manual allocations remain permanently marked
+	// and use the otherwise invalid value 1 as an until-free marker.
+	next uintptr
 
 	// layout holds the layout bitmap used to find pointers in the object.
 	layout gcLayout
@@ -482,6 +483,7 @@ func alloc(size uintptr, layout unsafe.Pointer) unsafe.Pointer {
 	// Create the object header.
 	size -= unsafe.Sizeof(objHeader{})
 	header := (*objHeader)(unsafe.Add(pointer, size))
+	header.next = 0
 	header.layout = parseGCLayout(layout)
 
 	// We've claimed this allocation, now we can unlock the heap.
@@ -500,42 +502,61 @@ func alloc(size uintptr, layout unsafe.Pointer) unsafe.Pointer {
 	return pointer
 }
 
-func realloc(ptr unsafe.Pointer, size uintptr) unsafe.Pointer {
-	if ptr == nil {
-		return alloc(size, gclayout.NoPtrs.AsPtr())
+// allocManual allocates pointer-free memory that remains live until freeManual.
+func allocManual(size uintptr) unsafe.Pointer {
+	if size == 0 {
+		return alloc_zero(size, gclayout.NoPtrs.AsPtr())
 	}
+	ptr := alloc(size, gclayout.NoPtrs.AsPtr())
 
-	// Find the first block of the original allocation.
-	firstBlock := blockFromAddr(uintptr(ptr))
-
-	// Find the last block of the original allocation.
-	lastBlock := firstBlock.findHead()
+	gcLock.Lock()
+	head := blockFromAddr(uintptr(ptr)).findHead()
+	head.setState(blockStateMark)
+	header := (*objHeader)(unsafe.Add(head.pointer(), bytesPerBlock-unsafe.Sizeof(objHeader{})))
+	header.next = 1
+	gcLock.Unlock()
+	return ptr
+}
 
-	// Calculate the size of the original allocation body.
-	oldSize := uintptr(lastBlock-firstBlock)*bytesPerBlock + (bytesPerBlock - unsafe.Sizeof(objHeader{}))
+func free(ptr unsafe.Pointer) {
+	if ptr == nil {
+		return
+	}
 
-	if size <= oldSize {
-		// The requested size is less than the old size.
-		// There are likely scenarios for this:
-		//  - The caller intended to grow the allocation, but the original size
-		//    was rounded up by alloc to a multiple of the block size.
-		//    The rounded size is already sufficient.
-		//  - The caller intended to shrink the allocation.
-		//    We currently ignore this case.
-		// Either way, the current allocation can be left alone.
-		return ptr
+	gcLock.Lock()
+	addr := uintptr(ptr)
+	if !isOnHeap(addr) || (addr-heapStart)%bytesPerBlock != 0 {
+		gcLock.Unlock()
+		runtimeFatal("free: invalid pointer")
 	}
 
-	// Create a new allocation and copy the old data.
-	newAlloc := alloc(size, gclayout.NoPtrs.AsPtr())
-	memcpy(newAlloc, ptr, oldSize)
-	free(ptr)
+	firstBlock := blockFromAddr(addr)
+	state := firstBlock.state()
+	if state != blockStateTail && state != blockStateHead && state != blockStateMark {
+		gcLock.Unlock()
+		runtimeFatal("free: invalid pointer")
+	}
 
-	return newAlloc
-}
+	allocationStart := firstBlock
+	for allocationStart != 0 && (allocationStart-1).state() == blockStateTail {
+		allocationStart--
+	}
+	if allocationStart != firstBlock {
+		gcLock.Unlock()
+		runtimeFatal("free: invalid pointer")
+	}
 
-func free(ptr unsafe.Pointer) {
-	// TODO: free blocks on request, when the compiler knows they're unused.
+	lastBlock := firstBlock.findHead()
+	header := (*objHeader)(unsafe.Add(lastBlock.pointer(), bytesPerBlock-unsafe.Sizeof(objHeader{})))
+	if header.next != 1 {
+		gcLock.Unlock()
+		runtimeFatal("free: invalid pointer")
+	}
+	for block := firstBlock; block <= lastBlock; block++ {
+		block.free()
+	}
+	insertFreeRange(firstBlock.pointer(), uintptr(lastBlock-firstBlock+1))
+	gcLock.Unlock()
 }
 
 // GC performs a garbage collection cycle.
@@ -666,7 +687,7 @@ func finishMark() {
 		if obj == nil {
 			return
 		}
-		scanList = obj.next
+		scanList = (*objHeader)(unsafe.Pointer(obj.next))
 
 		// Check if the object may contain pointers.
 		if obj.layout.pointerFree() {
@@ -724,7 +745,7 @@ func markRoot(addr, root uintptr) {
 
 	// Add the object to the scan list.
 	header := (*objHeader)(unsafe.Add(head.pointer(), bytesPerBlock-unsafe.Sizeof(objHeader{})))
-	header.next = scanList
+	header.next = uintptr(unsafe.Pointer(scanList))
 	scanList = header
 }
 
@@ -758,7 +779,10 @@ func sweep() uintptr {
 
 		// Unmark the next head.
 		block--
-		block.unmark()
+		header := (*objHeader)(unsafe.Add(block.pointer(), bytesPerBlock-unsafe.Sizeof(objHeader{})))
+		if header.next != 1 {
+			block.unmark()
+		}
 
 		// Skip the tail.
 		for block > 0 && (block-1).state() == blockStateTail {
@@ -903,5 +927,19 @@ func SetFinalizer(obj interface{}, finalizer interface{}) {
 		// A nil pointer has nothing to finalize.
 		return
 	}
+
+	gcLock.Lock()
+	addr := uintptr(objPtr)
+	manual := false
+	if isOnHeap(addr) {
+		head := blockFromAddr(addr).findHead()
+		header := (*objHeader)(unsafe.Add(head.pointer(), bytesPerBlock-unsafe.Sizeof(objHeader{})))
+		manual = header.next == 1
+	}
+	gcLock.Unlock()
+	if manual && finalizer != nil {
+		runtimeFatal("runtime.SetFinalizer: manual allocation")
+	}
+
 	registerFinalizer(uintptr(objPtr), finalizer)
 }