runtime: reduce typed Boehm allocation overhead

Canonicalize all-pointer layouts so Boehm can use conservative allocation
when it is exact. Bypass trivial allocation wrappers and skip typed-array
setup for scalar objects.

Author: jakebailey

compiler/llvm.go

@@ -252,14 +252,30 @@ func (c *compilerContext) createObjectLayout(t llvm.Type, pos token.Pos) llvm.Va
 	// Create the pointer bitmap.
 	objectSizeBytes := c.targetData.TypeAllocSize(t)
 	pointerAlignment := uint64(c.targetData.PrefTypeAlignment(c.dataPtrType))
+	pointerSize := c.targetData.TypeAllocSize(c.dataPtrType)
+	pointerBits := pointerSize * 8
 	bitmapLen := objectSizeBytes / pointerAlignment
 	bitmapBytes := (bitmapLen + 7) / 8
 	bitmap := make([]byte, bitmapBytes, max(bitmapBytes, 8))
 	c.buildPointerBitmap(bitmap, pointerAlignment, pos, t, 0)
 
+	// Use the one-pointer layout when every pointer-sized word is a pointer.
+	// Repeating this layout is equivalent for objects of any size.
+	pointerUnits := pointerSize / pointerAlignment
+	allPointers := bitmapLen != 0 && bitmapLen%pointerUnits == 0
+	for i := uint64(0); allPointers && i < bitmapLen; i++ {
+		isPointer := bitmap[i/8]&(1<<(i%8)) != 0
+		if isPointer != (i%pointerUnits == 0) {
+			allPointers = false
+		}
+	}
+	if allPointers {
+		layout := (pointerBits + pointerUnits) << 1
+		layout |= 1
+		return llvm.ConstIntToPtr(llvm.ConstInt(c.uintptrType, layout, false), c.dataPtrType)
+	}
+
 	// Try to encode the layout inline.
-	pointerSize := c.targetData.TypeAllocSize(c.dataPtrType)
-	pointerBits := pointerSize * 8
 	if bitmapLen < pointerBits {
 		rawMask := binary.LittleEndian.Uint64(bitmap[0:8])
 		layout := rawMask*pointerBits + bitmapLen

compiler/testdata/gc.ll

@@ -100,7 +100,7 @@ declare noalias nonnull ptr @runtime.alloc_zero(i32, ptr, ptr) #2
 define hidden ptr @main.newFuncValue(ptr %context) unnamed_addr #1 {
 entry:
   %stackalloc = alloca i8, align 1
-  %new = call align 4 dereferenceable(8) ptr @runtime.alloc(i32 8, ptr nonnull inttoptr (i32 197 to ptr), ptr undef) #3
+  %new = call align 4 dereferenceable(8) ptr @runtime.alloc(i32 8, ptr nonnull inttoptr (i32 67 to ptr), ptr undef) #3
   call void @runtime.trackPointer(ptr nonnull %new, ptr nonnull %stackalloc, ptr undef) #3
   ret ptr %new
 }

src/runtime/gc_boehm.go

@@ -27,6 +27,10 @@ import (
 const needsStaticHeap = false
 
 const boehmLayoutSizeBits = 4 + unsafe.Sizeof(uintptr(0))/4
+const (
+	boehmPtrFreeKind = 0
+	boehmNormalKind  = 1
+)
 
 var gcLock task.PMutex
 
@@ -77,35 +81,43 @@ func alloc(size uintptr, layout unsafe.Pointer) unsafe.Pointer {
 		// This object is entirely pointer free, for example make([]int, ...).
 		// Make sure the GC knows this so it doesn't scan the object
 		// unnecessarily to improve performance.
-		ptr = libgc_malloc_atomic(size)
-		// Memory returned from libgc_malloc_atomic has not been zeroed so we
-		// have to do that manually.
+		ptr = libgc_malloc_kind(size, boehmPtrFreeKind)
 		needsZero = true
 	case gclayout.Conservative.AsPtr():
 		// Stack storage does not have an ordinary repeating Go object layout.
-		ptr = libgc_malloc(size)
+		ptr = libgc_malloc_kind(size, boehmNormalKind)
+	case gclayout.Pointer.AsPtr(), gclayout.PointerPair.AsPtr():
+		// Conservative scanning is exact when every word is a pointer.
+		ptr = libgc_malloc_kind(size, boehmNormalKind)
 	default:
 		elementWords := boehmLayoutElementWords(layout)
-		pointerSize := unsafe.Sizeof(uintptr(0))
-		if elementWords == 0 || elementWords > size/pointerSize {
+		pointerAlign := unsafe.Alignof(uintptr(0))
+		if elementWords == 0 || elementWords > size/pointerAlign {
 			// This should not happen for compiler-generated Go allocations.
-			ptr = libgc_malloc(size)
+			ptr = libgc_malloc_kind(size, boehmNormalKind)
 			break
 		}
-		elementSize := elementWords * pointerSize
+		elementSize := elementWords * pointerAlign
 		if size%elementSize != 0 {
-			ptr = libgc_malloc(size)
+			ptr = libgc_malloc_kind(size, boehmNormalKind)
 			break
 		}
 
 		descriptor := libgc_make_descriptor(uintptr(layout))
 		if descriptor == 0 {
-			// Descriptor construction can fail under memory pressure. A
-			// conservative allocation remains correct in that case.
-			ptr = libgc_malloc(size)
+			// The bridge returns zero if its cache or bitmap allocation fails.
+			// It also rejects the no-pointer descriptor, which is handled above.
+			ptr = libgc_malloc_kind(size, boehmNormalKind)
 			break
 		}
-		ptr = libgc_calloc_explicitly_typed(size/elementSize, elementSize, descriptor)
+		elementCount := size / elementSize
+		if elementCount == 1 {
+			ptr = libgc_malloc_explicitly_typed(size, descriptor)
+		} else {
+			ptr = libgc_calloc_explicitly_typed(
+				elementCount, elementSize, descriptor,
+			)
+		}
 	}
 	gcResumeWorld()
 	gcLock.Unlock()
@@ -116,7 +128,6 @@ func alloc(size uintptr, layout unsafe.Pointer) unsafe.Pointer {
 	if needsZero {
 		memzero(ptr, size)
 	}
-
 	return ptr
 }
 
@@ -202,11 +213,8 @@ func SetFinalizer(obj interface{}, finalizer interface{}) {
 //export GC_init
 func libgc_init()
 
-//export GC_malloc
-func libgc_malloc(uintptr) unsafe.Pointer
-
-//export GC_malloc_atomic
-func libgc_malloc_atomic(uintptr) unsafe.Pointer
+//export GC_malloc_kind
+func libgc_malloc_kind(uintptr, int32) unsafe.Pointer
 
 //export GC_malloc_atomic_uncollectable
 func libgc_malloc_atomic_uncollectable(uintptr) unsafe.Pointer
@@ -217,6 +225,9 @@ func libgc_make_descriptor(uintptr) uintptr
 //export GC_calloc_explicitly_typed
 func libgc_calloc_explicitly_typed(uintptr, uintptr, uintptr) unsafe.Pointer
 
+//export GC_malloc_explicitly_typed
+func libgc_malloc_explicitly_typed(uintptr, uintptr) unsafe.Pointer
+
 //export GC_free
 func libgc_free(unsafe.Pointer)