perf: keep the emulator out of the qemu-user package for later builds

Co-Authored-By: Claude <noreply@anthropic.com>
Signed-off-by: karo <karolin.kostial@gmail.com>

Author: The127

internal/emulator/pin.go

@@ -15,3 +15,18 @@ var pins = map[string]download.Pin{
 		Digest: "sha256:c73711af02b97cd5e2667e735d9c06890c57165517110ca4e646c95a7083158d",
 	},
 }
+
+// emulators are the digests of the programs in each package, by the
+// builder's architecture and then the architecture a program runs the
+// programs of. A build keeps the one it takes out under its digest, and
+// later builds read it without opening the package again.
+var emulators = map[string]map[string]string{
+	"amd64": {
+		"amd64": "sha256:36d35ffe5363bd49bfa9c2b6a048e9fb2e798fea38285a5a45f52d34f9869089",
+		"arm64": "sha256:73cc2584f119f9e74c85c0a8549dbb6fd104ae3897a06a7500797f9e5b4d0943",
+	},
+	"arm64": {
+		"amd64": "sha256:9af86352fd463ad9ec25bdd6a4b891ac6b6a7f3d46a7db3da6f23e2c09094a59",
+		"arm64": "sha256:088500814a5a8e0e0ee3cad0f6c600b5dfa261f77cc0e6460f81b0b616e09509",
+	},
+}

internal/emulator/ready.go

@@ -1,9 +1,12 @@
 package emulator
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"io"
+	"os"
+	"time"
 
 	"github.com/The127/miso/internal/deb"
 	"github.com/The127/miso/internal/download"
@@ -30,12 +33,16 @@ func Ready(ctx context.Context, store *download.Store, builder, target string) (
 		return nil, fmt.Errorf("miso has no QEMU that runs programs of %s", target)
 	}
 
-	return ready(ctx, store, pin, target)
+	return ready(ctx, store, pin, target, emulators[builder][target])
 }
 
-// ready takes the program that runs the programs of target out of the
-// package a pin names.
-func ready(ctx context.Context, store *download.Store, pin download.Pin, target string) ([]byte, error) {
+// ready is the program that runs the programs of target, the one the store
+// keeps under digest, or else taken out of the package a pin names.
+func ready(ctx context.Context, store *download.Store, pin download.Pin, target, digest string) ([]byte, error) {
+	if kept, err := os.ReadFile(store.Path(digest)); err == nil {
+		return kept, store.Use(time.Now(), digest)
+	}
+
 	pkg, err := store.OpenPinned(ctx, pin)
 	if err != nil {
 		return nil, err
@@ -47,7 +54,24 @@ func ready(ctx context.Context, store *download.Store, pin download.Pin, target
 	files, failed := deb.Files(pkg)
 	for name, file := range files {
 		if name == wanted {
-			return io.ReadAll(file)
+			program, err := io.ReadAll(file)
+			if err != nil {
+				return nil, err
+			}
+
+			got, err := store.Keep(bytes.NewReader(program))
+			if err != nil {
+				return nil, err
+			}
+
+			if got != digest {
+				// a pin on the digest it sits under would later find it
+				_ = store.Remove(got)
+
+				return nil, fmt.Errorf("the package holds %s as %s, pinned to %s", wanted, got, digest)
+			}
+
+			return program, nil
 		}
 	}
 

internal/emulator/ready_test.go

@@ -2,8 +2,12 @@ package emulator_test
 
 import (
 	"context"
+	"crypto/sha256"
+	"encoding/hex"
 	"errors"
 	"net/http"
+	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -26,13 +30,64 @@ func TestTheEmulatorOfAnArm64ImageIsTakenFromThePinnedPackage(t *testing.T) {
 	store := download.Open(dir, http.DefaultClient)
 
 	// act
-	found, err := emulator.ReadyFrom(context.Background(), store, download.Pin{URL: "https://snapshot.invalid/qemu-user.deb", Digest: digest}, "arm64")
+	found, err := emulator.ReadyFrom(context.Background(), store, download.Pin{URL: "https://snapshot.invalid/qemu-user.deb", Digest: digest}, "arm64", digestOf("the emulator"))
 
 	// assert
 	require.NoError(t, err)
 	assert.Equal(t, []byte("the emulator"), found)
 }
 
+func TestAnEmulatorTheStoreKeepsIsReadWithoutThePackage(t *testing.T) {
+	// arrange
+	dir := t.TempDir()
+	digest := downloadtest.Kept(t, dir, []byte("the emulator"))
+	refuse := roundTrip(func(*http.Request) (*http.Response, error) {
+		return nil, errors.New("offline")
+	})
+	store := download.Open(dir, &http.Client{Transport: refuse})
+
+	// act
+	found, err := emulator.ReadyFrom(context.Background(), store, download.Pin{URL: "https://snapshot.invalid/qemu-user.deb", Digest: "sha256:notkept"}, "arm64", digest)
+
+	// assert
+	require.NoError(t, err)
+	assert.Equal(t, []byte("the emulator"), found)
+}
+
+func TestAnEmulatorTakenFromThePackageIsKeptUnderItsDigest(t *testing.T) {
+	// arrange
+	pkg := debtest.Package(t, map[string]string{"./usr/bin/qemu-aarch64": "the emulator"})
+	dir := t.TempDir()
+	digest := downloadtest.Kept(t, dir, pkg)
+	store := download.Open(dir, http.DefaultClient)
+	program := digestOf("the emulator")
+
+	// act
+	_, err := emulator.ReadyFrom(context.Background(), store, download.Pin{URL: "https://snapshot.invalid/qemu-user.deb", Digest: digest}, "arm64", program)
+
+	// assert
+	require.NoError(t, err)
+	kept, err := os.ReadFile(store.Path(program))
+	require.NoError(t, err)
+	assert.Equal(t, []byte("the emulator"), kept)
+}
+
+func TestAnEmulatorOtherThanItsPinIsRefusedAndNotKept(t *testing.T) {
+	// arrange
+	pkg := debtest.Package(t, map[string]string{"./usr/bin/qemu-aarch64": "another emulator"})
+	dir := t.TempDir()
+	digest := downloadtest.Kept(t, dir, pkg)
+	store := download.Open(dir, http.DefaultClient)
+	pinned := digestOf("the emulator")
+
+	// act
+	_, err := emulator.ReadyFrom(context.Background(), store, download.Pin{URL: "https://snapshot.invalid/qemu-user.deb", Digest: digest}, "arm64", pinned)
+
+	// assert
+	require.ErrorContains(t, err, pinned)
+	assert.NoFileExists(t, store.Path(digestOf("another emulator")))
+}
+
 func TestAnArm64BuilderRunsAnAmd64ImageWithQEMUOfItsOwnArchitecture(t *testing.T) {
 	// arrange
 	var asked string
@@ -51,6 +106,44 @@ func TestAnArm64BuilderRunsAnAmd64ImageWithQEMUOfItsOwnArchitecture(t *testing.T
 	assert.Contains(t, asked, "/qemu-user_10.0.13%2Bds-0%2Bdeb13u1_arm64.deb")
 }
 
+func TestAnArm64BuilderReadsTheAmd64EmulatorItKeptWithoutThePackage(t *testing.T) {
+	// arrange
+	dir := t.TempDir()
+	refuse := roundTrip(func(*http.Request) (*http.Response, error) {
+		return nil, errors.New("offline")
+	})
+	store := download.Open(dir, &http.Client{Transport: refuse})
+	kept := store.Path("sha256:9af86352fd463ad9ec25bdd6a4b891ac6b6a7f3d46a7db3da6f23e2c09094a59")
+	require.NoError(t, os.MkdirAll(filepath.Dir(kept), 0o750))
+	require.NoError(t, os.WriteFile(kept, []byte("the emulator"), 0o600))
+
+	// act
+	found, err := emulator.Ready(context.Background(), store, "arm64", "amd64")
+
+	// assert
+	require.NoError(t, err)
+	assert.Equal(t, []byte("the emulator"), found)
+}
+
+func TestAnAmd64BuilderReadsTheArm64EmulatorItKeptWithoutThePackage(t *testing.T) {
+	// arrange
+	dir := t.TempDir()
+	refuse := roundTrip(func(*http.Request) (*http.Response, error) {
+		return nil, errors.New("offline")
+	})
+	store := download.Open(dir, &http.Client{Transport: refuse})
+	kept := store.Path("sha256:73cc2584f119f9e74c85c0a8549dbb6fd104ae3897a06a7500797f9e5b4d0943")
+	require.NoError(t, os.MkdirAll(filepath.Dir(kept), 0o750))
+	require.NoError(t, os.WriteFile(kept, []byte("the emulator"), 0o600))
+
+	// act
+	found, err := emulator.Ready(context.Background(), store, "amd64", "arm64")
+
+	// assert
+	require.NoError(t, err)
+	assert.Equal(t, []byte("the emulator"), found)
+}
+
 func TestAnEmulatorForABuilderMisoHasNoQEMUForIsRefusedNamingIt(t *testing.T) {
 	// arrange
 	store := download.Open(t.TempDir(), http.DefaultClient)
@@ -86,3 +179,9 @@ type roundTrip func(*http.Request) (*http.Response, error)
 func (f roundTrip) RoundTrip(request *http.Request) (*http.Response, error) {
 	return f(request)
 }
+
+func digestOf(content string) string {
+	sum := sha256.Sum256([]byte(content))
+
+	return "sha256:" + hex.EncodeToString(sum[:])
+}