diff --git a/.goreleaser.yaml b/.goreleaser.yaml index add46e24..9272afe5 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -46,9 +46,13 @@ nfpms: deb: dependencies: - qemu-system-x86 + recommends: + - qemu-system-arm rpm: dependencies: - qemu-system-x86-core + recommends: + - qemu-system-aarch64-core - id: arm64 ids: [arm64] package_name: miso @@ -66,9 +70,13 @@ nfpms: deb: dependencies: - qemu-system-arm + recommends: + - qemu-system-x86 rpm: dependencies: - qemu-system-aarch64-core + recommends: + - qemu-system-x86-core checksum: name_template: checksums.txt diff --git a/arch-go.yml b/arch-go.yml index 64272821..daddccd2 100644 --- a/arch-go.yml +++ b/arch-go.yml @@ -452,6 +452,20 @@ dependenciesRules: - os/exec - net - syscall + # the emulator takes miso's pinned qemu-user package apart into what runs + # the programs of an image of another architecture. It has the download + # fetch the package, and never starts a process, opens a connection or + # touches the kernel + - package: '**.internal.emulator$' + shouldOnlyDependsOn: + internal: + - '**.internal.deb$' + - '**.internal.download$' + shouldNotDependsOn: + standard: + - os/exec + - net + - syscall # the builder kernel takes miso's pinned Debian package apart into the # kernel the builder VM boots and the modules its agent loads. It has the # download fetch the package, and never starts a process, opens a diff --git a/cmd/miso/arch_test.go b/cmd/miso/arch_test.go index 27a73fc2..f9d8fae3 100644 --- a/cmd/miso/arch_test.go +++ b/cmd/miso/arch_test.go @@ -1,12 +1,16 @@ package main_test import ( + "context" + "errors" + "net/http" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" main "github.com/The127/miso/cmd/miso" + "github.com/The127/miso/internal/download" ) func TestAHostMisoDoesNotBuildOnIsRefusedNamingItAndWhereMisoBuilds(t *testing.T) { @@ -18,3 +22,25 @@ func TestAHostMisoDoesNotBuildOnIsRefusedNamingItAndWhereMisoBuilds(t *testing.T assert.ErrorContains(t, err, "riscv64") assert.ErrorContains(t, err, "amd64 and arm64") } + +func TestABuildForTheBuildersOwnArchitectureFetchesNoEmulator(t *testing.T) { + // arrange + offline := &http.Client{Transport: roundTrip(func(*http.Request) (*http.Response, error) { + return nil, errors.New("offline") + })} + store := download.Open(t.TempDir(), offline) + + // act + emulators, err := main.EmulatorsFor(context.Background(), store, main.BuilderArch) + + // assert + require.NoError(t, err) + assert.Empty(t, emulators) +} + +// roundTrip is an HTTP transport that is a function. +type roundTrip func(*http.Request) (*http.Response, error) + +func (f roundTrip) RoundTrip(request *http.Request) (*http.Response, error) { + return f(request) +} diff --git a/cmd/miso/build.go b/cmd/miso/build.go index d440ea13..c20a4d29 100644 --- a/cmd/miso/build.go +++ b/cmd/miso/build.go @@ -2,7 +2,6 @@ package main import ( "context" - "fmt" "io" "os" @@ -22,14 +21,14 @@ var buildCommand = &cli.Command{ Flags: []cli.Flag{ fileFlag(), outputFlag(), - &cli.StringFlag{Name: "arch", Usage: "the architecture the image is built for", Value: builderArch}, + archFlag(), }, Action: runBuild, } func runBuild(ctx context.Context, command *cli.Command) error { - if arch := command.String("arch"); arch != builderArch { - return fmt.Errorf("--arch %q is not an architecture miso builds for, only %s", arch, builderArch) + if err := archRefusal(command); err != nil { + return err } return withBuilder(ctx, command, build.Requests, func(have prepared, boot booter) error { @@ -49,7 +48,7 @@ func runBuild(ctx context.Context, command *cli.Command) error { defer func() { _ = os.RemoveAll(scratch) }() - outputs, err := outputsOf(command, checker(ctx, have.blobs, string(have.dir), console), scratch) + outputs, err := outputsOf(command, checker(ctx, have.blobs, string(have.dir), console, targetArch(command)), scratch) if err != nil { return err } diff --git a/cmd/miso/cache.go b/cmd/miso/cache.go index 70e10682..d6cc3206 100644 --- a/cmd/miso/cache.go +++ b/cmd/miso/cache.go @@ -25,16 +25,16 @@ func cacheDir() (string, error) { return filepath.Join(dir, "miso"), nil } -// baseImages are the base images miso keeps in its cache for the builder's +// baseImages are the base images miso keeps in its cache for an // architecture, and the store their bytes live in. -func baseImages(cache string) (*download.Store, *baseimage.Cache, error) { +func baseImages(cache, arch string) (*download.Store, *baseimage.Cache, error) { if err := hostRefusal(builderArch); err != nil { return nil, nil, err } blobs := download.Open(filepath.Join(cache, "bases"), http.DefaultClient) - return blobs, baseimage.OpenFor(filepath.Join(cache, "bases"), builderArch, blobs), nil + return blobs, baseimage.OpenFor(filepath.Join(cache, "bases"), arch, blobs), nil } // cacheDiskSize is how large the cache disk may grow. The file is sparse, diff --git a/cmd/miso/checks.go b/cmd/miso/checks.go index 9eb6209f..50979bcd 100644 --- a/cmd/miso/checks.go +++ b/cmd/miso/checks.go @@ -24,8 +24,9 @@ const bootPatience = 5 * time.Minute // checker boots each checked output on a vsock of its own and runs its checks: // a rootfs with its kernel and initrd, any other output under UEFI. The boots -// write their scratch into dir and their console to console. -func checker(ctx context.Context, blobs *download.Store, dir string, console io.Writer) builder.Check { +// write their scratch into dir and their console to console. The outputs are +// of arch. +func checker(ctx context.Context, blobs *download.Store, dir string, console io.Writer, arch string) builder.Check { return func(disk string, request build.Request) error { namespace, err := vsockns.Open() if errors.Is(err, vsockns.ErrNotPrivate) { @@ -52,7 +53,7 @@ func checker(ctx context.Context, blobs *download.Store, dir string, console io. Dir: scratch, Console: console, Patience: bootPatience, - Arch: builderArch, + Arch: arch, } if err := bootsFrom(ctx, blobs, disk, request, &boot); err != nil { @@ -78,7 +79,7 @@ func checker(ctx context.Context, blobs *download.Store, dir string, console io. // a rootfs, the firmware for any other output. func bootsFrom(ctx context.Context, blobs *download.Store, disk string, request build.Request, boot *check.Boot) error { if request.Boot == nil { - found, err := firmware.Ready(ctx, blobs, builderArch) + found, err := firmware.Ready(ctx, blobs, boot.Arch) if err != nil { return err } diff --git a/cmd/miso/export_test.go b/cmd/miso/export_test.go index 6a00d840..13396fa2 100644 --- a/cmd/miso/export_test.go +++ b/cmd/miso/export_test.go @@ -5,3 +5,18 @@ var CachedKeys = cachedKeys // HostRefusal is hostRefusal for the tests. var HostRefusal = hostRefusal + +// EmulatorsFor is emulatorsFor for the tests. +var EmulatorsFor = emulatorsFor + +// BuilderArch is builderArch for the tests. +const BuilderArch = builderArch + +// WithShellHint is withShellHint for the tests. +var WithShellHint = withShellHint + +// FileFlag is fileFlag for the tests. +var FileFlag = fileFlag + +// ArchFlag is archFlag for the tests. +var ArchFlag = archFlag diff --git a/cmd/miso/plan.go b/cmd/miso/plan.go index 1a9da721..3c2993ab 100644 --- a/cmd/miso/plan.go +++ b/cmd/miso/plan.go @@ -2,8 +2,11 @@ package main import ( "context" + "fmt" "os" "path/filepath" + "slices" + "strings" "github.com/urfave/cli/v3" @@ -19,17 +22,22 @@ var planCommand = &cli.Command{ ArgsUsage: "[context]", Flags: []cli.Flag{ fileFlag(), + archFlag(), }, Action: listPlan, } func listPlan(ctx context.Context, command *cli.Command) error { + if err := archRefusal(command); err != nil { + return err + } + cache, err := cacheDir() if err != nil { return err } - blobs, bases, err := baseImages(cache) + blobs, bases, err := baseImages(cache, targetArch(command)) if err != nil { return err } @@ -65,7 +73,7 @@ func planOf(command *cli.Command, files *buildcontext.Dir, bases plan.Bases) (pl return plan.Plan{}, err } - planned, err := plan.New(stages, agentName(), builderArch, files, bases) + planned, err := plan.New(stages, agentName(), targetArch(command), files, bases) if err != nil { return plan.Plan{}, imagefile.InFile(file, err) } @@ -73,6 +81,27 @@ func planOf(command *cli.Command, files *buildcontext.Dir, bases plan.Bases) (pl return planned, nil } +// archFlag names the architecture a command builds for. Each command gets +// its own, since a flag keeps what was parsed into it. +func archFlag() *cli.StringFlag { + return &cli.StringFlag{Name: "arch", Usage: "the architecture the image is built for", Value: builderArch} +} + +// archRefusal is why miso cannot build for the architecture the command +// names, or nil when it can. +func archRefusal(command *cli.Command) error { + if arch := command.String("arch"); !slices.Contains(arches, arch) { + return fmt.Errorf("--arch %q is not an architecture miso builds for, only %s", arch, strings.Join(arches, " and ")) + } + + return nil +} + +// targetArch is the architecture the command builds for. +func targetArch(command *cli.Command) string { + return command.String("arch") +} + // fileFlagName is the flag that names the build file. const fileFlagName = "file" diff --git a/cmd/miso/prune.go b/cmd/miso/prune.go index 5ed9502e..0ace52f0 100644 --- a/cmd/miso/prune.go +++ b/cmd/miso/prune.go @@ -41,7 +41,7 @@ func runPrune(ctx context.Context, command *cli.Command) error { return err } - blobs, _, err := baseImages(cache) + blobs, _, err := baseImages(cache, builderArch) if err != nil { return err } @@ -77,7 +77,7 @@ func pruneLayers(ctx context.Context, command *cli.Command, dir cachedisk.Dir, b // askBuilder has the builder VM answer one request that reaches nothing // outside the VM, from the cache disk in dir, which the caller holds. func askBuilder(ctx context.Context, command *cli.Command, dir cachedisk.Dir, blobs *download.Store, request protocol.Message, out io.Writer) error { - boot, err := bootFiles(ctx, blobs) + boot, err := bootFiles(ctx, blobs, builderArch) if err != nil { return err } diff --git a/cmd/miso/shell.go b/cmd/miso/shell.go index 781ac6af..7f47c556 100644 --- a/cmd/miso/shell.go +++ b/cmd/miso/shell.go @@ -22,12 +22,17 @@ var shellCommand = &cli.Command{ Metadata: map[string]any{takesWords: true}, Flags: []cli.Flag{ fileFlag(), + archFlag(), &cli.IntFlag{Name: "before", Usage: "the line of the build file whose step the shell stands before, the end of the build if not given"}, }, Action: runShell, } func runShell(ctx context.Context, command *cli.Command) error { + if err := archRefusal(command); err != nil { + return err + } + shellOn := func(planned plan.Plan, network protocol.Network) ([]build.Request, error) { if !command.IsSet("before") { return build.After(planned, network) diff --git a/cmd/miso/shellhint.go b/cmd/miso/shellhint.go index 9e06844e..e38c334c 100644 --- a/cmd/miso/shellhint.go +++ b/cmd/miso/shellhint.go @@ -22,6 +22,10 @@ func withShellHint(command *cli.Command, requests []build.Request, err error) er } words := []string{"miso", "shell", "--before", strconv.Itoa(located.Line)} + if arch := targetArch(command); arch != builderArch { + words = append(words, "--arch", arch) + } + if file := command.String(fileFlagName); strings.HasPrefix(file, "-") { words = append(words, "--file="+file) } else if file != "" { diff --git a/cmd/miso/shellhint_test.go b/cmd/miso/shellhint_test.go new file mode 100644 index 00000000..b9240623 --- /dev/null +++ b/cmd/miso/shellhint_test.go @@ -0,0 +1,48 @@ +package main_test + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/urfave/cli/v3" + + main "github.com/The127/miso/cmd/miso" + "github.com/The127/miso/internal/imagefile" +) + +// hinted is the error of a failed step on line 2 with its hint, as a build +// run with the arguments gives it. +func hinted(t *testing.T, args ...string) error { + t.Helper() + + var got error + build := &cli.Command{ + Name: "build", + Flags: []cli.Flag{main.FileFlag(), main.ArchFlag()}, + Action: func(_ context.Context, command *cli.Command) error { + got = main.WithShellHint(command, nil, &imagefile.Error{Line: 2, Err: errors.New("RUN false: exit code 1")}) + + return nil + }, + } + require.NoError(t, build.Run(context.Background(), append([]string{"build"}, args...))) + + return got +} + +func TestTheShellAFailingStepOfABuildForTheOtherArchitectureSaysToOpenIsForThatArchitecture(t *testing.T) { + // arrange + other := "arm64" + if main.BuilderArch == "arm64" { + other = "amd64" + } + + // act + err := hinted(t, "--arch", other) + + // assert + assert.ErrorContains(t, err, "miso shell --before 2 --arch "+other) +} diff --git a/cmd/miso/vm.go b/cmd/miso/vm.go index d78b9fe3..cf40f477 100644 --- a/cmd/miso/vm.go +++ b/cmd/miso/vm.go @@ -8,39 +8,63 @@ import ( "os" "runtime" "slices" + "strings" "github.com/The127/miso/internal/builder" "github.com/The127/miso/internal/builderkernel" "github.com/The127/miso/internal/download" + "github.com/The127/miso/internal/emulator" + "github.com/The127/miso/internal/initramfs" "github.com/The127/miso/internal/qemu" "github.com/The127/miso/internal/reach" ) // builderArch is the architecture of the builder VM, always the host's, so -// it runs on KVM. Until cross builds, images are built only for it. +// it runs on KVM. const builderArch = runtime.GOARCH -// hostArches are the hosts miso has a builder kernel, a firmware and base -// images for. -var hostArches = []string{"amd64", "arm64"} +// arches are what miso has a builder kernel, a firmware, an emulator and +// base images for. It builds on each, for each. +var arches = []string{"amd64", "arm64"} // hostRefusal is why miso cannot run a builder on host, or nil when it can. func hostRefusal(host string) error { - if !slices.Contains(hostArches, host) { - return fmt.Errorf("miso builds on amd64 and arm64, not on %s", host) + if !slices.Contains(arches, host) { + return fmt.Errorf("miso builds on %s, not on %s", strings.Join(arches, " and "), host) } return nil } +// emulatorsFor are what the builder VM needs to run the programs of a build +// for target, nothing for a build of its own architecture. +func emulatorsFor(ctx context.Context, blobs *download.Store, target string) ([]initramfs.Emulator, error) { + if target == builderArch { + return nil, nil + } + + program, err := emulator.Ready(ctx, blobs, builderArch, target) + if err != nil { + return nil, err + } + + return []initramfs.Emulator{{Arch: target, Program: program}}, nil +} + // bootFiles writes the builder kernel and an initramfs with this very miso -// as its init into a directory of their own. -func bootFiles(ctx context.Context, blobs *download.Store) (builder.Boot, error) { +// as its init, and what runs the programs of a build for target, into a +// directory of their own. +func bootFiles(ctx context.Context, blobs *download.Store, target string) (builder.Boot, error) { kernel, err := builderkernel.Ready(ctx, blobs, builderArch) if err != nil { return builder.Boot{}, err } + emulators, err := emulatorsFor(ctx, blobs, target) + if err != nil { + return builder.Boot{}, err + } + self, err := os.Executable() if err != nil { return builder.Boot{}, err @@ -56,7 +80,7 @@ func bootFiles(ctx context.Context, blobs *download.Store) (builder.Boot, error) return builder.Boot{}, err } - boot, err := builder.WriteBoot(dir, builderArch, kernel, init) + boot, err := builder.WriteBoot(dir, builderArch, kernel, init, emulators...) if err != nil { return builder.Boot{}, errors.Join(err, os.RemoveAll(dir)) } diff --git a/cmd/miso/withbuilder.go b/cmd/miso/withbuilder.go index 7b2c2a37..b152ee99 100644 --- a/cmd/miso/withbuilder.go +++ b/cmd/miso/withbuilder.go @@ -49,7 +49,8 @@ func withBuilder(ctx context.Context, command *cli.Command, requests requestsOf, return err } - blobs, bases, err := baseImages(cache) + arch := targetArch(command) + blobs, bases, err := baseImages(cache, arch) if err != nil { return err } @@ -83,7 +84,7 @@ func withBuilder(ctx context.Context, command *cli.Command, requests requestsOf, defer func() { _ = held.Close() }() boot := func(work inVM) error { - bootable, err := bootFiles(ctx, blobs) + bootable, err := bootFiles(ctx, blobs, arch) if err != nil { return err } diff --git a/hack/test-vm.sh b/hack/test-vm.sh index e471c22a..e93676d8 100644 --- a/hack/test-vm.sh +++ b/hack/test-vm.sh @@ -194,6 +194,9 @@ for pkg in $packages; do dir=$work/vm/$pkg mkdir -p "$dir/root" cp -r "$work/root/modules" "$dir/root/" + # as the builder's initramfs, which has emulators only for another + # architecture + mkdir "$dir/root/emulators" CGO_ENABLED=0 go test -c -tags vmtest -o "$dir/root/init" "./$pkg" # a cache disk made the way a build makes it, and a disk with no file # system, which must never be taken for one. Each VM writes its own, @@ -201,7 +204,7 @@ for pkg in $packages; do # imported base, and sparse, so it takes only what the tests write go run ./hack/cachedisk "$dir/cache.img" $((4 << 30)) truncate -s 16M "$dir/blank.img" - (cd "$dir/root" && find init modules | cpio --quiet -o -H newc) > "$dir/initrd" + (cd "$dir/root" && find init modules emulators | cpio --quiet -o -H newc) > "$dir/initrd" done # one package shows its VM as it runs, several would mix their lines diff --git a/internal/agent/disk.go b/internal/agent/disk.go index 25b8465d..5d7d8ec9 100644 --- a/internal/agent/disk.go +++ b/internal/agent/disk.go @@ -4,7 +4,6 @@ import ( "context" "io" "path/filepath" - "runtime" "github.com/The127/miso/internal/place" "github.com/The127/miso/internal/protocol" @@ -19,10 +18,6 @@ type efi struct { Fallback string } -// imageEFI is the efi of the image. The image is of the builder VM's -// architecture, which is what the agent runs on. -var imageEFI = efiOf(runtime.GOARCH) - // efiOf is the efi of an architecture. func efiOf(arch string) efi { if arch == "arm64" { diff --git a/internal/agent/efi_test.go b/internal/agent/efi_test.go index ca1cbea7..4b25194e 100644 --- a/internal/agent/efi_test.go +++ b/internal/agent/efi_test.go @@ -1,11 +1,16 @@ package agent_test import ( + "os" + "path/filepath" "testing" + "testing/fstest" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/The127/miso/internal/agent" + "github.com/The127/miso/internal/kernel" ) func TestAnArm64ImageBootsThroughItsArm64SystemdBootAndStub(t *testing.T) { @@ -19,3 +24,38 @@ func TestAnArm64ImageBootsThroughItsArm64SystemdBootAndStub(t *testing.T) { Fallback: "BOOTAA64.EFI", }, efi) } + +func TestTheESPOfAnArm64ImageFallsBackToItsArm64SystemdBoot(t *testing.T) { + // arrange + image := fstest.MapFS{"usr/lib/systemd/boot/efi/systemd-bootaa64.efi": {Data: []byte("loader")}} + esp := filepath.Join(t.TempDir(), "esp") + + // act + err := agent.MakeESP(image, t.TempDir(), esp, "arm64") + + // assert + require.NoError(t, err) + loader, err := os.ReadFile(filepath.Join(esp, "efi", "EFI", "BOOT", "BOOTAA64.EFI")) + require.NoError(t, err) + assert.Equal(t, "loader", string(loader)) +} + +func TestTheUKIOfAnArm64ImageIsBuiltOnItsArm64Stub(t *testing.T) { + // arrange + image := fstest.MapFS{ + "boot/vmlinuz": {Data: []byte("linux")}, + "boot/initrd": {Data: []byte("initrd")}, + "etc/os-release": {Data: []byte("ID=debian\n")}, + "usr/lib/systemd/boot/efi/linuxaa64.efi.stub": {Data: []byte("stub")}, + } + parts := t.TempDir() + + // act + err := agent.CopyParts(image, kernel.Kernel{Linux: "boot/vmlinuz", Initrd: "boot/initrd"}, parts, "arm64") + + // assert + require.NoError(t, err) + stub, err := os.ReadFile(filepath.Join(parts, "stub")) + require.NoError(t, err) + assert.Equal(t, "stub", string(stub)) +} diff --git a/internal/agent/esp.go b/internal/agent/esp.go index 132118c7..325cc907 100644 --- a/internal/agent/esp.go +++ b/internal/agent/esp.go @@ -11,8 +11,9 @@ import ( // makeESP makes in a directory the layer that lies on the image, mounted at // root, with the ESP as its efi, where the image's repart definitions take -// it from, and with the image's systemd-boot as the ESP's fallback loader. -func makeESP(image fs.FS, root, esp string) error { +// it from, and with the image's systemd-boot of arch as the ESP's fallback +// loader. +func makeESP(image fs.FS, root, esp, arch string) error { fallback := filepath.Join(esp, "efi", "EFI", "BOOT") if err := os.MkdirAll(fallback, 0o755); err != nil { //nolint:gosec // an image's directories are open to all return err @@ -38,6 +39,7 @@ func makeESP(image fs.FS, root, esp string) error { return err } + imageEFI := efiOf(arch) err = copyPart(image, imageEFI.Loader, filepath.Join(fallback, imageEFI.Fallback)) if errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("the image has no systemd-boot at /%s", imageEFI.Loader) diff --git a/internal/agent/export_efi_test.go b/internal/agent/export_efi_test.go index 15e05df4..2eec88d6 100644 --- a/internal/agent/export_efi_test.go +++ b/internal/agent/export_efi_test.go @@ -6,3 +6,9 @@ type EFI = efi // EFIOf is the EFI of an architecture. var EFIOf = efiOf + +// MakeESP makes the ESP layer of an image of an architecture. +var MakeESP = makeESP + +// CopyParts copies what a UKI of an image of an architecture is built from. +var CopyParts = copyParts diff --git a/internal/agent/parts.go b/internal/agent/parts.go index 171bf9b9..464e4c13 100644 --- a/internal/agent/parts.go +++ b/internal/agent/parts.go @@ -30,7 +30,7 @@ func prepareBoot(imageFS fs.FS, image, scratch string, request protocol.Disk) (b } esp := filepath.Join(scratch, "esp") - if err := makeESP(imageFS, image, esp); err != nil { + if err := makeESP(imageFS, image, esp, request.Arch); err != nil { return boot{}, err } @@ -39,7 +39,7 @@ func prepareBoot(imageFS fs.FS, image, scratch string, request protocol.Disk) (b return boot{}, err } - if err := copyParts(imageFS, found, parts); err != nil { + if err := copyParts(imageFS, found, parts, request.Arch); err != nil { return boot{}, err } @@ -68,8 +68,9 @@ type part struct { // copyParts copies what a UKI of the kernel is built from out of the image // into a directory, so the tools read the image's files and nothing that a -// link of the image would reach in their own root. -func copyParts(image fs.FS, found kernel.Kernel, dir string) error { +// link of the image would reach in their own root. The stub is that of arch. +func copyParts(image fs.FS, found kernel.Kernel, dir, arch string) error { + imageEFI := efiOf(arch) parts := []part{ {name: "linux", path: found.Linux}, {name: "initrd", path: found.Initrd}, diff --git a/internal/boot/boot.go b/internal/boot/boot.go index 6aa75e7d..ce43d6ec 100644 --- a/internal/boot/boot.go +++ b/internal/boot/boot.go @@ -10,5 +10,9 @@ func Boot() error { return err } - return loadModules() + if err := loadModules(); err != nil { + return err + } + + return registerEmulators() } diff --git a/internal/boot/emulators.go b/internal/boot/emulators.go new file mode 100644 index 00000000..6e952c0f --- /dev/null +++ b/internal/boot/emulators.go @@ -0,0 +1,85 @@ +package boot + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "syscall" +) + +// magics are what the header of a program of an architecture looks like to +// binfmt_misc, and the mask it is compared under, as Debian's qemu-user +// registers them. +var magics = map[string]struct{ magic, mask string }{ + "amd64": { + magic: `\x7f\x45\x4c\x46\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x3e\x00`, + mask: `\xff\xff\xff\xff\xff\xfe\xfe\xfc\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff`, + }, + "arm64": { + magic: `\x7f\x45\x4c\x46\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\xb7\x00`, + mask: `\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff`, + }, +} + +// registerEmulators registers the emulators init was handed. Only a build +// for another architecture has one, and only it needs binfmt_misc. +func registerEmulators() error { + found, err := os.ReadDir("/emulators") + if err != nil || len(found) == 0 { + return err + } + + if err := syscall.Mount("binfmt_misc", binfmtMisc, "binfmt_misc", 0, ""); err != nil { + return fmt.Errorf("mount %s: %w", binfmtMisc, err) + } + + return register("/emulators", writeEntry) +} + +// writeEntry hands binfmt_misc an entry in one write, as it takes them, on +// a file it never truncates. +func writeEntry(entry string) error { + f, err := os.OpenFile(filepath.Join(binfmtMisc, "register"), os.O_WRONLY, 0) + if err != nil { + return err + } + + _, err = f.WriteString(entry) + + return errors.Join(err, f.Close()) +} + +// binfmtMisc is where the kernel takes binfmt_misc entries. +const binfmtMisc = "/proc/sys/fs/binfmt_misc" + +// register has each emulator in dir, named by its architecture, run the +// programs of that architecture, through a write of its binfmt_misc entry. +// Debian's QEMU keeps a program's argv[0] only when its own name ends in +// -binfmt-P, so it is registered through a link of that name. F opens it +// right away, so it runs in the root of a run too. +func register(dir string, write func(entry string) error) error { + found, err := os.ReadDir(dir) + if err != nil { + return err + } + + for _, emulator := range found { + arch := emulator.Name() + magic, known := magics[arch] + if !known { + return fmt.Errorf("no binfmt_misc entry for an emulator of %s", arch) + } + + link := filepath.Join(dir, arch+"-binfmt-P") + if err := os.Symlink(arch, link); err != nil { + return err + } + + if err := write(fmt.Sprintf(":%s:M::%s:%s:%s:OPF", arch, magic.magic, magic.mask, link)); err != nil { + return err + } + } + + return nil +} diff --git a/internal/boot/emulators_test.go b/internal/boot/emulators_test.go new file mode 100644 index 00000000..dc83f54b --- /dev/null +++ b/internal/boot/emulators_test.go @@ -0,0 +1,64 @@ +package boot_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/The127/miso/internal/boot" +) + +func TestAnArm64EmulatorIsRegisteredAsDebianRegistersQEMUForAArch64(t *testing.T) { + // arrange + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "arm64"), []byte("qemu"), 0o600)) + var written []string + + // act + err := boot.Register(dir, func(entry string) error { + written = append(written, entry) + + return nil + }) + + // assert + require.NoError(t, err) + assert.Equal(t, []string{ + `:arm64:M::\x7f\x45\x4c\x46\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\xb7\x00:\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff:` + filepath.Join(dir, "arm64-binfmt-P") + `:OPF`, + }, written) +} + +func TestAnAmd64EmulatorIsRegisteredAsDebianRegistersQEMUForX86_64(t *testing.T) { + // arrange + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "amd64"), []byte("qemu"), 0o600)) + var written []string + + // act + err := boot.Register(dir, func(entry string) error { + written = append(written, entry) + + return nil + }) + + // assert + require.NoError(t, err) + assert.Equal(t, []string{ + `:amd64:M::\x7f\x45\x4c\x46\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x3e\x00:\xff\xff\xff\xff\xff\xfe\xfe\xfc\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff:` + filepath.Join(dir, "amd64-binfmt-P") + `:OPF`, + }, written) +} + +func TestAnEmulatorOfAnArchitectureWithoutAnEntryIsRefusedNamingIt(t *testing.T) { + // arrange + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "sparc"), []byte("qemu"), 0o600)) + + // act + err := boot.Register(dir, func(string) error { return nil }) + + // assert + assert.ErrorContains(t, err, "sparc") +} diff --git a/internal/boot/export_test.go b/internal/boot/export_test.go new file mode 100644 index 00000000..2842552d --- /dev/null +++ b/internal/boot/export_test.go @@ -0,0 +1,5 @@ +package boot + +// Register registers the emulators in a directory through a write of each +// entry. +var Register = register diff --git a/internal/boot/root.go b/internal/boot/root.go index a9ab3549..e996f33f 100644 --- a/internal/boot/root.go +++ b/internal/boot/root.go @@ -31,6 +31,10 @@ func switchRoot() error { return err } + if err := os.CopyFS("/vm/emulators", os.DirFS("/emulators")); err != nil { + return err + } + if err := os.Chdir("/vm"); err != nil { return err } diff --git a/internal/build/before.go b/internal/build/before.go index 271abc10..9988cfdf 100644 --- a/internal/build/before.go +++ b/internal/build/before.go @@ -48,16 +48,16 @@ func After(planned plan.Plan, network protocol.Network) ([]Request, error) { // shellOf is a shell on the layers and the environment a step would run on, // with the network the step would have. -func shellOf(step plan.Step, under rootfs, network protocol.Network) Request { +func shellOf(step plan.Step, under rootfs, network protocol.Network, arch string) Request { line, written := imagefile.Written(step.Instruction) - return shellRequest(under, networkFor(step.Instruction, network), line, "shell before "+written) + return shellRequest(under, networkFor(step.Instruction, network), line, "shell before "+written, arch) } // shellRequest is a shell on a root file system. A shell at no line of the // build file has none to be named at. -func shellRequest(under rootfs, network *protocol.Network, line int, written string) Request { - return Request{Line: line, Written: written, Message: protocol.Shell{Layers: under.layers, Env: under.env, Network: network}} +func shellRequest(under rootfs, network *protocol.Network, line int, written, arch string) Request { + return Request{Line: line, Written: written, Message: protocol.Shell{Layers: under.layers, Env: envOf(under, arch), Network: network}} } // IsFrom tells whether the line of the build file is the FROM of a stage, diff --git a/internal/build/before_test.go b/internal/build/before_test.go index e621df25..1b49686f 100644 --- a/internal/build/before_test.go +++ b/internal/build/before_test.go @@ -56,7 +56,7 @@ func TestALineWithNoStepHasNoShell(t *testing.T) { func TestAShellBeforeAStepHasTheEnvironmentItWouldRunIn(t *testing.T) { // arrange - planned := plan.Plan{Stages: []plan.Stage{{ + planned := plan.Plan{Arch: "amd64", Stages: []plan.Stage{{ Base: "debian:13", BaseDigest: "sha256:image", BaseKey: "base", @@ -73,7 +73,28 @@ func TestAShellBeforeAStepHasTheEnvironmentItWouldRunIn(t *testing.T) { require.NoError(t, err) shell, isShell := requests[len(requests)-1].Message.(protocol.Shell) require.True(t, isShell) - assert.Equal(t, []string{"MODE=debug"}, shell.Env) + assert.Equal(t, []string{"TARGETARCH=amd64", "MODE=debug"}, shell.Env) +} + +func TestAShellSeesTheArchitectureItsStepBuildsFor(t *testing.T) { + // arrange + planned := plan.Plan{Arch: "arm64", Stages: []plan.Stage{{ + Base: "debian:13", + BaseDigest: "sha256:image", + BaseKey: "base", + Steps: []plan.Step{ + {Instruction: imagefile.Run{Line: 2, Command: "make"}, Key: "k1", BuiltOn: []string{"base"}}, + }, + }}} + + // act + requests, err := build.Before(planned, network, 2) + + // assert + require.NoError(t, err) + shell, isShell := requests[len(requests)-1].Message.(protocol.Shell) + require.True(t, isShell) + assert.Contains(t, shell.Env, "TARGETARCH=arm64") } func TestAShellBeforeAnOnlineStepHasTheNetworkOfTheHost(t *testing.T) { diff --git a/internal/build/env.go b/internal/build/env.go index 6f1e25be..4ba68958 100644 --- a/internal/build/env.go +++ b/internal/build/env.go @@ -21,3 +21,13 @@ func withVariable(env []string, variable imagefile.Env) []string { return set } + +// envOf is the environment of a run or a shell on a root file system for an +// image of arch. A TARGETARCH of the build file replaces miso's. +func envOf(under rootfs, arch string) []string { + if slices.ContainsFunc(under.env, func(set string) bool { return strings.HasPrefix(set, "TARGETARCH=") }) { + return slices.Clone(under.env) + } + + return append([]string{"TARGETARCH=" + arch}, under.env...) +} diff --git a/internal/build/env_test.go b/internal/build/env_test.go index ed7d4a0c..f06a1d44 100644 --- a/internal/build/env_test.go +++ b/internal/build/env_test.go @@ -13,7 +13,7 @@ import ( func TestARunKeepsTheVariablesSetBeforeItInOrder(t *testing.T) { // arrange - planned := plan.Plan{Stages: []plan.Stage{{ + planned := plan.Plan{Arch: "amd64", Stages: []plan.Stage{{ Base: "debian:13", BaseDigest: "sha256:image", BaseKey: "base", @@ -30,12 +30,12 @@ func TestARunKeepsTheVariablesSetBeforeItInOrder(t *testing.T) { // assert require.NoError(t, err) require.Len(t, runsOf(requests), 1) - assert.Equal(t, []string{"A=1", "B=2"}, runsOf(requests)[0].Env) + assert.Equal(t, []string{"TARGETARCH=amd64", "A=1", "B=2"}, runsOf(requests)[0].Env) } func TestAVariableSetTwiceIsSentOnceWithItsLastValue(t *testing.T) { // arrange - planned := plan.Plan{Stages: []plan.Stage{{ + planned := plan.Plan{Arch: "amd64", Stages: []plan.Stage{{ Base: "debian:13", BaseDigest: "sha256:image", BaseKey: "base", @@ -53,5 +53,46 @@ func TestAVariableSetTwiceIsSentOnceWithItsLastValue(t *testing.T) { // assert require.NoError(t, err) require.Len(t, runsOf(requests), 1) - assert.Equal(t, []string{"A=3", "B=2"}, runsOf(requests)[0].Env) + assert.Equal(t, []string{"TARGETARCH=amd64", "A=3", "B=2"}, runsOf(requests)[0].Env) +} + +func TestARunSeesTheArchitectureItBuildsFor(t *testing.T) { + // arrange + planned := plan.Plan{Arch: "arm64", Stages: []plan.Stage{{ + Base: "debian:13", + BaseDigest: "sha256:image", + BaseKey: "base", + Steps: []plan.Step{ + {Instruction: imagefile.Run{Line: 2, Command: "apt-get install linux-image-$TARGETARCH"}, Key: "k1", BuiltOn: []string{"base"}}, + }, + }}} + + // act + requests, err := build.Requests(planned, network) + + // assert + require.NoError(t, err) + require.Len(t, runsOf(requests), 1) + assert.Contains(t, runsOf(requests)[0].Env, "TARGETARCH=arm64") +} + +func TestARunSeesTheArchitectureTheBuildFileSetsOnlyOnce(t *testing.T) { + // arrange + planned := plan.Plan{Arch: "arm64", Stages: []plan.Stage{{ + Base: "debian:13", + BaseDigest: "sha256:image", + BaseKey: "base", + Steps: []plan.Step{ + {Instruction: imagefile.Env{Line: 2, Key: "TARGETARCH", Value: "aarch64"}, Key: "k1", BuiltOn: []string{"base"}}, + {Instruction: imagefile.Run{Line: 3, Command: "echo $TARGETARCH"}, Key: "k2", BuiltOn: []string{"k1"}}, + }, + }}} + + // act + requests, err := build.Requests(planned, network) + + // assert + require.NoError(t, err) + require.Len(t, runsOf(requests), 1) + assert.Equal(t, []string{"TARGETARCH=aarch64"}, runsOf(requests)[0].Env) } diff --git a/internal/build/output.go b/internal/build/output.go index 89d0eb28..7a99aa1d 100644 --- a/internal/build/output.go +++ b/internal/build/output.go @@ -36,7 +36,7 @@ func outputRequest(step plan.Step, instruction imagefile.Output, under rootfs, t return protocol.Rootfs{Key: step.Key, Layers: under.layers, Tools: tools.layers, Format: formatOf(instruction.Options), Name: instruction.Name, Wrap: wrapOf(instruction.Kind)}, nil } - return protocol.Disk{Key: step.Key, Layers: under.layers, Tools: tools.layers, ElTorito: instruction.Kind == plan.KindISO, Split: instruction.Kind == plan.KindUpdate, Partitions: slices.Clone(inputs.partitions), Cmdline: strings.Join(inputs.cmdline, " ")}, nil + return protocol.Disk{Key: step.Key, Layers: under.layers, Tools: tools.layers, ElTorito: instruction.Kind == plan.KindISO, Split: instruction.Kind == plan.KindUpdate, Partitions: slices.Clone(inputs.partitions), Cmdline: strings.Join(inputs.cmdline, " "), Arch: arch}, nil } // refusal is why miso cannot make an output for an image of arch, or nil diff --git a/internal/build/output_test.go b/internal/build/output_test.go index 52430c90..a5057c4a 100644 --- a/internal/build/output_test.go +++ b/internal/build/output_test.go @@ -49,9 +49,24 @@ func TestADiskOutputIsMadeOfTheLayersOfItsStageWithTheLayersOfTheToolsStageMisoA Key: image.Steps[1].Key, Layers: []string{image.BaseKey, image.Steps[0].Key}, Tools: []string{tools.BaseKey, tools.Steps[0].Key}, + Arch: source.Arch, }, disksOf(requests)[0]) } +func TestADiskIsMadeForTheArchitectureOfItsPlan(t *testing.T) { + // arrange + source := planned(t, "FROM debian:13\nOUTPUT disk os.raw\n") + source.Arch = "arm64" + + // act + requests, err := build.Requests(source, network) + + // assert + require.NoError(t, err) + require.Len(t, disksOf(requests), 1) + assert.Equal(t, "arm64", disksOf(requests)[0].Arch) +} + func TestAnISOOutputIsADiskThatBootsFromACD(t *testing.T) { // arrange source := planned(t, "FROM debian:13\nRUN apt-get install htop\nOUTPUT iso os.iso\n") @@ -68,6 +83,7 @@ func TestAnISOOutputIsADiskThatBootsFromACD(t *testing.T) { Layers: []string{image.BaseKey, image.Steps[0].Key}, Tools: []string{tools.BaseKey, tools.Steps[0].Key}, ElTorito: true, + Arch: source.Arch, }, disksOf(requests)[0]) } diff --git a/internal/build/request.go b/internal/build/request.go index 9428f228..3c9f6902 100644 --- a/internal/build/request.go +++ b/internal/build/request.go @@ -118,7 +118,7 @@ func requestsUntil(planned plan.Plan, network protocol.Network, shell shellAt) ( under := roots[step.BuiltOn[0]] if at, _ := imagefile.Written(step.Instruction); shell.line != 0 && at == shell.line { - return append(requests, shellOf(step, current, network)), true, nil + return append(requests, shellOf(step, current, network, planned.Arch)), true, nil } if shell.wanted() && buildsOutput(step.Instruction) { @@ -165,7 +165,7 @@ func requestsUntil(planned plan.Plan, network protocol.Network, shell shellAt) ( if shell.end && len(planned.Stages) > 0 { last := planned.Stages[len(planned.Stages)-1] - return append(requests, shellRequest(ends[last.Name], &network, 0, "")), true, nil + return append(requests, shellRequest(ends[last.Name], &network, 0, "", planned.Arch)), true, nil } return requests, false, nil @@ -181,7 +181,7 @@ func importOf(stage plan.Stage) Request { func messageOf(step plan.Step, under rootfs, ends map[string]rootfs, network protocol.Network, inputs diskInputs, arch string) (protocol.Message, error) { switch instruction := step.Instruction.(type) { case imagefile.Run: - return protocol.Run{Key: step.Key, Layers: under.layers, Env: under.env, Command: instruction.Command, Network: networkFor(instruction, network)}, nil + return protocol.Run{Key: step.Key, Layers: under.layers, Env: envOf(under, arch), Command: instruction.Command, Network: networkFor(instruction, network)}, nil case imagefile.Copy: return copyRequest(step, instruction, under, ends[instruction.From]) case imagefile.Output: diff --git a/internal/builder/boot.go b/internal/builder/boot.go index 107f7c8a..44ee8395 100644 --- a/internal/builder/boot.go +++ b/internal/builder/boot.go @@ -16,11 +16,11 @@ type Boot struct { } // WriteBoot writes the files the builder VM of an architecture boots from -// into a directory: the kernel, and an initramfs with init as its init and -// the kernel's modules. -func WriteBoot(dir, arch string, kernel builderkernel.Kernel, init []byte) (Boot, error) { +// into a directory: the kernel, and an initramfs with init as its init, the +// kernel's modules and the emulators of the architectures it builds for. +func WriteBoot(dir, arch string, kernel builderkernel.Kernel, init []byte, emulators ...initramfs.Emulator) (Boot, error) { var initrd bytes.Buffer - if err := initramfs.Write(&initrd, arch, init, kernel.Modules); err != nil { + if err := initramfs.Write(&initrd, arch, init, kernel.Modules, emulators...); err != nil { return Boot{}, err } diff --git a/internal/builder/boot_test.go b/internal/builder/boot_test.go index f8459db3..5b167310 100644 --- a/internal/builder/boot_test.go +++ b/internal/builder/boot_test.go @@ -47,6 +47,24 @@ func TestTheBootFilesHoldTheInitAndTheKernelsModules(t *testing.T) { assert.Equal(t, want.Bytes(), written) } +func TestTheBootFilesHoldTheEmulatorOfTheArchitectureTheBuildIsFor(t *testing.T) { + // arrange + kernel := builderkernel.Kernel{Image: []byte("the kernel")} + init := static(t) + emulator := initramfs.Emulator{Arch: "arm64", Program: []byte("qemu")} + var want bytes.Buffer + require.NoError(t, initramfs.Write(&want, "amd64", init, nil, emulator)) + + // act + boot, err := builder.WriteBoot(t.TempDir(), "amd64", kernel, init, emulator) + + // assert + require.NoError(t, err) + written, err := os.ReadFile(boot.Initramfs) + require.NoError(t, err) + assert.Equal(t, want.Bytes(), written) +} + // static is the start of a static program for x86-64, as much as an init // must be. func static(t *testing.T) []byte { diff --git a/internal/builderkernel/needs.go b/internal/builderkernel/needs.go index 246a0bb7..a012316f 100644 --- a/internal/builderkernel/needs.go +++ b/internal/builderkernel/needs.go @@ -31,4 +31,6 @@ var needs = []string{ "sch_ingress", "cls_flower", "act_gact", + // what runs a RUN of an image of another architecture through QEMU + "binfmt_misc", } diff --git a/internal/builderkernel/needs_test.go b/internal/builderkernel/needs_test.go index 69339f26..f90634dd 100644 --- a/internal/builderkernel/needs_test.go +++ b/internal/builderkernel/needs_test.go @@ -18,3 +18,14 @@ func TestTheBuilderVMLoadsTheDriverOfItsAgentsVirtioPort(t *testing.T) { // assert assert.Contains(t, loads, driver) } + +func TestTheBuilderVMLoadsWhatRunsProgramsOfAnotherArchitecture(t *testing.T) { + // arrange + module := "binfmt_misc" + + // act + loads := builderkernel.Needs + + // assert + assert.Contains(t, loads, module) +} diff --git a/internal/emulator/doc.go b/internal/emulator/doc.go new file mode 100644 index 00000000..a95a59fd --- /dev/null +++ b/internal/emulator/doc.go @@ -0,0 +1,4 @@ +// Package emulator readies the QEMU that runs the programs of an image of +// another architecture than the builder VM's: it takes the pinned Debian +// qemu-user package of the builder's architecture apart. +package emulator diff --git a/internal/emulator/export_test.go b/internal/emulator/export_test.go new file mode 100644 index 00000000..858b1496 --- /dev/null +++ b/internal/emulator/export_test.go @@ -0,0 +1,5 @@ +package emulator + +// ReadyFrom takes the program that runs the programs of an architecture out +// of the package a pin names. +var ReadyFrom = ready diff --git a/internal/emulator/pin.go b/internal/emulator/pin.go new file mode 100644 index 00000000..4526d528 --- /dev/null +++ b/internal/emulator/pin.go @@ -0,0 +1,17 @@ +package emulator + +import "github.com/The127/miso/internal/download" + +// The builder VM runs the programs of another architecture through one +// pinned Debian qemu-user build of its own architecture. The digest is the +// guarantee, not the archive the bytes come from. +var pins = map[string]download.Pin{ + "amd64": { + URL: "https://snapshot.debian.org/archive/debian/20260905T205002Z/pool/main/q/qemu/qemu-user_10.0.13%2Bds-0%2Bdeb13u1_amd64.deb", + Digest: "sha256:ca6ede739327a20ae5a3498230c8a3a9a072c586e131bc6bcc1201eb1725e336", + }, + "arm64": { + URL: "https://snapshot.debian.org/archive/debian/20260905T205002Z/pool/main/q/qemu/qemu-user_10.0.13%2Bds-0%2Bdeb13u1_arm64.deb", + Digest: "sha256:c73711af02b97cd5e2667e735d9c06890c57165517110ca4e646c95a7083158d", + }, +} diff --git a/internal/emulator/ready.go b/internal/emulator/ready.go new file mode 100644 index 00000000..14daced5 --- /dev/null +++ b/internal/emulator/ready.go @@ -0,0 +1,59 @@ +package emulator + +import ( + "context" + "fmt" + "io" + + "github.com/The127/miso/internal/deb" + "github.com/The127/miso/internal/download" +) + +// programs are where a qemu-user package keeps the program that runs the +// programs of an architecture. Debian builds them static, so they run in +// the builder VM, which has no libraries. +var programs = map[string]string{ + "amd64": "usr/bin/qemu-x86_64", + "arm64": "usr/bin/qemu-aarch64", +} + +// Ready fetches miso's pinned qemu-user package of the builder's +// architecture, through the store, and takes the program out of it that +// runs the programs of target. +func Ready(ctx context.Context, store *download.Store, builder, target string) ([]byte, error) { + pin, ok := pins[builder] + if !ok { + return nil, fmt.Errorf("miso has no QEMU for a builder on %s", builder) + } + + if _, ok := programs[target]; !ok { + return nil, fmt.Errorf("miso has no QEMU that runs programs of %s", target) + } + + return ready(ctx, store, pin, target) +} + +// ready takes the program that runs the programs of target out of the +// package a pin names. +func ready(ctx context.Context, store *download.Store, pin download.Pin, target string) ([]byte, error) { + pkg, err := store.OpenPinned(ctx, pin) + if err != nil { + return nil, err + } + + defer func() { _ = pkg.Close() }() + + wanted := programs[target] + files, failed := deb.Files(pkg) + for name, file := range files { + if name == wanted { + return io.ReadAll(file) + } + } + + if err := failed(); err != nil { + return nil, err + } + + return nil, fmt.Errorf("the package holds no %s", wanted) +} diff --git a/internal/emulator/ready_test.go b/internal/emulator/ready_test.go new file mode 100644 index 00000000..da70426a --- /dev/null +++ b/internal/emulator/ready_test.go @@ -0,0 +1,88 @@ +package emulator_test + +import ( + "context" + "errors" + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/The127/miso/internal/deb/debtest" + "github.com/The127/miso/internal/download" + "github.com/The127/miso/internal/download/downloadtest" + "github.com/The127/miso/internal/emulator" +) + +func TestTheEmulatorOfAnArm64ImageIsTakenFromThePinnedPackage(t *testing.T) { + // arrange + pkg := debtest.Package(t, map[string]string{ + "./usr/bin/qemu-aarch64": "the emulator", + "./usr/bin/qemu-x86_64": "another emulator", + }) + dir := t.TempDir() + digest := downloadtest.Kept(t, dir, pkg) + store := download.Open(dir, http.DefaultClient) + + // act + found, err := emulator.ReadyFrom(context.Background(), store, download.Pin{URL: "https://snapshot.invalid/qemu-user.deb", Digest: digest}, "arm64") + + // assert + require.NoError(t, err) + assert.Equal(t, []byte("the emulator"), found) +} + +func TestAnArm64BuilderRunsAnAmd64ImageWithQEMUOfItsOwnArchitecture(t *testing.T) { + // arrange + var asked string + refuse := roundTrip(func(request *http.Request) (*http.Response, error) { + asked = request.URL.String() + + return nil, errors.New("offline") + }) + store := download.Open(t.TempDir(), &http.Client{Transport: refuse}) + + // act + _, err := emulator.Ready(context.Background(), store, "arm64", "amd64") + + // assert + require.Error(t, err) + assert.Contains(t, asked, "/qemu-user_10.0.13%2Bds-0%2Bdeb13u1_arm64.deb") +} + +func TestAnEmulatorForABuilderMisoHasNoQEMUForIsRefusedNamingIt(t *testing.T) { + // arrange + store := download.Open(t.TempDir(), http.DefaultClient) + + // act + _, err := emulator.Ready(context.Background(), store, "sparc", "amd64") + + // assert + assert.ErrorContains(t, err, "sparc") +} + +func TestAnEmulatorForAnImageMisoHasNoQEMUForIsRefusedNamingItBeforeAnyFetch(t *testing.T) { + // arrange + var asked string + refuse := roundTrip(func(request *http.Request) (*http.Response, error) { + asked = request.URL.String() + + return nil, errors.New("offline") + }) + store := download.Open(t.TempDir(), &http.Client{Transport: refuse}) + + // act + _, err := emulator.Ready(context.Background(), store, "amd64", "sparc") + + // assert + assert.ErrorContains(t, err, "sparc") + assert.Empty(t, asked) +} + +// roundTrip is an HTTP transport that is a function. +type roundTrip func(*http.Request) (*http.Response, error) + +func (f roundTrip) RoundTrip(request *http.Request) (*http.Response, error) { + return f(request) +} diff --git a/internal/initramfs/emulator.go b/internal/initramfs/emulator.go new file mode 100644 index 00000000..6e39e4fb --- /dev/null +++ b/internal/initramfs/emulator.go @@ -0,0 +1,8 @@ +package initramfs + +// Emulator is the QEMU that runs the programs of an architecture other than +// the builder VM's. +type Emulator struct { + Arch string + Program []byte +} diff --git a/internal/initramfs/write.go b/internal/initramfs/write.go index b3d19652..a4c38c23 100644 --- a/internal/initramfs/write.go +++ b/internal/initramfs/write.go @@ -8,8 +8,9 @@ import ( ) // Write writes an initial ramfs for a builder VM of an architecture, whose -// init is the program given, with the modules in the order to load them. -func Write(w io.Writer, arch string, init []byte, modules []Module) error { +// init is the program given, with the modules in the order to load them and +// the emulators of the other architectures it builds for. +func Write(w io.Writer, arch string, init []byte, modules []Module, emulators ...Emulator) error { if err := checkInit(arch, init); err != nil { return err } @@ -21,6 +22,8 @@ func Write(w io.Writer, arch string, init []byte, modules []Module) error { // the agent copies it off the initial ramfs, also for a kernel that // has every module built in cpio.Directory("modules", 0o700), + // the agent copies it off too, also for a build of its own architecture + cpio.Directory("emulators", 0o700), } // the agent loads them in the order of their names @@ -29,6 +32,10 @@ func Write(w io.Writer, arch string, init []byte, modules []Module) error { records = append(records, cpio.StaticRecord(module.Content, cpio.Info{Name: name, Mode: cpio.S_IFREG | 0o600})) } + for _, emulator := range emulators { + records = append(records, cpio.StaticRecord(emulator.Program, cpio.Info{Name: "emulators/" + emulator.Arch, Mode: cpio.S_IFREG | 0o700})) + } + archive := cpio.Newc.Writer(w) if err := cpio.WriteRecordsAndDirs(archive, records); err != nil { return err diff --git a/internal/initramfs/write_test.go b/internal/initramfs/write_test.go index 19d0e1d3..5585a74a 100644 --- a/internal/initramfs/write_test.go +++ b/internal/initramfs/write_test.go @@ -112,3 +112,34 @@ func TestAnInitramfsForAKernelWithoutModulesStillHoldsTheirDirectory(t *testing. modules := record(t, archive.Bytes(), "modules") assert.Equal(t, uint64(cpio.S_IFDIR), modules.Mode&cpio.S_IFMT) } + +func TestAnInitramfsHoldsTheEmulatorOfAnotherArchitectureAsAProgramNamedByIt(t *testing.T) { + // arrange + var archive bytes.Buffer + emulator := initramfs.Emulator{Arch: "arm64", Program: []byte("qemu")} + + // act + err := initramfs.Write(&archive, "amd64", static(t), nil, emulator) + + // assert + require.NoError(t, err) + program := record(t, archive.Bytes(), "emulators/arm64") + assert.Equal(t, uint64(cpio.S_IFREG|0o700), program.Mode) + content := make([]byte, program.FileSize) + _, err = program.ReadAt(content, 0) + require.NoError(t, err) + assert.Equal(t, "qemu", string(content)) +} + +func TestAnInitramfsWithoutEmulatorsStillHoldsTheirDirectory(t *testing.T) { + // arrange + var archive bytes.Buffer + + // act + err := initramfs.Write(&archive, "amd64", static(t), nil) + + // assert + require.NoError(t, err) + emulators := record(t, archive.Bytes(), "emulators") + assert.Equal(t, uint64(cpio.S_IFDIR), emulators.Mode&cpio.S_IFMT) +} diff --git a/internal/plan/bases.go b/internal/plan/bases.go index dba5e75f..51f96b3d 100644 --- a/internal/plan/bases.go +++ b/internal/plan/bases.go @@ -30,8 +30,9 @@ func baseDigest(stage imagefile.Stage, bases Bases) (string, error) { } // baseKey holds the agent too, because a changed agent may build the same -// step differently. The name stays out, so that two names for one image -// share its layers. -func baseKey(agent string, digest string) string { - return hashed([]string{agent, digest}) +// step differently, and the architecture, because a scratch stage has no +// digest to tell two apart. The name stays out, so that two names for one +// image share its layers. +func baseKey(agent, arch, digest string) string { + return hashed([]string{agent, arch, digest}) } diff --git a/internal/plan/bases_test.go b/internal/plan/bases_test.go index e14ce509..dc75f410 100644 --- a/internal/plan/bases_test.go +++ b/internal/plan/bases_test.go @@ -36,6 +36,20 @@ func TestADifferentAgentChangesTheKeys(t *testing.T) { assert.NotEqual(t, lastKey(t, oldKeys), lastKey(t, newKeys)) } +func TestADifferentArchitectureChangesTheKeys(t *testing.T) { + // arrange + stages := parse(t, "FROM scratch\nRUN true\n") + + // act + amd64, amd64Err := plan.New(stages, anyAgent, "amd64", noFiles, noImages) + arm64, arm64Err := plan.New(stages, anyAgent, "arm64", noFiles, noImages) + + // assert + require.NoError(t, amd64Err) + require.NoError(t, arm64Err) + assert.NotEqual(t, amd64.Stages[0].Steps[0].Key, arm64.Stages[0].Steps[0].Key) +} + // images are base images in memory, by what a FROM calls them. An empty // digest is an image that is known but not fetched yet. type images map[string]string diff --git a/internal/plan/planner.go b/internal/plan/planner.go index 6dbe5451..88c92520 100644 --- a/internal/plan/planner.go +++ b/internal/plan/planner.go @@ -69,7 +69,7 @@ func (p *planner) start(stage imagefile.Stage) (key string, digest string, err e return "", "", nil } - return baseKey(p.agent, digest), digest, nil + return baseKey(p.agent, p.arch, digest), digest, nil } // steps also hands back where the stage's root file system ends, which for diff --git a/internal/protocol/disk.go b/internal/protocol/disk.go index 6346f05f..1989ccdd 100644 --- a/internal/protocol/disk.go +++ b/internal/protocol/disk.go @@ -24,6 +24,9 @@ type Disk struct { // the kernel command line of the disk, empty for the image's own Cmdline string + + // the architecture of the image, which names its loader and its stub + Arch string } // Partition is a definition for systemd-repart, named as its file is. diff --git a/internal/qemu/accel.go b/internal/qemu/accel.go index 87555862..7e2c5b71 100644 --- a/internal/qemu/accel.go +++ b/internal/qemu/accel.go @@ -1,6 +1,7 @@ package qemu import ( + "errors" "os" "golang.org/x/sys/unix" @@ -8,17 +9,28 @@ import ( const kvmDevice = "/dev/kvm" +// ErrOtherArch is a machine of another architecture than the host's, which +// KVM cannot run. +var ErrOtherArch = errors.New("KVM runs only machines of the host's architecture") + // kvmGetAPIVersion is KVM_GET_API_VERSION, _IO(0xAE, 0x00). Only KVM // answers it, so a device that merely opens is not taken for KVM. const kvmGetAPIVersion = 0xAE00 -// accel runs the machine on KVM when the host's KVM answers, and else on -// TCG, which is much slower, and says why. On KVM the host's own CPU model -// hands the guest every feature the host has, the builder VM never moves -// to another host. TCG has no host CPU to hand on, its most capable model -// comes closest. -func accel(device string) ([]string, error) { +// accel runs a machine of the host's architecture on KVM when the host's KVM +// answers, and else on TCG, which is much slower, and says why. On KVM the +// host's own CPU model hands the guest every feature the host has, the +// builder VM never moves to another host. TCG has no host CPU to hand on, +// its most capable model comes closest. +func accel(device, arch, host string) ([]string, error) { tcg := []string{"-accel", "tcg", "-cpu", "max"} + if arch == "" { + arch = "amd64" + } + + if arch != host { + return tcg, ErrOtherArch + } kvm, err := os.OpenFile(device, os.O_RDWR, 0) if err != nil { diff --git a/internal/qemu/accel_kvm_test.go b/internal/qemu/accel_kvm_test.go index 5cf13b95..104e5056 100644 --- a/internal/qemu/accel_kvm_test.go +++ b/internal/qemu/accel_kvm_test.go @@ -3,6 +3,7 @@ package qemu_test import ( + "runtime" "testing" "github.com/stretchr/testify/assert" @@ -16,7 +17,7 @@ func TestAHostWithKVMRunsTheMachineOnKVMWithTheHostsCPU(t *testing.T) { device := "/dev/kvm" // act - args, why := qemu.Accel(device) + args, why := qemu.Accel(device, runtime.GOARCH, runtime.GOARCH) // assert require.NoError(t, why) diff --git a/internal/qemu/accel_test.go b/internal/qemu/accel_test.go index ba454ad6..5e1f5bba 100644 --- a/internal/qemu/accel_test.go +++ b/internal/qemu/accel_test.go @@ -18,7 +18,7 @@ func TestAHostWithoutKVMRunsTheMachineOnTCGWithTheMostCapableCPU(t *testing.T) { missing := filepath.Join(t.TempDir(), "kvm") // act - args, why := qemu.Accel(missing) + args, why := qemu.Accel(missing, "amd64", "amd64") // assert assert.ErrorIs(t, why, fs.ErrNotExist) @@ -32,7 +32,7 @@ func TestAHostWhoseKVMDeviceIsNoKVMRunsTheMachineOnTCG(t *testing.T) { require.NoError(t, os.WriteFile(device, nil, 0o600)) // act - args, why := qemu.Accel(device) + args, why := qemu.Accel(device, "amd64", "amd64") // assert assert.ErrorIs(t, why, unix.ENOTTY) @@ -45,8 +45,31 @@ func TestAKVMDeviceThatIsNoKVMIsNamedInTheReason(t *testing.T) { require.NoError(t, os.WriteFile(device, nil, 0o600)) // act - _, why := qemu.Accel(device) + _, why := qemu.Accel(device, "amd64", "amd64") // assert assert.ErrorContains(t, why, device) } + +func TestAMachineOfAnotherArchitectureThanTheHostsRunsOnTCG(t *testing.T) { + // arrange + missing := filepath.Join(t.TempDir(), "kvm") + + // act + args, why := qemu.Accel(missing, "arm64", "amd64") + + // assert + assert.ErrorIs(t, why, qemu.ErrOtherArch) + assert.Equal(t, "tcg", valueOf(t, args, "-accel")) +} + +func TestAMachineThatNamesNoArchitectureRunsOnAnAmd64HostsKVM(t *testing.T) { + // arrange + missing := filepath.Join(t.TempDir(), "kvm") + + // act + _, why := qemu.Accel(missing, "", "amd64") + + // assert + assert.ErrorIs(t, why, fs.ErrNotExist) +} diff --git a/internal/qemu/driver.go b/internal/qemu/driver.go index e02b5fa7..b6f3266f 100644 --- a/internal/qemu/driver.go +++ b/internal/qemu/driver.go @@ -5,6 +5,7 @@ import ( "errors" "os" "os/exec" + "runtime" "slices" ) @@ -46,7 +47,7 @@ func (d Driver) start(ctx context.Context, machine Machine, kvm string) (*VM, er return nil, err } - accelerated, why := accel(kvm) + accelerated, why := accel(kvm, machine.Arch, runtime.GOARCH) if why != nil { d.withoutKVM(why) } diff --git a/internal/qemu/start_test.go b/internal/qemu/start_test.go index 8bbf5590..64da09d9 100644 --- a/internal/qemu/start_test.go +++ b/internal/qemu/start_test.go @@ -7,6 +7,7 @@ import ( "io/fs" "os" "path/filepath" + "runtime" "slices" "strings" "testing" @@ -35,7 +36,7 @@ func TestQEMURunsWithTheMachinesArgumentsAndItsVsockDevice(t *testing.T) { require.NoError(t, err) written, err := os.ReadFile(recorded) require.NoError(t, err) - accelerated, _ := qemu.Accel("/dev/kvm") + accelerated, _ := qemu.Accel("/dev/kvm", "amd64", runtime.GOARCH) assert.Equal(t, slices.Concat(want, accelerated, qemu.Vsock(vm.CID(), 3, qemu.Machine{})), strings.Split(string(written), "\n")) } @@ -167,7 +168,7 @@ func TestAMachineQEMUStoppedWithoutAWordSaysOnlyHow(t *testing.T) { func startOn(t *testing.T, driver qemu.Driver, kvm string) { t.Helper() - vm, err := qemu.StartOn(t.Context(), driver, qemu.Machine{}, kvm) + vm, err := qemu.StartOn(t.Context(), driver, qemu.Machine{Arch: runtime.GOARCH}, kvm) if err == nil { t.Cleanup(func() { <-vm.Done() }) } diff --git a/site/src/base-images.md b/site/src/base-images.md index c4fdc898..bbf3fdf8 100644 --- a/site/src/base-images.md +++ b/site/src/base-images.md @@ -10,7 +10,7 @@ The images are the `genericcloud` qcow2 images that Debian publishes at `cloud.debian.org`, for amd64 and arm64. miso fetches the image for the -architecture of the machine it runs on. A name that is not in the table is an +architecture the image is built for, see [`--arch`](cli.md#miso-build). A name that is not in the table is an error, and so are paths, URLs and container images. `FROM scratch` starts from an empty root file system, and `FROM ` starts diff --git a/site/src/caching.md b/site/src/caching.md index 25922500..3f54f5ae 100644 --- a/site/src/caching.md +++ b/site/src/caching.md @@ -25,6 +25,7 @@ The key of a step is made from: of every file it copies. For `COPY --from` it is the key of the stage it copies from. - the version of miso +- the architecture the image is built for A step whose key is in the cache is not run again. When a step changes, its key changes, and so do the keys of all the steps after it. The steps before it diff --git a/site/src/checks.md b/site/src/checks.md index 8a9a7d93..4463199b 100644 --- a/site/src/checks.md +++ b/site/src/checks.md @@ -100,8 +100,9 @@ there, or write `console=ttyAMA0`. Checks run on the machine that runs miso, not in the builder VM. They need: -- `qemu-system-x86_64` on amd64 or `qemu-system-aarch64` on arm64, and - access to `/dev/kvm` +- `qemu-system-x86_64` for an amd64 image or `qemu-system-aarch64` for an + arm64 image, and access to `/dev/kvm`. An image for another architecture + than the machine's boots without KVM, which is much slower - the firmware, which miso downloads and keeps in the cache - a vsock device. miso uses it to talk to the booted system. If there is none, the build stops with `CHECK needs a vsock of its own`. diff --git a/site/src/cli.md b/site/src/cli.md index 4a102d16..b64603fb 100644 --- a/site/src/cli.md +++ b/site/src/cli.md @@ -40,10 +40,12 @@ context. `COPY` always reads from the context. ## miso plan ``` -miso plan [-f ] [context] +miso plan [-f ] [--arch ] [context] ``` Reads the Imagefile and lists what a build would do, without building. +`--arch` plans the build for an architecture, as it does for +[`miso build`](#miso-build). The first line names the version of miso. A base image that is not in the cache yet is listed as `download `. Then every stage is listed with its @@ -77,7 +79,12 @@ run again. | --- | --- | | `-f`, `--file` | The build file. `Imagefile` in the context if not given. | | `-o`, `--output` | The directory the outputs are written to. It is created if it is missing. | -| `--arch` | The architecture the image is built for. The default is the architecture miso runs on, and it is the only one accepted. miso builds on `amd64` and `arm64`. | +| `--arch` | The architecture the image is built for, `amd64` or `arm64`. The default is the architecture miso runs on. | + +An image for the other architecture is built in the same builder VM. Its +`RUN` steps run through QEMU in user mode, which miso downloads and keeps in +the cache, and its checks boot without KVM. Both are much slower than a +build for the architecture miso runs on. Without `-o` no output is written. Disks with a `CHECK` are still built and booted, so the checks run. @@ -91,7 +98,7 @@ The builder VM needs access to `/dev/kvm`. ## miso shell ``` -miso shell [-f ] [--before ] [context] [-- ...] +miso shell [-f ] [--arch ] [--before ] [context] [-- ...] ``` Opens an interactive shell in the root file system as it is at some point of @@ -100,6 +107,7 @@ the build. Layers that are not built yet are built first. | Option | Meaning | | --- | --- | | `-f`, `--file` | The build file. `Imagefile` in the context if not given. | +| `--arch` | The architecture of the layers, as for [`miso build`](#miso-build). | | `--before` | The line of the Imagefile whose step the shell stands before. | Without `--before` the shell stands at the end of the last stage. With diff --git a/site/src/imagefile.md b/site/src/imagefile.md index 4369d8ab..b1aa0e6a 100644 --- a/site/src/imagefile.md +++ b/site/src/imagefile.md @@ -62,7 +62,7 @@ Starts a stage from a base. The base is one of: - The name of an earlier stage. The new stage starts where that stage ends, including its `ENV` values. - `debian:sid`, `debian:13` or `debian:trixie`. These are the Debian cloud - images for the architecture miso runs on. `debian:13` and `debian:trixie` + images for the architecture the image is built for. `debian:13` and `debian:trixie` are the same image. A name means whatever image is current at its address when it is first fetched. After that the fetched image is cached. @@ -85,8 +85,17 @@ the command, exactly as written. Every `RUN` runs as root in the builder VM, in its own process, mount and network namespaces, with `/` as the working directory. The environment is -`PATH`, `HOME=/root` and the values set by `ENV`. The environment of the -host is not passed on. +`PATH`, `HOME=/root`, `TARGETARCH` and the values set by `ENV`. The +environment of the host is not passed on. + +`TARGETARCH` is the architecture the image is built for, `amd64` or `arm64`, +so one Imagefile can install the kernel of either: + +``` +RUN apt-get install -y linux-image-$TARGETARCH +``` + +An `ENV` that sets `TARGETARCH` replaces it. A command that exits with a code other than 0 fails the build. diff --git a/test/build.bats b/test/build.bats index ee62d1a7..f83cdbe2 100644 --- a/test/build.bats +++ b/test/build.bats @@ -9,20 +9,27 @@ own_arch() { esac } -@test "a build for an arch it does not build for fails and names the flag and the arch it builds for" { +other_arch() { + case $(own_arch) in + amd64) echo arm64 ;; + *) echo amd64 ;; + esac +} + +@test "a build for an arch it does not build for fails and names the flag and the arches it builds for" { run "$MISO" build --arch sparc [ "$status" -eq 1 ] [[ "$output" == *'--arch "sparc"'* ]] - [[ "$output" == *"$(own_arch)"* ]] + [[ "$output" == *"amd64 and arm64"* ]] } -@test "a build for an empty arch fails and names the flag and the arch it builds for" { +@test "a build for an empty arch fails and names the flag and the arches it builds for" { run "$MISO" build --arch= [ "$status" -eq 1 ] [[ "$output" == *'--arch ""'* ]] - [[ "$output" == *"$(own_arch)"* ]] + [[ "$output" == *"amd64 and arm64"* ]] } @test "a build for the arch miso runs on is not refused for its arch" { @@ -36,6 +43,17 @@ own_arch() { [[ "$output" != *"--arch"* ]] } +@test "a build for the other arch miso builds for is not refused for its arch" { + # the cache of the test, which is not the user's + export XDG_CACHE_HOME="$BATS_TEST_TMPDIR/cache" + cd "$BATS_TEST_TMPDIR" + + run "$MISO" build --arch "$(other_arch)" + + # it fails later, on the missing Imagefile + [[ "$output" != *"--arch"* ]] +} + # only a host that is not amd64 tells the two runs apart, see the arm64 job @test "a build without --arch is a build for the arch miso runs on" { # the cache of the test, which is not the user's diff --git a/test/plan.bats b/test/plan.bats new file mode 100644 index 00000000..f5f1b9c6 --- /dev/null +++ b/test/plan.bats @@ -0,0 +1,10 @@ +#!/usr/bin/env bats +# MISO names the miso binary under test, see just test-cli. + +@test "a plan for an arch it does not build for fails and names the flag and the arches it builds for" { + run "$MISO" plan --arch sparc + + [ "$status" -eq 1 ] + [[ "$output" == *'--arch "sparc"'* ]] + [[ "$output" == *"amd64 and arm64"* ]] +} diff --git a/test/shell.bats b/test/shell.bats new file mode 100644 index 00000000..9a173319 --- /dev/null +++ b/test/shell.bats @@ -0,0 +1,10 @@ +#!/usr/bin/env bats +# MISO names the miso binary under test, see just test-cli. + +@test "a shell for an arch it does not build for fails and names the flag and the arches it builds for" { + run "$MISO" shell --arch sparc + + [ "$status" -eq 1 ] + [[ "$output" == *'--arch "sparc"'* ]] + [[ "$output" == *"amd64 and arm64"* ]] +}