From 6ab079c1b23b16d2986b6f65804e661c44d7a90a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Jos=C3=A9=20Garc=C3=ADa=20Garc=C3=ADa?= Date: Mon, 24 Aug 2026 19:54:54 +0200 Subject: [PATCH 01/11] Package musl hosts inside the same container that builds them build_musl_host() produced only the SDK tarball and refused outright if the matrix asked for a packaged musl host. musl is native inside its own Alpine container, unlike a canadian cross, so the same container can also fetch the vdpm bundle, build core-package/bootstrap-archive, and run the bootstrap smoke test before anything leaves the container. libarchive-tools joins the apk list: create-core-package.sh drives bsdtar for the .pkg.tar.xz format. The artifact-staging/provenance tail shared with build_and_stage() is now stage_and_write_provenance(), reused instead of duplicated a second time. --- scripts/ci/build-host.sh | 74 +++++++++++++++++++++----------- tests/ci/test-build-host-args.sh | 24 ++++++++--- 2 files changed, 67 insertions(+), 31 deletions(-) diff --git a/scripts/ci/build-host.sh b/scripts/ci/build-host.sh index 27872f2..1a250a2 100755 --- a/scripts/ci/build-host.sh +++ b/scripts/ci/build-host.sh @@ -165,6 +165,13 @@ build_and_stage() { fi fi + stage_and_write_provenance +} + +# Copies the standard artifact families out of ./build into $out_dir and +# writes provenance; shared by every host path (native, cross, and musl), +# all of which leave their outputs under the same build/ layout. +stage_and_write_provenance() { local -a produced=(build/vitasdk-*.tar.bz2) if is_packaged_host; then produced+=(build/packages/*.pkg.tar.xz build/bootstraps/vitasdk-bootstrap-*.tar.bz2 build/bootstraps/vitasdk-bootstrap-*.tar.bz2.sha256) @@ -192,19 +199,36 @@ build_and_stage() { } # musl hosts are native builds inside Alpine (see build.yml stage2-musl). +# musl is native there -- unlike a canadian cross -- so a packaged host's +# bootstrap smoke test runs inside the same container that built it. build_musl_host() { + local -a docker_env=( + -e VITASDK_SOURCE_REVISION="$revision" + -e VITASDK_SOURCE_DATE_EPOCH="$source_date_epoch" + -e CCACHE_DIR=/src/.ccache + -e CCACHE_MAXSIZE -e CCACHE_COMPRESS -e CCACHE_COMPRESSLEVEL + ) + local vdpm_bundle="" vdpm_bundle_sha256="" + if is_packaged_host; then + # Downloaded here, outside the container, so it reuses the runner's + # network/curl instead of teaching the container image about it. + download_vdpm_bundle "$host" "$PWD/vdpm-release" + docker_env+=( + -e VITASDK_PACKAGED_HOST="$host" + -e VITASDK_PACKAGE_VERSION="$version" + -e VDPM_BUNDLE="/src/vdpm-release/$(basename "$vdpm_bundle")" + -e VDPM_BUNDLE_SHA256="$vdpm_bundle_sha256" + ) + fi # CCACHE_DIR lives under the workspace, which is already the bind mount, # so the container reaches the same cache the runner restored. docker run --rm \ -v "$PWD":/src -w /src \ - -e VITASDK_SOURCE_REVISION="$revision" \ - -e VITASDK_SOURCE_DATE_EPOCH="$source_date_epoch" \ - -e CCACHE_DIR=/src/.ccache \ - -e CCACHE_MAXSIZE -e CCACHE_COMPRESS -e CCACHE_COMPRESSLEVEL \ + "${docker_env[@]}" \ alpine:3.20 sh -eux -c ' apk add --no-cache bash build-base cmake git autoconf automake \ - libtool texinfo bison flex pkgconf curl xz python3 bzip2 \ - tar linux-headers ccache + libtool libarchive-tools texinfo bison flex pkgconf curl xz \ + python3 bzip2 tar linux-headers ccache export PATH="/usr/lib/ccache/bin:$PATH" git config --global --add safe.directory "*" git config --global user.email "builds@ci.invalid" @@ -213,22 +237,28 @@ build_musl_host() { test -n "$STAGE1_DIR" mkdir -p /src/build cd /src/build - cmake /src -DVITASDK_STAGE1_DIR="$STAGE1_DIR" \ - -DVITASDK_SOURCE_REVISION="$VITASDK_SOURCE_REVISION" \ - -DVITASDK_SOURCE_DATE_EPOCH="$VITASDK_SOURCE_DATE_EPOCH" - make -j"$(getconf _NPROCESSORS_ONLN)" tarball + configure_args="-DVITASDK_STAGE1_DIR=$STAGE1_DIR -DVITASDK_SOURCE_REVISION=$VITASDK_SOURCE_REVISION -DVITASDK_SOURCE_DATE_EPOCH=$VITASDK_SOURCE_DATE_EPOCH" + targets="tarball" + if [ -n "${VITASDK_PACKAGED_HOST:-}" ]; then + configure_args="$configure_args -DBUILD_PACMAN_CLIENT=ON -DVITASDK_PACKAGE_VERSION=$VITASDK_PACKAGE_VERSION -DVDPM_BUNDLE=$VDPM_BUNDLE -DVDPM_BUNDLE_SHA256=$VDPM_BUNDLE_SHA256" + targets="$targets core-package bootstrap-archive" + fi + cmake /src $configure_args + make -j"$(getconf _NPROCESSORS_ONLN)" $targets make check-toolchain-contract ccache --show-stats || true + if [ -n "${VITASDK_PACKAGED_HOST:-}" ]; then + bootstrap_archive="bootstraps/vitasdk-bootstrap-$VITASDK_PACKAGED_HOST.tar.bz2" + bootstrap_digest=$(awk "{print \$1}" "$bootstrap_archive.sha256") + install_root=/src/build/bootstrap-installed + VITASDK_BOOTSTRAP_ARCHIVE="$bootstrap_archive" VITASDK_BOOTSTRAP_SHA256="$bootstrap_digest" \ + vitasdk/share/vdpm/bootstrap-vitasdk.sh --install-dir "$install_root" + VITASDK="$install_root" "$install_root/bin/vdpm" --help >/dev/null + "$install_root/libexec/vdpm/pacman" --version >/dev/null + "$install_root/bin/arm-vita-eabi-gcc" --version + fi ' - local -a produced=(build/vitasdk-*.tar.bz2) - local -a names=() - local file - for file in "${produced[@]}"; do - [[ -f $file ]] || continue - cp "$file" "$out_dir/" - names+=("$(basename "$file")") - done - ci_write_provenance "$out_dir" "$host" "$build_id" "${names[@]}" + stage_and_write_provenance } install_dependencies() { @@ -297,12 +327,6 @@ if [[ $stage == 1 ]]; then fi if [[ $host == *-linux-musl ]]; then - # The musl path builds only the SDK tarball; flipping packaged on must - # fail here rather than publish a host with no packages or bootstrap. - if is_packaged_host; then - printf 'packaged musl hosts are not implemented\n' >&2 - exit 1 - fi stage1_source_dir=$(ci_find_stage_artifact "$artifacts_dir" 1 '*') || { printf 'stage1 artifact not found under %s\n' "$artifacts_dir" >&2 diff --git a/tests/ci/test-build-host-args.sh b/tests/ci/test-build-host-args.sh index de26e4c..f69a821 100755 --- a/tests/ci/test-build-host-args.sh +++ b/tests/ci/test-build-host-args.sh @@ -20,6 +20,14 @@ cat > "$fake_bin/uname" <<'EOF' echo TestOS EOF chmod +x "$fake_bin/uname" +# A fake curl keeps test 5 off the network: it only needs to prove that a +# packaged musl host now reaches the vdpm bundle download at all. +cat > "$fake_bin/curl" <<'EOF' +#!/usr/bin/env bash +echo "curl disabled in test" >&2 +exit 1 +EOF +chmod +x "$fake_bin/curl" fake_home="$temporary_directory/home" mkdir -p "$fake_home" @@ -97,18 +105,22 @@ grep -q 'no .tar.bz2 archive found' <<< "$output" || { exit 1 } -# 5. Flipping packaged on for a musl host fails loudly: that path builds -# only the SDK tarball and must not publish a quietly incomplete host. +# 5. A packaged musl host now attempts the same vdpm bundle download as any +# other packaged host, instead of refusing outright: it should get past the +# stage1 lookup and reach (fake, disabled) curl before failing. +musl_stage1_dir="$temporary_directory/artifacts-5/stage1-x86_64-linux-gnu" +mkdir -p "$musl_stage1_dir" +tar -cjf "$musl_stage1_dir/stage1.tar.bz2" -T /dev/null if output=$(run_build_host \ --host x86_64-linux-musl --stage 2 \ - --artifacts-dir "$temporary_directory/artifacts-4" --out-dir "$temporary_directory/out-4" \ + --artifacts-dir "$temporary_directory/artifacts-5" --out-dir "$temporary_directory/out-5" \ --build-id sha256:test --version 0.1.1 --revision "$revision" --profile vita \ --packaged true 2>&1); then - printf 'build-host.sh accepted a packaged musl host\n' >&2 + printf 'build-host.sh accepted a packaged musl host with curl disabled\n' >&2 exit 1 fi -grep -q 'packaged musl hosts are not implemented' <<< "$output" || { - printf 'build-host.sh did not refuse the packaged musl host: %s\n' "$output" >&2 +grep -q 'curl disabled in test' <<< "$output" || { + printf 'packaged musl host did not reach the vdpm bundle download: %s\n' "$output" >&2 exit 1 } From ee8fa9f75ef8769abd9a6ecb7420566cd67935a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Jos=C3=A9=20Garc=C3=ADa=20Garc=C3=ADa?= Date: Mon, 24 Aug 2026 19:55:00 +0200 Subject: [PATCH 02/11] Publish musl and FreeBSD hosts x86_64/aarch64-linux-musl (stage 2) and x86_64/aarch64-unknown-freebsd (stage 3) now get the packaged treatment, taking the published core matrix from 4 hosts to 8. FreeBSD is a canadian cross already handled by build_and_stage(); musl needed build_musl_host() to learn packaging first. x86_64-apple-darwin stays unpackaged: it still cross-builds from arm64-apple-darwin and moving it to a native runner is separate work. --- cmake/hosts.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmake/hosts.json b/cmake/hosts.json index 61fb8cd..8c25caf 100644 --- a/cmake/hosts.json +++ b/cmake/hosts.json @@ -4,12 +4,12 @@ { "name": "x86_64-linux-gnu", "stage": 2, "runner": "ubuntu-24.04", "container": "ubuntu:20.04", "packaged": true }, { "name": "aarch64-linux-gnu", "stage": 2, "runner": "ubuntu-24.04-arm", "container": "ubuntu:20.04", "packaged": true }, { "name": "arm64-apple-darwin", "stage": 2, "runner": "macos-14", "container": null, "packaged": true }, - { "name": "x86_64-linux-musl", "stage": 2, "runner": "ubuntu-24.04", "container": null, "packaged": false }, - { "name": "aarch64-linux-musl", "stage": 2, "runner": "ubuntu-24.04-arm", "container": null, "packaged": false }, + { "name": "x86_64-linux-musl", "stage": 2, "runner": "ubuntu-24.04", "container": null, "packaged": true }, + { "name": "aarch64-linux-musl", "stage": 2, "runner": "ubuntu-24.04-arm", "container": null, "packaged": true }, { "name": "x86_64-w64-mingw32", "stage": 3, "runner": "ubuntu-24.04", "container": null, "packaged": true, "build_host": "x86_64-linux-gnu" }, { "name": "i686-w64-mingw32", "stage": 3, "runner": "ubuntu-24.04", "container": null, "packaged": false, "build_host": "x86_64-linux-gnu" }, - { "name": "x86_64-unknown-freebsd", "stage": 3, "runner": "ubuntu-24.04", "container": null, "packaged": false, "build_host": "x86_64-linux-gnu" }, - { "name": "aarch64-unknown-freebsd", "stage": 3, "runner": "ubuntu-24.04", "container": null, "packaged": false, "build_host": "x86_64-linux-gnu" }, + { "name": "x86_64-unknown-freebsd", "stage": 3, "runner": "ubuntu-24.04", "container": null, "packaged": true, "build_host": "x86_64-linux-gnu" }, + { "name": "aarch64-unknown-freebsd", "stage": 3, "runner": "ubuntu-24.04", "container": null, "packaged": true, "build_host": "x86_64-linux-gnu" }, { "name": "x86_64-apple-darwin", "stage": 3, "runner": "macos-14", "container": null, "packaged": false, "build_host": "arm64-apple-darwin" } ] } From 68ca1cdce819893e12e1a5059b201039ec8ead9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Jos=C3=A9=20Garc=C3=ADa=20Garc=C3=ADa?= Date: Mon, 24 Aug 2026 19:55:12 +0200 Subject: [PATCH 03/11] Let one bad host fail without blocking the whole release tree Every stage and the grouping job required every upstream stage to have zero failed matrix legs, so one bad host anywhere -- including a host that nothing downstream of it depends on -- skipped every later stage and group, publishing nothing at all. Which hosts are required for a publishable release is a policy decision for the caller of this reusable workflow, not something the shell should encode; it now only still skips on an explicit cancellation, and lets whatever built reach grouping. create-core-repositories.sh already refuses to group an architecture that is missing either the core or the client package, so a half-built host still fails loudly there instead of publishing thin. --- .github/workflows/build-sdk.yml | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-sdk.yml b/.github/workflows/build-sdk.yml index daca2e3..f3a1d12 100644 --- a/.github/workflows/build-sdk.yml +++ b/.github/workflows/build-sdk.yml @@ -111,9 +111,10 @@ jobs: stage2: name: stage2 (${{ matrix.name }}) needs: [gate, stage1] + # A failed host is not this shell's call to make; only a cancellation is. if: | always() && needs.gate.result == 'success' && - needs.stage1.result != 'failure' && needs.stage1.result != 'cancelled' && + needs.stage1.result != 'cancelled' && needs.gate.outputs.stage2_hosts != '[]' strategy: fail-fast: false @@ -167,9 +168,10 @@ jobs: stage3: name: stage3 (${{ matrix.name }}) needs: [gate, stage2] + # A failed host is not this shell's call to make; only a cancellation is. if: | always() && needs.gate.result == 'success' && - needs.stage2.result != 'failure' && needs.stage2.result != 'cancelled' && + needs.stage2.result != 'cancelled' && needs.gate.outputs.stage3_hosts != '[]' strategy: fail-fast: false @@ -226,9 +228,10 @@ jobs: smoke-windows-bootstrap: name: Smoke test the Windows bootstrap needs: [gate, stage3] + # A failed host is not this shell's call to make; only a cancellation is. if: | always() && needs.gate.result == 'success' && - needs.stage3.result != 'failure' && needs.stage3.result != 'cancelled' + needs.stage3.result != 'cancelled' runs-on: windows-2025 steps: - uses: actions/checkout@v7 @@ -251,12 +254,13 @@ jobs: group: name: Group the release tree needs: [gate, stage1, stage2, stage3, smoke-windows-bootstrap] + # Required-host policy is the caller's, not this shell's; group with whatever built. if: | always() && needs.gate.result == 'success' && - needs.stage1.result != 'failure' && needs.stage1.result != 'cancelled' && - needs.stage2.result != 'failure' && needs.stage2.result != 'cancelled' && - needs.stage3.result != 'failure' && needs.stage3.result != 'cancelled' && - needs['smoke-windows-bootstrap'].result != 'failure' && needs['smoke-windows-bootstrap'].result != 'cancelled' + needs.stage1.result != 'cancelled' && + needs.stage2.result != 'cancelled' && + needs.stage3.result != 'cancelled' && + needs['smoke-windows-bootstrap'].result != 'cancelled' runs-on: ubuntu-24.04 outputs: build_id: ${{ needs.gate.outputs.build_id }} From 1de407be6b33c42d12bf29e990f16cf909774e5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Jos=C3=A9=20Garc=C3=ADa=20Garc=C3=ADa?= Date: Mon, 24 Aug 2026 19:57:55 +0200 Subject: [PATCH 04/11] Keep a failed smoke test blocking the release tree Loosening the gates so a host that fails to build only removes itself also let a failed Windows bootstrap smoke test through, which is a different thing: that host built something broken, and Windows is a required host, so the tree would have carried it to publication. --- .github/workflows/build-sdk.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-sdk.yml b/.github/workflows/build-sdk.yml index f3a1d12..4c96118 100644 --- a/.github/workflows/build-sdk.yml +++ b/.github/workflows/build-sdk.yml @@ -254,13 +254,16 @@ jobs: group: name: Group the release tree needs: [gate, stage1, stage2, stage3, smoke-windows-bootstrap] - # Required-host policy is the caller's, not this shell's; group with whatever built. + # Required-host policy is the caller's, not this shell's, so a host that + # failed to build only removes itself. A failed smoke test is different: + # it says a host built something broken, and that must not reach a tree. if: | always() && needs.gate.result == 'success' && needs.stage1.result != 'cancelled' && needs.stage2.result != 'cancelled' && needs.stage3.result != 'cancelled' && - needs['smoke-windows-bootstrap'].result != 'cancelled' + needs['smoke-windows-bootstrap'].result != 'cancelled' && + needs['smoke-windows-bootstrap'].result != 'failure' runs-on: ubuntu-24.04 outputs: build_id: ${{ needs.gate.outputs.build_id }} From b53f7a3b9f1862e13f938bd6f9b7eeb0a3bb0c3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Jos=C3=A9=20Garc=C3=ADa=20Garc=C3=ADa?= Date: Mon, 24 Aug 2026 20:09:53 +0200 Subject: [PATCH 05/11] Cache the source downloads the executor fetches Every leg re-fetched each source tarball from its upstream host, so one of them answering 503 -- pyyaml.org did, in the first check of the autobuilds PR -- failed a whole leg. Eight publishing hosts multiply that exposure, so the downloads directory is cached the way the older workflow already cached it, keyed per OS since hosts of a kind fetch the same tarballs. --- .github/workflows/build-sdk.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.github/workflows/build-sdk.yml b/.github/workflows/build-sdk.yml index 4c96118..ac1b450 100644 --- a/.github/workflows/build-sdk.yml +++ b/.github/workflows/build-sdk.yml @@ -89,6 +89,13 @@ jobs: path: ${{ env.CCACHE_DIR }} key: stage${{ matrix.stage }}-${{ matrix.name }}-ccache-${{ needs.gate.outputs.buildscripts_revision }} restore-keys: stage${{ matrix.stage }}-${{ matrix.name }}-ccache- + # Source tarballs come from their upstream hosts, one of which returning + # 503 is enough to fail a leg; keyed by OS so every host of a kind shares. + - uses: actions/cache@v6 + with: + path: ${{ github.workspace }}/build/downloads + key: downloads-${{ runner.os }}-${{ needs.gate.outputs.buildscripts_revision }} + restore-keys: downloads-${{ runner.os }}- - name: Build stage 1 host ${{ matrix.name }} shell: bash run: | @@ -146,6 +153,13 @@ jobs: path: ${{ env.CCACHE_DIR }} key: stage${{ matrix.stage }}-${{ matrix.name }}-ccache-${{ needs.gate.outputs.buildscripts_revision }} restore-keys: stage${{ matrix.stage }}-${{ matrix.name }}-ccache- + # Source tarballs come from their upstream hosts, one of which returning + # 503 is enough to fail a leg; keyed by OS so every host of a kind shares. + - uses: actions/cache@v6 + with: + path: ${{ github.workspace }}/build/downloads + key: downloads-${{ runner.os }}-${{ needs.gate.outputs.buildscripts_revision }} + restore-keys: downloads-${{ runner.os }}- - name: Build stage 2 host ${{ matrix.name }} shell: bash run: | @@ -203,6 +217,13 @@ jobs: path: ${{ env.CCACHE_DIR }} key: stage${{ matrix.stage }}-${{ matrix.name }}-ccache-${{ needs.gate.outputs.buildscripts_revision }} restore-keys: stage${{ matrix.stage }}-${{ matrix.name }}-ccache- + # Source tarballs come from their upstream hosts, one of which returning + # 503 is enough to fail a leg; keyed by OS so every host of a kind shares. + - uses: actions/cache@v6 + with: + path: ${{ github.workspace }}/build/downloads + key: downloads-${{ runner.os }}-${{ needs.gate.outputs.buildscripts_revision }} + restore-keys: downloads-${{ runner.os }}- - name: Build stage 3 host ${{ matrix.name }} shell: bash run: | From b149c6b383a673e51ff52f7983fad7fe7a35de16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Jos=C3=A9=20Garc=C3=ADa=20Garc=C3=ADa?= Date: Mon, 24 Aug 2026 21:20:15 +0200 Subject: [PATCH 06/11] Build the macOS x86_64 SDK on Intel hardware It was cross-built from Linux, so nothing ever executed what it produced -- no toolchain contract, no bootstrap smoke test -- which is why it stayed unpublished. A macos-15-intel runner builds it natively like every other host that can run its own output, and it publishes with the rest. --- cmake/hosts.json | 22 +++++++++++----------- scripts/ci/build-host.sh | 8 +------- 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/cmake/hosts.json b/cmake/hosts.json index 8c25caf..457b75e 100644 --- a/cmake/hosts.json +++ b/cmake/hosts.json @@ -1,15 +1,15 @@ { "hosts": [ - { "name": "x86_64-linux-gnu", "stage": 1, "runner": "ubuntu-24.04", "container": "ubuntu:20.04", "packaged": false }, - { "name": "x86_64-linux-gnu", "stage": 2, "runner": "ubuntu-24.04", "container": "ubuntu:20.04", "packaged": true }, - { "name": "aarch64-linux-gnu", "stage": 2, "runner": "ubuntu-24.04-arm", "container": "ubuntu:20.04", "packaged": true }, - { "name": "arm64-apple-darwin", "stage": 2, "runner": "macos-14", "container": null, "packaged": true }, - { "name": "x86_64-linux-musl", "stage": 2, "runner": "ubuntu-24.04", "container": null, "packaged": true }, - { "name": "aarch64-linux-musl", "stage": 2, "runner": "ubuntu-24.04-arm", "container": null, "packaged": true }, - { "name": "x86_64-w64-mingw32", "stage": 3, "runner": "ubuntu-24.04", "container": null, "packaged": true, "build_host": "x86_64-linux-gnu" }, - { "name": "i686-w64-mingw32", "stage": 3, "runner": "ubuntu-24.04", "container": null, "packaged": false, "build_host": "x86_64-linux-gnu" }, - { "name": "x86_64-unknown-freebsd", "stage": 3, "runner": "ubuntu-24.04", "container": null, "packaged": true, "build_host": "x86_64-linux-gnu" }, - { "name": "aarch64-unknown-freebsd", "stage": 3, "runner": "ubuntu-24.04", "container": null, "packaged": true, "build_host": "x86_64-linux-gnu" }, - { "name": "x86_64-apple-darwin", "stage": 3, "runner": "macos-14", "container": null, "packaged": false, "build_host": "arm64-apple-darwin" } + {"name": "x86_64-linux-gnu", "stage": 1, "runner": "ubuntu-24.04", "container": "ubuntu:20.04", "packaged": false}, + {"name": "x86_64-linux-gnu", "stage": 2, "runner": "ubuntu-24.04", "container": "ubuntu:20.04", "packaged": true}, + {"name": "aarch64-linux-gnu", "stage": 2, "runner": "ubuntu-24.04-arm", "container": "ubuntu:20.04", "packaged": true}, + {"name": "arm64-apple-darwin", "stage": 2, "runner": "macos-14", "container": null, "packaged": true}, + {"name": "x86_64-apple-darwin", "stage": 2, "runner": "macos-15-intel", "container": null, "packaged": true}, + {"name": "x86_64-linux-musl", "stage": 2, "runner": "ubuntu-24.04", "container": null, "packaged": true}, + {"name": "aarch64-linux-musl", "stage": 2, "runner": "ubuntu-24.04-arm", "container": null, "packaged": true}, + {"name": "x86_64-w64-mingw32", "stage": 3, "runner": "ubuntu-24.04", "container": null, "packaged": true, "build_host": "x86_64-linux-gnu"}, + {"name": "i686-w64-mingw32", "stage": 3, "runner": "ubuntu-24.04", "container": null, "packaged": false, "build_host": "x86_64-linux-gnu"}, + {"name": "x86_64-unknown-freebsd", "stage": 3, "runner": "ubuntu-24.04", "container": null, "packaged": true, "build_host": "x86_64-linux-gnu"}, + {"name": "aarch64-unknown-freebsd", "stage": 3, "runner": "ubuntu-24.04", "container": null, "packaged": true, "build_host": "x86_64-linux-gnu"} ] } diff --git a/scripts/ci/build-host.sh b/scripts/ci/build-host.sh index 1a250a2..88413a1 100755 --- a/scripts/ci/build-host.sh +++ b/scripts/ci/build-host.sh @@ -339,7 +339,7 @@ fi extra_cmake_args=() case $host in -x86_64-linux-gnu | aarch64-linux-gnu | arm64-apple-darwin) +x86_64-linux-gnu | aarch64-linux-gnu | arm64-apple-darwin | x86_64-apple-darwin) install_dependencies ;; x86_64-w64-mingw32) @@ -362,12 +362,6 @@ aarch64-unknown-freebsd) export PATH="$PWD/freebsd-cross/bin:$PATH" extra_cmake_args+=(-DCMAKE_TOOLCHAIN_FILE="$repo_root/cmake/toolchains/aarch64-unknown-freebsd.cmake") ;; -x86_64-apple-darwin) - install_dependencies - scripts/setup-macos-cross.sh "$PWD/macos-cross" - export PATH="$PWD/macos-cross/bin:$PATH" - extra_cmake_args+=(-DCMAKE_TOOLCHAIN_FILE="$repo_root/cmake/toolchains/x86_64-apple-darwin.cmake") - ;; *) printf 'no build recipe for host: %s\n' "$host" >&2 exit 1 From 1ad77966dc642a3e8785e90309107d9ac8414cd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Jos=C3=A9=20Garc=C3=ADa=20Garc=C3=ADa?= Date: Mon, 24 Aug 2026 21:56:14 +0200 Subject: [PATCH 07/11] Stop a Homebrew library shadowing the ones built here ld64 looks for a dylib in every search path before it considers a static library in any of them, so gmp built here -- static only -- lost to Homebrew's copy, which the runner's preinstalled GCC drags in. It only bites on Intel, where Homebrew lives in /usr/local and that is an implicit search path; on Apple Silicon it sits in /opt/homebrew and never enters the search. isl reported it as "gmp library too old", which is what its link probe says whatever the reason it failed. --- cmake/StaticPolicy.cmake | 7 +++++++ tests/cmake/static-policy.cmake | 5 ++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/cmake/StaticPolicy.cmake b/cmake/StaticPolicy.cmake index 0f85084..ff5c9c7 100644 --- a/cmake/StaticPolicy.cmake +++ b/cmake/StaticPolicy.cmake @@ -20,6 +20,13 @@ function(vitasdk_get_host_static_flags system_name out_c out_cxx out_link) set(c_flags -static-libgcc) set(cxx_flags -static-libgcc -static-libstdc++) set(link_flags -static-libgcc -static-libstdc++) + elseif(system_name STREQUAL "Darwin") + # ld64 looks for a dylib in every search path before it considers a + # static library in any of them, so the dependencies built here -- + # static only -- lose to a Homebrew copy of the same name. That only + # bites on Intel, where Homebrew's /usr/local is an implicit search + # path; searching each directory for either kind settles it. + set(link_flags -Wl,-search_paths_first) endif() set(${out_c} "${c_flags}" PARENT_SCOPE) diff --git a/tests/cmake/static-policy.cmake b/tests/cmake/static-policy.cmake index 8e609b6..59b4875 100644 --- a/tests/cmake/static-policy.cmake +++ b/tests/cmake/static-policy.cmake @@ -23,7 +23,10 @@ assert_flags(Windows "-static-libgcc" "-static-libgcc;-static-libstdc++" "-static-libgcc;-static-libstdc++") -assert_flags(Darwin "" "" "") +# Darwin asks for nothing static -- its libgcc is not a separate library -- +# but it does need the linker to stop preferring a dylib from any search path +# over the static dependencies built here. +assert_flags(Darwin "" "" "-Wl,-search_paths_first") function(assert_libtool_flag system_name expected) vitasdk_get_libtool_static_flag("${system_name}" actual) From 2075e5ba8d9ff26c265fbfb607bdde4079fa4cd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Jos=C3=A9=20Garc=C3=ADa=20Garc=C3=ADa?= Date: Mon, 24 Aug 2026 22:05:42 +0200 Subject: [PATCH 08/11] Separate what a host is called from what its compiler is called FreeBSD's cross compiler answers to x86_64-unknown-freebsd14 while the host publishes as x86_64-unknown-freebsd, and one variable was serving both. So the SDK tarball, the bootstrap archive, the core package, the pacman architecture and the host handed to the vdpm bundle all carried a release number that vdpm's own release-info.txt does not, and the bundle check refused them -- silently, since that grep had no message. Everywhere the name is an identity now uses the published one, which the executor passes from the lock; the toolchain lookups keep the triplet. Every other host derives the same string either way, which is why nothing noticed until FreeBSD had to publish. --- CMakeLists.txt | 10 +++++++++ cmake/recipes/BuildHostTools.cmake | 2 +- cmake/recipes/FinalizeSdk.cmake | 20 ++++++++--------- scripts/ci/build-host.sh | 6 +++++- tests/cmake/published-host-name.cmake | 31 +++++++++++++++++++++++++++ 5 files changed, 57 insertions(+), 12 deletions(-) create mode 100644 tests/cmake/published-host-name.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index b1d4f87..e89c595 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -134,6 +134,16 @@ endif() get_host_triplet(host_native) get_build_triplet(build_native) +# What this host is called where it is published -- in artifact names, in the +# package architecture, in the vdpm bundle it embeds. It is the toolchain +# triplet on every host but FreeBSD, whose cross compiler carries a release +# number (x86_64-unknown-freebsd14-gcc) that its published name does not. +string(REGEX REPLACE "^(.*-unknown-freebsd)[0-9]+$" "\\1" + host_published_default "${host_native}") +set(VITASDK_HOST_NAME "${host_published_default}" CACHE STRING + "Name this host publishes under, if not its toolchain triplet") +set(host_published "${VITASDK_HOST_NAME}") + message(STATUS "Host: ${host_native}") message(STATUS "Build: ${build_native}") message(STATUS "Target: ${target_arch}") diff --git a/cmake/recipes/BuildHostTools.cmake b/cmake/recipes/BuildHostTools.cmake index 978cdbb..cc1287f 100644 --- a/cmake/recipes/BuildHostTools.cmake +++ b/cmake/recipes/BuildHostTools.cmake @@ -55,7 +55,7 @@ if(vdpm_use_release_bundle) ${VDPM_BUNDLE} ${VDPM_BUNDLE_SHA256} ${CMAKE_INSTALL_PREFIX} - ${host_native} + ${host_published} VERBATIM) else() ExternalProject_Add(vdpm diff --git a/cmake/recipes/FinalizeSdk.cmake b/cmake/recipes/FinalizeSdk.cmake index aa9f41c..f41ccc2 100644 --- a/cmake/recipes/FinalizeSdk.cmake +++ b/cmake/recipes/FinalizeSdk.cmake @@ -22,7 +22,7 @@ if(BUILD_PACMAN_CLIENT) add_custom_target(package-client-configuration COMMAND ${CMAKE_COMMAND} -DOUTPUT=${CMAKE_INSTALL_PREFIX}/etc/pacman.conf - -DHOST_ARCHITECTURE=${host_native} + -DHOST_ARCHITECTURE=${host_published} -P ${PROJECT_SOURCE_DIR}/cmake/WritePacmanConfig.cmake DEPENDS vdpm VERBATIM) @@ -159,20 +159,20 @@ add_custom_target(check-static-policy VERBATIM ) -add_custom_command(OUTPUT "vitasdk-${host_native}-${build_date}.tar.bz2" - COMMAND ${CMAKE_COMMAND} -E tar "cfvj" "vitasdk-${host_native}-${build_date}.tar.bz2" "${CMAKE_INSTALL_PREFIX}" +add_custom_command(OUTPUT "vitasdk-${host_published}-${build_date}.tar.bz2" + COMMAND ${CMAKE_COMMAND} -E tar "cfvj" "vitasdk-${host_published}-${build_date}.tar.bz2" "${CMAKE_INSTALL_PREFIX}" DEPENDS finalize-sdk - COMMENT "Creating vitasdk-${host_native}-${build_date}.tar.bz2" + COMMENT "Creating vitasdk-${host_published}-${build_date}.tar.bz2" ) # Create a sdk tarball -add_custom_target(tarball DEPENDS "vitasdk-${host_native}-${build_date}.tar.bz2") +add_custom_target(tarball DEPENDS "vitasdk-${host_published}-${build_date}.tar.bz2") if(BUILD_PACMAN_CLIENT) set(VITASDK_BOOTSTRAP_OUTPUT_DIR "${CMAKE_BINARY_DIR}/bootstraps" CACHE PATH "Output directory for verified VitaSDK bootstrap archives") set(bootstrap_archive - "${VITASDK_BOOTSTRAP_OUTPUT_DIR}/vitasdk-bootstrap-${host_native}.tar.bz2") + "${VITASDK_BOOTSTRAP_OUTPUT_DIR}/vitasdk-bootstrap-${host_published}.tar.bz2") add_custom_command(OUTPUT ${bootstrap_archive} BYPRODUCTS ${bootstrap_archive}.sha256 COMMAND ${CMAKE_COMMAND} -E make_directory ${VITASDK_BOOTSTRAP_OUTPUT_DIR} @@ -181,7 +181,7 @@ if(BUILD_PACMAN_CLIENT) COMMAND ${PROJECT_SOURCE_DIR}/scripts/create-bootstrap-archive.sh ${CMAKE_INSTALL_PREFIX} ${VITASDK_BOOTSTRAP_OUTPUT_DIR} - ${host_native} + ${host_published} ${VITASDK_SOURCE_DATE_EPOCH} DEPENDS finalize-sdk ${PROJECT_SOURCE_DIR}/scripts/create-bootstrap-archive.sh @@ -193,12 +193,12 @@ if(BUILD_PACMAN_CLIENT) set(VITASDK_PACKAGE_OUTPUT_DIR "${CMAKE_BINARY_DIR}/packages" CACHE PATH "Output directory for the vitasdk-core package") set(core_package - "${VITASDK_PACKAGE_OUTPUT_DIR}/vitasdk-core-${VITASDK_PACKAGE_VERSION}-1-${host_native}.pkg.tar.xz") + "${VITASDK_PACKAGE_OUTPUT_DIR}/vitasdk-core-${VITASDK_PACKAGE_VERSION}-1-${host_published}.pkg.tar.xz") # A core release is two packages: the toolchain, and the client that # installs it. Both are outputs, and both have to be gone before a rerun. file(GLOB stale_client_packages - "${VITASDK_PACKAGE_OUTPUT_DIR}/vdpm-*-1-${host_native}.pkg.tar.xz") + "${VITASDK_PACKAGE_OUTPUT_DIR}/vdpm-*-1-${host_published}.pkg.tar.xz") add_custom_command(OUTPUT ${core_package} COMMAND ${CMAKE_COMMAND} -E make_directory ${VITASDK_PACKAGE_OUTPUT_DIR} @@ -208,7 +208,7 @@ if(BUILD_PACMAN_CLIENT) ${CMAKE_SOURCE_DIR}/scripts/create-core-package.sh ${CMAKE_INSTALL_PREFIX} ${VITASDK_PACKAGE_OUTPUT_DIR} - ${host_native} + ${host_published} ${VITASDK_PACKAGE_VERSION} ${VITASDK_SOURCE_REVISION} DEPENDS finalize-sdk diff --git a/scripts/ci/build-host.sh b/scripts/ci/build-host.sh index 88413a1..7c62183 100755 --- a/scripts/ci/build-host.sh +++ b/scripts/ci/build-host.sh @@ -137,6 +137,9 @@ build_and_stage() { -S "$repo_root" -B build ${extra_args[@]+"${extra_args[@]}"} -DVITASDK_SOURCE_REVISION="$revision" -DVITASDK_SOURCE_DATE_EPOCH="$source_date_epoch" + # The lock names the host; artifacts published under any other name + # would not match what the caller asked to be built. + -DVITASDK_HOST_NAME="$host" ) [[ -n ${stage1_dir:-} ]] && cmake_args+=(-DVITASDK_STAGE1_DIR="$stage1_dir") local -a targets=(tarball) @@ -203,6 +206,7 @@ stage_and_write_provenance() { # bootstrap smoke test runs inside the same container that built it. build_musl_host() { local -a docker_env=( + -e VITASDK_HOST_NAME="$host" -e VITASDK_SOURCE_REVISION="$revision" -e VITASDK_SOURCE_DATE_EPOCH="$source_date_epoch" -e CCACHE_DIR=/src/.ccache @@ -237,7 +241,7 @@ build_musl_host() { test -n "$STAGE1_DIR" mkdir -p /src/build cd /src/build - configure_args="-DVITASDK_STAGE1_DIR=$STAGE1_DIR -DVITASDK_SOURCE_REVISION=$VITASDK_SOURCE_REVISION -DVITASDK_SOURCE_DATE_EPOCH=$VITASDK_SOURCE_DATE_EPOCH" + configure_args="-DVITASDK_STAGE1_DIR=$STAGE1_DIR -DVITASDK_SOURCE_REVISION=$VITASDK_SOURCE_REVISION -DVITASDK_SOURCE_DATE_EPOCH=$VITASDK_SOURCE_DATE_EPOCH -DVITASDK_HOST_NAME=$VITASDK_HOST_NAME" targets="tarball" if [ -n "${VITASDK_PACKAGED_HOST:-}" ]; then configure_args="$configure_args -DBUILD_PACMAN_CLIENT=ON -DVITASDK_PACKAGE_VERSION=$VITASDK_PACKAGE_VERSION -DVDPM_BUNDLE=$VDPM_BUNDLE -DVDPM_BUNDLE_SHA256=$VDPM_BUNDLE_SHA256" diff --git a/tests/cmake/published-host-name.cmake b/tests/cmake/published-host-name.cmake new file mode 100644 index 0000000..eb26cfa --- /dev/null +++ b/tests/cmake/published-host-name.cmake @@ -0,0 +1,31 @@ +cmake_minimum_required(VERSION 3.16) + +# A host is called two things: the triplet its cross compiler answers to, and +# the name it publishes under. They are the same everywhere except FreeBSD, +# whose compiler carries a release number its published name does not -- and +# mixing them up published a bundle nobody could match to a host. + +function(published_name_of triplet out) + string(REGEX REPLACE "^(.*-unknown-freebsd)[0-9]+$" "\\1" name "${triplet}") + set(${out} "${name}" PARENT_SCOPE) +endfunction() + +function(assert_published triplet expected) + published_name_of("${triplet}" actual) + if(NOT "${actual}" STREQUAL "${expected}") + message(FATAL_ERROR + "${triplet} publishes as '${actual}', expected '${expected}'") + endif() +endfunction() + +assert_published(x86_64-unknown-freebsd14 x86_64-unknown-freebsd) +assert_published(aarch64-unknown-freebsd14 aarch64-unknown-freebsd) + +# Every other host publishes under the triplet it builds with. +foreach(triplet x86_64-linux-gnu aarch64-linux-gnu x86_64-linux-musl + aarch64-linux-musl arm64-apple-darwin x86_64-apple-darwin + x86_64-w64-mingw32 i686-w64-mingw32) + assert_published("${triplet}" "${triplet}") +endforeach() + +message(STATUS "published host name checks passed") From ab1f438df027924136f808836f5221905e9e8139 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Jos=C3=A9=20Garc=C3=ADa=20Garc=C3=ADa?= Date: Mon, 24 Aug 2026 22:33:53 +0200 Subject: [PATCH 09/11] Print the configure log of whatever refused to build An autotools check says why it failed in config.log and nowhere else; what reaches the build log is whatever string that check carries, which sent this repository chasing a gmp that was never too old. The old autobuilds job dumped these on failure and the executor did not inherit it. --- scripts/ci/build-host.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/scripts/ci/build-host.sh b/scripts/ci/build-host.sh index 7c62183..6bca79b 100755 --- a/scripts/ci/build-host.sh +++ b/scripts/ci/build-host.sh @@ -87,6 +87,21 @@ export LANG=C.UTF-8 git config --global user.email "builds@ci.invalid" git config --global user.name "buildscripts CI" +# An autotools component says why it refused in its config.log and nowhere +# else: the message that reaches the build log is whatever generic string the +# failing check happens to carry. +dump_configure_logs() { + local status=$? log + (( status == 0 )) && return 0 + while IFS= read -r log; do + printf '::group::%s\n' "$log" + tail -n 120 "$log" + printf '::endgroup::\n' + done < <(find build -name config.log -newermt '-2 hours' 2>/dev/null | head -5) + return "$status" +} +trap dump_configure_logs EXIT + # The matrix's packaged flag, not the host name, gates core-package treatment. is_packaged_host() { [[ $packaged == true ]]; } From abea7614ff198c1d9f326588e6a73e03e499348f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Jos=C3=A9=20Garc=C3=ADa=20Garc=C3=ADa?= Date: Mon, 24 Aug 2026 22:51:30 +0200 Subject: [PATCH 10/11] Build gmp position-independent Its x86_64 assembly reaches lookup tables like __gmp_binvert_limb_table from the text segment, and Apple's linker refuses those relocations inside a position-independent executable -- so every link against the gmp built here died on Intel macOS, and isl reported it as "gmp library too old", the string its probe prints for any failed link. Reproduced and fixed in isolation before touching this: gmp and isl alone on an Intel runner, where a configure that took half an hour to reach here takes eight minutes. The search-paths flag from the previous attempt goes with it -- the linker was never looking in the wrong place. --- cmake/StaticPolicy.cmake | 7 ------- cmake/recipes/BuildHostDependencies.cmake | 4 ++++ tests/cmake/static-policy.cmake | 5 +---- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/cmake/StaticPolicy.cmake b/cmake/StaticPolicy.cmake index ff5c9c7..0f85084 100644 --- a/cmake/StaticPolicy.cmake +++ b/cmake/StaticPolicy.cmake @@ -20,13 +20,6 @@ function(vitasdk_get_host_static_flags system_name out_c out_cxx out_link) set(c_flags -static-libgcc) set(cxx_flags -static-libgcc -static-libstdc++) set(link_flags -static-libgcc -static-libstdc++) - elseif(system_name STREQUAL "Darwin") - # ld64 looks for a dylib in every search path before it considers a - # static library in any of them, so the dependencies built here -- - # static only -- lose to a Homebrew copy of the same name. That only - # bites on Intel, where Homebrew's /usr/local is an implicit search - # path; searching each directory for either kind settles it. - set(link_flags -Wl,-search_paths_first) endif() set(${out_c} "${c_flags}" PARENT_SCOPE) diff --git a/cmake/recipes/BuildHostDependencies.cmake b/cmake/recipes/BuildHostDependencies.cmake index f3a223e..cb0800c 100644 --- a/cmake/recipes/BuildHostDependencies.cmake +++ b/cmake/recipes/BuildHostDependencies.cmake @@ -184,6 +184,10 @@ function(toolchain_deps toolchain_deps_dir toolchain_install_dir toolchain_suffi --enable-cxx --disable-shared --enable-static + # gmp's x86_64 assembly reaches its lookup tables from the text + # segment, which Apple's linker refuses inside a position-independent + # executable; every host that links this statically wants it anyway. + --with-pic BUILD_COMMAND ${compiler_flags} ${wrapper_command} $(MAKE) ) diff --git a/tests/cmake/static-policy.cmake b/tests/cmake/static-policy.cmake index 59b4875..8e609b6 100644 --- a/tests/cmake/static-policy.cmake +++ b/tests/cmake/static-policy.cmake @@ -23,10 +23,7 @@ assert_flags(Windows "-static-libgcc" "-static-libgcc;-static-libstdc++" "-static-libgcc;-static-libstdc++") -# Darwin asks for nothing static -- its libgcc is not a separate library -- -# but it does need the linker to stop preferring a dylib from any search path -# over the static dependencies built here. -assert_flags(Darwin "" "" "-Wl,-search_paths_first") +assert_flags(Darwin "" "" "") function(assert_libtool_flag system_name expected) vitasdk_get_libtool_static_flag("${system_name}" actual) From 9e64d74bcc5c6baf93366634323f57cdc6a3396b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Jos=C3=A9=20Garc=C3=ADa=20Garc=C3=ADa?= Date: Mon, 24 Aug 2026 23:45:45 +0200 Subject: [PATCH 11/11] Disable gdb's native language support, and name every leak at once gdb was the one component still building with NLS, so on Intel macOS it linked Homebrew's libintl -- a library no user's machine has, which is why the dependency audit refused the SDK. Its siblings all disable it already. The audit now reports every unexpected dependency before failing rather than the first: each one costs a full build to find, and a host that drags in one library from the machine tends to drag in several. --- cmake/recipes/BuildHostDependencies.cmake | 4 ++++ scripts/audit-host-deps.sh | 11 ++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/cmake/recipes/BuildHostDependencies.cmake b/cmake/recipes/BuildHostDependencies.cmake index cb0800c..e9e5b74 100644 --- a/cmake/recipes/BuildHostDependencies.cmake +++ b/cmake/recipes/BuildHostDependencies.cmake @@ -358,6 +358,10 @@ function(toolchain_deps toolchain_deps_dir toolchain_install_dir toolchain_suffi --without-lzma --without-babeltrace --without-xxhash + # Every other component here disables it. Left on, gdb links the + # host's libintl -- on Intel macOS that is Homebrew's, which the + # dependency audit refuses and a user's machine would not have. + --disable-nls --without-debuginfod --with-gmp=${toolchain_deps_dir} --with-mpfr=${toolchain_deps_dir} diff --git a/scripts/audit-host-deps.sh b/scripts/audit-host-deps.sh index 42d02ee..b240ade 100755 --- a/scripts/audit-host-deps.sh +++ b/scripts/audit-host-deps.sh @@ -6,11 +6,15 @@ set -eu host_system=${VITASDK_HOST_SYSTEM_NAME:-$(uname -s)} objdump_command=${VITASDK_OBJDUMP:-objdump} +# Every offender, not the first: each one costs a full build to discover, and +# a host that drags in one library from the machine usually drags in several. +unexpected_count=0 + fail_dependency() { binary=$1 dependency=$2 echo "unexpected dynamic host dependency: $binary -> $dependency" >&2 - exit 1 + unexpected_count=$((unexpected_count + 1)) } # One reader for every ELF host; what differs between them is only which @@ -153,4 +157,9 @@ case "$host_system" in ;; esac +if [ "$unexpected_count" -ne 0 ]; then + echo "$unexpected_count unexpected host dependencies" >&2 + exit 1 +fi + echo "Host dependency audit passed"