From 44706ecaa5135b6e9d2ec33ea70391847216e976 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Jos=C3=A9=20Garc=C3=ADa=20Garc=C3=ADa?= Date: Wed, 26 Aug 2026 10:10:23 +0200 Subject: [PATCH 1/3] Give the Intel Mac host back the recipe that cross-builds it hosts.json says x86_64-apple-darwin is stage 3, cross-built on the arm64 runner; build-host.sh has called it native since #161, which made it so in both places at once. db4d1a593 turned the lock back into a cross and left the recipe alone, so stage 3 configured with the machine's own compiler: "-- Host: arm64-apple-darwin", every component at --host=aarch64-apple- darwin, and an arm64 SDK published under the Intel name. It stayed green through the SDK, the static validation and the toolchain contract -- the contract runs run.sh under arch -x86_64, and a Rosetta process execs an arm64 binary natively, so it never demanded the slice. What demanded it was the bootstrap smoke test, 18 minutes in: arch -x86_64 bootstrap-installed/bin/arm-vita-eabi-gcc, Bad CPU type in executable. The recipe restored here is the one #161 deleted, unchanged: the wrappers from setup-macos-cross.sh in PATH and the toolchain file that pins the triplet. The test is what was missing. The lock and the case block are two halves of one decision and nothing tied them together, so it runs the real case block for every host in hosts.json and fails when one that names a build_host gets no toolchain file, or one that names none gets a toolchain file. --- scripts/ci/build-host.sh | 8 ++- tests/ci/test-cross-host-recipe.sh | 92 ++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 1 deletion(-) create mode 100755 tests/ci/test-cross-host-recipe.sh diff --git a/scripts/ci/build-host.sh b/scripts/ci/build-host.sh index 2cb72c2..5fa88ff 100755 --- a/scripts/ci/build-host.sh +++ b/scripts/ci/build-host.sh @@ -386,7 +386,7 @@ fi extra_cmake_args=() case $host in -x86_64-linux-gnu | aarch64-linux-gnu | arm64-apple-darwin | x86_64-apple-darwin) +x86_64-linux-gnu | aarch64-linux-gnu | arm64-apple-darwin) install_dependencies ;; x86_64-w64-mingw32) @@ -409,6 +409,12 @@ aarch64-unknown-freebsd) export PATH="$PWD/freebsd-cross/bin:$PATH" extra_cmake_args+=(-DCMAKE_TOOLCHAIN_FILE="$repo_root/cmake/toolchains/aarch64-unknown-freebsd.cmake") ;; +x86_64-apple-darwin) + install_dependencies + scripts/setup-macos-cross.sh "$PWD/macos-cross" + export PATH="$PWD/macos-cross/bin:$PATH" + extra_cmake_args+=(-DCMAKE_TOOLCHAIN_FILE="$repo_root/cmake/toolchains/x86_64-apple-darwin.cmake") + ;; *) printf 'no build recipe for host: %s\n' "$host" >&2 exit 1 diff --git a/tests/ci/test-cross-host-recipe.sh b/tests/ci/test-cross-host-recipe.sh new file mode 100755 index 0000000..9ecb75f --- /dev/null +++ b/tests/ci/test-cross-host-recipe.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# The lock and the build recipe are two halves of one decision. +# +# hosts.json says whether a host is cross-built; the case block in +# build-host.sh says how. Nothing tied them together, so #161 made +# x86_64-apple-darwin native in both and db4d1a593 made it a cross again in +# the lock alone: the job then built an arm64 SDK, published it under the +# Intel name, and stayed green until the bootstrap smoke test, 18 minutes in. + +set -euo pipefail + +repository_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P) +script="$repository_root/scripts/ci/build-host.sh" + +failures=0 + +# The case block is run as the script runs it, not re-implemented here. The +# setup scripts it calls are stubbed by name from a scratch directory: what +# is under test is the dispatch, not what a cross environment costs to build. +stubs=$(mktemp -d) +trap 'rm -rf "$stubs"' EXIT +mkdir -p "$stubs/scripts" +for stub in setup-macos-cross.sh setup-freebsd-cross.sh; do + printf '#!/bin/sh\nexit 0\n' >"$stubs/scripts/$stub" + chmod +x "$stubs/scripts/$stub" +done + +toolchain_file_for() +{ + ( + cd "$stubs" || exit 1 + host=$1 repo_root=$repository_root bash -c ' + set -euo pipefail + host=$host + repo_root=$repo_root + install_dependencies() { :; } + '"$(sed -n '/^extra_cmake_args=()/,/^esac/p' "$script")"' + printf "%s" "${extra_cmake_args[*]:-}" + ' + ) +} + +check() +{ + local host=$1 build_host=$2 args status expected + args=$(toolchain_file_for "$host") && status=0 || status=$? + if ((status != 0)); then + printf 'FAIL: %s has no build recipe in build-host.sh\n' "$host" >&2 + failures=$((failures + 1)) + return + fi + if [[ -n $build_host ]]; then + expected="-DCMAKE_TOOLCHAIN_FILE=$repository_root/cmake/toolchains/$host.cmake" + if [[ $args != *"$expected"* ]]; then + printf 'FAIL: %s is cross-built from %s but its recipe does not use %s.cmake\n recipe: %s\n' \ + "$host" "$build_host" "$host" "${args:-}" >&2 + failures=$((failures + 1)) + fi + if [[ ! -f $repository_root/cmake/toolchains/$host.cmake ]]; then + printf 'FAIL: %s names a toolchain file that does not exist\n' "$host" >&2 + failures=$((failures + 1)) + fi + elif [[ $args == *CMAKE_TOOLCHAIN_FILE* ]]; then + printf 'FAIL: %s builds natively in the lock but its recipe cross-compiles\n recipe: %s\n' \ + "$host" "$args" >&2 + failures=$((failures + 1)) + fi +} + +# musl hosts never reach the case block: they build inside Alpine, where the +# build is native, and build-host.sh dispatches them before it. +while read -r host build_host; do + check "$host" "$build_host" +done < <(python3 - "$repository_root/cmake/hosts.json" <<'PYTHON' +import json, sys + +seen = set() +for host in json.load(open(sys.argv[1]))["hosts"]: + name = host["name"] + if name.endswith("-linux-musl") or name in seen: + continue + seen.add(name) + print(name, host.get("build_host", "")) +PYTHON +) + +if ((failures)); then + printf '%d cross-host recipe check(s) failed\n' "$failures" >&2 + exit 1 +fi + +printf 'cross host recipe tests passed\n' From a90b8597ec318f0dd50270e9ec5b3237b0b735e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Jos=C3=A9=20Garc=C3=ADa=20Garc=C3=ADa?= Date: Wed, 26 Aug 2026 10:10:35 +0200 Subject: [PATCH 2/3] Refuse to publish a host under a name it was not built for VITASDK_HOST_NAME is what the artifacts are called; the toolchain triplet is what they are. Nothing compared the two, so the build whose toolchain file had gone missing named one host and produced another, and everything downstream agreed with the name: the tarball, the package architecture, the bootstrap, the provenance. The check runs as soon as both are known, which costs seconds instead of the 18 minutes the bootstrap smoke test needed to reach the same conclusion by running a binary. arm64 and aarch64 are one machine here: Apple writes the first, config.sub the second, and both reach this line depending on who was asked. The name derivation moves to cmake/PublishedHostName.cmake so its test can call it instead of keeping a second copy of the FreeBSD rule. --- CMakeLists.txt | 9 ++-- cmake/PublishedHostName.cmake | 40 ++++++++++++++++++ tests/cmake/published-host-name-check.cmake | 5 +++ tests/cmake/published-host-name.cmake | 47 ++++++++++++++++----- 4 files changed, 86 insertions(+), 15 deletions(-) create mode 100644 cmake/PublishedHostName.cmake create mode 100644 tests/cmake/published-host-name-check.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index e89c595..299078d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -113,6 +113,7 @@ endif() include(Utils) include(GetTriplet) +include(PublishedHostName) include(ExternalProject) set(vitasdk_base_c_flags "${CMAKE_C_FLAGS}") @@ -135,14 +136,12 @@ get_host_triplet(host_native) get_build_triplet(build_native) # What this host is called where it is published -- in artifact names, in the -# package architecture, in the vdpm bundle it embeds. It is the toolchain -# triplet on every host but FreeBSD, whose cross compiler carries a release -# number (x86_64-unknown-freebsd14-gcc) that its published name does not. -string(REGEX REPLACE "^(.*-unknown-freebsd)[0-9]+$" "\\1" - host_published_default "${host_native}") +# package architecture, in the vdpm bundle it embeds. +vitasdk_published_host_name("${host_native}" host_published_default) set(VITASDK_HOST_NAME "${host_published_default}" CACHE STRING "Name this host publishes under, if not its toolchain triplet") set(host_published "${VITASDK_HOST_NAME}") +vitasdk_check_published_host_name("${host_published}" "${host_native}") message(STATUS "Host: ${host_native}") message(STATUS "Build: ${build_native}") diff --git a/cmake/PublishedHostName.cmake b/cmake/PublishedHostName.cmake new file mode 100644 index 0000000..1652845 --- /dev/null +++ b/cmake/PublishedHostName.cmake @@ -0,0 +1,40 @@ +# +# Copyright(c) 2016 codestation +# Distributed under the MIT License (http://opensource.org/licenses/MIT) +# + +# A host is called two things: the triplet its cross compiler answers to, and +# the name it publishes under. Keeping them apart is what lets a host be +# published under a name it was not built for, which is silent -- the +# artifacts are well formed, they just belong to another machine. + +include_guard(GLOBAL) + +# The name a host triplet publishes under. The same everywhere except +# FreeBSD, whose cross compiler carries a release number (x86_64-unknown- +# freebsd14-gcc) that its published name does not. +function(vitasdk_published_host_name triplet out_name) + string(REGEX REPLACE "^(.*-unknown-freebsd)[0-9]+$" "\\1" name "${triplet}") + set(${out_name} "${name}" PARENT_SCOPE) +endfunction() + +# arm64 and aarch64 name one machine: Apple writes the first, config.sub the +# second, and both reach here depending on who was asked. +function(vitasdk_canonical_host_name name out_name) + string(REGEX REPLACE "^arm64-" "aarch64-" canonical "${name}") + set(${out_name} "${canonical}" PARENT_SCOPE) +endfunction() + +# Fail unless published is the name triplet is entitled to publish under. +function(vitasdk_check_published_host_name published triplet) + vitasdk_published_host_name("${triplet}" expected) + vitasdk_canonical_host_name("${expected}" expected_canonical) + vitasdk_canonical_host_name("${published}" published_canonical) + if(published_canonical STREQUAL expected_canonical) + return() + endif() + message(FATAL_ERROR + "this build publishes as ${published} but its toolchain builds " + "${triplet}. Either it is missing the toolchain file that would make " + "it cross-compile, or VITASDK_HOST_NAME names the wrong host.") +endfunction() diff --git a/tests/cmake/published-host-name-check.cmake b/tests/cmake/published-host-name-check.cmake new file mode 100644 index 0000000..ddd3eb9 --- /dev/null +++ b/tests/cmake/published-host-name-check.cmake @@ -0,0 +1,5 @@ +cmake_minimum_required(VERSION 3.16) + +include("${CMAKE_CURRENT_LIST_DIR}/../../cmake/PublishedHostName.cmake") + +vitasdk_check_published_host_name("${PUBLISHED}" "${TRIPLET}") diff --git a/tests/cmake/published-host-name.cmake b/tests/cmake/published-host-name.cmake index eb26cfa..75e7bac 100644 --- a/tests/cmake/published-host-name.cmake +++ b/tests/cmake/published-host-name.cmake @@ -1,17 +1,9 @@ cmake_minimum_required(VERSION 3.16) -# A host is called two things: the triplet its cross compiler answers to, and -# the name it publishes under. They are the same everywhere except FreeBSD, -# whose compiler carries a release number its published name does not -- and -# mixing them up published a bundle nobody could match to a host. - -function(published_name_of triplet out) - string(REGEX REPLACE "^(.*-unknown-freebsd)[0-9]+$" "\\1" name "${triplet}") - set(${out} "${name}" PARENT_SCOPE) -endfunction() +include("${CMAKE_CURRENT_LIST_DIR}/../../cmake/PublishedHostName.cmake") function(assert_published triplet expected) - published_name_of("${triplet}" actual) + vitasdk_published_host_name("${triplet}" actual) if(NOT "${actual}" STREQUAL "${expected}") message(FATAL_ERROR "${triplet} publishes as '${actual}', expected '${expected}'") @@ -28,4 +20,39 @@ foreach(triplet x86_64-linux-gnu aarch64-linux-gnu x86_64-linux-musl assert_published("${triplet}" "${triplet}") endforeach() +# The gate: a name may only be published by the toolchain that earns it. +function(assert_gate published triplet expected_result) + execute_process( + COMMAND ${CMAKE_COMMAND} + -DPUBLISHED=${published} -DTRIPLET=${triplet} + -P "${CMAKE_CURRENT_LIST_DIR}/published-host-name-check.cmake" + RESULT_VARIABLE result + OUTPUT_QUIET + ERROR_VARIABLE error) + if(expected_result STREQUAL "accepts" AND NOT result EQUAL 0) + message(FATAL_ERROR "${triplet} must publish as ${published}: ${error}") + endif() + if(expected_result STREQUAL "rejects") + if(result EQUAL 0) + message(FATAL_ERROR "${triplet} must not publish as ${published}") + endif() + if(NOT error MATCHES "${published}" OR NOT error MATCHES "${triplet}") + message(FATAL_ERROR "the rejection must name both, got: ${error}") + endif() + endif() +endfunction() + +assert_gate(x86_64-linux-gnu x86_64-linux-gnu accepts) +assert_gate(x86_64-unknown-freebsd x86_64-unknown-freebsd14 accepts) +# Apple's spelling of the machine config.sub calls aarch64. +assert_gate(arm64-apple-darwin aarch64-apple-darwin accepts) +assert_gate(aarch64-apple-darwin arm64-apple-darwin accepts) + +# What shipped an arm64 SDK under the Intel name: the cross host lost its +# toolchain file, so the build named one host and produced another. +assert_gate(x86_64-apple-darwin arm64-apple-darwin rejects) +assert_gate(arm64-apple-darwin x86_64-apple-darwin rejects) +assert_gate(x86_64-w64-mingw32 x86_64-linux-gnu rejects) +assert_gate(aarch64-linux-gnu x86_64-linux-gnu rejects) + message(STATUS "published host name checks passed") From 0cc11d114ba5ae2b96d974b43b399be18854a661 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Jos=C3=A9=20Garc=C3=ADa=20Garc=C3=ADa?= Date: Wed, 26 Aug 2026 10:10:35 +0200 Subject: [PATCH 3/3] See which machine a Mach-O is for, not just that it is one The format guard exists because a staged build assembles one SDK out of two machines, and a file belonging to the producer can reach the consumer's tree without anything else noticing. On Apple it could not see that: both machines emit Mach-O, so arm64 and x86_64 were the same answer -- and the one host pair staged across two architectures, x86_64-apple-darwin cross-built on the arm64 runner, was exactly the blind spot. The cputype sits four bytes behind the magic, in the byte order the magic announces. A universal binary carries one per slice and none of its own, so it stays unclassified rather than guessed at. The test drives the check with the real cmake binary against the triplet of the machine it is not, which exercises the Mach-O path on macOS and the ELF one on Linux, and the helper it calls now takes the host to check against instead of hardcoding Windows. --- cmake/HostBinaryFormat.cmake | 35 +++++++++++-- tests/cmake/host-binary-format-check.cmake | 2 +- tests/cmake/host-binary-format.cmake | 57 +++++++++++++++++++++- 3 files changed, 87 insertions(+), 7 deletions(-) diff --git a/cmake/HostBinaryFormat.cmake b/cmake/HostBinaryFormat.cmake index 5fff1cc..7d9c518 100644 --- a/cmake/HostBinaryFormat.cmake +++ b/cmake/HostBinaryFormat.cmake @@ -12,8 +12,9 @@ include_guard(GLOBAL) # Decode the leading bytes of an executable, given as an uppercase hex string. -# Sets format to ELF, PE, MachO or the empty string, and machine to the ELF -# e_machine value (decimal) when the header carries one. +# Sets format to ELF, PE, MachO or the empty string, and machine to whichever +# machine number the header carries (decimal): the ELF e_machine, or the +# Mach-O cputype. Empty when the format has no room for one. function(vitasdk_decode_binary_format hex out_format out_machine) set(format "") set(machine "") @@ -38,6 +39,22 @@ function(vitasdk_decode_binary_format hex out_format out_machine) set(format PE) elseif(hex MATCHES "^(FEEDFACE|FEEDFACF|CEFAEDFE|CFFAEDFE|CAFEBABE)") set(format MachO) + string(SUBSTRING "${hex}" 0 8 magic) + string(LENGTH "${hex}" hex_length) + # A universal binary carries one cputype per slice and none of its + # own, so it is left unclassified rather than guessed at. + if(hex_length GREATER_EQUAL 16 AND NOT magic STREQUAL "CAFEBABE") + string(SUBSTRING "${hex}" 8 8 cpu_hex) + if(magic MATCHES "^(CEFAEDFE|CFFAEDFE)$") + # cputype is stored in the byte order the magic announced. + string(SUBSTRING "${cpu_hex}" 0 2 cpu_byte0) + string(SUBSTRING "${cpu_hex}" 2 2 cpu_byte1) + string(SUBSTRING "${cpu_hex}" 4 2 cpu_byte2) + string(SUBSTRING "${cpu_hex}" 6 2 cpu_byte3) + set(cpu_hex "${cpu_byte3}${cpu_byte2}${cpu_byte1}${cpu_byte0}") + endif() + math(EXPR machine "0x${cpu_hex}") + endif() endif() set(${out_format} "${format}" PARENT_SCOPE) @@ -58,6 +75,16 @@ function(vitasdk_expected_binary_format system_name host_triple out_format out_m set(format PE) elseif(system_name STREQUAL "Darwin") set(format MachO) + # CPU_ARCH_ABI64 | CPU_TYPE_X86, and the same for CPU_TYPE_ARM. Both + # Macs produce Mach-O, so this is the only thing that tells the two + # halves of a staged Apple build apart. + if(host_triple MATCHES "^(x86_64|amd64)") + set(machine 16777223) + elseif(host_triple MATCHES "^i[3-6]86") + set(machine 7) + elseif(host_triple MATCHES "^(aarch64|arm64)") + set(machine 16777228) + endif() else() set(format ELF) if(host_triple MATCHES "^(x86_64|amd64)") @@ -101,8 +128,8 @@ function(vitasdk_check_binary_directory directory system_name host_triple) if(NOT expected_machine STREQUAL "" AND NOT machine STREQUAL "" AND NOT machine EQUAL expected_machine) message(FATAL_ERROR - "${entry} is built for ELF machine ${machine}, but this SDK " - "targets ${host_triple} (machine ${expected_machine})") + "${entry} is built for ${format} machine ${machine}, but this " + "SDK targets ${host_triple} (machine ${expected_machine})") endif() endforeach() endfunction() diff --git a/tests/cmake/host-binary-format-check.cmake b/tests/cmake/host-binary-format-check.cmake index fe08fac..d9bfea2 100644 --- a/tests/cmake/host-binary-format-check.cmake +++ b/tests/cmake/host-binary-format-check.cmake @@ -5,4 +5,4 @@ cmake_minimum_required(VERSION 3.16) include("${CMAKE_CURRENT_LIST_DIR}/../../cmake/HostBinaryFormat.cmake") -vitasdk_check_binary_directory("${DIRECTORY}" Windows x86_64-w64-mingw32) +vitasdk_check_binary_directory("${DIRECTORY}" "${SYSTEM}" "${TRIPLE}") diff --git a/tests/cmake/host-binary-format.cmake b/tests/cmake/host-binary-format.cmake index 97c0e8b..175ce16 100644 --- a/tests/cmake/host-binary-format.cmake +++ b/tests/cmake/host-binary-format.cmake @@ -19,7 +19,12 @@ assert_decode("7F454C4601010100000000000000000002002800" ELF 40) # ELF big endian keeps e_machine in its own byte order. assert_decode("7F454C4602020100000000000000000000020016" ELF 22) assert_decode("4D5A90000300000004000000FFFF0000B8000000" PE "") -assert_decode("CFFAEDFE0C000001000000000200000013000000" MachO "") +# Mach-O 64, little endian: the cputype follows the magic in its byte order. +assert_decode("CFFAEDFE0C000001000000000200000013000000" MachO 16777228) +assert_decode("CFFAEDFE07000001030000000200000013000000" MachO 16777223) +assert_decode("FEEDFACF010000070000000300000002000000A0" MachO 16777223) +# A universal binary has a cputype per slice and none of its own. +assert_decode("CAFEBABE0000000201000007000000030000C000" MachO "") # A shell script is in no executable format and must decode as nothing. assert_decode("23212F62696E2F73680A6563686F206869" "" "") @@ -33,7 +38,9 @@ function(assert_expected system triple expected_format expected_machine) endfunction() assert_expected(Windows x86_64-w64-mingw32 PE "") -assert_expected(Darwin arm64-apple-darwin MachO "") +assert_expected(Darwin arm64-apple-darwin MachO 16777228) +assert_expected(Darwin aarch64-apple-darwin MachO 16777228) +assert_expected(Darwin x86_64-apple-darwin MachO 16777223) assert_expected(Linux x86_64-linux-gnu ELF 62) assert_expected(Linux i686-linux-gnu ELF 3) assert_expected(Linux aarch64-linux-musl ELF 183) @@ -67,6 +74,7 @@ file(COPY "${real_cmake}" DESTINATION "${scratch}/bin") execute_process( COMMAND ${CMAKE_COMMAND} -DDIRECTORY=${scratch}/bin + -DSYSTEM=Windows -DTRIPLE=x86_64-w64-mingw32 -P "${CMAKE_CURRENT_LIST_DIR}/host-binary-format-check.cmake" RESULT_VARIABLE check_result OUTPUT_QUIET @@ -79,4 +87,49 @@ if(NOT check_error MATCHES "in a PE SDK") endif() file(REMOVE_RECURSE "${scratch}") +# The machine half of the same failure, which format alone cannot see: both +# Macs produce Mach-O, so on the one host pair that is staged across two +# architectures nothing else would catch it. The fixture is the real cmake, +# so the bytes are a machine's own rather than a hand-written header, and the +# foreign triple is picked by what it is not -- script mode knows the host +# system name but not its processor. +set(foreign_triple "") +if(NOT machine STREQUAL "") + foreach(arch x86_64 aarch64) + if(CMAKE_HOST_SYSTEM_NAME STREQUAL "Darwin") + set(candidate ${arch}-apple-darwin) + elseif(CMAKE_HOST_SYSTEM_NAME STREQUAL "Linux") + set(candidate ${arch}-linux-gnu) + else() + break() + endif() + vitasdk_expected_binary_format("${CMAKE_HOST_SYSTEM_NAME}" "${candidate}" + candidate_format candidate_machine) + if(NOT candidate_machine EQUAL machine) + set(foreign_triple ${candidate}) + break() + endif() + endforeach() +endif() +if(NOT foreign_triple STREQUAL "") + set(scratch "${temp_root}/vitasdk-host-binary-machine-${fixture_id}") + file(COPY "${real_cmake}" DESTINATION "${scratch}/bin") + execute_process( + COMMAND ${CMAKE_COMMAND} + -DDIRECTORY=${scratch}/bin + -DSYSTEM=${CMAKE_HOST_SYSTEM_NAME} + -DTRIPLE=${foreign_triple} + -P "${CMAKE_CURRENT_LIST_DIR}/host-binary-format-check.cmake" + RESULT_VARIABLE check_result + OUTPUT_QUIET + ERROR_VARIABLE check_error) + if(check_result EQUAL 0) + message(FATAL_ERROR "a binary for another machine must be rejected") + endif() + if(NOT check_error MATCHES "machine ${machine}" OR NOT check_error MATCHES "${foreign_triple}") + message(FATAL_ERROR "the rejection must name the machine, got: ${check_error}") + endif() + file(REMOVE_RECURSE "${scratch}") +endif() + message(STATUS "Host binary format checks passed")