diff --git a/CMakeLists.txt b/CMakeLists.txt index e89c595..299078d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -113,6 +113,7 @@ endif() include(Utils) include(GetTriplet) +include(PublishedHostName) include(ExternalProject) set(vitasdk_base_c_flags "${CMAKE_C_FLAGS}") @@ -135,14 +136,12 @@ get_host_triplet(host_native) get_build_triplet(build_native) # What this host is called where it is published -- in artifact names, in the -# package architecture, in the vdpm bundle it embeds. It is the toolchain -# triplet on every host but FreeBSD, whose cross compiler carries a release -# number (x86_64-unknown-freebsd14-gcc) that its published name does not. -string(REGEX REPLACE "^(.*-unknown-freebsd)[0-9]+$" "\\1" - host_published_default "${host_native}") +# package architecture, in the vdpm bundle it embeds. +vitasdk_published_host_name("${host_native}" host_published_default) set(VITASDK_HOST_NAME "${host_published_default}" CACHE STRING "Name this host publishes under, if not its toolchain triplet") set(host_published "${VITASDK_HOST_NAME}") +vitasdk_check_published_host_name("${host_published}" "${host_native}") message(STATUS "Host: ${host_native}") message(STATUS "Build: ${build_native}") diff --git a/cmake/HostBinaryFormat.cmake b/cmake/HostBinaryFormat.cmake index 5fff1cc..7d9c518 100644 --- a/cmake/HostBinaryFormat.cmake +++ b/cmake/HostBinaryFormat.cmake @@ -12,8 +12,9 @@ include_guard(GLOBAL) # Decode the leading bytes of an executable, given as an uppercase hex string. -# Sets format to ELF, PE, MachO or the empty string, and machine to the ELF -# e_machine value (decimal) when the header carries one. +# Sets format to ELF, PE, MachO or the empty string, and machine to whichever +# machine number the header carries (decimal): the ELF e_machine, or the +# Mach-O cputype. Empty when the format has no room for one. function(vitasdk_decode_binary_format hex out_format out_machine) set(format "") set(machine "") @@ -38,6 +39,22 @@ function(vitasdk_decode_binary_format hex out_format out_machine) set(format PE) elseif(hex MATCHES "^(FEEDFACE|FEEDFACF|CEFAEDFE|CFFAEDFE|CAFEBABE)") set(format MachO) + string(SUBSTRING "${hex}" 0 8 magic) + string(LENGTH "${hex}" hex_length) + # A universal binary carries one cputype per slice and none of its + # own, so it is left unclassified rather than guessed at. + if(hex_length GREATER_EQUAL 16 AND NOT magic STREQUAL "CAFEBABE") + string(SUBSTRING "${hex}" 8 8 cpu_hex) + if(magic MATCHES "^(CEFAEDFE|CFFAEDFE)$") + # cputype is stored in the byte order the magic announced. + string(SUBSTRING "${cpu_hex}" 0 2 cpu_byte0) + string(SUBSTRING "${cpu_hex}" 2 2 cpu_byte1) + string(SUBSTRING "${cpu_hex}" 4 2 cpu_byte2) + string(SUBSTRING "${cpu_hex}" 6 2 cpu_byte3) + set(cpu_hex "${cpu_byte3}${cpu_byte2}${cpu_byte1}${cpu_byte0}") + endif() + math(EXPR machine "0x${cpu_hex}") + endif() endif() set(${out_format} "${format}" PARENT_SCOPE) @@ -58,6 +75,16 @@ function(vitasdk_expected_binary_format system_name host_triple out_format out_m set(format PE) elseif(system_name STREQUAL "Darwin") set(format MachO) + # CPU_ARCH_ABI64 | CPU_TYPE_X86, and the same for CPU_TYPE_ARM. Both + # Macs produce Mach-O, so this is the only thing that tells the two + # halves of a staged Apple build apart. + if(host_triple MATCHES "^(x86_64|amd64)") + set(machine 16777223) + elseif(host_triple MATCHES "^i[3-6]86") + set(machine 7) + elseif(host_triple MATCHES "^(aarch64|arm64)") + set(machine 16777228) + endif() else() set(format ELF) if(host_triple MATCHES "^(x86_64|amd64)") @@ -101,8 +128,8 @@ function(vitasdk_check_binary_directory directory system_name host_triple) if(NOT expected_machine STREQUAL "" AND NOT machine STREQUAL "" AND NOT machine EQUAL expected_machine) message(FATAL_ERROR - "${entry} is built for ELF machine ${machine}, but this SDK " - "targets ${host_triple} (machine ${expected_machine})") + "${entry} is built for ${format} machine ${machine}, but this " + "SDK targets ${host_triple} (machine ${expected_machine})") endif() endforeach() endfunction() diff --git a/cmake/PublishedHostName.cmake b/cmake/PublishedHostName.cmake new file mode 100644 index 0000000..1652845 --- /dev/null +++ b/cmake/PublishedHostName.cmake @@ -0,0 +1,40 @@ +# +# Copyright(c) 2016 codestation +# Distributed under the MIT License (http://opensource.org/licenses/MIT) +# + +# A host is called two things: the triplet its cross compiler answers to, and +# the name it publishes under. Keeping them apart is what lets a host be +# published under a name it was not built for, which is silent -- the +# artifacts are well formed, they just belong to another machine. + +include_guard(GLOBAL) + +# The name a host triplet publishes under. The same everywhere except +# FreeBSD, whose cross compiler carries a release number (x86_64-unknown- +# freebsd14-gcc) that its published name does not. +function(vitasdk_published_host_name triplet out_name) + string(REGEX REPLACE "^(.*-unknown-freebsd)[0-9]+$" "\\1" name "${triplet}") + set(${out_name} "${name}" PARENT_SCOPE) +endfunction() + +# arm64 and aarch64 name one machine: Apple writes the first, config.sub the +# second, and both reach here depending on who was asked. +function(vitasdk_canonical_host_name name out_name) + string(REGEX REPLACE "^arm64-" "aarch64-" canonical "${name}") + set(${out_name} "${canonical}" PARENT_SCOPE) +endfunction() + +# Fail unless published is the name triplet is entitled to publish under. +function(vitasdk_check_published_host_name published triplet) + vitasdk_published_host_name("${triplet}" expected) + vitasdk_canonical_host_name("${expected}" expected_canonical) + vitasdk_canonical_host_name("${published}" published_canonical) + if(published_canonical STREQUAL expected_canonical) + return() + endif() + message(FATAL_ERROR + "this build publishes as ${published} but its toolchain builds " + "${triplet}. Either it is missing the toolchain file that would make " + "it cross-compile, or VITASDK_HOST_NAME names the wrong host.") +endfunction() diff --git a/scripts/ci/build-host.sh b/scripts/ci/build-host.sh index 2cb72c2..5fa88ff 100755 --- a/scripts/ci/build-host.sh +++ b/scripts/ci/build-host.sh @@ -386,7 +386,7 @@ fi extra_cmake_args=() case $host in -x86_64-linux-gnu | aarch64-linux-gnu | arm64-apple-darwin | x86_64-apple-darwin) +x86_64-linux-gnu | aarch64-linux-gnu | arm64-apple-darwin) install_dependencies ;; x86_64-w64-mingw32) @@ -409,6 +409,12 @@ aarch64-unknown-freebsd) export PATH="$PWD/freebsd-cross/bin:$PATH" extra_cmake_args+=(-DCMAKE_TOOLCHAIN_FILE="$repo_root/cmake/toolchains/aarch64-unknown-freebsd.cmake") ;; +x86_64-apple-darwin) + install_dependencies + scripts/setup-macos-cross.sh "$PWD/macos-cross" + export PATH="$PWD/macos-cross/bin:$PATH" + extra_cmake_args+=(-DCMAKE_TOOLCHAIN_FILE="$repo_root/cmake/toolchains/x86_64-apple-darwin.cmake") + ;; *) printf 'no build recipe for host: %s\n' "$host" >&2 exit 1 diff --git a/tests/ci/test-cross-host-recipe.sh b/tests/ci/test-cross-host-recipe.sh new file mode 100755 index 0000000..9ecb75f --- /dev/null +++ b/tests/ci/test-cross-host-recipe.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# The lock and the build recipe are two halves of one decision. +# +# hosts.json says whether a host is cross-built; the case block in +# build-host.sh says how. Nothing tied them together, so #161 made +# x86_64-apple-darwin native in both and db4d1a593 made it a cross again in +# the lock alone: the job then built an arm64 SDK, published it under the +# Intel name, and stayed green until the bootstrap smoke test, 18 minutes in. + +set -euo pipefail + +repository_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P) +script="$repository_root/scripts/ci/build-host.sh" + +failures=0 + +# The case block is run as the script runs it, not re-implemented here. The +# setup scripts it calls are stubbed by name from a scratch directory: what +# is under test is the dispatch, not what a cross environment costs to build. +stubs=$(mktemp -d) +trap 'rm -rf "$stubs"' EXIT +mkdir -p "$stubs/scripts" +for stub in setup-macos-cross.sh setup-freebsd-cross.sh; do + printf '#!/bin/sh\nexit 0\n' >"$stubs/scripts/$stub" + chmod +x "$stubs/scripts/$stub" +done + +toolchain_file_for() +{ + ( + cd "$stubs" || exit 1 + host=$1 repo_root=$repository_root bash -c ' + set -euo pipefail + host=$host + repo_root=$repo_root + install_dependencies() { :; } + '"$(sed -n '/^extra_cmake_args=()/,/^esac/p' "$script")"' + printf "%s" "${extra_cmake_args[*]:-}" + ' + ) +} + +check() +{ + local host=$1 build_host=$2 args status expected + args=$(toolchain_file_for "$host") && status=0 || status=$? + if ((status != 0)); then + printf 'FAIL: %s has no build recipe in build-host.sh\n' "$host" >&2 + failures=$((failures + 1)) + return + fi + if [[ -n $build_host ]]; then + expected="-DCMAKE_TOOLCHAIN_FILE=$repository_root/cmake/toolchains/$host.cmake" + if [[ $args != *"$expected"* ]]; then + printf 'FAIL: %s is cross-built from %s but its recipe does not use %s.cmake\n recipe: %s\n' \ + "$host" "$build_host" "$host" "${args:-}" >&2 + failures=$((failures + 1)) + fi + if [[ ! -f $repository_root/cmake/toolchains/$host.cmake ]]; then + printf 'FAIL: %s names a toolchain file that does not exist\n' "$host" >&2 + failures=$((failures + 1)) + fi + elif [[ $args == *CMAKE_TOOLCHAIN_FILE* ]]; then + printf 'FAIL: %s builds natively in the lock but its recipe cross-compiles\n recipe: %s\n' \ + "$host" "$args" >&2 + failures=$((failures + 1)) + fi +} + +# musl hosts never reach the case block: they build inside Alpine, where the +# build is native, and build-host.sh dispatches them before it. +while read -r host build_host; do + check "$host" "$build_host" +done < <(python3 - "$repository_root/cmake/hosts.json" <<'PYTHON' +import json, sys + +seen = set() +for host in json.load(open(sys.argv[1]))["hosts"]: + name = host["name"] + if name.endswith("-linux-musl") or name in seen: + continue + seen.add(name) + print(name, host.get("build_host", "")) +PYTHON +) + +if ((failures)); then + printf '%d cross-host recipe check(s) failed\n' "$failures" >&2 + exit 1 +fi + +printf 'cross host recipe tests passed\n' diff --git a/tests/cmake/host-binary-format-check.cmake b/tests/cmake/host-binary-format-check.cmake index fe08fac..d9bfea2 100644 --- a/tests/cmake/host-binary-format-check.cmake +++ b/tests/cmake/host-binary-format-check.cmake @@ -5,4 +5,4 @@ cmake_minimum_required(VERSION 3.16) include("${CMAKE_CURRENT_LIST_DIR}/../../cmake/HostBinaryFormat.cmake") -vitasdk_check_binary_directory("${DIRECTORY}" Windows x86_64-w64-mingw32) +vitasdk_check_binary_directory("${DIRECTORY}" "${SYSTEM}" "${TRIPLE}") diff --git a/tests/cmake/host-binary-format.cmake b/tests/cmake/host-binary-format.cmake index 97c0e8b..175ce16 100644 --- a/tests/cmake/host-binary-format.cmake +++ b/tests/cmake/host-binary-format.cmake @@ -19,7 +19,12 @@ assert_decode("7F454C4601010100000000000000000002002800" ELF 40) # ELF big endian keeps e_machine in its own byte order. assert_decode("7F454C4602020100000000000000000000020016" ELF 22) assert_decode("4D5A90000300000004000000FFFF0000B8000000" PE "") -assert_decode("CFFAEDFE0C000001000000000200000013000000" MachO "") +# Mach-O 64, little endian: the cputype follows the magic in its byte order. +assert_decode("CFFAEDFE0C000001000000000200000013000000" MachO 16777228) +assert_decode("CFFAEDFE07000001030000000200000013000000" MachO 16777223) +assert_decode("FEEDFACF010000070000000300000002000000A0" MachO 16777223) +# A universal binary has a cputype per slice and none of its own. +assert_decode("CAFEBABE0000000201000007000000030000C000" MachO "") # A shell script is in no executable format and must decode as nothing. assert_decode("23212F62696E2F73680A6563686F206869" "" "") @@ -33,7 +38,9 @@ function(assert_expected system triple expected_format expected_machine) endfunction() assert_expected(Windows x86_64-w64-mingw32 PE "") -assert_expected(Darwin arm64-apple-darwin MachO "") +assert_expected(Darwin arm64-apple-darwin MachO 16777228) +assert_expected(Darwin aarch64-apple-darwin MachO 16777228) +assert_expected(Darwin x86_64-apple-darwin MachO 16777223) assert_expected(Linux x86_64-linux-gnu ELF 62) assert_expected(Linux i686-linux-gnu ELF 3) assert_expected(Linux aarch64-linux-musl ELF 183) @@ -67,6 +74,7 @@ file(COPY "${real_cmake}" DESTINATION "${scratch}/bin") execute_process( COMMAND ${CMAKE_COMMAND} -DDIRECTORY=${scratch}/bin + -DSYSTEM=Windows -DTRIPLE=x86_64-w64-mingw32 -P "${CMAKE_CURRENT_LIST_DIR}/host-binary-format-check.cmake" RESULT_VARIABLE check_result OUTPUT_QUIET @@ -79,4 +87,49 @@ if(NOT check_error MATCHES "in a PE SDK") endif() file(REMOVE_RECURSE "${scratch}") +# The machine half of the same failure, which format alone cannot see: both +# Macs produce Mach-O, so on the one host pair that is staged across two +# architectures nothing else would catch it. The fixture is the real cmake, +# so the bytes are a machine's own rather than a hand-written header, and the +# foreign triple is picked by what it is not -- script mode knows the host +# system name but not its processor. +set(foreign_triple "") +if(NOT machine STREQUAL "") + foreach(arch x86_64 aarch64) + if(CMAKE_HOST_SYSTEM_NAME STREQUAL "Darwin") + set(candidate ${arch}-apple-darwin) + elseif(CMAKE_HOST_SYSTEM_NAME STREQUAL "Linux") + set(candidate ${arch}-linux-gnu) + else() + break() + endif() + vitasdk_expected_binary_format("${CMAKE_HOST_SYSTEM_NAME}" "${candidate}" + candidate_format candidate_machine) + if(NOT candidate_machine EQUAL machine) + set(foreign_triple ${candidate}) + break() + endif() + endforeach() +endif() +if(NOT foreign_triple STREQUAL "") + set(scratch "${temp_root}/vitasdk-host-binary-machine-${fixture_id}") + file(COPY "${real_cmake}" DESTINATION "${scratch}/bin") + execute_process( + COMMAND ${CMAKE_COMMAND} + -DDIRECTORY=${scratch}/bin + -DSYSTEM=${CMAKE_HOST_SYSTEM_NAME} + -DTRIPLE=${foreign_triple} + -P "${CMAKE_CURRENT_LIST_DIR}/host-binary-format-check.cmake" + RESULT_VARIABLE check_result + OUTPUT_QUIET + ERROR_VARIABLE check_error) + if(check_result EQUAL 0) + message(FATAL_ERROR "a binary for another machine must be rejected") + endif() + if(NOT check_error MATCHES "machine ${machine}" OR NOT check_error MATCHES "${foreign_triple}") + message(FATAL_ERROR "the rejection must name the machine, got: ${check_error}") + endif() + file(REMOVE_RECURSE "${scratch}") +endif() + message(STATUS "Host binary format checks passed") diff --git a/tests/cmake/published-host-name-check.cmake b/tests/cmake/published-host-name-check.cmake new file mode 100644 index 0000000..ddd3eb9 --- /dev/null +++ b/tests/cmake/published-host-name-check.cmake @@ -0,0 +1,5 @@ +cmake_minimum_required(VERSION 3.16) + +include("${CMAKE_CURRENT_LIST_DIR}/../../cmake/PublishedHostName.cmake") + +vitasdk_check_published_host_name("${PUBLISHED}" "${TRIPLET}") diff --git a/tests/cmake/published-host-name.cmake b/tests/cmake/published-host-name.cmake index eb26cfa..75e7bac 100644 --- a/tests/cmake/published-host-name.cmake +++ b/tests/cmake/published-host-name.cmake @@ -1,17 +1,9 @@ cmake_minimum_required(VERSION 3.16) -# A host is called two things: the triplet its cross compiler answers to, and -# the name it publishes under. They are the same everywhere except FreeBSD, -# whose compiler carries a release number its published name does not -- and -# mixing them up published a bundle nobody could match to a host. - -function(published_name_of triplet out) - string(REGEX REPLACE "^(.*-unknown-freebsd)[0-9]+$" "\\1" name "${triplet}") - set(${out} "${name}" PARENT_SCOPE) -endfunction() +include("${CMAKE_CURRENT_LIST_DIR}/../../cmake/PublishedHostName.cmake") function(assert_published triplet expected) - published_name_of("${triplet}" actual) + vitasdk_published_host_name("${triplet}" actual) if(NOT "${actual}" STREQUAL "${expected}") message(FATAL_ERROR "${triplet} publishes as '${actual}', expected '${expected}'") @@ -28,4 +20,39 @@ foreach(triplet x86_64-linux-gnu aarch64-linux-gnu x86_64-linux-musl assert_published("${triplet}" "${triplet}") endforeach() +# The gate: a name may only be published by the toolchain that earns it. +function(assert_gate published triplet expected_result) + execute_process( + COMMAND ${CMAKE_COMMAND} + -DPUBLISHED=${published} -DTRIPLET=${triplet} + -P "${CMAKE_CURRENT_LIST_DIR}/published-host-name-check.cmake" + RESULT_VARIABLE result + OUTPUT_QUIET + ERROR_VARIABLE error) + if(expected_result STREQUAL "accepts" AND NOT result EQUAL 0) + message(FATAL_ERROR "${triplet} must publish as ${published}: ${error}") + endif() + if(expected_result STREQUAL "rejects") + if(result EQUAL 0) + message(FATAL_ERROR "${triplet} must not publish as ${published}") + endif() + if(NOT error MATCHES "${published}" OR NOT error MATCHES "${triplet}") + message(FATAL_ERROR "the rejection must name both, got: ${error}") + endif() + endif() +endfunction() + +assert_gate(x86_64-linux-gnu x86_64-linux-gnu accepts) +assert_gate(x86_64-unknown-freebsd x86_64-unknown-freebsd14 accepts) +# Apple's spelling of the machine config.sub calls aarch64. +assert_gate(arm64-apple-darwin aarch64-apple-darwin accepts) +assert_gate(aarch64-apple-darwin arm64-apple-darwin accepts) + +# What shipped an arm64 SDK under the Intel name: the cross host lost its +# toolchain file, so the build named one host and produced another. +assert_gate(x86_64-apple-darwin arm64-apple-darwin rejects) +assert_gate(arm64-apple-darwin x86_64-apple-darwin rejects) +assert_gate(x86_64-w64-mingw32 x86_64-linux-gnu rejects) +assert_gate(aarch64-linux-gnu x86_64-linux-gnu rejects) + message(STATUS "published host name checks passed")