Give the Intel Mac host back the recipe that cross-builds it

CMakeLists.txt

@@ -113,6 +113,7 @@ endif()
 
 include(Utils)
 include(GetTriplet)
+include(PublishedHostName)
 include(ExternalProject)
 
 set(vitasdk_base_c_flags "${CMAKE_C_FLAGS}")
@@ -135,14 +136,12 @@ get_host_triplet(host_native)
 get_build_triplet(build_native)
 
 # What this host is called where it is published -- in artifact names, in the
-# package architecture, in the vdpm bundle it embeds. It is the toolchain
-# triplet on every host but FreeBSD, whose cross compiler carries a release
-# number (x86_64-unknown-freebsd14-gcc) that its published name does not.
-string(REGEX REPLACE "^(.*-unknown-freebsd)[0-9]+$" "\\1"
-    host_published_default "${host_native}")
+# package architecture, in the vdpm bundle it embeds.
+vitasdk_published_host_name("${host_native}" host_published_default)
 set(VITASDK_HOST_NAME "${host_published_default}" CACHE STRING
     "Name this host publishes under, if not its toolchain triplet")
 set(host_published "${VITASDK_HOST_NAME}")
+vitasdk_check_published_host_name("${host_published}" "${host_native}")
 
 message(STATUS "Host:   ${host_native}")
 message(STATUS "Build:  ${build_native}")

cmake/HostBinaryFormat.cmake

@@ -12,8 +12,9 @@
 include_guard(GLOBAL)
 
 # Decode the leading bytes of an executable, given as an uppercase hex string.
-# Sets format to ELF, PE, MachO or the empty string, and machine to the ELF
-# e_machine value (decimal) when the header carries one.
+# Sets format to ELF, PE, MachO or the empty string, and machine to whichever
+# machine number the header carries (decimal): the ELF e_machine, or the
+# Mach-O cputype. Empty when the format has no room for one.
 function(vitasdk_decode_binary_format hex out_format out_machine)
     set(format "")
     set(machine "")
@@ -38,6 +39,22 @@ function(vitasdk_decode_binary_format hex out_format out_machine)
         set(format PE)
     elseif(hex MATCHES "^(FEEDFACE|FEEDFACF|CEFAEDFE|CFFAEDFE|CAFEBABE)")
         set(format MachO)
+        string(SUBSTRING "${hex}" 0 8 magic)
+        string(LENGTH "${hex}" hex_length)
+        # A universal binary carries one cputype per slice and none of its
+        # own, so it is left unclassified rather than guessed at.
+        if(hex_length GREATER_EQUAL 16 AND NOT magic STREQUAL "CAFEBABE")
+            string(SUBSTRING "${hex}" 8 8 cpu_hex)
+            if(magic MATCHES "^(CEFAEDFE|CFFAEDFE)$")
+                # cputype is stored in the byte order the magic announced.
+                string(SUBSTRING "${cpu_hex}" 0 2 cpu_byte0)
+                string(SUBSTRING "${cpu_hex}" 2 2 cpu_byte1)
+                string(SUBSTRING "${cpu_hex}" 4 2 cpu_byte2)
+                string(SUBSTRING "${cpu_hex}" 6 2 cpu_byte3)
+                set(cpu_hex "${cpu_byte3}${cpu_byte2}${cpu_byte1}${cpu_byte0}")
+            endif()
+            math(EXPR machine "0x${cpu_hex}")
+        endif()
     endif()
 
     set(${out_format} "${format}" PARENT_SCOPE)
@@ -58,6 +75,16 @@ function(vitasdk_expected_binary_format system_name host_triple out_format out_m
         set(format PE)
     elseif(system_name STREQUAL "Darwin")
         set(format MachO)
+        # CPU_ARCH_ABI64 | CPU_TYPE_X86, and the same for CPU_TYPE_ARM. Both
+        # Macs produce Mach-O, so this is the only thing that tells the two
+        # halves of a staged Apple build apart.
+        if(host_triple MATCHES "^(x86_64|amd64)")
+            set(machine 16777223)
+        elseif(host_triple MATCHES "^i[3-6]86")
+            set(machine 7)
+        elseif(host_triple MATCHES "^(aarch64|arm64)")
+            set(machine 16777228)
+        endif()
     else()
         set(format ELF)
         if(host_triple MATCHES "^(x86_64|amd64)")
@@ -101,8 +128,8 @@ function(vitasdk_check_binary_directory directory system_name host_triple)
         if(NOT expected_machine STREQUAL "" AND NOT machine STREQUAL ""
                 AND NOT machine EQUAL expected_machine)
             message(FATAL_ERROR
-                "${entry} is built for ELF machine ${machine}, but this SDK "
-                "targets ${host_triple} (machine ${expected_machine})")
+                "${entry} is built for ${format} machine ${machine}, but this "
+                "SDK targets ${host_triple} (machine ${expected_machine})")
         endif()
     endforeach()
 endfunction()

cmake/PublishedHostName.cmake

@@ -0,0 +1,40 @@
+#
+# Copyright(c) 2016 codestation
+# Distributed under the MIT License (http://opensource.org/licenses/MIT)
+#
+
+# A host is called two things: the triplet its cross compiler answers to, and
+# the name it publishes under. Keeping them apart is what lets a host be
+# published under a name it was not built for, which is silent -- the
+# artifacts are well formed, they just belong to another machine.
+
+include_guard(GLOBAL)
+
+# The name a host triplet publishes under. The same everywhere except
+# FreeBSD, whose cross compiler carries a release number (x86_64-unknown-
+# freebsd14-gcc) that its published name does not.
+function(vitasdk_published_host_name triplet out_name)
+    string(REGEX REPLACE "^(.*-unknown-freebsd)[0-9]+$" "\\1" name "${triplet}")
+    set(${out_name} "${name}" PARENT_SCOPE)
+endfunction()
+
+# arm64 and aarch64 name one machine: Apple writes the first, config.sub the
+# second, and both reach here depending on who was asked.
+function(vitasdk_canonical_host_name name out_name)
+    string(REGEX REPLACE "^arm64-" "aarch64-" canonical "${name}")
+    set(${out_name} "${canonical}" PARENT_SCOPE)
+endfunction()
+
+# Fail unless published is the name triplet is entitled to publish under.
+function(vitasdk_check_published_host_name published triplet)
+    vitasdk_published_host_name("${triplet}" expected)
+    vitasdk_canonical_host_name("${expected}" expected_canonical)
+    vitasdk_canonical_host_name("${published}" published_canonical)
+    if(published_canonical STREQUAL expected_canonical)
+        return()
+    endif()
+    message(FATAL_ERROR
+        "this build publishes as ${published} but its toolchain builds "
+        "${triplet}. Either it is missing the toolchain file that would make "
+        "it cross-compile, or VITASDK_HOST_NAME names the wrong host.")
+endfunction()

scripts/ci/build-host.sh

@@ -386,7 +386,7 @@ fi
 
 extra_cmake_args=()
 case $host in
-x86_64-linux-gnu | aarch64-linux-gnu | arm64-apple-darwin | x86_64-apple-darwin)
+x86_64-linux-gnu | aarch64-linux-gnu | arm64-apple-darwin)
 	install_dependencies
 	;;
 x86_64-w64-mingw32)
@@ -409,6 +409,12 @@ aarch64-unknown-freebsd)
 	export PATH="$PWD/freebsd-cross/bin:$PATH"
 	extra_cmake_args+=(-DCMAKE_TOOLCHAIN_FILE="$repo_root/cmake/toolchains/aarch64-unknown-freebsd.cmake")
 	;;
+x86_64-apple-darwin)
+	install_dependencies
+	scripts/setup-macos-cross.sh "$PWD/macos-cross"
+	export PATH="$PWD/macos-cross/bin:$PATH"
+	extra_cmake_args+=(-DCMAKE_TOOLCHAIN_FILE="$repo_root/cmake/toolchains/x86_64-apple-darwin.cmake")
+	;;
 *)
 	printf 'no build recipe for host: %s\n' "$host" >&2
 	exit 1

tests/ci/test-cross-host-recipe.sh

@@ -0,0 +1,92 @@
+#!/usr/bin/env bash
+# The lock and the build recipe are two halves of one decision.
+#
+# hosts.json says whether a host is cross-built; the case block in
+# build-host.sh says how. Nothing tied them together, so #161 made
+# x86_64-apple-darwin native in both and db4d1a593 made it a cross again in
+# the lock alone: the job then built an arm64 SDK, published it under the
+# Intel name, and stayed green until the bootstrap smoke test, 18 minutes in.
+
+set -euo pipefail
+
+repository_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)
+script="$repository_root/scripts/ci/build-host.sh"
+
+failures=0
+
+# The case block is run as the script runs it, not re-implemented here. The
+# setup scripts it calls are stubbed by name from a scratch directory: what
+# is under test is the dispatch, not what a cross environment costs to build.
+stubs=$(mktemp -d)
+trap 'rm -rf "$stubs"' EXIT
+mkdir -p "$stubs/scripts"
+for stub in setup-macos-cross.sh setup-freebsd-cross.sh; do
+	printf '#!/bin/sh\nexit 0\n' >"$stubs/scripts/$stub"
+	chmod +x "$stubs/scripts/$stub"
+done
+
+toolchain_file_for()
+{
+	(
+		cd "$stubs" || exit 1
+		host=$1 repo_root=$repository_root bash -c '
+			set -euo pipefail
+			host=$host
+			repo_root=$repo_root
+			install_dependencies() { :; }
+			'"$(sed -n '/^extra_cmake_args=()/,/^esac/p' "$script")"'
+			printf "%s" "${extra_cmake_args[*]:-}"
+		'
+	)
+}
+
+check()
+{
+	local host=$1 build_host=$2 args status expected
+	args=$(toolchain_file_for "$host") && status=0 || status=$?
+	if ((status != 0)); then
+		printf 'FAIL: %s has no build recipe in build-host.sh\n' "$host" >&2
+		failures=$((failures + 1))
+		return
+	fi
+	if [[ -n $build_host ]]; then
+		expected="-DCMAKE_TOOLCHAIN_FILE=$repository_root/cmake/toolchains/$host.cmake"
+		if [[ $args != *"$expected"* ]]; then
+			printf 'FAIL: %s is cross-built from %s but its recipe does not use %s.cmake\n  recipe: %s\n' \
+				"$host" "$build_host" "$host" "${args:-<none>}" >&2
+			failures=$((failures + 1))
+		fi
+		if [[ ! -f $repository_root/cmake/toolchains/$host.cmake ]]; then
+			printf 'FAIL: %s names a toolchain file that does not exist\n' "$host" >&2
+			failures=$((failures + 1))
+		fi
+	elif [[ $args == *CMAKE_TOOLCHAIN_FILE* ]]; then
+		printf 'FAIL: %s builds natively in the lock but its recipe cross-compiles\n  recipe: %s\n' \
+			"$host" "$args" >&2
+		failures=$((failures + 1))
+	fi
+}
+
+# musl hosts never reach the case block: they build inside Alpine, where the
+# build is native, and build-host.sh dispatches them before it.
+while read -r host build_host; do
+	check "$host" "$build_host"
+done < <(python3 - "$repository_root/cmake/hosts.json" <<'PYTHON'
+import json, sys
+
+seen = set()
+for host in json.load(open(sys.argv[1]))["hosts"]:
+    name = host["name"]
+    if name.endswith("-linux-musl") or name in seen:
+        continue
+    seen.add(name)
+    print(name, host.get("build_host", ""))
+PYTHON
+)
+
+if ((failures)); then
+	printf '%d cross-host recipe check(s) failed\n' "$failures" >&2
+	exit 1
+fi
+
+printf 'cross host recipe tests passed\n'

tests/cmake/host-binary-format-check.cmake

@@ -5,4 +5,4 @@ cmake_minimum_required(VERSION 3.16)
 
 include("${CMAKE_CURRENT_LIST_DIR}/../../cmake/HostBinaryFormat.cmake")
 
-vitasdk_check_binary_directory("${DIRECTORY}" Windows x86_64-w64-mingw32)
+vitasdk_check_binary_directory("${DIRECTORY}" "${SYSTEM}" "${TRIPLE}")

tests/cmake/host-binary-format.cmake

@@ -19,7 +19,12 @@ assert_decode("7F454C4601010100000000000000000002002800" ELF 40)
 # ELF big endian keeps e_machine in its own byte order.
 assert_decode("7F454C4602020100000000000000000000020016" ELF 22)
 assert_decode("4D5A90000300000004000000FFFF0000B8000000" PE "")
-assert_decode("CFFAEDFE0C000001000000000200000013000000" MachO "")
+# Mach-O 64, little endian: the cputype follows the magic in its byte order.
+assert_decode("CFFAEDFE0C000001000000000200000013000000" MachO 16777228)
+assert_decode("CFFAEDFE07000001030000000200000013000000" MachO 16777223)
+assert_decode("FEEDFACF010000070000000300000002000000A0" MachO 16777223)
+# A universal binary has a cputype per slice and none of its own.
+assert_decode("CAFEBABE0000000201000007000000030000C000" MachO "")
 # A shell script is in no executable format and must decode as nothing.
 assert_decode("23212F62696E2F73680A6563686F206869" "" "")
 
@@ -33,7 +38,9 @@ function(assert_expected system triple expected_format expected_machine)
 endfunction()
 
 assert_expected(Windows x86_64-w64-mingw32 PE "")
-assert_expected(Darwin arm64-apple-darwin MachO "")
+assert_expected(Darwin arm64-apple-darwin MachO 16777228)
+assert_expected(Darwin aarch64-apple-darwin MachO 16777228)
+assert_expected(Darwin x86_64-apple-darwin MachO 16777223)
 assert_expected(Linux x86_64-linux-gnu ELF 62)
 assert_expected(Linux i686-linux-gnu ELF 3)
 assert_expected(Linux aarch64-linux-musl ELF 183)
@@ -67,6 +74,7 @@ file(COPY "${real_cmake}" DESTINATION "${scratch}/bin")
 execute_process(
     COMMAND ${CMAKE_COMMAND}
         -DDIRECTORY=${scratch}/bin
+        -DSYSTEM=Windows -DTRIPLE=x86_64-w64-mingw32
         -P "${CMAKE_CURRENT_LIST_DIR}/host-binary-format-check.cmake"
     RESULT_VARIABLE check_result
     OUTPUT_QUIET
@@ -79,4 +87,49 @@ if(NOT check_error MATCHES "in a PE SDK")
 endif()
 file(REMOVE_RECURSE "${scratch}")
 
+# The machine half of the same failure, which format alone cannot see: both
+# Macs produce Mach-O, so on the one host pair that is staged across two
+# architectures nothing else would catch it. The fixture is the real cmake,
+# so the bytes are a machine's own rather than a hand-written header, and the
+# foreign triple is picked by what it is not -- script mode knows the host
+# system name but not its processor.
+set(foreign_triple "")
+if(NOT machine STREQUAL "")
+    foreach(arch x86_64 aarch64)
+        if(CMAKE_HOST_SYSTEM_NAME STREQUAL "Darwin")
+            set(candidate ${arch}-apple-darwin)
+        elseif(CMAKE_HOST_SYSTEM_NAME STREQUAL "Linux")
+            set(candidate ${arch}-linux-gnu)
+        else()
+            break()
+        endif()
+        vitasdk_expected_binary_format("${CMAKE_HOST_SYSTEM_NAME}" "${candidate}"
+            candidate_format candidate_machine)
+        if(NOT candidate_machine EQUAL machine)
+            set(foreign_triple ${candidate})
+            break()
+        endif()
+    endforeach()
+endif()
+if(NOT foreign_triple STREQUAL "")
+    set(scratch "${temp_root}/vitasdk-host-binary-machine-${fixture_id}")
+    file(COPY "${real_cmake}" DESTINATION "${scratch}/bin")
+    execute_process(
+        COMMAND ${CMAKE_COMMAND}
+            -DDIRECTORY=${scratch}/bin
+            -DSYSTEM=${CMAKE_HOST_SYSTEM_NAME}
+            -DTRIPLE=${foreign_triple}
+            -P "${CMAKE_CURRENT_LIST_DIR}/host-binary-format-check.cmake"
+        RESULT_VARIABLE check_result
+        OUTPUT_QUIET
+        ERROR_VARIABLE check_error)
+    if(check_result EQUAL 0)
+        message(FATAL_ERROR "a binary for another machine must be rejected")
+    endif()
+    if(NOT check_error MATCHES "machine ${machine}" OR NOT check_error MATCHES "${foreign_triple}")
+        message(FATAL_ERROR "the rejection must name the machine, got: ${check_error}")
+    endif()
+    file(REMOVE_RECURSE "${scratch}")
+endif()
+
 message(STATUS "Host binary format checks passed")

tests/cmake/published-host-name-check.cmake

@@ -0,0 +1,5 @@
+cmake_minimum_required(VERSION 3.16)
+
+include("${CMAKE_CURRENT_LIST_DIR}/../../cmake/PublishedHostName.cmake")
+
+vitasdk_check_published_host_name("${PUBLISHED}" "${TRIPLET}")

tests/cmake/published-host-name.cmake

@@ -1,17 +1,9 @@
 cmake_minimum_required(VERSION 3.16)
 
-# A host is called two things: the triplet its cross compiler answers to, and
-# the name it publishes under. They are the same everywhere except FreeBSD,
-# whose compiler carries a release number its published name does not -- and
-# mixing them up published a bundle nobody could match to a host.
-
-function(published_name_of triplet out)
-    string(REGEX REPLACE "^(.*-unknown-freebsd)[0-9]+$" "\\1" name "${triplet}")
-    set(${out} "${name}" PARENT_SCOPE)
-endfunction()
+include("${CMAKE_CURRENT_LIST_DIR}/../../cmake/PublishedHostName.cmake")
 
 function(assert_published triplet expected)
-    published_name_of("${triplet}" actual)
+    vitasdk_published_host_name("${triplet}" actual)
     if(NOT "${actual}" STREQUAL "${expected}")
         message(FATAL_ERROR
             "${triplet} publishes as '${actual}', expected '${expected}'")
@@ -28,4 +20,39 @@ foreach(triplet x86_64-linux-gnu aarch64-linux-gnu x86_64-linux-musl
     assert_published("${triplet}" "${triplet}")
 endforeach()
 
+# The gate: a name may only be published by the toolchain that earns it.
+function(assert_gate published triplet expected_result)
+    execute_process(
+        COMMAND ${CMAKE_COMMAND}
+            -DPUBLISHED=${published} -DTRIPLET=${triplet}
+            -P "${CMAKE_CURRENT_LIST_DIR}/published-host-name-check.cmake"
+        RESULT_VARIABLE result
+        OUTPUT_QUIET
+        ERROR_VARIABLE error)
+    if(expected_result STREQUAL "accepts" AND NOT result EQUAL 0)
+        message(FATAL_ERROR "${triplet} must publish as ${published}: ${error}")
+    endif()
+    if(expected_result STREQUAL "rejects")
+        if(result EQUAL 0)
+            message(FATAL_ERROR "${triplet} must not publish as ${published}")
+        endif()
+        if(NOT error MATCHES "${published}" OR NOT error MATCHES "${triplet}")
+            message(FATAL_ERROR "the rejection must name both, got: ${error}")
+        endif()
+    endif()
+endfunction()
+
+assert_gate(x86_64-linux-gnu x86_64-linux-gnu accepts)
+assert_gate(x86_64-unknown-freebsd x86_64-unknown-freebsd14 accepts)
+# Apple's spelling of the machine config.sub calls aarch64.
+assert_gate(arm64-apple-darwin aarch64-apple-darwin accepts)
+assert_gate(aarch64-apple-darwin arm64-apple-darwin accepts)
+
+# What shipped an arm64 SDK under the Intel name: the cross host lost its
+# toolchain file, so the build named one host and produced another.
+assert_gate(x86_64-apple-darwin arm64-apple-darwin rejects)
+assert_gate(arm64-apple-darwin x86_64-apple-darwin rejects)
+assert_gate(x86_64-w64-mingw32 x86_64-linux-gnu rejects)
+assert_gate(aarch64-linux-gnu x86_64-linux-gnu rejects)
+
 message(STATUS "published host name checks passed")