See which machine a Mach-O is for, not just that it is one

The format guard exists because a staged build assembles one SDK out of two
machines, and a file belonging to the producer can reach the consumer's tree
without anything else noticing. On Apple it could not see that: both
machines emit Mach-O, so arm64 and x86_64 were the same answer -- and the
one host pair staged across two architectures, x86_64-apple-darwin
cross-built on the arm64 runner, was exactly the blind spot.

The cputype sits four bytes behind the magic, in the byte order the magic
announces. A universal binary carries one per slice and none of its own, so
it stays unclassified rather than guessed at.

The test drives the check with the real cmake binary against the triplet of
the machine it is not, which exercises the Mach-O path on macOS and the ELF
one on Linux, and the helper it calls now takes the host to check against
instead of hardcoding Windows.

Author: frangarcj

cmake/HostBinaryFormat.cmake

@@ -12,8 +12,9 @@
 include_guard(GLOBAL)
 
 # Decode the leading bytes of an executable, given as an uppercase hex string.
-# Sets format to ELF, PE, MachO or the empty string, and machine to the ELF
-# e_machine value (decimal) when the header carries one.
+# Sets format to ELF, PE, MachO or the empty string, and machine to whichever
+# machine number the header carries (decimal): the ELF e_machine, or the
+# Mach-O cputype. Empty when the format has no room for one.
 function(vitasdk_decode_binary_format hex out_format out_machine)
     set(format "")
     set(machine "")
@@ -38,6 +39,22 @@ function(vitasdk_decode_binary_format hex out_format out_machine)
         set(format PE)
     elseif(hex MATCHES "^(FEEDFACE|FEEDFACF|CEFAEDFE|CFFAEDFE|CAFEBABE)")
         set(format MachO)
+        string(SUBSTRING "${hex}" 0 8 magic)
+        string(LENGTH "${hex}" hex_length)
+        # A universal binary carries one cputype per slice and none of its
+        # own, so it is left unclassified rather than guessed at.
+        if(hex_length GREATER_EQUAL 16 AND NOT magic STREQUAL "CAFEBABE")
+            string(SUBSTRING "${hex}" 8 8 cpu_hex)
+            if(magic MATCHES "^(CEFAEDFE|CFFAEDFE)$")
+                # cputype is stored in the byte order the magic announced.
+                string(SUBSTRING "${cpu_hex}" 0 2 cpu_byte0)
+                string(SUBSTRING "${cpu_hex}" 2 2 cpu_byte1)
+                string(SUBSTRING "${cpu_hex}" 4 2 cpu_byte2)
+                string(SUBSTRING "${cpu_hex}" 6 2 cpu_byte3)
+                set(cpu_hex "${cpu_byte3}${cpu_byte2}${cpu_byte1}${cpu_byte0}")
+            endif()
+            math(EXPR machine "0x${cpu_hex}")
+        endif()
     endif()
 
     set(${out_format} "${format}" PARENT_SCOPE)
@@ -58,6 +75,16 @@ function(vitasdk_expected_binary_format system_name host_triple out_format out_m
         set(format PE)
     elseif(system_name STREQUAL "Darwin")
         set(format MachO)
+        # CPU_ARCH_ABI64 | CPU_TYPE_X86, and the same for CPU_TYPE_ARM. Both
+        # Macs produce Mach-O, so this is the only thing that tells the two
+        # halves of a staged Apple build apart.
+        if(host_triple MATCHES "^(x86_64|amd64)")
+            set(machine 16777223)
+        elseif(host_triple MATCHES "^i[3-6]86")
+            set(machine 7)
+        elseif(host_triple MATCHES "^(aarch64|arm64)")
+            set(machine 16777228)
+        endif()
     else()
         set(format ELF)
         if(host_triple MATCHES "^(x86_64|amd64)")
@@ -101,8 +128,8 @@ function(vitasdk_check_binary_directory directory system_name host_triple)
         if(NOT expected_machine STREQUAL "" AND NOT machine STREQUAL ""
                 AND NOT machine EQUAL expected_machine)
             message(FATAL_ERROR
-                "${entry} is built for ELF machine ${machine}, but this SDK "
-                "targets ${host_triple} (machine ${expected_machine})")
+                "${entry} is built for ${format} machine ${machine}, but this "
+                "SDK targets ${host_triple} (machine ${expected_machine})")
         endif()
     endforeach()
 endfunction()

tests/cmake/host-binary-format-check.cmake

@@ -5,4 +5,4 @@ cmake_minimum_required(VERSION 3.16)
 
 include("${CMAKE_CURRENT_LIST_DIR}/../../cmake/HostBinaryFormat.cmake")
 
-vitasdk_check_binary_directory("${DIRECTORY}" Windows x86_64-w64-mingw32)
+vitasdk_check_binary_directory("${DIRECTORY}" "${SYSTEM}" "${TRIPLE}")

tests/cmake/host-binary-format.cmake

@@ -19,7 +19,12 @@ assert_decode("7F454C4601010100000000000000000002002800" ELF 40)
 # ELF big endian keeps e_machine in its own byte order.
 assert_decode("7F454C4602020100000000000000000000020016" ELF 22)
 assert_decode("4D5A90000300000004000000FFFF0000B8000000" PE "")
-assert_decode("CFFAEDFE0C000001000000000200000013000000" MachO "")
+# Mach-O 64, little endian: the cputype follows the magic in its byte order.
+assert_decode("CFFAEDFE0C000001000000000200000013000000" MachO 16777228)
+assert_decode("CFFAEDFE07000001030000000200000013000000" MachO 16777223)
+assert_decode("FEEDFACF010000070000000300000002000000A0" MachO 16777223)
+# A universal binary has a cputype per slice and none of its own.
+assert_decode("CAFEBABE0000000201000007000000030000C000" MachO "")
 # A shell script is in no executable format and must decode as nothing.
 assert_decode("23212F62696E2F73680A6563686F206869" "" "")
 
@@ -33,7 +38,9 @@ function(assert_expected system triple expected_format expected_machine)
 endfunction()
 
 assert_expected(Windows x86_64-w64-mingw32 PE "")
-assert_expected(Darwin arm64-apple-darwin MachO "")
+assert_expected(Darwin arm64-apple-darwin MachO 16777228)
+assert_expected(Darwin aarch64-apple-darwin MachO 16777228)
+assert_expected(Darwin x86_64-apple-darwin MachO 16777223)
 assert_expected(Linux x86_64-linux-gnu ELF 62)
 assert_expected(Linux i686-linux-gnu ELF 3)
 assert_expected(Linux aarch64-linux-musl ELF 183)
@@ -67,6 +74,7 @@ file(COPY "${real_cmake}" DESTINATION "${scratch}/bin")
 execute_process(
     COMMAND ${CMAKE_COMMAND}
         -DDIRECTORY=${scratch}/bin
+        -DSYSTEM=Windows -DTRIPLE=x86_64-w64-mingw32
         -P "${CMAKE_CURRENT_LIST_DIR}/host-binary-format-check.cmake"
     RESULT_VARIABLE check_result
     OUTPUT_QUIET
@@ -79,4 +87,49 @@ if(NOT check_error MATCHES "in a PE SDK")
 endif()
 file(REMOVE_RECURSE "${scratch}")
 
+# The machine half of the same failure, which format alone cannot see: both
+# Macs produce Mach-O, so on the one host pair that is staged across two
+# architectures nothing else would catch it. The fixture is the real cmake,
+# so the bytes are a machine's own rather than a hand-written header, and the
+# foreign triple is picked by what it is not -- script mode knows the host
+# system name but not its processor.
+set(foreign_triple "")
+if(NOT machine STREQUAL "")
+    foreach(arch x86_64 aarch64)
+        if(CMAKE_HOST_SYSTEM_NAME STREQUAL "Darwin")
+            set(candidate ${arch}-apple-darwin)
+        elseif(CMAKE_HOST_SYSTEM_NAME STREQUAL "Linux")
+            set(candidate ${arch}-linux-gnu)
+        else()
+            break()
+        endif()
+        vitasdk_expected_binary_format("${CMAKE_HOST_SYSTEM_NAME}" "${candidate}"
+            candidate_format candidate_machine)
+        if(NOT candidate_machine EQUAL machine)
+            set(foreign_triple ${candidate})
+            break()
+        endif()
+    endforeach()
+endif()
+if(NOT foreign_triple STREQUAL "")
+    set(scratch "${temp_root}/vitasdk-host-binary-machine-${fixture_id}")
+    file(COPY "${real_cmake}" DESTINATION "${scratch}/bin")
+    execute_process(
+        COMMAND ${CMAKE_COMMAND}
+            -DDIRECTORY=${scratch}/bin
+            -DSYSTEM=${CMAKE_HOST_SYSTEM_NAME}
+            -DTRIPLE=${foreign_triple}
+            -P "${CMAKE_CURRENT_LIST_DIR}/host-binary-format-check.cmake"
+        RESULT_VARIABLE check_result
+        OUTPUT_QUIET
+        ERROR_VARIABLE check_error)
+    if(check_result EQUAL 0)
+        message(FATAL_ERROR "a binary for another machine must be rejected")
+    endif()
+    if(NOT check_error MATCHES "machine ${machine}" OR NOT check_error MATCHES "${foreign_triple}")
+        message(FATAL_ERROR "the rejection must name the machine, got: ${check_error}")
+    endif()
+    file(REMOVE_RECURSE "${scratch}")
+endif()
+
 message(STATUS "Host binary format checks passed")