Add Mach-O universal binary support

A universal binary could not be read at all, so an image shipping for both Intel and Apple Silicon was out of reach even though each slice inside it was already readable. Adds reading and writing of the container, exposing one image per architecture.

The header and its slice table are big-endian whatever the architectures inside are, which is the one place the format departs from the image's own byte order, so they are read through explicit big-endian primitives rather than by copying a structure. Each slice is read through a view bounded to it, so a malformed slice cannot reach into its neighbours.

Slices are written back at the offsets recorded for them and the gaps between them are left zeroed. Every universal binary examined pads with zeros, and slices are page aligned rather than packed, so the gaps hold nothing worth preserving.

- src/LibObjectFile/MachO/MachOFatFile.cs: the container, its slice table and the bounded reads
- src/LibObjectFile/MachO/MachOFatSlice.cs: one architecture's placement and image

Author: ProjectSynchro

src/LibObjectFile.Tests/MachO/MachOSimpleTests.cs

@@ -200,6 +200,35 @@ public void ReadsRelocations()
         Assert.AreEqual((MachORelocation.ScatteredMask | (2u << 28) | (1u << 24) | 0x123, 0x4567u), scattered.Encode());
     }
 
+    /// <summary>
+    /// A universal binary stores its header big-endian whatever the architectures inside are,
+    /// which is the one place the format departs from the image's own byte order.
+    /// </summary>
+    [TestMethod]
+    public void ReadsAndWritesAUniversalBinary()
+    {
+        var original = File.ReadAllBytes(GetFile("helloworld_fat"));
+
+        using var input = new MemoryStream(original);
+        Assert.IsTrue(MachOFatFile.IsFat(input));
+        Assert.IsFalse(MachOFile.IsMachO(input), "a universal binary is not itself a Mach-O image");
+
+        var fat = MachOFatFile.Read(input);
+        Assert.IsFalse(fat.Is64BitOffsets);
+        CollectionAssert.AreEqual(new[] { MachOCpuType.X86_64, MachOCpuType.Arm64 }, fat.Slices.Select(s => s.CpuType).ToArray());
+
+        foreach (var slice in fat.Slices)
+        {
+            Assert.IsNotNull(slice.File);
+            Assert.AreEqual(slice.CpuType, slice.File.CpuType, "the slice table has to agree with the image it points at");
+            Assert.AreEqual(0ul, slice.FileOffset % slice.Alignment, "a slice is mapped directly, so it has to be page aligned");
+        }
+
+        var output = new MemoryStream();
+        fat.Write(output);
+        ByteArrayAssert.AreEqual(original, output.ToArray(), "Invalid binary diff for helloworld_fat after read -> write");
+    }
+
     [TestMethod]
     public void RejectsANonMachOStream()
     {

src/LibObjectFile/MachO/MachOFatFile.cs

@@ -0,0 +1,244 @@
+// Copyright (c) Alexandre Mutel. All rights reserved.
+// This file is licensed under the BSD-Clause 2 license.
+// See the license.txt file in the project root for more information.
+
+using System;
+using System.Buffers.Binary;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.IO;
+using System.Linq;
+using LibObjectFile.Diagnostics;
+using LibObjectFile.IO;
+
+namespace LibObjectFile.MachO;
+
+/// <summary>
+/// A universal binary, holding one <see cref="MachOFile"/> per architecture.
+/// </summary>
+/// <remarks>
+/// The header and its slice table are stored big-endian whatever the architectures inside are,
+/// which is the one place the format does not follow the image's own byte order.
+/// <para>
+/// Slices are aligned to the page size of their architecture so the loader can map one straight
+/// out of the containing file, which is why a universal binary is larger than its slices put
+/// together.
+/// </para>
+/// </remarks>
+public sealed class MachOFatFile
+{
+    /// <summary>
+    /// Gets the slices of this universal binary, in the order the header lists them.
+    /// </summary>
+    public List<MachOFatSlice> Slices { get; } = [];
+
+    /// <summary>
+    /// Gets or sets whether slice offsets are stored as 64-bit values, which is needed once a
+    /// slice starts beyond 4GB.
+    /// </summary>
+    public bool Is64BitOffsets { get; set; }
+
+    /// <summary>
+    /// Checks whether a stream starts with a universal binary magic, without consuming it.
+    /// </summary>
+    /// <param name="stream">The stream to inspect.</param>
+    /// <returns><c>true</c> if the stream starts with a universal binary magic.</returns>
+    /// <exception cref="ArgumentNullException"><paramref name="stream"/> is null.</exception>
+    public static bool IsFat(Stream stream)
+    {
+        ArgumentNullException.ThrowIfNull(stream);
+
+        var position = stream.Position;
+        try
+        {
+            Span<byte> magic = stackalloc byte[4];
+            if (stream.Read(magic) != 4) return false;
+            var value = BinaryPrimitives.ReadUInt32BigEndian(magic);
+            return value is MachOMagic.FatMagic or MachOMagic.FatMagic64;
+        }
+        finally
+        {
+            stream.Position = position;
+        }
+    }
+
+    /// <summary>
+    /// Reads a universal binary from a stream.
+    /// </summary>
+    /// <param name="stream">The stream positioned at the start of the file.</param>
+    /// <param name="options">Options controlling how each slice is read, or null for the defaults.</param>
+    /// <returns>The universal binary read.</returns>
+    /// <exception cref="ArgumentNullException"><paramref name="stream"/> is null.</exception>
+    /// <exception cref="ObjectFileException">The stream does not contain a readable universal binary.</exception>
+    public static MachOFatFile Read(Stream stream, MachOReaderOptions? options = null)
+    {
+        if (!TryRead(stream, out var file, out var diagnostics, options))
+        {
+            throw new ObjectFileException($"Unexpected error while reading the Mach-O universal binary", diagnostics);
+        }
+        return file;
+    }
+
+    /// <summary>
+    /// Tries to read a universal binary from a stream.
+    /// </summary>
+    /// <param name="stream">The stream positioned at the start of the file.</param>
+    /// <param name="file">The universal binary read, if reading succeeded.</param>
+    /// <param name="diagnostics">The diagnostics collected, if reading failed.</param>
+    /// <param name="options">Options controlling how each slice is read, or null for the defaults.</param>
+    /// <returns><c>true</c> if the file was read; otherwise <c>false</c>.</returns>
+    /// <exception cref="ArgumentNullException"><paramref name="stream"/> is null.</exception>
+    public static bool TryRead(Stream stream, [NotNullWhen(true)] out MachOFatFile? file, [NotNullWhen(false)] out DiagnosticBag? diagnostics, MachOReaderOptions? options = null)
+    {
+        ArgumentNullException.ThrowIfNull(stream);
+
+        var bag = new DiagnosticBag();
+        file = new MachOFatFile();
+        diagnostics = bag;
+
+        var basePosition = stream.Position;
+        Span<byte> header = stackalloc byte[8];
+        stream.Position = basePosition;
+        if (stream.Read(header) != 8)
+        {
+            bag.Error(DiagnosticId.MACHO_ERR_InvalidFatHeader, "The stream is too short to hold a universal binary header");
+            file = null;
+            return false;
+        }
+
+        var magic = BinaryPrimitives.ReadUInt32BigEndian(header);
+        if (magic is not (MachOMagic.FatMagic or MachOMagic.FatMagic64))
+        {
+            bag.Error(DiagnosticId.MACHO_ERR_InvalidFatHeader, $"Invalid universal binary magic 0x{magic:X8}");
+            file = null;
+            return false;
+        }
+
+        file.Is64BitOffsets = magic == MachOMagic.FatMagic64;
+        var count = BinaryPrimitives.ReadUInt32BigEndian(header.Slice(4));
+
+        var entrySize = file.Is64BitOffsets ? 32 : 20;
+        var entries = new byte[count * (uint)entrySize];
+        try
+        {
+            stream.ReadExactly(entries);
+        }
+        catch (EndOfStreamException)
+        {
+            bag.Error(DiagnosticId.MACHO_ERR_InvalidFatHeader, $"The universal binary header claims {count} architectures, which do not fit in the file");
+            file = null;
+            return false;
+        }
+
+        for (var i = 0; i < count; i++)
+        {
+            var entry = entries.AsSpan(i * entrySize);
+            var slice = new MachOFatSlice
+            {
+                CpuType = (MachOCpuType)BinaryPrimitives.ReadUInt32BigEndian(entry),
+                CpuSubType = BinaryPrimitives.ReadUInt32BigEndian(entry.Slice(4)),
+            };
+
+            if (file.Is64BitOffsets)
+            {
+                slice.FileOffset = BinaryPrimitives.ReadUInt64BigEndian(entry.Slice(8));
+                slice.Size = BinaryPrimitives.ReadUInt64BigEndian(entry.Slice(16));
+                slice.AlignLog2 = BinaryPrimitives.ReadUInt32BigEndian(entry.Slice(24));
+            }
+            else
+            {
+                slice.FileOffset = BinaryPrimitives.ReadUInt32BigEndian(entry.Slice(8));
+                slice.Size = BinaryPrimitives.ReadUInt32BigEndian(entry.Slice(12));
+                slice.AlignLog2 = BinaryPrimitives.ReadUInt32BigEndian(entry.Slice(16));
+            }
+
+            if (slice.FileOffset + slice.Size > (ulong)stream.Length)
+            {
+                bag.Error(DiagnosticId.MACHO_ERR_InvalidFatArchRange, $"Slice {i} spans [0x{slice.FileOffset:X}, 0x{slice.FileOffset + slice.Size:X}) which extends past the end of the file");
+                file = null;
+                return false;
+            }
+
+            // Each slice is a complete image, so it is read through a view bounded to the slice.
+            // Handing it the whole stream would let a malformed slice read its neighbours.
+            var sliceStream = new SubStream(stream, basePosition + (long)slice.FileOffset, (long)slice.Size);
+            if (!MachOFile.TryRead(sliceStream, out var sliceFile, out var sliceDiagnostics, options))
+            {
+                foreach (var message in sliceDiagnostics.Messages)
+                {
+                    bag.Log(message);
+                }
+                file = null;
+                return false;
+            }
+
+            slice.File = sliceFile;
+            file.Slices.Add(slice);
+        }
+
+        diagnostics = null;
+        return true;
+    }
+
+    /// <summary>
+    /// Writes this universal binary to a stream.
+    /// </summary>
+    /// <param name="stream">The stream to write to.</param>
+    /// <exception cref="ArgumentNullException"><paramref name="stream"/> is null.</exception>
+    /// <exception cref="ObjectFileException">A slice could not be written.</exception>
+    /// <remarks>
+    /// Slices are written at the offsets already recorded for them, and the space between them
+    /// is left zeroed. Every universal binary examined pads with zeros, and a slice is aligned
+    /// rather than packed, so there is nothing meaningful in the gaps to preserve.
+    /// </remarks>
+    public void Write(Stream stream)
+    {
+        ArgumentNullException.ThrowIfNull(stream);
+
+        var basePosition = stream.Position;
+        var entrySize = Is64BitOffsets ? 32 : 20;
+
+        var header = new byte[8 + Slices.Count * entrySize];
+        BinaryPrimitives.WriteUInt32BigEndian(header, Is64BitOffsets ? MachOMagic.FatMagic64 : MachOMagic.FatMagic);
+        BinaryPrimitives.WriteUInt32BigEndian(header.AsSpan(4), (uint)Slices.Count);
+
+        for (var i = 0; i < Slices.Count; i++)
+        {
+            var slice = Slices[i];
+            var entry = header.AsSpan(8 + i * entrySize);
+            BinaryPrimitives.WriteUInt32BigEndian(entry, (uint)slice.CpuType);
+            BinaryPrimitives.WriteUInt32BigEndian(entry.Slice(4), slice.CpuSubType);
+
+            if (Is64BitOffsets)
+            {
+                BinaryPrimitives.WriteUInt64BigEndian(entry.Slice(8), slice.FileOffset);
+                BinaryPrimitives.WriteUInt64BigEndian(entry.Slice(16), slice.Size);
+                BinaryPrimitives.WriteUInt32BigEndian(entry.Slice(24), slice.AlignLog2);
+            }
+            else
+            {
+                BinaryPrimitives.WriteUInt32BigEndian(entry.Slice(8), (uint)slice.FileOffset);
+                BinaryPrimitives.WriteUInt32BigEndian(entry.Slice(12), (uint)slice.Size);
+                BinaryPrimitives.WriteUInt32BigEndian(entry.Slice(16), slice.AlignLog2);
+            }
+        }
+
+        stream.Write(header);
+
+        foreach (var slice in Slices)
+        {
+            if (slice.File is null) continue;
+            stream.Position = basePosition + (long)slice.FileOffset;
+            slice.File.Write(stream);
+        }
+
+        var end = Slices.Count == 0 ? 0 : Slices.Max(s => (long)(s.FileOffset + s.Size));
+        if (stream.Length < basePosition + end)
+        {
+            stream.SetLength(basePosition + end);
+        }
+    }
+
+    /// <inheritdoc />
+    public override string ToString() => $"{nameof(MachOFatFile)} {{ Slices = {Slices.Count} }}";
+}

src/LibObjectFile/MachO/MachOFatSlice.cs

@@ -0,0 +1,54 @@
+// Copyright (c) Alexandre Mutel. All rights reserved.
+// This file is licensed under the BSD-Clause 2 license.
+// See the license.txt file in the project root for more information.
+
+using System;
+
+namespace LibObjectFile.MachO;
+
+/// <summary>
+/// One architecture's image inside a <see cref="MachOFatFile"/>.
+/// </summary>
+public sealed class MachOFatSlice
+{
+    /// <summary>
+    /// Gets or sets the CPU architecture of this slice. It repeats what the slice's own header
+    /// says, so that a loader can pick a slice without parsing it.
+    /// </summary>
+    public MachOCpuType CpuType { get; set; }
+
+    /// <summary>
+    /// Gets or sets the CPU subtype of this slice.
+    /// </summary>
+    public uint CpuSubType { get; set; }
+
+    /// <summary>
+    /// Gets or sets the alignment of this slice as a power of two. It is the page size of the
+    /// architecture, because the slice is mapped directly out of the containing file.
+    /// </summary>
+    public uint AlignLog2 { get; set; }
+
+    /// <summary>
+    /// Gets or sets the offset of this slice in the containing file.
+    /// </summary>
+    public ulong FileOffset { get; set; }
+
+    /// <summary>
+    /// Gets or sets the size of this slice in the containing file.
+    /// </summary>
+    public ulong Size { get; set; }
+
+    /// <summary>
+    /// Gets or sets the image this slice holds.
+    /// </summary>
+    public MachOFile? File { get; set; }
+
+    /// <summary>
+    /// Gets the alignment of this slice in bytes.
+    /// </summary>
+    public ulong Alignment => 1ul << (int)AlignLog2;
+
+    /// <inheritdoc />
+    public override string ToString()
+        => $"{nameof(MachOFatSlice)} {{ {CpuType}, FileOffset = 0x{FileOffset:X}, Size = 0x{Size:X}, Align = 2^{AlignLog2} }}";
+}