Detect Ghosts luafiles; add inspector/tests #65

Add detection and handling of Ghosts 'luafile' assets alongside the existing zlib-wrapped headers. GhostsAssetWalker now locates luafiles, pairs luafile pool entries in a separate pass, and surfaces luafiles as rawfile-like nodes using a new BuildLuaFileNode; wrapped asset node builder was renamed to BuildWrappedNode. GhostsZoneLayout gains a GhostsLuaFile struct, a LocateAllLuaFiles scanner (flat 16-byte header + Lua 5.1 signature), and a permissive pool-pointer check to accept IW6-style 0x40-flagged pointers. Introduce LuaBytecodeInspector to parse the Lua 5.1 header, extract printable strings, and format a summary used in the viewer. Tests updated/added to cover pool parsing, luafile discovery, and the bytecode inspector behavior.

Author: primetime43

Call of Duty FastFile Editor/ZoneParsers/GhostsAssetWalker.cs

@@ -28,30 +28,35 @@ public static WalkResult Walk(byte[] zoneData, List<ZoneAssetRecord>? records, i
             var result = new WalkResult();
             if (zoneData == null || zoneData.Length == 0) return result;
 
-            var headers = GhostsZoneLayout.LocateAllHeaders(zoneData, scanStart < 0 ? 0 : scanStart);
+            int scan = scanStart < 0 ? 0 : scanStart;
+            var headers  = GhostsZoneLayout.LocateAllHeaders(zoneData, scan);
+            var luaFiles = GhostsZoneLayout.LocateAllLuaFiles(zoneData, scan);
 
             // Base zones: no pool to pair against. Surface every located header
-            // as a rawfile (per docs/Ghosts_FastFile_Format.md, "almost all"
-            // wrapped assets in base zones are rawfiles).
+            // (wrapped + luafile) as a rawfile-like node so the editor's rawfile
+            // tree still populates. Per docs/Ghosts_FastFile_Format.md, "almost
+            // all" wrapped assets in base zones are rawfiles.
             if (records == null || records.Count == 0)
             {
                 foreach (var h in headers)
-                    result.RawFileNodes.Add(BuildRawFileNode(zoneData, h));
-                result.UnpairedHeaders = headers.Count;
-                Debug.WriteLine($"[GhostsAssetWalker] No pool — surfaced {headers.Count} located headers as rawfiles");
+                    result.RawFileNodes.Add(BuildWrappedNode(zoneData, h));
+                foreach (var lua in luaFiles)
+                    result.RawFileNodes.Add(BuildLuaFileNode(zoneData, lua));
+                result.UnpairedHeaders = headers.Count + luaFiles.Count;
+                Debug.WriteLine($"[GhostsAssetWalker] No pool — surfaced {headers.Count} wrapped + {luaFiles.Count} luafile headers as rawfiles");
                 return result;
             }
 
-            // Lift to library DTOs for pairing.
+            // Pair wrapped types (rawfile/scriptfile/mptype/aitype) with the
+            // located zlib-wrapped headers.
             var poolDtos = new List<GhostsPoolEntry>(records.Count);
             foreach (var r in records)
             {
                 poolDtos.Add(new GhostsPoolEntry(
                     recordOffset: r.AssetPoolRecordOffset,
                     type: r.AssetType_Ghosts,
-                    pointerKind: GhostsPointerKind.Placeholder)); // pointer kind isn't used during pairing
+                    pointerKind: GhostsPointerKind.Placeholder));
             }
-
             var pairing = GhostsZoneLayout.PairPoolWithHeaders(poolDtos, headers);
 
             for (int i = 0; i < records.Count; i++)
@@ -72,18 +77,44 @@ public static WalkResult Walk(byte[] zoneData, List<ZoneAssetRecord>? records, i
                 records[i] = record;
 
                 if (record.AssetType_Ghosts == GhostsAssetTypePS3.rawfile)
-                    result.RawFileNodes.Add(BuildRawFileNode(zoneData, h));
+                    result.RawFileNodes.Add(BuildWrappedNode(zoneData, h));
+            }
+
+            // Pair luafile pool entries with located luafiles — separate pass
+            // because the on-disk format is different (flat 16-byte header, no
+            // zlib wrapper). Same positional rule: i-th luafile pool entry ↔
+            // i-th located luafile body.
+            int luaIdx = 0;
+            int luaPaired = 0;
+            for (int i = 0; i < records.Count && luaIdx < luaFiles.Count; i++)
+            {
+                if (records[i].AssetType_Ghosts != GhostsAssetTypePS3.luafile) continue;
+                var lua = luaFiles[luaIdx++];
+                var record = records[i];
+
+                record.HeaderStartOffset      = lua.HeaderOffset;
+                record.HeaderEndOffset        = lua.BodyStart;
+                record.AssetDataStartPosition = lua.BodyStart;
+                record.AssetDataEndOffset     = lua.BodyEnd;
+                record.AssetRecordEndOffset   = lua.BodyEnd;
+                record.Name                   = lua.Name;
+                record.Size                   = lua.ByteCodeLen;
+                record.AdditionalData         = "Ghosts luafile (flat header)";
+                records[i] = record;
+                luaPaired++;
+
+                result.RawFileNodes.Add(BuildLuaFileNode(zoneData, lua));
             }
 
-            result.Resolved        = pairing.PairedCount;
-            result.Unresolved      = Math.Max(0, records.Count - pairing.PairedCount);
-            result.UnpairedHeaders = pairing.UnpairedHeaders;
+            result.Resolved        = pairing.PairedCount + luaPaired;
+            result.Unresolved      = Math.Max(0, records.Count - result.Resolved);
+            result.UnpairedHeaders = pairing.UnpairedHeaders + Math.Max(0, luaFiles.Count - luaIdx);
 
-            Debug.WriteLine($"[GhostsAssetWalker] Paired {result.Resolved}/{records.Count} pool entries with {headers.Count} located headers ({result.UnpairedHeaders} unpaired); {result.RawFileNodes.Count} rawfiles");
+            Debug.WriteLine($"[GhostsAssetWalker] Paired {pairing.PairedCount} wrapped + {luaPaired} luafile pool entries; {result.RawFileNodes.Count} rawfile-like nodes");
             return result;
         }
 
-        private static RawFileNode BuildRawFileNode(byte[] zone, GhostsAssetHeader h)
+        private static RawFileNode BuildWrappedNode(byte[] zone, GhostsAssetHeader h)
         {
             byte[] body = new byte[h.DecompressedLen];
             Buffer.BlockCopy(zone, h.BodyStart, body, 0, h.DecompressedLen);
@@ -102,6 +133,30 @@ private static RawFileNode BuildRawFileNode(byte[] zone, GhostsAssetHeader h)
             };
         }
 
+        private static RawFileNode BuildLuaFileNode(byte[] zone, GhostsLuaFile lua)
+        {
+            byte[] body = new byte[lua.ByteCodeLen];
+            Buffer.BlockCopy(zone, lua.BodyStart, body, 0, lua.ByteCodeLen);
+            // The body is Lua 5.1 bytecode (compiled, not source). Surface a
+            // header + extracted-strings summary in the text viewer so the
+            // user gets useful signal about each file's content; the original
+            // .lua source isn't recoverable without a decompiler.
+            var summary = LuaBytecodeInspector.Inspect(body);
+            string content = LuaBytecodeInspector.FormatSummaryText(lua.Name, summary);
+            return new RawFileNode
+            {
+                FileName          = lua.Name,
+                StartOfFileHeader = lua.HeaderOffset,
+                HeaderSize        = lua.BodyStart - lua.HeaderOffset,
+                MaxSize           = lua.ByteCodeLen,
+                CompressedSize    = lua.ByteCodeLen,
+                IsCompressed      = false,
+                RawFileBytes      = body,
+                RawFileContent    = content,
+                AdditionalData    = "Ghosts luafile (Lua 5.1 bytecode)",
+            };
+        }
+
         private static bool LooksTextual(byte[] body)
         {
             if (body.Length == 0) return true;

FastFileCLI.Tests/GhostsZoneLayoutTests.cs

@@ -1,3 +1,4 @@
+using System.Text;
 using FastFileLib;
 using FastFileLib.GameDefinitions;
 using Xunit;
@@ -302,4 +303,141 @@ public void IsWrappedType_OnlyKnownWrappedTypes()
         Assert.False(GhostsZoneLayout.IsWrappedType(GhostsAssetTypePS3.techset));
         Assert.False(GhostsZoneLayout.IsWrappedType(GhostsAssetTypePS3.weapon));
     }
+
+    // =========================================================
+    // Permissive pointer convention (0x40-flagged)
+    // =========================================================
+
+    [Fact]
+    public void WalkPool_AcceptsZero40FlaggedPointer()
+    {
+        // IW6 patch_ui_mp.zone has pool entries like [40 1F DF 85][00 00 00 05]
+        // (material ptr with the 0x40000000 flag bit set). Verify they parse.
+        byte[] flaggedPtr = { 0x40, 0x1F, 0xDF, 0x85 };
+        byte[] zone = Concat(
+            Header(0, 2),
+            PoolRecord(0x05, flaggedPtr), // material with 0x40-flagged ptr
+            PoolRecord(0x09, FF4),        // image with placeholder
+            new byte[256]);
+
+        var entries = GhostsZoneLayout.ParsePool(zone, out int poolStart, out _);
+        Assert.Equal(0x38, poolStart);
+        Assert.Equal(2, entries.Count);
+        Assert.Equal(GhostsPointerKind.Resolved, entries[0].PointerKind);
+        Assert.Equal(GhostsAssetTypePS3.material, entries[0].Type);
+    }
+
+    [Fact]
+    public void WalkPool_HeaderCountCapsAtAssetCount()
+    {
+        // Make the bytes past `assetCount` entries also look like a pool entry.
+        // The header-driven cap should stop the walk at the declared count.
+        byte[] zone = Concat(
+            Header(0, 2),
+            PoolRecord(0x28, FF4),
+            PoolRecord(0x29, Zero4),
+            PoolRecord(0x05, FF4), // extra would be walkable but capped
+            new byte[256]);
+
+        var entries = GhostsZoneLayout.ParsePool(zone, out _, out _);
+        Assert.Equal(2, entries.Count);
+    }
+
+    // =========================================================
+    // Luafile scan
+    // =========================================================
+
+    private static byte[] LuaFile(string name, byte[] body)
+    {
+        var nameBytes = Encoding.ASCII.GetBytes(name);
+        var nullTerm = new byte[] { 0 };
+        var unk = Be32(0x02000000); // observed-fixed value in real zones
+        return Concat(FF4, Be32((uint)body.Length), unk, FF4, nameBytes, nullTerm, body);
+    }
+
+    private static byte[] LuaBytecode(int totalSize)
+    {
+        var b = new byte[totalSize];
+        b[0] = 0x1B; b[1] = (byte)'L'; b[2] = (byte)'u'; b[3] = (byte)'a'; b[4] = 0x51;
+        return b;
+    }
+
+    [Fact]
+    public void LocateAllLuaFiles_FindsSingleEntry()
+    {
+        byte[] zone = Concat(new byte[0x100], LuaFile("ui/main.lua", LuaBytecode(128)), new byte[256]);
+
+        var lua = GhostsZoneLayout.LocateAllLuaFiles(zone, 0x100);
+
+        Assert.Single(lua);
+        Assert.Equal("ui/main.lua", lua[0].Name);
+        Assert.Equal(128, lua[0].ByteCodeLen);
+        Assert.Equal(0x100, lua[0].HeaderOffset);
+    }
+
+    [Fact]
+    public void LocateAllLuaFiles_FindsConsecutiveEntries()
+    {
+        // Three back-to-back luafiles. Stride should walk from one to the next.
+        byte[] zone = Concat(
+            new byte[0x40],
+            LuaFile("a.lua", LuaBytecode(64)),
+            LuaFile("ui/b.lua", LuaBytecode(96)),
+            LuaFile("c.lua", LuaBytecode(48)),
+            new byte[16]);
+
+        var lua = GhostsZoneLayout.LocateAllLuaFiles(zone, 0x40);
+
+        Assert.Equal(3, lua.Count);
+        Assert.Equal("a.lua",    lua[0].Name);
+        Assert.Equal("ui/b.lua", lua[1].Name);
+        Assert.Equal("c.lua",    lua[2].Name);
+    }
+
+    [Fact]
+    public void LocateAllLuaFiles_RejectsNonLuaSuffix()
+    {
+        // The 16-byte header pattern looks like luafile but the name doesn't
+        // end in .lua, so it must be rejected (would otherwise collide with
+        // other flat-header asset types in the same zone).
+        byte[] zone = Concat(new byte[0x40], LuaFile("ui/main.txt", LuaBytecode(64)), new byte[256]);
+
+        var lua = GhostsZoneLayout.LocateAllLuaFiles(zone, 0x40);
+
+        Assert.Empty(lua);
+    }
+
+    [Fact]
+    public void LocateAllLuaFiles_RejectsMissingLuaSignature()
+    {
+        // Header + name look fine but the body doesn't start with \x1B LuaQ.
+        // The signature check is the primary defense against false positives.
+        byte[] nameBytes = Encoding.ASCII.GetBytes("ui/main.lua");
+        byte[] notLua = new byte[64]; // all zeros, no Lua magic
+        byte[] entry = Concat(FF4, Be32(64), Be32(0x02000000), FF4, nameBytes, new byte[] { 0 }, notLua);
+        byte[] zone = Concat(new byte[0x40], entry, new byte[256]);
+
+        var lua = GhostsZoneLayout.LocateAllLuaFiles(zone, 0x40);
+
+        Assert.Empty(lua);
+    }
+
+    [Fact]
+    public void LocateAllLuaFiles_SkipsPastNonMatchBytes()
+    {
+        // Pad with random bytes between two luafiles. The scan must byte-walk
+        // through the gap and recover the second entry.
+        byte[] zone = Concat(
+            new byte[0x40],
+            LuaFile("a.lua", LuaBytecode(32)),
+            new byte[] { 0xAB, 0xCD, 0xEF, 0x12, 0x34 }, // 5 bytes of junk
+            LuaFile("b.lua", LuaBytecode(48)),
+            new byte[16]);
+
+        var lua = GhostsZoneLayout.LocateAllLuaFiles(zone, 0x40);
+
+        Assert.Equal(2, lua.Count);
+        Assert.Equal("a.lua", lua[0].Name);
+        Assert.Equal("b.lua", lua[1].Name);
+    }
 }