Merge pull request #395 from nuskey8/codex/fix-standard-library-contracts

Fix Lua 5.2 library contracts and bytecode interoperability

Author: akeit0

README.md

@@ -773,6 +773,10 @@ Paths and names passed to `ILuaFileSystem`, `ILuaFileLoader` and `ILuaModuleLoad
 
 Files, temporary files and redirected standard I/O preserve bytes, including a BOM read as data. Raw line reads stop at LF and retain CR; `*L` also retains the LF. Invalid UTF-8 is replaced when decoding with `DecodeUtf8`, so keep binary data in byte strings or byte arrays. Passing unconverted .NET characters above U+00FF to raw output writes `?` for those characters.
 
+Lua's `load` accepts source strings and reader functions, including readers that return a binary chunk in pieces. `string.dump` serializes a Lua function's prototype; it does not preserve captured upvalue values. The resulting string holds raw bytes in both string modes. Use byte-mode streams when writing it from Lua, or `LuaByteString.ToBytes` when writing it from C#. UTF-16 streams encode text as UTF-8 and cannot preserve arbitrary binary strings.
+
+`Prototype.StringMode` records its string representation, so `ToBytecode` and `WriteBytecode` preserve byte-mode constants. `Prototype.FromBytecode(bytes, name, stringMode)` selects the representation when loading without a state; `LuaState.Load` uses the state's mode. Bytecode uses the Lua 5.2 function-header layout. Regenerate older Lua-CSharp chunks that used a different field order.
+
 ### Garbage Collection
 
 Since Lua-CSharp is implemented in C#, it relies on .NET's garbage collector. As a result, memory management behavior differs from regular Lua.

README_JA.md

@@ -667,6 +667,10 @@ print(l)
 
 Stringライブラリの関数はすべて文字列をUTF-16として扱う実装に変更されていることに注意してください。
 
+`load`は文字列とreader関数に対応し、バイナリチャンクを分割して返すreaderも利用できます。`string.dump`はLua関数のプロトタイプを保存しますが、キャプチャしたupvalueの値は保存しません。戻り値はどちらの文字列モードでも生のバイト列です。Luaからファイルに書く場合はバイトモードのストリームを、C#から書く場合は`LuaByteString.ToBytes`を使用してください。UTF-16モードのストリームはUTF-8でテキストを書き込むため、任意のバイナリ文字列を保持できません。
+
+`Prototype.StringMode`は文字列表現を保持し、`ToBytecode`と`WriteBytecode`でもバイトモードの定数を保持します。stateを使わずに読み込む場合は`Prototype.FromBytecode(bytes, name, stringMode)`で表現を指定できます。`LuaState.Load`はstateのモードを使用します。バイトコードはLua 5.2の関数ヘッダー順に従います。異なる順序で生成された古いLua-CSharpのチャンクは再生成してください。
+
 ### ガベージコレクション
 
 Lua-CSharpはC#で実装されているため.NETのGCに依存しています。そのため、メモリ管理に関する動作が通常のLuaとは異なります。

sandbox/Benchmark/BytecodeInteropVerification.cs

@@ -0,0 +1,120 @@
+using Lua;
+using Lua.Platforms;
+using Lua.Standard;
+using NuLua;
+using NuLua.Lua52;
+using LuaFunction = Lua.LuaFunction;
+using LuaTable = Lua.LuaTable;
+
+static class BytecodeInteropVerification
+{
+    static readonly ComparisonCase Closures = new(
+        "dump-closures",
+        "Bytecode",
+        """
+        local value=10
+        local function make(step)
+            return function(...)
+                value=value+step
+                local first=...
+                return value+select('#',...)+first
+            end
+        end
+        local a,b=make(3),make(5)
+        assert(a(4)==18 and b(7,8)==27)
+        return a(20)
+        """,
+        42
+    );
+
+    static readonly ComparisonCase BinaryStrings = new(
+        "dump-binary-strings",
+        "Bytecode",
+        """
+        local bytes='\0\128\255'
+        local utf8='\225\186\191'
+        local function check()
+            assert(#bytes==3 and bytes:byte(1)==0 and bytes:byte(2)==128 and bytes:byte(3)==255)
+            assert(#utf8==3 and utf8:byte(1)==225 and utf8:byte(2)==186 and utf8:byte(3)==191)
+            assert(''==string.sub(bytes,1,0))
+            assert(true and not false and nil==nil)
+            return 42
+        end
+        return check()
+        """,
+        42
+    );
+
+    public static async Task VerifyAsync()
+    {
+        var buffer = new byte[1024 * 1024];
+        foreach (
+            var workload in ComparisonWorkloads
+                .Cases.Concat(ComparisonWorkloads.CompilationCases)
+                .Append(Closures)
+                .Append(BinaryStrings)
+                .DistinctBy(c => c.Name)
+        )
+        {
+            using var native = Lua52State.Create();
+            native.OpenLibraries();
+            native.RegisterFunction(
+                "host_add",
+                (vm, args) =>
+                {
+                    vm.PushNumber(args[0].Read<double>() + args[1].Read<double>());
+                    return 1;
+                }
+            );
+            native.LoadString(workload.Source, "@bytecode");
+            if (!native.TryDump(1, false, buffer, out var written))
+                throw new InvalidOperationException(
+                    $"Native dump exceeds verification buffer: {workload.Name}"
+                );
+            native.SetTop(0);
+
+            foreach (var mode in new[] { LuaStringMode.Utf16, LuaStringMode.Bytes })
+            {
+                // Arbitrary bytes have the same semantics as C Lua in byte mode.
+                if (workload == BinaryStrings && mode != LuaStringMode.Bytes)
+                    continue;
+                using var state = LuaState.Create(
+                    mode == LuaStringMode.Bytes
+                        ? LuaPlatform.DefaultByteStrings
+                        : LuaPlatform.Default
+                );
+                state.OpenStandardLibraries();
+                state.Environment["host_add"] = new LuaFunction(
+                    (context, _) =>
+                        new(
+                            context.Return(
+                                context.GetArgument<double>(0) + context.GetArgument<double>(1)
+                            )
+                        )
+                );
+
+                var loaded = state.Load(buffer.AsSpan(0, written), "native", "b");
+                var results = await state.ExecuteAsync(loaded);
+                ComparisonWorkloads.Check(
+                    workload,
+                    $"NuLua dump -> Lua-CSharp {mode}",
+                    results.Single().Read<double>()
+                );
+
+                var original = state.Load(workload.Source, "@bytecode");
+                var dump = state.Environment["string"].Read<LuaTable>()["dump"].Read<LuaFunction>();
+                var dumped = (await state.CallAsync(dump, [original])).Single().Read<string>();
+                native.LoadString("return assert(load(...,'managed','b'))()", "@bytecode");
+                native.PushString(LuaByteString.ToBytes(dumped));
+                native.Call(1, 1);
+                ComparisonWorkloads.Check(
+                    workload,
+                    $"Lua-CSharp {mode} dump -> NuLua",
+                    native.ToNumber(-1)
+                );
+                native.SetTop(0);
+            }
+            Console.WriteLine($"PASS Bytecode     {workload.Name, -20} NuLua <-> Lua-CSharp");
+        }
+    }
+}

sandbox/Benchmark/Program.cs

@@ -16,6 +16,7 @@
         $"NuLua Lua52 assembly: {typeof(NuLua.Lua52.Lua52State).Assembly.GetName().Version}"
     );
     InterpreterComparison.Verify();
+    await BytecodeInteropVerification.VerifyAsync();
     return;
 }
 

src/Lua/CodeAnalysis/Compilation/Dump.cs

@@ -5,6 +5,7 @@
 using System.Runtime.InteropServices;
 using System.Text;
 using Lua.Internal;
+using Lua.IO;
 using Lua.Runtime;
 
 namespace Lua.CodeAnalysis.Compilation;
@@ -96,8 +97,14 @@ public void Validate(ReadOnlySpan<char> name)
     }
 }
 
-unsafe ref struct DumpState(IBufferWriter<byte> writer, bool reversedEndian)
+unsafe ref struct DumpState(
+    IBufferWriter<byte> writer,
+    bool reversedEndian,
+    LuaStringMode stringMode
+)
 {
+    readonly Encoding encoding =
+        stringMode == LuaStringMode.Bytes ? LuaByteEncoding.Instance : Encoding.UTF8;
     public readonly IBufferWriter<byte> Writer = writer;
     Span<byte> unWritten;
 
@@ -142,8 +149,8 @@ void DumpFunction(Prototype prototype)
         WriteInt(prototype.LineDefined); // 4
         WriteInt(prototype.LastLineDefined); // 4
         WriteByte((byte)prototype.ParameterCount); // 1
-        WriteByte((byte)prototype.MaxStackSize); // 1
         WriteByte((byte)(prototype.HasVariableArguments ? 1 : 0)); // 1
+        WriteByte((byte)prototype.MaxStackSize); // 1
         WriteIntSpanWithLength(MemoryMarshal.Cast<Instruction, int>(prototype.Code)); // 4
         WriteConstants(prototype.Constants); // 4
         WritePrototypes(prototype.ChildPrototypes); // 4
@@ -215,12 +222,8 @@ void WriteBool(bool v)
 
     void WriteString(string v)
     {
-        var bytes = Encoding.UTF8.GetBytes(v);
-        var len = bytes.Length;
-        if (bytes.Length != 0)
-        {
-            len++;
-        }
+        var bytes = encoding.GetBytes(v);
+        var len = bytes.Length + 1;
 
         if (sizeof(IntPtr) == 8)
         {
@@ -231,32 +234,38 @@ void WriteString(string v)
             WriteInt(len);
         }
 
-        if (len != 0)
-        {
-            Write(bytes);
-            WriteByte(0);
-        }
+        Write(bytes);
+        WriteByte(0);
     }
 
     void WriteConstants(ReadOnlySpan<LuaValue> constants)
     {
         WriteInt(constants.Length);
         foreach (var c in constants)
         {
-            WriteByte((byte)c.Type);
+            // Lua bytecode tags differ from LuaValueType for strings.
             switch (c.Type)
             {
                 case LuaValueType.Nil:
+                    WriteByte(0);
                     break;
                 case LuaValueType.Boolean:
+                    WriteByte(1);
                     WriteBool(c.UnsafeReadDouble() != 0);
                     break;
                 case LuaValueType.Number:
+                    WriteByte(3);
                     WriteDouble(c.UnsafeReadDouble());
                     break;
                 case LuaValueType.String:
+                    WriteByte(4);
                     WriteString(c.UnsafeRead<string>());
                     break;
+                default:
+                    throw new ArgumentException(
+                        "Unsupported bytecode constant type",
+                        nameof(constants)
+                    );
             }
         }
     }
@@ -295,9 +304,12 @@ void WriteUpValues(ReadOnlySpan<UpValueDesc> upValues)
 unsafe ref struct UndumpState(
     ReadOnlySpan<byte> span,
     ReadOnlySpan<char> name,
-    StringInternPool internPool
+    StringInternPool internPool,
+    LuaStringMode stringMode
 )
 {
+    readonly Encoding encoding =
+        stringMode == LuaStringMode.Bytes ? LuaByteEncoding.Instance : Encoding.UTF8;
     public ReadOnlySpan<byte> Unread = span;
     bool otherEndian;
     int pointerSize;
@@ -412,8 +424,8 @@ Prototype UndumpFunction(int depth)
         var lineDefined = ReadInt(); // 4
         var lastLineDefined = ReadInt(); // 4
         var parameterCount = ReadByte(); // 1
-        var maxStackSize = ReadByte(); // 1
         var isVarArg = ReadBool(); // 1
+        var maxStackSize = ReadByte(); // 1
         var codeLength = ReadCount(sizeof(int));
         var code = new Instruction[codeLength];
         ReadInToIntSpan(MemoryMarshal.Cast<Instruction, int>(code));
@@ -422,7 +434,7 @@ Prototype UndumpFunction(int depth)
         var upValues = ReadUpValues();
 
         // Debug
-        var source = ReadString();
+        var source = ReadString("=?");
         var lineInfoLength = ReadCount(sizeof(int));
         if (lineInfoLength != 0 && lineInfoLength != codeLength)
             Throw("invalid line info in");
@@ -450,7 +462,10 @@ Prototype UndumpFunction(int depth)
             lineInfo,
             localVariables,
             upValues
-        );
+        )
+        {
+            StringMode = stringMode,
+        };
         new BytecodeVerifier(prototype, name).Verify();
         return prototype;
     }
@@ -465,12 +480,12 @@ void ReadInToIntSpan(Span<int> toWrite)
         }
     }
 
-    string ReadString()
+    string ReadString(string nullValue = "")
     {
         var serializedLength = pointerSize == 4 ? ReadInt() : ReadLong();
         if (serializedLength == 0)
         {
-            return "";
+            return nullValue;
         }
 
         if (serializedLength < 0 || serializedLength > Unread.Length)
@@ -489,7 +504,7 @@ string ReadString()
                 len <= 128
                     ? stackalloc char[len]
                     : (charArrayPooled = ArrayPool<char>.Shared.Rent(len));
-            var count = Encoding.UTF8.GetChars(bytes, chars);
+            var count = encoding.GetChars(bytes, chars);
             return internPool.Intern(chars[..count]);
         }
         finally
@@ -507,18 +522,18 @@ LuaValue[] ReadConstants()
         var constants = new LuaValue[count];
         for (var i = 0; i < count; i++)
         {
-            var type = (LuaValueType)ReadByte();
+            var type = ReadByte();
             switch (type)
             {
-                case LuaValueType.Nil:
+                case 0:
                     break;
-                case LuaValueType.Boolean:
+                case 1:
                     constants[i] = ReadBool();
                     break;
-                case LuaValueType.Number:
+                case 3:
                     constants[i] = ReadDouble();
                     break;
-                case LuaValueType.String:
+                case 4:
                     constants[i] = ReadString();
                     break;
                 default:

src/Lua/CodeAnalysis/Compilation/Parser.cs

@@ -979,7 +979,7 @@ public static Prototype Parse(LuaState l, TextReader r, string name)
         f.Proto.IsVarArg = true;
         f.Proto.LineDefined = 0;
         p.MainFunction();
-        var prototype = f.Proto.CreatePrototypeAndRelease();
+        var prototype = f.Proto.CreatePrototypeAndRelease(l.Platform.StringMode);
         f.Release();
         return prototype;
     }
@@ -990,7 +990,11 @@ public static void Dump(
         bool useLittleEndian = true
     )
     {
-        DumpState state = new(writer, useLittleEndian ^ BitConverter.IsLittleEndian);
+        DumpState state = new(
+            writer,
+            useLittleEndian ^ BitConverter.IsLittleEndian,
+            prototype.StringMode
+        );
         state.Dump(prototype);
     }
 
@@ -1001,7 +1005,11 @@ public static byte[] Dump(Prototype prototype, bool useLittleEndian = true)
         return writer.WrittenSpan.ToArray();
     }
 
-    public static Prototype Undump(ReadOnlySpan<byte> span, ReadOnlySpan<char> name)
+    public static Prototype Undump(
+        ReadOnlySpan<byte> span,
+        ReadOnlySpan<char> name,
+        LuaStringMode stringMode = LuaStringMode.Utf16
+    )
     {
         if (name.Length > 0)
         {
@@ -1014,7 +1022,7 @@ public static Prototype Undump(ReadOnlySpan<byte> span, ReadOnlySpan<char> name)
         }
 
         using var internPool = new StringInternPool(4);
-        UndumpState state = new(span, name, internPool);
+        UndumpState state = new(span, name, internPool, stringMode);
         return state.Undump();
     }
 }