diff --git a/tools/DifferentialProbe/.gitignore b/tools/DifferentialProbe/.gitignore new file mode 100644 index 00000000..cd42ee34 --- /dev/null +++ b/tools/DifferentialProbe/.gitignore @@ -0,0 +1,2 @@ +bin/ +obj/ diff --git a/tools/DifferentialProbe/Cases.cs b/tools/DifferentialProbe/Cases.cs new file mode 100644 index 00000000..239ec05c --- /dev/null +++ b/tools/DifferentialProbe/Cases.cs @@ -0,0 +1,648 @@ +using System.Globalization; + +record Case(string Family, string Name, string Source); + +static class Cases +{ + public static IEnumerable Generate() + { + string[] strings = ["''", "'a'", "'abc'", "'a b a'", "'a\\0b'", "'12ab34'"]; + string[] indices = + [ + "-2147483649", + "-8", + "-4", + "-1.9", + "-0.5", + "0", + "0.5", + "1", + "1.9", + "3", + "4", + "8", + "2147483648", + "'2'", + "'-1.9'", + "nil", + ]; + foreach (var s in strings) + foreach (var first in indices) + { + foreach (var last in indices) + foreach (var method in new[] { "sub", "byte" }) + yield return new( + "string.slice", + $"{method}/{s}/{first}/{last}", + $"return string.{method}({s},{first},{last})" + ); + + foreach (var pattern in new[] { "''", "'a'", "'.'", "'()'", "'%z'" }) + foreach (var method in new[] { "find", "match" }) + yield return new( + "string.search", + $"{method}/{s}/{first}/{pattern}", + $"return string.{method}({s},{pattern},{first})" + ); + } + + foreach (var c in Enumerable.Range(0, 33).Append(127)) + foreach ( + var suffix in new[] + { + "''", + "'a'", + "'0'", + "'1'", + "'2'", + "'3'", + "'4'", + "'5'", + "'6'", + "'7'", + "'8'", + "'9'", + } + ) + { + var setup = $"local s=string.char({c})..{suffix}; local q=string.format('%q',s); "; + yield return new("string.quote", $"spelling/{c}/{suffix}", setup + "return q"); + yield return new( + "string.quote-roundtrip", + $"{c}/{suffix}", + setup + "local f=assert(load('return '..q)); return f()==s" + ); + } + + foreach ( + var n in new[] + { + "-1.9", + "-0.5", + "0", + "0.5", + "1.9", + "65.9", + "255.9", + "256", + "'65.9'", + "nil", + "false", + } + ) + yield return new("string.char", n, $"return string.char({n})"); + + foreach ( + var value in new[] + { + "true", + "false", + "nil", + "{}", + "function() end", + "coroutine.create(function() end)", + } + ) + { + yield return new( + "string.quote-types", + $"first/{value}", + $"return string.format('%q',{value})" + ); + yield return new( + "string.quote-types", + $"later/{value}", + $"return string.format('%s:%q','prefix',{value})" + ); + } + yield return new( + "string.quote-types", + "tostring-metamethod", + "local calls=0; local x=setmetatable({}, {__tostring=function() calls=calls+1; return 'converted' end}); local ok=pcall(string.format,'%q',x); return ok,calls,string.format('%s',x),calls" + ); + + foreach ( + var value in new[] + { + "0", + "-0.0", + "12", + "-12", + "1.5", + "1/3", + "1e20", + "1e-20", + "2^53", + "0x1p-1074", + "-0x1p-1074", + "0x1.fffffffffffffp1023", + "1.234567890123456", + } + ) + foreach (var format in new[] { "%q", "%20q", "%+.2q" }) + yield return new( + "string.quote-number", + $"{format}/{value}", + $"local x={value}; local q=string.format('{format}',x); return q,type(assert(load('return '..q))())" + ); + + foreach ( + var body in new[] + { + "return", + "return nil", + "return true", + "return {}", + "return 12", + "return 'first','extra'", + } + ) + foreach (var hooks in new[] { false, true }) + { + var setup = hooks ? "debug.sethook(function() end,'cr'); " : ""; + yield return new( + "callback.results", + $"tostring/{body}/{hooks}", + setup + + $"local x=setmetatable({{}},{{__tostring=function() {body} end}}); local r=table.pack(tostring(x)); return r.n,type(r[1]),type(r[1])=='string' and r[1] or nil" + ); + } + yield return new( + "callback.results", + "sethook/enable", + "return select('#',debug.sethook(function() end,'cr'))" + ); + yield return new( + "callback.results", + "sethook/disable", + "return select('#',debug.sethook(nil))" + ); + yield return new( + "callback.results", + "upvaluejoin", + "local a,b=1,2; local function f() return a end; local function g() return b end; local n=select('#',debug.upvaluejoin(f,1,g,1)); return n,f()" + ); + + string[] patterns = + [ + "''", + "'.'", + "'.*'", + "'.-'", + "'a*'", + "'a-'", + "'a?'", + "'a+'", + "'^'", + "'$'", + "'^$'", + "'()'", + "'(a*)'", + "'(.)%1'", + "'((a)b)'", + "'%b()'", + "'%f[%a]%a+'", + "'[%a%d]+'", + "'[^a]*'", + "'%z'", + "'['", + "'('", + "')'", + "'%'", + "'%1'", + "'()%1'", + "'(a)%2'", + "'a**'", + ]; + foreach (var s in strings.Concat(["'(a)(b)'", "'^a^^b'", "'aaab abba'"])) + foreach (var pattern in patterns) + { + foreach (var method in new[] { "find", "match" }) + yield return new( + "pattern.search", + $"{method}/{s}/{pattern}", + $"return string.{method}({s},{pattern})" + ); + foreach ( + var repl in new[] + { + "'x'", + "'%0'", + "'%1'", + "function(...) return select('#',...) end", + } + ) + yield return new( + "pattern.gsub", + $"{s}/{pattern}/{repl}", + $"return string.gsub({s},{pattern},{repl})" + ); + yield return new( + "pattern.gmatch", + $"{s}/{pattern}", + $"local t={{}}; for x in string.gmatch({s},{pattern}) do t[#t+1]=tostring(x) end; return table.concat(t,'|')" + ); + } + + string[] numbers = + [ + "'-0'", + "' 2.5 '", + "'0x1.8p2'", + "'1e309'", + "'nan'", + "'NaN'", + "'inf'", + "'Infinity'", + "' +12'", + "'12x'", + "''", + "'1_2'", + "'0x'", + "'0x1p'", + "'1e'", + "'2\\0'", + "'\\1602'", + "'2\\160'", + "'0x2\\160'", + "'\\v2\\f'", + ]; + foreach (var n in numbers) + { + yield return new("number.parse", n, $"return tonumber({n})"); + yield return new("number.coercion", n, $"local x={n}; return x+0, -x"); + foreach (var b in new[] { "2", "8", "10", "16", "36", "16.9", "'16'" }) + yield return new("number.base", $"{n}/{b}", $"return tonumber({n},{b})"); + } + + foreach ( + var value in new[] + { + "1/3", + "1e20", + "-0.0", + "0.000012345", + "1e-20", + "2^53", + "1.234567890123456", + } + ) + yield return new( + "number.text", + value, + $"local x={value}; return tostring(x),string.format('%s',x),x..''" + ); + + foreach (var value in new[] { "1/3", "1e14", "1e20", "0x1p-1074", "-0.0", "1.5" }) + { + foreach (var replacement in new[] { "x", "function() return x end", "{a=x}" }) + yield return new( + "number.replacement", + $"{value}/{replacement}", + $"local x={value}; return string.gsub('aba','a',{replacement})" + ); + foreach (var radix in new[] { 2, 8, 10, 16, 36 }) + yield return new( + "number.base-numeric", + $"{value}/{radix}", + $"return tonumber({value},{radix})" + ); + } + + foreach ( + var method in new[] + { + "abs", + "ceil", + "floor", + "sqrt", + "exp", + "log", + "frexp", + "modf", + "deg", + "rad", + "sin", + "atan", + } + ) + foreach ( + var n in new[] + { + "-1/0", + "-1e300", + "-2.5", + "-0.5", + "-0.0", + "0", + "0.5", + "1", + "2.5", + "1e300", + "1/0", + "0/0", + "'2.5'", + } + ) + yield return new("math", $"{method}/{n}", $"return math.{method}({n})"); + + foreach ( + var n in new[] + { + "0x1p-1074", + "-0x1p-1074", + "0x0.fffffffffffffp-1022", + "0x1p-1022", + "0x1.fffffffffffffp1023", + "-0x1.fffffffffffffp1023", + "math.pi", + "-math.pi", + "0x1p-1021", + "0x1p1000", + } + ) + foreach (var method in new[] { "deg", "rad", "frexp" }) + yield return new("math", $"boundary/{method}/{n}", $"return math.{method}({n})"); + foreach ( + var exponent in new[] { -1074, -1073, -1023, -1022, -1021, -53, -1, 0, 1, 52, 53, 1023 } + ) + foreach (var sign in new[] { "", "-" }) + yield return new( + "math", + $"frexp/power/{sign}/{exponent}", + $"return math.frexp({sign}0x1p{exponent})" + ); + + foreach ( + var specifier in new[] { "d", "i", "u", "x", "X", "o", "c", "f", "e", "E", "g", "G" } + ) + foreach (var flags in new[] { "", "-", "+", " ", "0", "#", "+0", " 0", "-0", "++" }) + foreach (var precision in new[] { "", ".0", ".3" }) + foreach ( + var value in new[] + { + "-12.75", + "-0.5", + "-0.0", + "0", + "1.5", + "12.75", + "12345", + "0.000012345", + "'2.5'", + } + ) + { + var format = "%" + flags + "8" + precision + specifier; + var family = + specifier == "c" ? "format.char" + : specifier is "f" or "e" or "E" or "g" or "G" ? "format.float" + : "format.integer"; + yield return new( + family, + $"{format}/{value}", + $"return string.format('{format}',{value})" + ); + } + + // Keep allocation-producing table ranges small and valid for native Lua. + var tableIndices = indices.Where(i => i != "nil" && !i.Contains("214748364")); + foreach (var position in tableIndices) + { + var setup = "local t={'a','b','c'}; "; + yield return new( + "table", + $"insert/{position}", + setup + $"table.insert(t,{position},'x'); return #t,table.concat(t,',')" + ); + yield return new( + "table", + $"remove/{position}", + setup + $"local x=table.remove(t,{position}); return x,#t,table.concat(t,',')" + ); + foreach (var last in tableIndices) + { + yield return new( + "table", + $"unpack/{position}/{last}", + setup + $"return table.unpack(t,{position},{last})" + ); + yield return new( + "table", + $"concat/{position}/{last}", + setup + $"return table.concat(t,',',{position},{last})" + ); + } + } + + foreach ( + var value in new[] + { + "1/3", + "1e20", + "-0.0", + "0.000012345", + "1e-20", + "2^53", + "1.234567890123456", + } + ) + yield return new( + "table", + $"concat-number/{value}", + $"return table.concat({{{value},'x'}},',')" + ); + + foreach (var op in new[] { "+", "-", "*", "/", "%", "^", "<", "<=", "==", "~=" }) + foreach (var a in new[] { "-3", "-0.0", "0", "0.5", "2", "'2'", "'x'", "false", "nil" }) + foreach (var b in new[] { "-2", "0", "0.5", "3", "'3'", "'y'", "true", "nil" }) + yield return new( + "vm.operators", + $"{a}/{op}/{b}", + $"local a,b={a},{b}; return a {op} b" + ); + + for (var n = 0; n < 20; n++) + { + var size = n.ToString(CultureInfo.InvariantCulture); + yield return new( + "compiler", + $"closures/{n}", + $"local t={{}}; for i=1,{size} do local x=i; t[i]=function(y) return x+y end end; local s=0; for i=1,#t do s=s+t[i](i) end; return s" + ); + yield return new( + "compiler", + $"repeat/{n}", + $"local x=0; repeat local y=x+1; x=y until y>={size}; return x" + ); + yield return new( + "compiler", + $"varargs/{n}", + $"local function f(...) return select('#',...),... end; local t={{}}; for i=1,{size} do t[i]=i end; return f(table.unpack(t))" + ); + yield return new( + "compiler", + $"assignment/{n}", + $"local t={{1,2,3}}; local i=1; i,t[i]=2,{size}; t[i],i=i,3; return i,table.unpack(t)" + ); + yield return new( + "compiler", + $"table-conflict/{n}", + $"local t={{}}; local old=t; t.x,t={size},{{}}; return old.x,t.x" + ); + yield return new( + "compiler", + $"upvalue-conflict/{n}", + $"local t={{}}; local old=t; local function f() t.x,t={size},{{}} end; f(); return old.x,t.x" + ); + yield return new( + "compiler", + $"two-conflicts/{n}", + $"local a,b={{}},{{}}; local i=1; a[i],b[i],i={size},{size}+1,2; return a[1],a[2],b[1],b[2],i" + ); + yield return new( + "compiler", + $"rhs-call/{n}", + $"local t={{}}; local i=1; local function f() return {size},2 end; t[i],i=f(); return t[1],t[2],i" + ); + } + + foreach ( + var number in new[] + { + "0", + "1", + "1.", + ".5", + "1e2", + "0x1", + "0x1.8p2", + "0x.8", + "0x1p-2", + "0x1+2", + "0x1-2", + "1..2", + "1e", + "0x", + "0xp1", + "0x1p", + "0x1p+", + "0x1g", + "0x1.2.3", + } + ) + yield return new("scanner.number", number, "return " + number); + + for (var code = 0; code < 256; code++) + { + var escape = "\\" + code.ToString("D3", CultureInfo.InvariantCulture); + yield return new( + "scanner.escape", + $"decimal/{code}", + $"return string.byte('{escape}')" + ); + escape = "\\x" + code.ToString("x2", CultureInfo.InvariantCulture); + yield return new("scanner.escape", $"hex/{code}", $"return string.byte('{escape}')"); + } + + foreach ( + var word in new[] + { + "-4294967297", + "-1.5", + "-1", + "-0.5", + "0", + "0.5", + "1", + "1.5", + "255", + "4294967295", + "4294967296", + } + ) + { + foreach ( + var count in new[] + { + "-64", + "-33", + "-32", + "-31", + "-1.9", + "0", + "1.9", + "31", + "32", + "33", + "64", + } + ) + foreach (var method in new[] { "lshift", "rshift", "arshift", "lrotate", "rrotate" }) + yield return new( + "bit32", + $"{method}/{word}/{count}", + $"return bit32.{method}({word},{count})" + ); + foreach (var second in new[] { "-1.5", "0", "0.5", "1", "255", "4294967295" }) + foreach (var method in new[] { "band", "bor", "bxor", "btest" }) + yield return new( + "bit32", + $"{method}/{word}/{second}", + $"return bit32.{method}({word},{second})" + ); + } + + for (var n = 0; n < 30; n++) + { + yield return new( + "vm.metamethod", + $"index-chain/{n}", + $"local a={{x={n}}}; local b=setmetatable({{}},{{__index=a}}); local c=setmetatable({{}},{{__index=b}}); return c.x,c.y" + ); + yield return new( + "vm.metamethod", + $"newindex-chain/{n}", + $"local a={{}}; local b=setmetatable({{}},{{__newindex=a}}); local c=setmetatable({{}},{{__newindex=b}}); c.x={n}; return a.x,rawget(b,'x'),rawget(c,'x')" + ); + yield return new( + "vm.metamethod", + $"call/{n}", + $"local x=setmetatable({{n={n}}},{{__call=function(self,...) return self.n,select('#',...),... end}}); return x(1,nil,3)" + ); + yield return new( + "vm.metamethod", + $"le-fallback/{n}", + $"local mt={{__lt=function(a,b) return a.x + + Exe + net10.0 + enable + enable + + + + + $(LuaAssemblyPath) + + + + diff --git a/tools/DifferentialProbe/Program.cs b/tools/DifferentialProbe/Program.cs new file mode 100644 index 00000000..252af8a2 --- /dev/null +++ b/tools/DifferentialProbe/Program.cs @@ -0,0 +1,127 @@ +using System.Globalization; +using System.Text.Json; +using Lua; +using Lua.Platforms; +using Lua.Standard; +using NuLua; +using NuLua.Lua52; + +if (args.Length is < 1 or > 3) +{ + Console.Error.WriteLine( + "Usage: DifferentialProbe [family-prefix] [bytes|utf16]" + ); + return 2; +} + +var mode = args.Length > 2 ? args[2] : "bytes"; +if (mode is not ("bytes" or "utf16")) +{ + Console.Error.WriteLine("Expected bytes or utf16."); + return 2; +} +var platform = mode == "bytes" ? LuaPlatform.DefaultByteStrings : LuaPlatform.Default; +var counts = new SortedDictionary(); +var outputPath = Path.GetFullPath(args[0]); +Directory.CreateDirectory(Path.GetDirectoryName(outputPath)!); +using var output = new StreamWriter(outputPath); +var timer = System.Diagnostics.Stopwatch.StartNew(); +foreach ( + var test in Cases + .Generate() + .Where(c => args.Length < 2 || c.Family.StartsWith(args[1], StringComparison.Ordinal)) +) +{ + // Pcall tests report success/results or just failure. Diagnostics are retained + // for top-level failures but not compared: engines have different tracebacks. + var source = + "local function run() " + + test.Source + + " end; " + + "local r=table.pack(pcall(run)); if not r[1] then return false end; " + + + // Encode in Lua before crossing NuLua's UTF-8 string boundary so raw + // bytes (including NUL and invalid UTF-8) compare without decoding loss. + "for i=2,r.n do if type(r[i])=='string' then local bytes={}; " + + "for j=1,#r[i] do bytes[j]=string.byte(r[i],j) end; " + + "r[i]=table.concat(bytes,',') end end; return table.unpack(r,1,r.n)"; + Result managed; + try + { + using var state = LuaState.Create(platform); + state.OpenStandardLibraries(); + managed = new( + (await state.DoStringAsync(source)) + .Select(v => + v.Type + + ":" + + (v.Type == Lua.LuaValueType.Number ? Number(v.Read()) : v.ToString()) + ) + .ToArray() + ); + } + catch (Exception e) + { + managed = new(null, e.GetType().Name, e.Message); + } + + Result native; + try + { + using var state = Lua52State.Create(); + state.OpenLibraries(); + native = new( + state + .DoString(source) + .Select(v => + v.Type + + ":" + + (v.Type.ToString() == "Number" ? Number(v.Read()) : v.ToString()) + ) + .ToArray() + ); + } + catch (Exception e) + { + native = new(null, e.GetType().Name, e.Message); + } + + // Unexpected host exceptions must never be counted as equivalent Lua errors. + var equal = + managed.Values is not null && native.Values is not null + ? managed.Values.SequenceEqual(native.Values) + : managed.Error is "LuaParseException" or "LuaCompileException" or "LuaRuntimeException" + && native.Error == "LuaException"; + await output.WriteLineAsync( + JsonSerializer.Serialize( + new + { + test.Family, + test.Name, + test.Source, + Mode = mode, + Equal = equal, + LuaCSharp = managed, + NuLua52 = native, + } + ) + ); + var count = counts.GetValueOrDefault(test.Family); + counts[test.Family] = (count.Total + 1, count.Different + (equal ? 0 : 1)); +} + +foreach (var (family, count) in counts) + Console.WriteLine($"{family, -24} {count.Total, 5} cases; {count.Different, 4} differences"); +if (counts.Count == 0) +{ + Console.Error.WriteLine("No cases matched the family prefix."); + return 2; +} +Console.WriteLine( + $"Total: {counts.Values.Sum(c => c.Total)} cases, {counts.Values.Sum(c => c.Different)} differences; {timer.Elapsed.TotalSeconds:F1}s; {mode}" +); +return counts.Values.Any(c => c.Different != 0) ? 1 : 0; + +static string Number(double value) => value.ToString("R", CultureInfo.InvariantCulture); + +record Result(string[]? Values, string? Error = null, string? Message = null); diff --git a/tools/DifferentialProbe/README.md b/tools/DifferentialProbe/README.md new file mode 100644 index 00000000..0abfa78e --- /dev/null +++ b/tools/DifferentialProbe/README.md @@ -0,0 +1,52 @@ +# Lua 5.2 differential checks + +Compare deterministic edge cases in fresh Lua-CSharp and native Lua 5.2 states +through NuLua.Lua52 0.1.0. Run from this directory to select the .NET 10 SDK: + +```powershell +dotnet restore +dotnet run -c Release -- ../../artifacts/differential/all.jsonl +dotnet run -c Release -- ../../artifacts/differential/compiler.jsonl compiler +dotnet run -c Release -- ../../artifacts/differential/patterns.jsonl pattern +dotnet run -c Release -- ../../artifacts/differential/characters.jsonl format.char utf16 +``` + +Arguments are output path, optional family prefix, and optional `bytes`/`utf16` +mode (default `bytes`). The output directory is created automatically. Exit +codes: 0 means every case agrees; 1 means differences were recorded; 2 means +invalid arguments or no matching family. Each JSONL row retains the source, +case/family, mode, both results and equality flag. Differences are not filtered. + +The 14,176 cases cover string slices/searches/quoting, escapes, numerals, +numeric coercion/base conversion/text, math, bit32, patterns and replacements, +table ranges, operators/metamethods/coroutines, closures, varargs, assignment +conflicts, callback results, 3,240 finite printf combinations and 1,000 seeded +expression trees. Numeric replacement cases check direct, callback and table +forms; numeric-input base cases expose number-to-text coercion before parsing. + +## Interpreting differences + +Current main on Windows x64 records 58 byte-mode differences: 53 retained +managed numeric-text/coercion results, three non-finite `frexp` exponents and +two one-ULP `deg` results. These are visible policy/platform differences, not +new crash or state-corruption findings. The program still returns 1 for them. +The exact count can differ with the native C library. UTF-16 mode adds character +range differences because a .NET character is not a Lua byte. Use an agreeing +family for a focused check, and inspect full results when changing semantics. + +Counts, types and primitive values compare exactly, including signed zero; +NaNs compare by numeric category. Strings are encoded as numeric byte/code-unit +lists **inside Lua**, preserving NUL and undecodable bytes before NuLua marshals +them. This avoids the pinned package's Windows string-decoding behavior without +discarding binary data. Exact C NaN payload spellings, undefined integer casts, +huge allocation requests, random sequences and ambiguous table-hole lengths +are outside this corpus. + +Runtime calls use `pcall`; error wording and tracebacks are retained only for +top-level failures and are not compared. Unexpected host exception types do not +match native Lua errors. This is a finite behavioral corpus, not proof of full +language or diagnostic compatibility. + +To run the same corpus against a frozen engine DLL, build with +`-p:LuaAssemblyPath=C:/absolute/path/Lua.dll`. This changes only the engine +reference; keep the probe source and native package version identical. diff --git a/tools/DifferentialProbe/global.json b/tools/DifferentialProbe/global.json new file mode 100644 index 00000000..512142d2 --- /dev/null +++ b/tools/DifferentialProbe/global.json @@ -0,0 +1,6 @@ +{ + "sdk": { + "version": "10.0.100", + "rollForward": "latestFeature" + } +} diff --git a/tools/PerformanceProbe/.gitignore b/tools/PerformanceProbe/.gitignore new file mode 100644 index 00000000..502a49c3 --- /dev/null +++ b/tools/PerformanceProbe/.gitignore @@ -0,0 +1,3 @@ +bin/ +obj/ +BenchmarkDotNet.Artifacts/ diff --git a/tools/PerformanceProbe/PerformanceProbe.csproj b/tools/PerformanceProbe/PerformanceProbe.csproj new file mode 100644 index 00000000..61b238db --- /dev/null +++ b/tools/PerformanceProbe/PerformanceProbe.csproj @@ -0,0 +1,18 @@ + + + Exe + net10.0 + enable + enable + + + + + $(LuaAssemblyPath) + + + + + + + diff --git a/tools/PerformanceProbe/Program.cs b/tools/PerformanceProbe/Program.cs new file mode 100644 index 00000000..42f8a261 --- /dev/null +++ b/tools/PerformanceProbe/Program.cs @@ -0,0 +1,218 @@ +using System.Diagnostics; +using System.Globalization; +using System.Reflection; +using System.Runtime.InteropServices; +using System.Security.Cryptography; +using System.Text.Json; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains.InProcess.NoEmit; +using Lua; +using Lua.Runtime; + +if (args.Contains("--benchmark")) +{ + // Run in this process so a probe built against a baseline Lua.dll cannot + // silently rebuild its benchmarks against the current working tree. + var config = DefaultConfig.Instance.AddJob( + Job.ShortRun.WithToolchain(InProcessNoEmitToolchain.Instance) + ); + BenchmarkSwitcher + .FromTypes([typeof(InterpreterBenchmarks)]) + .Run(args.Where(a => a != "--benchmark").ToArray(), config); + return 0; +} + +var options = new Dictionary(StringComparer.Ordinal); +var switches = new HashSet(StringComparer.Ordinal); +string[] cases; +int samples, + warmup, + duration; +try +{ + for (var i = 0; i < args.Length; i++) + { + var name = args[i]; + if (name is "--bytes" or "--compile") + { + if (!switches.Add(name)) + throw new ArgumentException($"Duplicate option: {name}"); + } + else if (name is "--case" or "--samples" or "--warmup-ms" or "--sample-ms" or "--eval") + { + if (++i == args.Length) + throw new ArgumentException($"Missing value for {name}"); + if (!options.TryAdd(name, args[i])) + throw new ArgumentException($"Duplicate option: {name}"); + } + else + throw new ArgumentException($"Unknown option: {name}"); + } + + var selected = options.GetValueOrDefault("--case", "all"); + cases = selected == "all" ? Workload.Sources.Keys.ToArray() : selected.Split(','); + if (cases.Distinct(StringComparer.Ordinal).Count() != cases.Length) + throw new ArgumentException("Duplicate workload names"); + foreach (var name in cases) + if (!Workload.Sources.ContainsKey(name)) + throw new ArgumentException($"Unknown workload: {name}"); + samples = ReadCount("--samples", 5, 1); + warmup = ReadCount("--warmup-ms", 300, 0); + duration = ReadCount("--sample-ms", 60, 1); +} +catch (ArgumentException e) +{ + Console.Error.WriteLine(e.Message); + Console.Error.WriteLine( + "Usage: PerformanceProbe [--case name[,name]] [--samples n] [--warmup-ms n] [--sample-ms n] [--bytes] [--compile] | --eval source | --benchmark [BenchmarkDotNet options]" + ); + return 2; +} + +int ReadCount(string name, int fallback, int minimum) +{ + if (!options.TryGetValue(name, out var text)) + return fallback; + if ( + !int.TryParse(text, NumberStyles.Integer, CultureInfo.InvariantCulture, out var value) + || value < minimum + ) + throw new ArgumentException($"{name} must be an integer >= {minimum}"); + return value; +} + +if (options.TryGetValue("--eval", out var source)) +{ + using var state = Lua.LuaState.Create(); + Lua.Standard.OpenLibsExtensions.OpenStandardLibraries(state); + var values = state.DoStringAsync(source).AsTask().GetAwaiter().GetResult(); + Console.WriteLine(JsonSerializer.Serialize(values.Select(v => v.ToString()).ToArray())); + return 0; +} +var method = typeof(LuaVirtualMachine).GetMethod( + "MoveNext", + BindingFlags.Static | BindingFlags.NonPublic +)!; +var body = method.GetMethodBody()!; +Console.WriteLine( + JsonSerializer.Serialize( + new + { + runtime = RuntimeInformation.FrameworkDescription, + vmIlBytes = body.GetILAsByteArray()!.Length, + vmLocals = body.LocalVariables.Count, + vmEhRegions = body.ExceptionHandlingClauses.Count, + pgo = Environment.GetEnvironmentVariable("DOTNET_TieredPGO"), + tiered = Environment.GetEnvironmentVariable("DOTNET_TieredCompilation"), + mode = args.Contains("--compile") ? "compilation" : "execution", + stringMode = switches.Contains("--bytes") ? "bytes" : "utf16", + luaAssemblySha256 = Convert.ToHexString( + SHA256.HashData(File.ReadAllBytes(typeof(LuaState).Assembly.Location)) + ), + } + ) +); + +foreach (var name in cases) +{ + var setupTimer = Stopwatch.StartNew(); + var setupAllocated = GC.GetTotalAllocatedBytes(precise: true); + using var workload = new Workload(name, args.Contains("--bytes"), args.Contains("--compile")); + setupAllocated = GC.GetTotalAllocatedBytes(precise: true) - setupAllocated; + var setupNs = setupTimer.Elapsed.TotalNanoseconds; + var bytecode = workload.Closure.Proto.ToBytecode(); + var firstAllocated = GC.GetTotalAllocatedBytes(precise: true); + setupTimer.Restart(); + var expected = workload.Run(); + var firstExecutionNs = setupTimer.Elapsed.TotalNanoseconds; + firstAllocated = GC.GetTotalAllocatedBytes(precise: true) - firstAllocated; + var timer = Stopwatch.StartNew(); + do + { + workload.Run(); + } while (timer.ElapsedMilliseconds < warmup); + var times = new double[samples]; + var allocations = new double[samples]; + for (var sample = 0; sample < samples; sample++) + { + // Total allocated bytes also covers work resumed on other threads. + var before = GC.GetTotalAllocatedBytes(precise: true); + long count = 0; + timer.Restart(); + do + { + var result = workload.Run(); + if (result != expected) + throw new InvalidOperationException( + $"Unstable result for {name}: {result} != {expected}" + ); + count++; + } while (timer.ElapsedMilliseconds < duration); + var elapsed = timer.Elapsed.TotalNanoseconds; + allocations[sample] = (GC.GetTotalAllocatedBytes(precise: true) - before) / (double)count; + times[sample] = elapsed / count; + } + Console.WriteLine( + JsonSerializer.Serialize( + new + { + name, + ns = Median(times), + bytes = Median(allocations), + checksum = expected, + bytecodeBytes = bytecode.Length, + bytecodeHash = Convert.ToHexString(SHA256.HashData(bytecode)), + setupNs, + setupAllocated, + firstExecutionNs, + firstAllocated, + } + ) + ); +} +return 0; + +static double Median(double[] values) +{ + Array.Sort(values); + var middle = values.Length / 2; + return values.Length % 2 == 0 ? (values[middle - 1] + values[middle]) / 2 : values[middle]; +} + +[MemoryDiagnoser] +public class InterpreterBenchmarks +{ + [Params( + "numeric-for", + "numeric-eq", + "comparison-meta", + "concat-pair", + "concat-many", + "api-add-sync", + "api-add-async", + "arithmetic", + "lua-calls", + "math", + "strings", + "string-rep", + "string-sub", + "table-concat", + "table-sort", + "patterns", + "nbody" + )] + public string Case { get; set; } = "numeric-for"; + Workload workload = null!; + + [GlobalSetup] + public void Setup() => workload = new Workload(Case); + + [GlobalCleanup] + public void Cleanup() => workload.Dispose(); + + [Benchmark] + public double Execute() => workload.Run(); +} diff --git a/tools/PerformanceProbe/README.md b/tools/PerformanceProbe/README.md new file mode 100644 index 00000000..d43fec4b --- /dev/null +++ b/tools/PerformanceProbe/README.md @@ -0,0 +1,76 @@ +# Focused .NET 10 performance probe + +Use this tool for alternating Lua-CSharp revision comparisons, compiler +allocation checks and JIT diagnostics. Run from this directory: + +```powershell +dotnet restore +dotnet build -c Release -o ../../artifacts/performance/candidate +dotnet ../../artifacts/performance/candidate/PerformanceProbe.dll --case numeric-for,math +``` + +The probe loads source once, warms execution, then reports the median of five +timed batches as JSON Lines. Each row includes per-call time/allocation, result +checksum, bytecode length/SHA-256, and separate setup/first-call measurements. +The header records the runtime, engine DLL hash, string mode, tiering/PGO +settings and VM IL statistics. Every sample checks result stability. These +checks detect build differences; they do not independently prove correctness. + +`--compile` instead measures repeated source loads in an existing state; +its checksum is instruction count. `--bytes` selects byte strings. Select cases +with `--case name[,name]`; default is all. `--samples`, `--warmup-ms` and +`--sample-ms` control sampling. Invalid/missing options, duplicate cases and +non-positive sample counts/durations return exit code 2. Warmup may be zero. + +Sources cover arithmetic/coercion, Lua and native calls, tail calls/varargs, +metamethods, hooks, coroutines, table access/sort/unpack/moves, strings, +patterns, numeric conversion, bit32, and n-body. `format-hex` measures `%a`; +`format-number-quote` measures numeric `%q`. Compiler cases separate keywords, +identifiers, plain strings and escaped strings. + +## Comparing revisions + +Build both probes from the same source into different output directories. +`LuaAssemblyPath` selects a previously built Release/net10.0 engine without +rebuilding it from the current checkout: + +```powershell +dotnet build -c Release -o ../../artifacts/performance/baseline ` + -p:LuaAssemblyPath=C:/absolute/baseline/Lua.dll +./compare.ps1 ` + -Baseline ../../artifacts/performance/baseline/PerformanceProbe.dll ` + -Candidate ../../artifacts/performance/candidate/PerformanceProbe.dll ` + -Output ../../artifacts/performance/comparison -Cases 'math,strings,nbody' +``` + +The script alternates process order across rounds, restores the caller's PGO +environment, retains raw JSONL and exports CSV medians. It rejects different +measurement/string modes, workload sets, missing/duplicate process results, +and bytecode/checksum differences. Such differences need investigation before +treating a timing change as an optimization. + +Use `-Compile` for compiler comparisons and `-Bytes` for byte strings. Profiles +are `pgo-on` (default), `pgo-off`, and `tiered-off`. `-Rounds`, `-WarmupMs` and +`-SampleMs` narrow a run. Run timing processes serially on an otherwise idle +machine. Process-wide GC counters include async continuations and can include +background-runtime allocation noise; small changes need independent repeats. +Setup combines state creation, library opening and compilation; it is not a +parser-only metric. First-call measurements include relevant first-use JIT. + +`--benchmark` runs a focused BenchmarkDotNet suite with an in-process no-emit +toolchain. This prevents generated benchmark projects from silently rebuilding +a frozen baseline against current source. Set the process's PGO profile before +launching each binary; it measures warmed execution, not compilation/startup. + +## JIT diagnostics + +```powershell +$env:DOTNET_TieredCompilation = '0' +$env:DOTNET_JitDisasm = 'Lua.Runtime.LuaVirtualMachine:MoveNext' +$env:DOTNET_JitStdOutFile = 'C:/absolute/candidate-jit.txt' +dotnet ../../artifacts/performance/candidate/PerformanceProbe.dll --case numeric-for +``` + +Use identical cases/settings for both binaries and restore the environment +after capture. `--eval 'return math.abs(-3)'` executes a diagnostic script and +prints returned values without running timing samples. diff --git a/tools/PerformanceProbe/Workloads.cs b/tools/PerformanceProbe/Workloads.cs new file mode 100644 index 00000000..fc69e9a6 --- /dev/null +++ b/tools/PerformanceProbe/Workloads.cs @@ -0,0 +1,199 @@ +using Lua; +using Lua.Platforms; +using Lua.Runtime; +using Lua.Standard; + +public sealed class Workload : IDisposable +{ + public static readonly Dictionary Sources = new() + { + ["numeric-for"] = "local s=0; for i=1,20000 do s=s+i end; return s", + ["numeric-while"] = "local s,i=0,0; while i<10000 do s=s+i; i=i+1 end; return s", + ["concat-pair"] = + "local a,b='abc','def'; local n=0; for i=1,3000 do n=n+#(a..b) end; return n", + ["concat-many"] = + "local a,b,c,d='abc','def','ghi','jkl'; local n=0; for i=1,2000 do n=n+#(a..b..c..d..a..b..c..d) end; return n", + ["concat-empty"] = + "local a,b='','abc'; local n=0; for i=1,3000 do n=n+#(a..b..a..a) end; return n", + ["concat-mixed"] = + "local a,b='abc',42; local n=0; for i=1,2000 do n=n+#(a..b..a..b) end; return n", + ["concat-meta"] = + "local mt={__concat=function(a,b) return a.x..b.x end}; local a,b=setmetatable({x='a'},mt),setmetatable({x='b'},mt); local n=0; for i=1,1000 do n=n+#(a..b) end; return n", + ["api-add-sync"] = "local f=host_api; local n=0; for i=1,2000 do n=n+f() end; return n", + ["api-add-async"] = "local f=host_api; local n=0; for i=1,100 do n=n+f() end; return n", + ["numeric-eq"] = + "local s=0; for i=1,10000 do if i==0 then s=s-1 else s=s+i end end; return s", + ["table-eq"] = + "local a,b={},{}; local s=0; for i=1,3000 do if a==b then s=s-1 else s=s+i end end; return s", + ["generic-for"] = + "local t={}; for i=1,2000 do t[i]=i end; local s=0; for _,v in ipairs(t) do s=s+v end; return s", + ["comparison-meta"] = + "local mt={__lt=function(a,b) return a.x + $"do local returnValue{i},functionName{i},ordinaryIdentifier{i}=1,2,3; total=total+returnValue{i}+functionName{i}+ordinaryIdentifier{i} end;" + ) + ) + + "return total", + ["compile-plain-strings"] = + "local total=0;" + + string.Concat( + Enumerable + .Range(0, 80) + .Select(i => + $"do local value='ordinary string value {i}: the quick brown fox jumps over the lazy dog'; total=total+#value end;" + ) + ) + + "return total", + ["compile-escaped-strings"] = + "local total=0;" + + string.Concat( + Enumerable + .Range(0, 80) + .Select(i => + $"do local value='escaped value {i}:\\tcolumn\\nline\\r\\a\\b\\f\\v\\\\\\\"\\\'\\x41\\065\\z suffix'; total=total+#value end;" + ) + ) + + "return total", + }; + + static string LoadNBody() + { + using var source = typeof(Workload).Assembly.GetManifestResourceStream("n-body.lua")!; + using var reader = new StreamReader(source); + return reader.ReadToEnd().Replace("or 1000", "or 100") + "\nreturn energy(bodies, nbody)"; + } + + public LuaState State { get; } + public LuaClosure Closure { get; } + readonly LuaValue[] results = new LuaValue[1]; + readonly bool compile; + readonly string name; + + public Workload(string name, bool bytes = false, bool compile = false) + { + this.name = name; + this.compile = compile; + State = LuaState.Create(bytes ? LuaPlatform.DefaultByteStrings : LuaPlatform.Default); + State.OpenStandardLibraries(); + if (name is "api-add-sync" or "api-add-async") + AddApiOperation(State, name == "api-add-async"); + Closure = State.Load(Sources[name], name); + } + + static void AddApiOperation(LuaState state, bool suspend) + { + var operand = new LuaTable { Metatable = new LuaTable() }; + operand.Metatable["__add"] = new LuaFunction( + async (context, token) => + { + if (suspend) + await Task.Yield(); + return context.Return(42); + } + ); + state.Environment["host_api"] = new LuaFunction( + async (context, token) => + { + var result = await state.AddAsync(operand, operand, token); + return context.Return(result); + } + ); + } + + public double Run() + { + if (compile) + return State.Load(Sources[name], name).Proto.Code.Length; + var task = State.ExecuteAsync(Closure, results); + if (!task.IsCompletedSuccessfully) + task.AsTask().GetAwaiter().GetResult(); + else + task.GetAwaiter().GetResult(); + return results[0].Read(); + } + + public void Dispose() => State.Dispose(); +} diff --git a/tools/PerformanceProbe/compare.ps1 b/tools/PerformanceProbe/compare.ps1 new file mode 100644 index 00000000..6723d020 --- /dev/null +++ b/tools/PerformanceProbe/compare.ps1 @@ -0,0 +1,84 @@ +param( + [Parameter(Mandatory)][string]$Baseline, + [Parameter(Mandatory)][string]$Candidate, + [string]$Output = 'artifacts/performance/comparison', + [ValidateSet('pgo-on', 'pgo-off', 'tiered-off')][string]$Config = 'pgo-on', + [string]$Cases = 'all', + [ValidateRange(1, 2147483647)][int]$Rounds = 5, + [ValidateRange(0, 2147483647)][int]$WarmupMs = 500, + [ValidateRange(1, 2147483647)][int]$SampleMs = 80, + [switch]$Bytes, + [switch]$Compile +) +$ErrorActionPreference = 'Stop' +$Baseline = (Resolve-Path -LiteralPath $Baseline).Path +$Candidate = (Resolve-Path -LiteralPath $Candidate).Path +$null = New-Item -ItemType Directory -Force -Path $Output +$oldPgo = $env:DOTNET_TieredPGO +$oldTiered = $env:DOTNET_TieredCompilation +function Median($values) { + $sorted = @($values | Sort-Object) + $middle = [int][Math]::Floor($sorted.Count / 2) + if ($sorted.Count % 2 -eq 0) { return ($sorted[$middle - 1] + $sorted[$middle]) / 2 } + return $sorted[$middle] +} +$results = @{ baseline = @{}; candidate = @{} } +try { + $env:DOTNET_TieredPGO = if ($Config -eq 'pgo-on') { '1' } else { '0' } + $env:DOTNET_TieredCompilation = if ($Config -eq 'tiered-off') { '0' } else { '1' } + for ($round = 0; $round -lt $Rounds; $round++) { + $order = if ($round % 2 -eq 0) { @('baseline', 'candidate') } else { @('candidate', 'baseline') } + foreach ($side in $order) { + $dll = if ($side -eq 'baseline') { $Baseline } else { $Candidate } + $probeArgs = @($dll, '--case', $Cases, '--warmup-ms', $WarmupMs, '--sample-ms', $SampleMs) + if ($Bytes) { $probeArgs += '--bytes' } + if ($Compile) { $probeArgs += '--compile' } + $lines = @(& dotnet @probeArgs) + if ($LASTEXITCODE -ne 0) { throw "Probe failed: $side, round $round" } + $header = $lines[0] | ConvertFrom-Json + $expectedMode = if ($Compile) { 'compilation' } else { 'execution' } + if ($header.mode -ne $expectedMode) { throw "Probe mode mismatch: $side; rebuild both probes from the same source" } + $expectedStringMode = if ($Bytes) { 'bytes' } else { 'utf16' } + if ($header.stringMode -ne $expectedStringMode) { throw "Probe string mode mismatch: $side" } + $lines | Set-Content -LiteralPath (Join-Path $Output "$Config-$round-$side.jsonl") + foreach ($line in $lines) { + $row = $line | ConvertFrom-Json + if (!$row.name) { continue } + if (!$results[$side].ContainsKey($row.name)) { $results[$side][$row.name] = @() } + $results[$side][$row.name] += $row + } + } + Write-Host "Completed $Config round $($round + 1)/$Rounds" + } + if ($results.baseline.Count -eq 0) { throw 'No workload results were recorded' } + if (Compare-Object @($results.baseline.Keys) @($results.candidate.Keys) -CaseSensitive) { + throw 'Baseline and candidate reported different workload sets' + } + $summary = foreach ($name in $results.baseline.Keys | Sort-Object) { + $base = $results.baseline[$name] + $candidateRows = $results.candidate[$name] + if ($base.Count -ne $Rounds -or $candidateRows.Count -ne $Rounds) { + throw "Missing or duplicate process results: $name" + } + foreach ($row in @($base) + @($candidateRows)) { + if ($row.bytecodeHash -ne $base[0].bytecodeHash -or $row.checksum -ne $base[0].checksum) { + throw "Bytecode/result mismatch: $name" + } + } + $baseNs = Median $base.ns + $candidateNs = Median $candidateRows.ns + [pscustomobject]@{ + Case = $name + BaselineNs = [Math]::Round($baseNs, 1) + CandidateNs = [Math]::Round($candidateNs, 1) + ChangePercent = [Math]::Round(100 * ($candidateNs / $baseNs - 1), 2) + BaselineBytes = [Math]::Round((Median $base.bytes), 1) + CandidateBytes = [Math]::Round((Median $candidateRows.bytes), 1) + } + } + $summary | Export-Csv -LiteralPath (Join-Path $Output "$Config-summary.csv") -NoTypeInformation + $summary | Format-Table -AutoSize +} finally { + $env:DOTNET_TieredPGO = $oldPgo + $env:DOTNET_TieredCompilation = $oldTiered +} diff --git a/tools/PerformanceProbe/global.json b/tools/PerformanceProbe/global.json new file mode 100644 index 00000000..512142d2 --- /dev/null +++ b/tools/PerformanceProbe/global.json @@ -0,0 +1,6 @@ +{ + "sdk": { + "version": "10.0.100", + "rollForward": "latestFeature" + } +}