|
| 1 | +// |
| 2 | +// Copyright (c) 2019-2026 Angouri. |
| 3 | +// AngouriMath is licensed under MIT. |
| 4 | +// Details: https://github.com/asc-community/AngouriMath/blob/master/LICENSE.md. |
| 5 | +// Website: https://am.angouri.org. |
| 6 | +// |
| 7 | + |
| 8 | +using System; |
| 9 | +using System.Collections.Generic; |
| 10 | +using System.IO; |
| 11 | +using System.Linq; |
| 12 | +using System.Text.RegularExpressions; |
| 13 | +using System.Xml.Linq; |
| 14 | +using Microsoft.CodeAnalysis.CSharp.Scripting; |
| 15 | +using Microsoft.CodeAnalysis.Scripting; |
| 16 | +using Xunit; |
| 17 | + |
| 18 | +namespace AngouriMath.Tests.Convenience |
| 19 | +{ |
| 20 | + /// <summary> |
| 21 | + /// Every worked example in the XML documentation, compiled, run, and its output compared with |
| 22 | + /// the output it says it prints. |
| 23 | + /// </summary> |
| 24 | + /// <remarks> |
| 25 | + /// <para> |
| 26 | + /// <b>The guarantee <see cref="ExtensionsExampleTest"/> gives one namespace, given to all of |
| 27 | + /// them.</b> That class says it exists so that "a change of answer then fails a test instead of |
| 28 | + /// leaving a wrong claim standing in the XML docs", and it does that for |
| 29 | + /// <c>AngouriMath.Extensions</c> by transcribing each documented output into an assertion by |
| 30 | + /// hand. Transcription is the part that does not scale and can itself drift: this reads the |
| 31 | + /// documentation as the build emits it, so the thing checked is the thing a user reads. |
| 32 | + /// </para> |
| 33 | + /// <para> |
| 34 | + /// <b>The convention it relies on</b> is the one the examples already follow: a |
| 35 | + /// <c><code></c> block, the word <c>Prints</c>, and a second <c><code></c> block |
| 36 | + /// holding the expected output. An example in any other shape is counted and skipped rather |
| 37 | + /// than guessed at. |
| 38 | + /// </para> |
| 39 | + /// <para> |
| 40 | + /// <b>What it found on the commit that added it:</b> of 254 examples in that shape, 187 printed |
| 41 | + /// what they claimed, 56 printed something else and 11 did not compile or threw. Most of the 56 |
| 42 | + /// are the documentation trailing a deliberate change — <c>sqrt(x ^ 2)</c> stopped collapsing to |
| 43 | + /// <c>x</c> with <a href="https://github.com/asc-community/AngouriMath/issues/752">#752</a>, |
| 44 | + /// <c>tan(x) * cotan(x)</c> gained the conditions that make it true — and every one of them is a |
| 45 | + /// promise the library was not keeping. They are listed below rather than fixed here, so that |
| 46 | + /// the mechanism and the corrections are separately reviewable. |
| 47 | + /// </para> |
| 48 | + /// <para> |
| 49 | + /// Snippets are compiled with <c>AngouriMath.MathS</c> and <c>AngouriMath.Entity</c> imported |
| 50 | + /// statically, because that is how they are written and how the README and wiki open them. |
| 51 | + /// Without that, <c>pi</c> and <c>Providedf</c> read as undefined and the harness reports the |
| 52 | + /// documentation broken when it is the harness that is. |
| 53 | + /// </para> |
| 54 | + /// </remarks> |
| 55 | + [Trait("Area", "Convenience")] |
| 56 | + public sealed class DocumentationExamplesRunTest |
| 57 | + { |
| 58 | + /// <summary> |
| 59 | + /// The examples that do not print what they say, or do not run at all, as the member names |
| 60 | + /// the XML uses. <b>Every entry here is a defect in the documentation, not an exemption |
| 61 | + /// that has been argued for</b> — the list exists so that the count cannot grow quietly |
| 62 | + /// while they are corrected in batches. |
| 63 | + /// </summary> |
| 64 | + private static readonly HashSet<string> Failing = new() |
| 65 | + { |
| 66 | + "M:AngouriMath.Entity.Expand(System.Int32)", |
| 67 | + "M:AngouriMath.Entity.Factorize(System.Int32)", |
| 68 | + "M:AngouriMath.Entity.Latexize", |
| 69 | + "M:AngouriMath.Entity.Simplify(System.Int32)", |
| 70 | + "M:AngouriMath.Entity.Solve(AngouriMath.Entity.Variable)", |
| 71 | + "M:AngouriMath.Entity.Substitute(System.ValueTuple{AngouriMath.Entity,AngouriMath.Entity,AngouriMath.Entity,AngouriMath.Entity},System.ValueTuple{AngouriMath.Entity,AngouriMath.Entity,AngouriMath.Entity,AngouriMath.Entity})", |
| 72 | + "M:AngouriMath.Entity.ToSymPy(System.Boolean)", |
| 73 | + "M:AngouriMath.Extensions.AngouriMathExtensions.Factorize(System.String)", |
| 74 | + "M:AngouriMath.MathS.Apply(AngouriMath.Entity,AngouriMath.Entity[])", |
| 75 | + "M:AngouriMath.MathS.Arccos(AngouriMath.Entity)", |
| 76 | + "M:AngouriMath.MathS.Arccosec(AngouriMath.Entity)", |
| 77 | + "M:AngouriMath.MathS.Arccotan(AngouriMath.Entity)", |
| 78 | + "M:AngouriMath.MathS.Arcsec(AngouriMath.Entity)", |
| 79 | + "M:AngouriMath.MathS.Arcsin(AngouriMath.Entity)", |
| 80 | + "M:AngouriMath.MathS.Arctan(AngouriMath.Entity)", |
| 81 | + "M:AngouriMath.MathS.Cbrt(AngouriMath.Entity)", |
| 82 | + "M:AngouriMath.MathS.Compute.SymbolicFormOfCosine(AngouriMath.Entity)", |
| 83 | + "M:AngouriMath.MathS.Compute.SymbolicFormOfSine(AngouriMath.Entity)", |
| 84 | + "M:AngouriMath.MathS.Cotan(AngouriMath.Entity)", |
| 85 | + "M:AngouriMath.MathS.Derivative(AngouriMath.Entity,AngouriMath.Entity)", |
| 86 | + "M:AngouriMath.MathS.Derivative(AngouriMath.Entity,AngouriMath.Entity,System.Int32)", |
| 87 | + "M:AngouriMath.MathS.Equality(AngouriMath.Entity,AngouriMath.Entity)", |
| 88 | + "M:AngouriMath.MathS.Equations(System.Collections.Generic.IEnumerable{AngouriMath.Entity})", |
| 89 | + "M:AngouriMath.MathS.Factorial(AngouriMath.Entity)", |
| 90 | + "M:AngouriMath.MathS.FromBaseN(System.String,System.Int32)", |
| 91 | + "M:AngouriMath.MathS.GreaterOrEqualThan(AngouriMath.Entity,AngouriMath.Entity)", |
| 92 | + "M:AngouriMath.MathS.GreaterThan(AngouriMath.Entity,AngouriMath.Entity)", |
| 93 | + "M:AngouriMath.MathS.Hyperbolic.Cosech(AngouriMath.Entity)", |
| 94 | + "M:AngouriMath.MathS.Hyperbolic.Cotanh(AngouriMath.Entity)", |
| 95 | + "M:AngouriMath.MathS.Hyperbolic.Sech(AngouriMath.Entity)", |
| 96 | + "M:AngouriMath.MathS.Hyperbolic.Tanh(AngouriMath.Entity)", |
| 97 | + "M:AngouriMath.MathS.Integral(AngouriMath.Entity,AngouriMath.Entity)", |
| 98 | + "M:AngouriMath.MathS.Integral(AngouriMath.Entity,AngouriMath.Entity,AngouriMath.Entity,AngouriMath.Entity)", |
| 99 | + "M:AngouriMath.MathS.Lambda(AngouriMath.Entity.Variable,AngouriMath.Entity)", |
| 100 | + "M:AngouriMath.MathS.Latex(AngouriMath.Core.ILatexizeable)", |
| 101 | + "M:AngouriMath.MathS.LessOrEqualThan(AngouriMath.Entity,AngouriMath.Entity)", |
| 102 | + "M:AngouriMath.MathS.LessThan(AngouriMath.Entity,AngouriMath.Entity)", |
| 103 | + "M:AngouriMath.MathS.Limit(AngouriMath.Entity,AngouriMath.Entity,AngouriMath.Entity,AngouriMath.Core.ApproachFrom)", |
| 104 | + "M:AngouriMath.MathS.Multithreading.SetLocalCancellationToken(System.Threading.CancellationToken)", |
| 105 | + "M:AngouriMath.MathS.Piecewise(System.Collections.Generic.IEnumerable{AngouriMath.Entity.Providedf},AngouriMath.Entity)", |
| 106 | + "M:AngouriMath.MathS.Piecewise(System.ValueTuple{AngouriMath.Entity,AngouriMath.Entity}[])", |
| 107 | + "M:AngouriMath.MathS.Pow(AngouriMath.Entity,AngouriMath.Entity)", |
| 108 | + "M:AngouriMath.MathS.Series.Maclaurin(AngouriMath.Entity,System.Int32,AngouriMath.Entity.Variable[])", |
| 109 | + "M:AngouriMath.MathS.Series.Taylor(AngouriMath.Entity,System.Int32,System.ValueTuple{AngouriMath.Entity.Variable,AngouriMath.Entity.Variable,AngouriMath.Entity}[])", |
| 110 | + "M:AngouriMath.MathS.Series.Taylor(AngouriMath.Entity,System.Int32,System.ValueTuple{AngouriMath.Entity.Variable,AngouriMath.Entity}[])", |
| 111 | + "M:AngouriMath.MathS.Series.TaylorTerms(AngouriMath.Entity,System.ValueTuple{AngouriMath.Entity.Variable,AngouriMath.Entity.Variable,AngouriMath.Entity}[])", |
| 112 | + "M:AngouriMath.MathS.Signum(AngouriMath.Entity)", |
| 113 | + "M:AngouriMath.MathS.SolveOde(AngouriMath.Entity,AngouriMath.Entity.Variable,AngouriMath.Entity.Variable)", |
| 114 | + "M:AngouriMath.MathS.Sqr(AngouriMath.Entity)", |
| 115 | + "M:AngouriMath.MathS.Sqrt(AngouriMath.Entity)", |
| 116 | + "M:AngouriMath.MathS.Tan(AngouriMath.Entity)", |
| 117 | + "M:AngouriMath.MathS.ToBaseN(AngouriMath.Entity.Number.Real,System.Int32)", |
| 118 | + "M:AngouriMath.MathS.ToSympyCode(AngouriMath.Entity)", |
| 119 | + "M:AngouriMath.MathS.TryPolynomial(AngouriMath.Entity,AngouriMath.Entity.Variable,AngouriMath.Entity@)", |
| 120 | + "M:AngouriMath.MathS.Utils.TryGetPolyLinear(AngouriMath.Entity,AngouriMath.Entity.Variable,AngouriMath.Entity@,AngouriMath.Entity@)", |
| 121 | + "M:AngouriMath.MathS.Utils.TryGetPolyQuadratic(AngouriMath.Entity,AngouriMath.Entity.Variable,AngouriMath.Entity@,AngouriMath.Entity@,AngouriMath.Entity@)", |
| 122 | + "M:AngouriMath.MathS.Utils.TryGetPolynomial(AngouriMath.Entity,AngouriMath.Entity.Variable,System.Collections.Generic.Dictionary{PeterO.Numbers.EInteger,AngouriMath.Entity}@)", |
| 123 | + "P:AngouriMath.Entity.FreeVariables", |
| 124 | + "P:AngouriMath.Entity.Vars", |
| 125 | + "P:AngouriMath.Entity.VarsAndConsts", |
| 126 | + "P:AngouriMath.MathS.DecimalConst.e", |
| 127 | + "P:AngouriMath.MathS.DecimalConst.pi", |
| 128 | + "P:AngouriMath.MathS.Settings.AllowNewton", |
| 129 | + "P:AngouriMath.MathS.Settings.FloatToRationalIterCount", |
| 130 | + "P:AngouriMath.MathS.Settings.MaxAbsNumeratorOrDenominatorValue", |
| 131 | + "T:AngouriMath.Core.ILatexizeable", |
| 132 | + "T:AngouriMath.Core.MatrixBuilder", |
| 133 | + }; |
| 134 | + |
| 135 | + private static string DocumentationPath() |
| 136 | + { |
| 137 | + var beside = Path.GetDirectoryName(typeof(MathS).Assembly.Location)!; |
| 138 | + return Path.Combine(beside, "AngouriMath.xml"); |
| 139 | + } |
| 140 | + |
| 141 | + private static ScriptOptions Options() => ScriptOptions.Default |
| 142 | + .WithReferences( |
| 143 | + typeof(MathS).Assembly, |
| 144 | + typeof(global::AngouriMath.Extensions.AngouriMathExtensions).Assembly, |
| 145 | + typeof(PeterO.Numbers.EDecimal).Assembly, |
| 146 | + typeof(GenericTensor.Core.GenTensor<,>).Assembly, |
| 147 | + typeof(object).Assembly, |
| 148 | + typeof(Console).Assembly, |
| 149 | + typeof(Enumerable).Assembly) |
| 150 | + .WithImports( |
| 151 | + "System", "System.Linq", "System.Collections.Generic", "System.Numerics", |
| 152 | + "AngouriMath", "AngouriMath.Extensions", "AngouriMath.Core", |
| 153 | + "AngouriMath.Core.Compilation.IntoLinq", |
| 154 | + "PeterO.Numbers", |
| 155 | + "AngouriMath.MathS", "AngouriMath.Entity"); |
| 156 | + |
| 157 | + /// <summary> |
| 158 | + /// One example: the program it shows, and what it says that program prints. |
| 159 | + /// </summary> |
| 160 | + private readonly record struct Example(string Member, string Program, string Prints); |
| 161 | + |
| 162 | + private static List<Example> Examples() |
| 163 | + { |
| 164 | + var found = new List<Example>(); |
| 165 | + foreach (var member in XDocument.Load(DocumentationPath()).Descendants("member")) |
| 166 | + { |
| 167 | + var name = member.Attribute("name")?.Value; |
| 168 | + if (name is null) continue; |
| 169 | + foreach (var example in member.Elements("example")) |
| 170 | + { |
| 171 | + var codes = example.Elements("code").ToList(); |
| 172 | + var prose = string.Concat(example.Nodes().OfType<XText>().Select(t => t.Value)); |
| 173 | + if (codes.Count != 2 || !prose.Contains("Prints")) continue; |
| 174 | + found.Add(new Example(name, codes[0].Value, Lines(codes[1].Value))); |
| 175 | + } |
| 176 | + } |
| 177 | + return found; |
| 178 | + } |
| 179 | + |
| 180 | + /// <summary>Trimmed and blank-stripped, so that indentation in the XML is not the subject.</summary> |
| 181 | + private static string Lines(string text) => string.Join("\n", text |
| 182 | + .Replace("\r\n", "\n") |
| 183 | + .Split('\n') |
| 184 | + .Select(line => line.Trim()) |
| 185 | + .Where(line => line.Length > 0)); |
| 186 | + |
| 187 | + /// <summary> |
| 188 | + /// Runs the snippet and answers what it printed, or the error that stopped it. |
| 189 | + /// </summary> |
| 190 | + /// <remarks> |
| 191 | + /// <c>using var x = ...;</c> is not legal at a script's top level and several examples open |
| 192 | + /// a settings scope that way, so the keyword is dropped and the scope lasts to the end of |
| 193 | + /// the snippet — which is what those examples mean by it. |
| 194 | + /// </remarks> |
| 195 | + private static (string? Printed, string? Error) Run(string program, ScriptOptions options) |
| 196 | + { |
| 197 | + var script = Regex.Replace(program, @"(?m)^(\s*)using\s+var\s", "$1var "); |
| 198 | + var captured = new StringWriter(); |
| 199 | + var restore = Console.Out; |
| 200 | + Console.SetOut(captured); |
| 201 | + try |
| 202 | + { |
| 203 | + CSharpScript.EvaluateAsync(script, options).GetAwaiter().GetResult(); |
| 204 | + return (Lines(captured.ToString()), null); |
| 205 | + } |
| 206 | + catch (CompilationErrorException error) |
| 207 | + { |
| 208 | + return (null, "did not compile: " + error.Message.Split('\n')[0]); |
| 209 | + } |
| 210 | + catch (Exception error) |
| 211 | + { |
| 212 | + return (null, error.GetType().Name + ": " + error.Message.Split('\n')[0]); |
| 213 | + } |
| 214 | + finally { Console.SetOut(restore); } |
| 215 | + } |
| 216 | + |
| 217 | + [Fact] |
| 218 | + public void EveryDocumentedExamplePrintsWhatItSaysItPrints() |
| 219 | + { |
| 220 | + var options = Options(); |
| 221 | + var wrong = new List<string>(); |
| 222 | + var offending = new HashSet<string>(); |
| 223 | + |
| 224 | + foreach (var example in Examples()) |
| 225 | + { |
| 226 | + var (printed, error) = Run(example.Program, options); |
| 227 | + if (error is null && printed == example.Prints) continue; |
| 228 | + offending.Add(example.Member); |
| 229 | + wrong.Add(error is not null |
| 230 | + ? $"{example.Member}\n {error}" |
| 231 | + : $"{example.Member}\n says: {example.Prints.Replace("\n", " | ")}" |
| 232 | + + $"\n does: {printed!.Replace("\n", " | ")}"); |
| 233 | + } |
| 234 | + |
| 235 | + var unexpected = offending.Except(Failing).ToList(); |
| 236 | + Assert.True(unexpected.Count == 0, |
| 237 | + $"{unexpected.Count} documented examples newly disagree with what they print:\n" |
| 238 | + + string.Join("\n", wrong.Where(w => unexpected.Any(w.StartsWith)).Take(20))); |
| 239 | + |
| 240 | + // The other direction, so a corrected example is deleted from the list above rather |
| 241 | + // than left there excusing nothing. |
| 242 | + var fixedSince = Failing.Except(offending).ToList(); |
| 243 | + Assert.True(fixedSince.Count == 0, |
| 244 | + $"{fixedSince.Count} examples in the list now print what they say and should be " |
| 245 | + + "deleted from it:\n" + string.Join("\n", fixedSince)); |
| 246 | + } |
| 247 | + |
| 248 | + /// <summary> |
| 249 | + /// The corpus is the whole strength of the check above, and it does not fail when the |
| 250 | + /// corpus shrinks. Asserted so that it cannot. |
| 251 | + /// </summary> |
| 252 | + [Fact] |
| 253 | + public void TheDocumentedExamplesAreStillThere() |
| 254 | + { |
| 255 | + var examples = Examples(); |
| 256 | + Assert.True(examples.Count >= 254, |
| 257 | + $"only {examples.Count} examples are in the two-block 'Prints' shape, and there " |
| 258 | + + "were 254. An example that stops following the convention stops being checked."); |
| 259 | + } |
| 260 | + } |
| 261 | +} |
0 commit comments