Skip to content

Commit c6777f7

Browse files
Cover the fourth test leg, and watch the SymPy snapshot for drift (#746 items 35, 73) (#1038)
* Cover the fourth leg, and watch the SymPy snapshot for drift (#746 items 35, 73) Item 35: three of the four test legs report coverage and the C++ one reports none. It cannot report the same measurement -- the library reaches those tests as a Native AOT shared object, the published artefact holds no managed assembly, and ctest runs a native binary rather than a host the VSTest collector can attach to. What it does have is the C++ wrapper in AngouriMath.CPP.Importing, which nothing measured: the Linux job now builds it with --coverage and uploads a gcovr Cobertura report under the `cpp` flag, filtered to the wrapper so that neither googletest nor the test file inflates it. The step fails on an empty report rather than uploading one. Docs/Contributing/Coverage.md says what each flag covers and why this one is not the kernel. Item 73: the SymPy comparison is a hand-taken snapshot against one version and cannot tell you it has gone stale. A weekly job re-runs the 80 SymPy probes against whatever pip installs that day and fails when the version or any answer has moved from the committed baseline, uploading the new answers so re-taking the snapshot is a copy. Only the SymPy half runs here; the half that asks this library the same questions is not in this repository, and Maxima and SageMath are not touched, because nothing measures them. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Bjumi5K7fg8yx6UK1mZTQd * Name the report codecov actually uploads v5 of codecov-action takes `files`, not `file`. Given `file` it warns about an unexpected input, ignores the path and searches the workspace instead: on the C++ leg that search found 96 reports, the whole of googletest and of /usr/include/c++/13 among them, so the flag carried everything the gcovr filter exists to exclude. `disable_search` and `plugins: noop` hold it to the report gcovr wrote. CSharpTest.yml has the same input and the same consequence, so it is corrected here too; its search is left alone, not having been measured. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Bjumi5K7fg8yx6UK1mZTQd --------- Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
1 parent 46ac7c4 commit c6777f7

8 files changed

Lines changed: 1138 additions & 3 deletions

File tree

‎.github/sympy-parity/.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# Written by check.py on every run; the committed snapshot is sympy-baseline.json.
2+
sympy-current.json

‎.github/sympy-parity/check.py‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
# Re-runs the SymPy half of the parity survey against whatever SymPy pip installs today and
2+
# compares it with the snapshot in sympy-baseline.json.
3+
#
4+
# Only SymPy runs here. The survey's other half asks AngouriMath the same questions, and that
5+
# half is not in this repository, so nothing below compares the two libraries or restates a
6+
# verdict about either. What it answers is the narrower question that the published snapshot
7+
# cannot answer about itself: is SymPy still the version, and still giving the answers, that
8+
# the snapshot was taken from.
9+
#
10+
# Exit code 1 means the snapshot no longer describes SymPy, and a human has to re-take it.
11+
# https://github.com/asc-community/AngouriMath/issues/746 item 73.
12+
13+
import json
14+
import os
15+
import signal
16+
import sys
17+
18+
import sympy
19+
from sympy import * # noqa: F401,F403 the probes are written against the flat namespace
20+
from sympy.abc import x, y, k, n, s, a, b, c # noqa: F401
21+
22+
# Every import below serves at least one probe. A release that moves or removes one of these is
23+
# itself a finding, so an import failure is recorded and left to surface as the probes that need
24+
# it failing, rather than taking the whole run down before anything is measured.
25+
FAILED_IMPORTS = []
26+
for statement in (
27+
"import sympy.ntheory.modular",
28+
"from sympy.stats import Die, P",
29+
"from sympy.logic.boolalg import to_cnf",
30+
"from sympy.logic.inference import satisfiable",
31+
"from sympy.combinatorics import PermutationGroup",
32+
"from sympy.vector import CoordSys3D",
33+
"from sympy.tensor.tensor import TensorIndexType",
34+
"from sympy.diffgeom import Manifold",
35+
"from sympy.holonomic import expr_to_holonomic",
36+
"from sympy.physics.units import convert_to",
37+
"from sympy.printing.mathml import mathml",
38+
"from sympy.discrete import convolution, fft",
39+
"from sympy.simplify.sqrtdenest import sqrtdenest",
40+
"from sympy.polys.numberfields import minimal_polynomial",
41+
"from sympy.ntheory.continued_fraction import continued_fraction_iterator",
42+
"from sympy.sets.conditionset import ConditionSet",
43+
"from sympy.integrals.transforms import laplace_transform, fourier_transform",
44+
):
45+
try:
46+
exec(statement)
47+
except Exception as e: # noqa: BLE001 any import failure is data
48+
FAILED_IMPORTS.append(f"{statement} -- {type(e).__name__}: {e}")
49+
50+
51+
class Timeout(Exception):
52+
pass
53+
54+
55+
def _alarm(sig, frm):
56+
raise Timeout()
57+
58+
59+
signal.signal(signal.SIGALRM, _alarm)
60+
61+
here = os.path.dirname(os.path.abspath(__file__))
62+
probes = json.load(open(os.path.join(here, "probes.json")))
63+
baseline = json.load(open(os.path.join(here, "sympy-baseline.json")))
64+
65+
answers = []
66+
for p in probes:
67+
signal.alarm(20)
68+
try:
69+
value = str(eval(p["sympy"]))
70+
except Timeout:
71+
value = "*** timeout (>20s) ***"
72+
except Exception as e: # noqa: BLE001 an exception is the answer
73+
value = f"!!! {type(e).__name__}: {e}".split("\n")[0]
74+
finally:
75+
signal.alarm(0)
76+
answers.append({"area": p["area"], "what": p["what"], "sympy": value})
77+
78+
current = {"sympy": sympy.__version__, "answers": answers}
79+
with open(os.path.join(here, "sympy-current.json"), "w") as f:
80+
json.dump(current, f, indent=1)
81+
82+
was = {(r["area"], r["what"]): r["sympy"] for r in baseline["answers"]}
83+
now = {(r["area"], r["what"]): r["sympy"] for r in answers}
84+
85+
moved = [(k2, was[k2], now[k2]) for k2 in now if k2 in was and was[k2] != now[k2]]
86+
added = sorted(k2 for k2 in now if k2 not in was)
87+
gone = sorted(k2 for k2 in was if k2 not in now)
88+
version_moved = baseline["sympy"] != sympy.__version__
89+
90+
91+
def cell(t, at=70):
92+
t = t.replace("|", "\\|").replace("\n", " ")
93+
return t if len(t) <= at else t[: at - 1] + "…"
94+
95+
96+
lines = [
97+
"## SymPy parity watch",
98+
"",
99+
f"- baseline: **SymPy {baseline['sympy']}**, taken {baseline['measured']}",
100+
f"- installed today: **SymPy {sympy.__version__}**",
101+
f"- {len(answers)} probes run, **{len(moved)}** answer{'' if len(moved) == 1 else 's'} moved",
102+
"",
103+
]
104+
if FAILED_IMPORTS:
105+
lines += ["Imports that no longer resolve:", ""] + [f"- `{i}`" for i in FAILED_IMPORTS] + [""]
106+
if moved:
107+
lines += ["| area | capability | was | is now |", "|---|---|---|---|"]
108+
lines += [
109+
f"| {area} | {what} | `{cell(old)}` | `{cell(new)}` |" for (area, what), old, new in sorted(moved)
110+
]
111+
lines.append("")
112+
if added or gone:
113+
lines += [f"Probes added since the baseline: {added}", f"Probes gone since the baseline: {gone}", ""]
114+
115+
summary = "\n".join(lines)
116+
print(summary)
117+
118+
step_summary = os.environ.get("GITHUB_STEP_SUMMARY")
119+
if step_summary:
120+
with open(step_summary, "a") as f:
121+
f.write(summary + "\n")
122+
123+
if not (version_moved or moved or added or gone):
124+
print(f"The snapshot still describes SymPy {sympy.__version__}.")
125+
sys.exit(0)
126+
127+
# A version bump on its own is a failure because the published comparison names the version it
128+
# measured; a version it does not name is a version it did not measure, whether or not any of
129+
# these 80 answers happens to have moved with it.
130+
print(
131+
"\nThe snapshot no longer describes SymPy. Re-run work/sympyparity end to end against "
132+
f"SymPy {sympy.__version__}, republish the comparison, and replace sympy-baseline.json "
133+
"with the sympy-current.json this run uploaded.",
134+
file=sys.stderr,
135+
)
136+
sys.exit(1)

0 commit comments

Comments
 (0)