From b2a397a12ad50ed9f7ce9c69c1bc1e8c698906d3 Mon Sep 17 00:00:00 2001 From: Jae Hoon Seo Date: Sat, 7 Nov 2026 22:13:35 +0900 Subject: [PATCH] ci: run the suite and the example decks on the Prescott BLAS kernel A result that depends on the BLAS kernel passes on the runners' own AVX2 and AVX-512 kernels. Add a Linux Release job, separate from the required ones, that runs the full CTest with OPENBLAS_CORETYPE=Prescott and then every shipped example deck through tests/check_example_fallbacks.py. The scan fails on a non-zero exit, a timeout and a fallback note, and lists the offending decks. It matches the wording the solver prints for an equilibrium accepted above the requested tolerance ("residual floor above"), which does not contain "looser tolerance". OPENBLAS_CORETYPE is ignored by an OpenBLAS without dynamic architecture support and an unknown name falls back to detection, so the job first has OpenBLAS name its kernel (OPENBLAS_VERBOSE=2) and stops on any other kernel or none. example_fallback_scan proves the scan fails where it should. Signed-off-by: Jae Hoon Seo --- .github/workflows/fortran.yml | 52 +++++++ CMakeLists.txt | 11 ++ DEVELOPMENT.md | 13 ++ tests/check_example_fallbacks.py | 206 ++++++++++++++++++++++++++ tests/test_check_example_fallbacks.py | 152 +++++++++++++++++++ 5 files changed, 434 insertions(+) create mode 100644 tests/check_example_fallbacks.py create mode 100644 tests/test_check_example_fallbacks.py diff --git a/.github/workflows/fortran.yml b/.github/workflows/fortran.yml index edd730ad..40e46d0f 100644 --- a/.github/workflows/fortran.yml +++ b/.github/workflows/fortran.yml @@ -89,3 +89,55 @@ jobs: OMP_NUM_THREADS: "1" OPENBLAS_NUM_THREADS: "1" run: ctest --test-dir build --output-on-failure --parallel 4 + + # Round-off fragility: a result that depends on the BLAS kernel fails here although every + # job above, on the runner's own kernel, passes. Prescott, an SSE3 kernel of the conda-forge + # OpenBLAS, sums in another order than the AVX2 and AVX-512 kernels the runners select. + # This job is advisory: branch protection names the jobs above, not this one. + roundoff: + name: round-off kernel (conda gfortran, ubuntu-latest, Prescott) + runs-on: ubuntu-latest + env: + OPENBLAS_CORETYPE: Prescott + defaults: + run: + shell: bash -el {0} # login shell so the conda env is active + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up the conda toolchain (gfortran, CMake, OpenBLAS, LAPACK) + uses: mamba-org/setup-micromamba@f457c30a868e4760d3a6fcea5f25dc655b8edf39 # v3.2.1 + with: + environment-file: environment.yml + cache-environment: true + + - name: Show toolchain + run: gfortran --version && cmake --version + + - name: Configure + run: cmake -S . -B build -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release + + - name: Build + run: cmake --build build --parallel + + - name: Confirm the OpenBLAS kernel + # OPENBLAS_CORETYPE works only on a dynamic-architecture OpenBLAS and is otherwise + # ignored without a word: stop here rather than pass on the runner's own kernel. + run: >- + python tests/check_example_fallbacks.py --driver build/cabledyn --examples examples + --out build/kernel_check --coretype "$OPENBLAS_CORETYPE" --kernel-only + + - name: Test (CTest) + # As in the job above: four tests at once, each single-threaded. The few tests that + # pin a kernel of their own set OPENBLAS_CORETYPE themselves. + env: + OMP_NUM_THREADS: "1" + OPENBLAS_NUM_THREADS: "1" + run: ctest --test-dir build --output-on-failure --parallel 4 + + - name: Example decks without a fallback note + run: >- + python tests/check_example_fallbacks.py --driver build/cabledyn --examples examples + --out build/example_scan --coretype "$OPENBLAS_CORETYPE" --jobs 4 diff --git a/CMakeLists.txt b/CMakeLists.txt index ad56bfd6..ed223549 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2756,6 +2756,17 @@ if(Python3_Interpreter_FOUND) SKIP_RETURN_CODE 77 ENVIRONMENT "${_cabledyn_python_test_environment}") endif() +# The scan of the round-off CI job: it must flag a fallback note, a failed run and a forced +# BLAS kernel that did not take effect, or a clean scan would prove nothing. +if(Python3_Interpreter_FOUND) + add_test(NAME example_fallback_scan + COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_check_example_fallbacks.py + $ ${CMAKE_CURRENT_SOURCE_DIR}/examples) + set_tests_properties(example_fallback_scan PROPERTIES + LABELS "python;driver;integration" + TIMEOUT 120 + ENVIRONMENT "${_cabledyn_python_test_environment}") +endif() # A clamped finite-EI cable with a buoyant section on a Rigid6 body: the body static solve # holds the objects at their deck poses, warns once, and reports no balanced clamped end loads. add_test(NAME driver_clamped_buoyant_held diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 704a8b10..6cc9d3d4 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -86,6 +86,19 @@ stops at the first mismatch. The Fortran CI workflow builds Release and Debug on Ubuntu and Windows and runs the full suite on every pull request and push to `main`. +The `roundoff` job of the same workflow repeats the Linux Release build and the full +CTest on the SSE3 OpenBLAS kernel (`OPENBLAS_CORETYPE=Prescott`), then runs every shipped +example deck through `tests/check_example_fallbacks.py`, which fails on a non-zero exit and on a +fallback note where a deck should converge cleanly, and lists the offending decks. The job stops +first if OpenBLAS does not report that kernel. It is advisory, not a required check. To repeat it +locally, on Linux or Windows: + +```bash +OPENBLAS_CORETYPE=Prescott ctest --test-dir build --output-on-failure +python tests/check_example_fallbacks.py --driver build/cabledyn --examples examples \ + --out build/example_scan --coretype Prescott +``` + ## Python package The `cabledyn` package (`python/`) loads the C-ABI shared library built by diff --git a/tests/check_example_fallbacks.py b/tests/check_example_fallbacks.py new file mode 100644 index 00000000..b61d1dfd --- /dev/null +++ b/tests/check_example_fallbacks.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Run every shipped example deck and fail on a solver fallback note. + +Each deck of the examples directory runs on the given driver, single-threaded, with its output +root under a scratch directory. A deck offends when the driver does not exit with 0, runs past +the timeout, or writes a fallback note to the console: a deck that should converge cleanly must +not reach its result through a failed Newton solve, a stall, a looser tolerance or a sub-step that +gave up. Such a note marks round-off fragility, which a different BLAS kernel can turn into a +failure. + +With --coretype every run sets OPENBLAS_CORETYPE to that kernel and asks OpenBLAS to name the +kernel it selected. The check fails when OpenBLAS names another kernel, or when no run names one +at all, as with a build of OpenBLAS without dynamic architecture support, which ignores the +variable. --kernel-only runs the first deck alone for that check, before a long test run. + +Usage: check_example_fallbacks.py --driver --examples + --out [--coretype ] + [--kernel-only] [--jobs ] [--timeout ] +Exit status: 0 clean; 1 an offending deck or the wrong kernel; 2 no decks or bad arguments. +""" + +from __future__ import annotations + +import argparse +import os +import re +import subprocess +import sys +import time +from collections.abc import Sequence +from concurrent.futures import ThreadPoolExecutor, as_completed +from dataclasses import dataclass, field +from pathlib import Path + +# The console notes of a fallback. A note may be wrapped, so whitespace runs match any spacing. +FALLBACK_NOTES = ( + ("failed Newton solve", re.compile(r"after\s+a\s+failed\s+Newton\s+solve", re.IGNORECASE)), + ("stall", re.compile(r"\bstalled\b", re.IGNORECASE)), + ("looser tolerance", re.compile(r"\blooser\s+tolerance\b", re.IGNORECASE)), + # The note of an equilibrium accepted above the requested tolerance. + ("looser tolerance", re.compile(r"\bresidual\s+floor\s+above\b", re.IGNORECASE)), + ("sub-step", re.compile(r"\bresolving\s+sub-step\s+did\s+not\s+converge", re.IGNORECASE)), +) +# What OpenBLAS prints at start-up under OPENBLAS_VERBOSE=2, in a dynamic-architecture build only. +CORE_LINE = re.compile(r"^\s*Core:\s*(\S+)\s*$", re.MULTILINE) + + +@dataclass +class DeckRun: + """The outcome of one deck.""" + + deck: str + returncode: int | None = None + seconds: float = 0.0 + notes: list[str] = field(default_factory=list) + cores: set[str] = field(default_factory=set) + + @property + def offends(self) -> bool: + return self.returncode != 0 or bool(self.notes) + + +def fallback_notes(console: str) -> list[str]: + """The console text of every fallback note, one entry per note found.""" + found: list[str] = [] + for name, pattern in FALLBACK_NOTES: + for match in pattern.finditer(console): + start = console.rfind("\n", 0, match.start()) + 1 + stop = console.find("\n", match.end()) + line = console[start : stop if stop >= 0 else len(console)].strip() + found.append(f"{name}: {line[:200]}") + return found + + +def run_deck( + command: Sequence[str], + deck: Path, + out: Path, + timeout: float, + coretype: str | None, +) -> DeckRun: + """Run one deck from the directory of the deck, whose relative paths it may use.""" + env = dict(os.environ, OMP_NUM_THREADS="1", OPENBLAS_NUM_THREADS="1") + if coretype is not None: + env.update(OPENBLAS_CORETYPE=coretype, OPENBLAS_VERBOSE="2") + result = DeckRun(deck.name) + start = time.monotonic() + try: + process = subprocess.run( + [*command, deck.name, str(out / deck.stem)], + cwd=deck.parent, + env=env, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=timeout, + check=False, + ) + except subprocess.TimeoutExpired: + result.seconds = time.monotonic() - start + result.notes.append(f"timeout: still running after {timeout:g} s") + return result + result.seconds = time.monotonic() - start + result.returncode = process.returncode + console = process.stdout + "\n" + process.stderr + (out / f"{deck.stem}.console.log").write_text(console, encoding="utf-8") + result.notes = fallback_notes(console) + result.cores = set(CORE_LINE.findall(console)) + return result + + +def kernel_problem(runs: Sequence[DeckRun], coretype: str) -> str | None: + """Why the runs did not use the kernel ``coretype``, or None if they did.""" + cores = set().union(*(run.cores for run in runs)) if runs else set() + if not cores: + return ( + f"OpenBLAS named no kernel, so OPENBLAS_CORETYPE={coretype} was not shown to take " + "effect: the driver is not on a dynamic-architecture OpenBLAS" + ) + wrong = sorted(core for core in cores if core.lower() != coretype.lower()) + if wrong: + return f"OPENBLAS_CORETYPE={coretype} did not take effect: OpenBLAS ran {', '.join(wrong)}" + return None + + +def scan( + command: Sequence[str], + decks: Sequence[Path], + out: Path, + *, + jobs: int, + timeout: float, + coretype: str | None, +) -> list[DeckRun]: + """Run the decks, printing each result as it arrives.""" + out.mkdir(parents=True, exist_ok=True) + runs: list[DeckRun] = [] + with ThreadPoolExecutor(jobs) as pool: + pending = [ + pool.submit(run_deck, command, deck, out, timeout, coretype) for deck in decks + ] + for future in as_completed(pending): + run = future.result() + runs.append(run) + state = "OFFENDS" if run.offends else "ok" + print(f"{state:8}{run.seconds:9.1f} s {run.deck}", flush=True) + return sorted(runs, key=lambda run: run.deck) + + +def report(runs: Sequence[DeckRun], coretype: str | None) -> int: + """Print the offending decks and the verdict; return the exit status.""" + offending = [run for run in runs if run.offends] + if offending: + print(f"\n{len(offending)} of {len(runs)} example decks offend:") + for run in offending: + exit_text = "no exit" if run.returncode is None else f"exit {run.returncode}" + print(f" {run.deck} ({exit_text})") + for note in run.notes: + print(f" {note}") + problem = kernel_problem(runs, coretype) if coretype is not None else None + if problem is not None: + print(f"\nFAIL: {problem}") + if offending or problem is not None: + return 1 + kernel = f" on the {coretype} kernel" if coretype is not None else "" + print(f"\n{len(runs)} example decks ran{kernel} with no fallback note") + return 0 + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Fail on a fallback note of an example deck.") + parser.add_argument("--driver", required=True, type=Path, help="the cabledyn executable") + parser.add_argument("--examples", required=True, type=Path, help="the examples directory") + parser.add_argument("--out", required=True, type=Path, help="scratch output directory") + parser.add_argument("--coretype", help="the OpenBLAS kernel to force and to confirm") + parser.add_argument( + "--kernel-only", action="store_true", help="only confirm --coretype, on the first deck" + ) + parser.add_argument("--jobs", type=int, default=os.cpu_count() or 1, help="parallel decks") + parser.add_argument("--timeout", type=float, default=3600.0, help="seconds for each deck") + args = parser.parse_args(argv) + if args.kernel_only and args.coretype is None: + parser.error("--kernel-only needs --coretype") + + examples = args.examples.resolve() + decks = sorted(path for path in examples.glob("*.dat") if path.is_file()) + if not decks: + print(f"FAIL: no example decks in {examples}") + return 2 + command = [str(args.driver.resolve())] + out = args.out.resolve() + if args.kernel_only: + runs = scan(command, decks[:1], out, jobs=1, timeout=args.timeout, coretype=args.coretype) + problem = kernel_problem(runs, args.coretype) + print(f"FAIL: {problem}" if problem else f"OpenBLAS ran the {args.coretype} kernel") + return 1 if problem else 0 + runs = scan( + command, decks, out, jobs=args.jobs, timeout=args.timeout, coretype=args.coretype + ) + return report(runs, args.coretype) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_check_example_fallbacks.py b/tests/test_check_example_fallbacks.py new file mode 100644 index 00000000..059b3933 --- /dev/null +++ b/tests/test_check_example_fallbacks.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""The example scan of the round-off CI job must fail where it is meant to. + +A stand-in driver, whose behaviour each deck selects, exercises tests/check_example_fallbacks.py +as a child process would: every fallback note fails the scan and names its deck, also when wrapped +over two lines; a non-zero exit and a timeout fail it; a clean deck passes. With a forced kernel, +a run that names another kernel or none at all fails it. The built driver then shows the same on +the real OpenBLAS: an unknown kernel name is not reported as taken. + +Usage: test_check_example_fallbacks.py +""" + +from __future__ import annotations + +import contextlib +import importlib.util +import io +import sys +import tempfile +from pathlib import Path + +HERE = Path(__file__).resolve().parent +spec = importlib.util.spec_from_file_location("scan", HERE / "check_example_fallbacks.py") +assert spec is not None and spec.loader is not None +scan = importlib.util.module_from_spec(spec) +sys.modules["scan"] = scan # dataclasses look the module up by name +spec.loader.exec_module(scan) + +# Each deck holds what the driver prints to stderr, then a last line with its exit code. +FAKE_DRIVER = """ +import sys +import time +text = open(sys.argv[1], encoding="utf-8").read().split("|") +if text[-1] == "hang": + time.sleep(60) +sys.stderr.write(text[0]) +sys.exit(int(text[-1])) +""" + +NOTES = { + "newton": " Note: line 1: the discrete attachment equilibrium was reached after a failed " + "Newton solve (pseudo-transient continuation).\n", + "wrapped": " Note: line 1: reached after a failed\n Newton solve.\n", + "stalled": " Note: the Newton iteration stalled at the load fraction 0.5.\n", + "looser": " Note: a looser tolerance is needed.\n", + "floor": " Note: line 1: static residual floor above the dynamic tolerance 1.00E-08; " + "equilibrium accepted at the relative residual 2.00E-07.\n", + "torsion_floor": " Note: line 1: torsion static residual floor above the tolerance " + "1.00E-08; twisted equilibrium accepted at 2.00E-07.\n", + "substep": " Note: line 1: 3 step(s) kept the whole step because a resolving sub-step did " + "not converge.\n", +} +# Notes of a healthy run that must not trip the scan. +CLEAN = ( + " Note: LINE 1: dtM = 0.1 s resolves its axial period of 1 s in 10 steps.\n" + " Note: line 1: 2 attachment(s) lumped at the nearest nodes.\n" + " CableDyn initialization completed.\n" +) + + +def write_deck(directory: Path, name: str, console: str, status: str = "0") -> Path: + deck = directory / f"{name}.dat" + deck.write_text(f"{console}|{status}", encoding="utf-8") + return deck + + +def run_scan(command, decks, out, *, coretype=None, timeout=20.0): + """The exit status and the printed report of a scan.""" + text = io.StringIO() + with contextlib.redirect_stdout(text): + runs = scan.scan(command, decks, out, jobs=4, timeout=timeout, coretype=coretype) + status = scan.report(runs, coretype) + return status, text.getvalue() + + +def check(condition: bool, what: str, failures: list[str], detail: str = "") -> None: + print(f"{'ok' if condition else 'FAIL'}: {what}") + if not condition: + failures.append(f"{what}\n{detail}") + + +def main() -> int: + driver, examples = Path(sys.argv[1]), Path(sys.argv[2]) + failures: list[str] = [] + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + decks = root / "decks" + decks.mkdir() + fake = root / "fake_driver.py" + fake.write_text(FAKE_DRIVER, encoding="utf-8") + command = [sys.executable, str(fake)] + + clean = write_deck(decks, "clean", CLEAN) + status, text = run_scan(command, [clean], root / "clean") + check(status == 0 and "OFFENDS" not in text, "a clean deck passes", failures, text) + + for name, note in NOTES.items(): + deck = write_deck(decks, name, CLEAN + note) + status, text = run_scan(command, [clean, deck], root / name) + listed = f" {deck.name} (exit 0)" in text and f" {clean.name} (" not in text + check(status == 1 and listed, f"the note {name} fails its deck", failures, text) + + failed = write_deck(decks, "failed", CLEAN, "3") + status, text = run_scan(command, [failed], root / "failed") + check(status == 1 and "failed.dat (exit 3)" in text, "an exit status fails", failures, text) + + hung = write_deck(decks, "hung", CLEAN, "hang") + status, text = run_scan(command, [hung], root / "hung", timeout=2.0) + check(status == 1 and "hung.dat (no exit)" in text, "a timeout fails", failures, text) + + taken = write_deck(decks, "taken", "Core: Prescott\n" + CLEAN) + status, text = run_scan(command, [taken], root / "taken", coretype="Prescott") + check(status == 0, "the forced kernel passes", failures, text) + status, text = run_scan(command, [taken], root / "case", coretype="PRESCOTT") + check(status == 0, "kernel names match in any case", failures, text) + + other = write_deck(decks, "other", "Core not found: Prescott\nCore: Haswell\n" + CLEAN) + status, text = run_scan(command, [other], root / "other", coretype="Prescott") + check(status == 1 and "ran Haswell" in text, "another kernel fails", failures, text) + + status, text = run_scan(command, [clean], root / "none", coretype="Prescott") + check(status == 1 and "named no kernel" in text, "no kernel fails", failures, text) + + status, text = run_scan(command, [taken, other], root / "mixed", coretype="Prescott") + check(status == 1 and "ran Haswell" in text, "one wrong run fails", failures, text) + + # The real driver and OpenBLAS: a kernel that does not exist cannot be confirmed. + real = io.StringIO() + with contextlib.redirect_stdout(real): + status = scan.main( + [ + "--driver", + str(driver), + "--examples", + str(examples), + "--out", + str(root / "real"), + "--coretype", + "NoSuchKernel", + "--kernel-only", + ] + ) + check(status == 1, "an unknown kernel fails on the real driver", failures, real.getvalue()) + + for failure in failures: + print(f"\nFAILED: {failure}") + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main())