diff --git a/.github/workflows/fortran.yml b/.github/workflows/fortran.yml index edd730ad..40e46d0f 100644 --- a/.github/workflows/fortran.yml +++ b/.github/workflows/fortran.yml @@ -89,3 +89,55 @@ jobs: OMP_NUM_THREADS: "1" OPENBLAS_NUM_THREADS: "1" run: ctest --test-dir build --output-on-failure --parallel 4 + + # Round-off fragility: a result that depends on the BLAS kernel fails here although every + # job above, on the runner's own kernel, passes. Prescott, an SSE3 kernel of the conda-forge + # OpenBLAS, sums in another order than the AVX2 and AVX-512 kernels the runners select. + # This job is advisory: branch protection names the jobs above, not this one. + roundoff: + name: round-off kernel (conda gfortran, ubuntu-latest, Prescott) + runs-on: ubuntu-latest + env: + OPENBLAS_CORETYPE: Prescott + defaults: + run: + shell: bash -el {0} # login shell so the conda env is active + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up the conda toolchain (gfortran, CMake, OpenBLAS, LAPACK) + uses: mamba-org/setup-micromamba@f457c30a868e4760d3a6fcea5f25dc655b8edf39 # v3.2.1 + with: + environment-file: environment.yml + cache-environment: true + + - name: Show toolchain + run: gfortran --version && cmake --version + + - name: Configure + run: cmake -S . -B build -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release + + - name: Build + run: cmake --build build --parallel + + - name: Confirm the OpenBLAS kernel + # OPENBLAS_CORETYPE works only on a dynamic-architecture OpenBLAS and is otherwise + # ignored without a word: stop here rather than pass on the runner's own kernel. + run: >- + python tests/check_example_fallbacks.py --driver build/cabledyn --examples examples + --out build/kernel_check --coretype "$OPENBLAS_CORETYPE" --kernel-only + + - name: Test (CTest) + # As in the job above: four tests at once, each single-threaded. The few tests that + # pin a kernel of their own set OPENBLAS_CORETYPE themselves. + env: + OMP_NUM_THREADS: "1" + OPENBLAS_NUM_THREADS: "1" + run: ctest --test-dir build --output-on-failure --parallel 4 + + - name: Example decks without a fallback note + run: >- + python tests/check_example_fallbacks.py --driver build/cabledyn --examples examples + --out build/example_scan --coretype "$OPENBLAS_CORETYPE" --jobs 4 diff --git a/CMakeLists.txt b/CMakeLists.txt index ad56bfd6..ed223549 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2756,6 +2756,17 @@ if(Python3_Interpreter_FOUND) SKIP_RETURN_CODE 77 ENVIRONMENT "${_cabledyn_python_test_environment}") endif() +# The scan of the round-off CI job: it must flag a fallback note, a failed run and a forced +# BLAS kernel that did not take effect, or a clean scan would prove nothing. +if(Python3_Interpreter_FOUND) + add_test(NAME example_fallback_scan + COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_check_example_fallbacks.py + $ ${CMAKE_CURRENT_SOURCE_DIR}/examples) + set_tests_properties(example_fallback_scan PROPERTIES + LABELS "python;driver;integration" + TIMEOUT 120 + ENVIRONMENT "${_cabledyn_python_test_environment}") +endif() # A clamped finite-EI cable with a buoyant section on a Rigid6 body: the body static solve # holds the objects at their deck poses, warns once, and reports no balanced clamped end loads. add_test(NAME driver_clamped_buoyant_held diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 704a8b10..6cc9d3d4 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -86,6 +86,19 @@ stops at the first mismatch. The Fortran CI workflow builds Release and Debug on Ubuntu and Windows and runs the full suite on every pull request and push to `main`. +The `roundoff` job of the same workflow repeats the Linux Release build and the full +CTest on the SSE3 OpenBLAS kernel (`OPENBLAS_CORETYPE=Prescott`), then runs every shipped +example deck through `tests/check_example_fallbacks.py`, which fails on a non-zero exit and on a +fallback note where a deck should converge cleanly, and lists the offending decks. The job stops +first if OpenBLAS does not report that kernel. It is advisory, not a required check. To repeat it +locally, on Linux or Windows: + +```bash +OPENBLAS_CORETYPE=Prescott ctest --test-dir build --output-on-failure +python tests/check_example_fallbacks.py --driver build/cabledyn --examples examples \ + --out build/example_scan --coretype Prescott +``` + ## Python package The `cabledyn` package (`python/`) loads the C-ABI shared library built by diff --git a/tests/check_example_fallbacks.py b/tests/check_example_fallbacks.py new file mode 100644 index 00000000..b61d1dfd --- /dev/null +++ b/tests/check_example_fallbacks.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Run every shipped example deck and fail on a solver fallback note. + +Each deck of the examples directory runs on the given driver, single-threaded, with its output +root under a scratch directory. A deck offends when the driver does not exit with 0, runs past +the timeout, or writes a fallback note to the console: a deck that should converge cleanly must +not reach its result through a failed Newton solve, a stall, a looser tolerance or a sub-step that +gave up. Such a note marks round-off fragility, which a different BLAS kernel can turn into a +failure. + +With --coretype every run sets OPENBLAS_CORETYPE to that kernel and asks OpenBLAS to name the +kernel it selected. The check fails when OpenBLAS names another kernel, or when no run names one +at all, as with a build of OpenBLAS without dynamic architecture support, which ignores the +variable. --kernel-only runs the first deck alone for that check, before a long test run. + +Usage: check_example_fallbacks.py --driver --examples + --out [--coretype ] + [--kernel-only] [--jobs ] [--timeout ] +Exit status: 0 clean; 1 an offending deck or the wrong kernel; 2 no decks or bad arguments. +""" + +from __future__ import annotations + +import argparse +import os +import re +import subprocess +import sys +import time +from collections.abc import Sequence +from concurrent.futures import ThreadPoolExecutor, as_completed +from dataclasses import dataclass, field +from pathlib import Path + +# The console notes of a fallback. A note may be wrapped, so whitespace runs match any spacing. +FALLBACK_NOTES = ( + ("failed Newton solve", re.compile(r"after\s+a\s+failed\s+Newton\s+solve", re.IGNORECASE)), + ("stall", re.compile(r"\bstalled\b", re.IGNORECASE)), + ("looser tolerance", re.compile(r"\blooser\s+tolerance\b", re.IGNORECASE)), + # The note of an equilibrium accepted above the requested tolerance. + ("looser tolerance", re.compile(r"\bresidual\s+floor\s+above\b", re.IGNORECASE)), + ("sub-step", re.compile(r"\bresolving\s+sub-step\s+did\s+not\s+converge", re.IGNORECASE)), +) +# What OpenBLAS prints at start-up under OPENBLAS_VERBOSE=2, in a dynamic-architecture build only. +CORE_LINE = re.compile(r"^\s*Core:\s*(\S+)\s*$", re.MULTILINE) + + +@dataclass +class DeckRun: + """The outcome of one deck.""" + + deck: str + returncode: int | None = None + seconds: float = 0.0 + notes: list[str] = field(default_factory=list) + cores: set[str] = field(default_factory=set) + + @property + def offends(self) -> bool: + return self.returncode != 0 or bool(self.notes) + + +def fallback_notes(console: str) -> list[str]: + """The console text of every fallback note, one entry per note found.""" + found: list[str] = [] + for name, pattern in FALLBACK_NOTES: + for match in pattern.finditer(console): + start = console.rfind("\n", 0, match.start()) + 1 + stop = console.find("\n", match.end()) + line = console[start : stop if stop >= 0 else len(console)].strip() + found.append(f"{name}: {line[:200]}") + return found + + +def run_deck( + command: Sequence[str], + deck: Path, + out: Path, + timeout: float, + coretype: str | None, +) -> DeckRun: + """Run one deck from the directory of the deck, whose relative paths it may use.""" + env = dict(os.environ, OMP_NUM_THREADS="1", OPENBLAS_NUM_THREADS="1") + if coretype is not None: + env.update(OPENBLAS_CORETYPE=coretype, OPENBLAS_VERBOSE="2") + result = DeckRun(deck.name) + start = time.monotonic() + try: + process = subprocess.run( + [*command, deck.name, str(out / deck.stem)], + cwd=deck.parent, + env=env, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=timeout, + check=False, + ) + except subprocess.TimeoutExpired: + result.seconds = time.monotonic() - start + result.notes.append(f"timeout: still running after {timeout:g} s") + return result + result.seconds = time.monotonic() - start + result.returncode = process.returncode + console = process.stdout + "\n" + process.stderr + (out / f"{deck.stem}.console.log").write_text(console, encoding="utf-8") + result.notes = fallback_notes(console) + result.cores = set(CORE_LINE.findall(console)) + return result + + +def kernel_problem(runs: Sequence[DeckRun], coretype: str) -> str | None: + """Why the runs did not use the kernel ``coretype``, or None if they did.""" + cores = set().union(*(run.cores for run in runs)) if runs else set() + if not cores: + return ( + f"OpenBLAS named no kernel, so OPENBLAS_CORETYPE={coretype} was not shown to take " + "effect: the driver is not on a dynamic-architecture OpenBLAS" + ) + wrong = sorted(core for core in cores if core.lower() != coretype.lower()) + if wrong: + return f"OPENBLAS_CORETYPE={coretype} did not take effect: OpenBLAS ran {', '.join(wrong)}" + return None + + +def scan( + command: Sequence[str], + decks: Sequence[Path], + out: Path, + *, + jobs: int, + timeout: float, + coretype: str | None, +) -> list[DeckRun]: + """Run the decks, printing each result as it arrives.""" + out.mkdir(parents=True, exist_ok=True) + runs: list[DeckRun] = [] + with ThreadPoolExecutor(jobs) as pool: + pending = [ + pool.submit(run_deck, command, deck, out, timeout, coretype) for deck in decks + ] + for future in as_completed(pending): + run = future.result() + runs.append(run) + state = "OFFENDS" if run.offends else "ok" + print(f"{state:8}{run.seconds:9.1f} s {run.deck}", flush=True) + return sorted(runs, key=lambda run: run.deck) + + +def report(runs: Sequence[DeckRun], coretype: str | None) -> int: + """Print the offending decks and the verdict; return the exit status.""" + offending = [run for run in runs if run.offends] + if offending: + print(f"\n{len(offending)} of {len(runs)} example decks offend:") + for run in offending: + exit_text = "no exit" if run.returncode is None else f"exit {run.returncode}" + print(f" {run.deck} ({exit_text})") + for note in run.notes: + print(f" {note}") + problem = kernel_problem(runs, coretype) if coretype is not None else None + if problem is not None: + print(f"\nFAIL: {problem}") + if offending or problem is not None: + return 1 + kernel = f" on the {coretype} kernel" if coretype is not None else "" + print(f"\n{len(runs)} example decks ran{kernel} with no fallback note") + return 0 + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Fail on a fallback note of an example deck.") + parser.add_argument("--driver", required=True, type=Path, help="the cabledyn executable") + parser.add_argument("--examples", required=True, type=Path, help="the examples directory") + parser.add_argument("--out", required=True, type=Path, help="scratch output directory") + parser.add_argument("--coretype", help="the OpenBLAS kernel to force and to confirm") + parser.add_argument( + "--kernel-only", action="store_true", help="only confirm --coretype, on the first deck" + ) + parser.add_argument("--jobs", type=int, default=os.cpu_count() or 1, help="parallel decks") + parser.add_argument("--timeout", type=float, default=3600.0, help="seconds for each deck") + args = parser.parse_args(argv) + if args.kernel_only and args.coretype is None: + parser.error("--kernel-only needs --coretype") + + examples = args.examples.resolve() + decks = sorted(path for path in examples.glob("*.dat") if path.is_file()) + if not decks: + print(f"FAIL: no example decks in {examples}") + return 2 + command = [str(args.driver.resolve())] + out = args.out.resolve() + if args.kernel_only: + runs = scan(command, decks[:1], out, jobs=1, timeout=args.timeout, coretype=args.coretype) + problem = kernel_problem(runs, args.coretype) + print(f"FAIL: {problem}" if problem else f"OpenBLAS ran the {args.coretype} kernel") + return 1 if problem else 0 + runs = scan( + command, decks, out, jobs=args.jobs, timeout=args.timeout, coretype=args.coretype + ) + return report(runs, args.coretype) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_check_example_fallbacks.py b/tests/test_check_example_fallbacks.py new file mode 100644 index 00000000..059b3933 --- /dev/null +++ b/tests/test_check_example_fallbacks.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""The example scan of the round-off CI job must fail where it is meant to. + +A stand-in driver, whose behaviour each deck selects, exercises tests/check_example_fallbacks.py +as a child process would: every fallback note fails the scan and names its deck, also when wrapped +over two lines; a non-zero exit and a timeout fail it; a clean deck passes. With a forced kernel, +a run that names another kernel or none at all fails it. The built driver then shows the same on +the real OpenBLAS: an unknown kernel name is not reported as taken. + +Usage: test_check_example_fallbacks.py +""" + +from __future__ import annotations + +import contextlib +import importlib.util +import io +import sys +import tempfile +from pathlib import Path + +HERE = Path(__file__).resolve().parent +spec = importlib.util.spec_from_file_location("scan", HERE / "check_example_fallbacks.py") +assert spec is not None and spec.loader is not None +scan = importlib.util.module_from_spec(spec) +sys.modules["scan"] = scan # dataclasses look the module up by name +spec.loader.exec_module(scan) + +# Each deck holds what the driver prints to stderr, then a last line with its exit code. +FAKE_DRIVER = """ +import sys +import time +text = open(sys.argv[1], encoding="utf-8").read().split("|") +if text[-1] == "hang": + time.sleep(60) +sys.stderr.write(text[0]) +sys.exit(int(text[-1])) +""" + +NOTES = { + "newton": " Note: line 1: the discrete attachment equilibrium was reached after a failed " + "Newton solve (pseudo-transient continuation).\n", + "wrapped": " Note: line 1: reached after a failed\n Newton solve.\n", + "stalled": " Note: the Newton iteration stalled at the load fraction 0.5.\n", + "looser": " Note: a looser tolerance is needed.\n", + "floor": " Note: line 1: static residual floor above the dynamic tolerance 1.00E-08; " + "equilibrium accepted at the relative residual 2.00E-07.\n", + "torsion_floor": " Note: line 1: torsion static residual floor above the tolerance " + "1.00E-08; twisted equilibrium accepted at 2.00E-07.\n", + "substep": " Note: line 1: 3 step(s) kept the whole step because a resolving sub-step did " + "not converge.\n", +} +# Notes of a healthy run that must not trip the scan. +CLEAN = ( + " Note: LINE 1: dtM = 0.1 s resolves its axial period of 1 s in 10 steps.\n" + " Note: line 1: 2 attachment(s) lumped at the nearest nodes.\n" + " CableDyn initialization completed.\n" +) + + +def write_deck(directory: Path, name: str, console: str, status: str = "0") -> Path: + deck = directory / f"{name}.dat" + deck.write_text(f"{console}|{status}", encoding="utf-8") + return deck + + +def run_scan(command, decks, out, *, coretype=None, timeout=20.0): + """The exit status and the printed report of a scan.""" + text = io.StringIO() + with contextlib.redirect_stdout(text): + runs = scan.scan(command, decks, out, jobs=4, timeout=timeout, coretype=coretype) + status = scan.report(runs, coretype) + return status, text.getvalue() + + +def check(condition: bool, what: str, failures: list[str], detail: str = "") -> None: + print(f"{'ok' if condition else 'FAIL'}: {what}") + if not condition: + failures.append(f"{what}\n{detail}") + + +def main() -> int: + driver, examples = Path(sys.argv[1]), Path(sys.argv[2]) + failures: list[str] = [] + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + decks = root / "decks" + decks.mkdir() + fake = root / "fake_driver.py" + fake.write_text(FAKE_DRIVER, encoding="utf-8") + command = [sys.executable, str(fake)] + + clean = write_deck(decks, "clean", CLEAN) + status, text = run_scan(command, [clean], root / "clean") + check(status == 0 and "OFFENDS" not in text, "a clean deck passes", failures, text) + + for name, note in NOTES.items(): + deck = write_deck(decks, name, CLEAN + note) + status, text = run_scan(command, [clean, deck], root / name) + listed = f" {deck.name} (exit 0)" in text and f" {clean.name} (" not in text + check(status == 1 and listed, f"the note {name} fails its deck", failures, text) + + failed = write_deck(decks, "failed", CLEAN, "3") + status, text = run_scan(command, [failed], root / "failed") + check(status == 1 and "failed.dat (exit 3)" in text, "an exit status fails", failures, text) + + hung = write_deck(decks, "hung", CLEAN, "hang") + status, text = run_scan(command, [hung], root / "hung", timeout=2.0) + check(status == 1 and "hung.dat (no exit)" in text, "a timeout fails", failures, text) + + taken = write_deck(decks, "taken", "Core: Prescott\n" + CLEAN) + status, text = run_scan(command, [taken], root / "taken", coretype="Prescott") + check(status == 0, "the forced kernel passes", failures, text) + status, text = run_scan(command, [taken], root / "case", coretype="PRESCOTT") + check(status == 0, "kernel names match in any case", failures, text) + + other = write_deck(decks, "other", "Core not found: Prescott\nCore: Haswell\n" + CLEAN) + status, text = run_scan(command, [other], root / "other", coretype="Prescott") + check(status == 1 and "ran Haswell" in text, "another kernel fails", failures, text) + + status, text = run_scan(command, [clean], root / "none", coretype="Prescott") + check(status == 1 and "named no kernel" in text, "no kernel fails", failures, text) + + status, text = run_scan(command, [taken, other], root / "mixed", coretype="Prescott") + check(status == 1 and "ran Haswell" in text, "one wrong run fails", failures, text) + + # The real driver and OpenBLAS: a kernel that does not exist cannot be confirmed. + real = io.StringIO() + with contextlib.redirect_stdout(real): + status = scan.main( + [ + "--driver", + str(driver), + "--examples", + str(examples), + "--out", + str(root / "real"), + "--coretype", + "NoSuchKernel", + "--kernel-only", + ] + ) + check(status == 1, "an unknown kernel fails on the real driver", failures, real.getvalue()) + + for failure in failures: + print(f"\nFAILED: {failure}") + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main())