From 12992a9eff2f36fdd4a14c9d5c2d7a09cec26a6d Mon Sep 17 00:00:00 2001 From: Jae Hoon Seo Date: Mon, 5 Oct 2026 09:46:24 +0900 Subject: [PATCH 1/6] Report every abnormal end of the standalone driver A process that is interrupted or hits a fatal fault now writes one line to stderr naming the cause and the simulated time of the last committed step, then lets the event continue to the runtime's own handler, so the exit status is unchanged. On Windows a console-control handler, a vectored handler for fatal faults and an unhandled-exception filter (with a stack guarantee, so a stack overflow is reported) are used; on Linux and macOS signal handlers on an alternate stack. Only the driver installs them; the shared library leaves the host process alone. Every non-zero exit the driver makes itself now ends stderr with the closing line "CableDyn_driver: ended with exit code ". A process ended from outside (taskkill /F, End task, kill -9) runs none of its own code and cannot report anything; the missing closing line identifies it, since taskkill /F leaves exit code 1, the code of a refused input. The Python runner reports such a run as ended early, with the time its output reached. Tests: a stack overflow on a 1 MiB stack, a null-pointer write and SIGTERM each end the process with the report; the march records the time of its last committed step; the driver's failure checks require the closing line. --- ARCHITECTURE.md | 2 + CHANGELOG.md | 14 + CMakeLists.txt | 33 ++ app/cabledyn.f90 | 52 +++- doc/api_reference.md | 4 + doc/cli.rst | 7 +- doc/standalone_driver.rst | 49 ++- doc/troubleshooting.rst | 6 + python/cabledyn/driver.py | 58 +++- python/tests/conftest.py | 1 + python/tests/test_cli.py | 5 +- python/tests/test_driver.py | 81 +++++ src/CableDyn_DeckDriver.f90 | 8 +- src/CableDyn_FatalReport.f90 | 42 +++ src/cabledyn_fatal.c | 396 ++++++++++++++++++++++++ tests/check_driver_nonconvergence.cmake | 4 + tests/check_driver_output.cmake | 7 + tests/check_fatal_report.cmake | 41 +++ tests/fatal_report_faults.c | 29 ++ tests/test_fatal_report.f90 | 99 ++++++ 20 files changed, 912 insertions(+), 26 deletions(-) create mode 100644 src/CableDyn_FatalReport.f90 create mode 100644 src/cabledyn_fatal.c create mode 100644 tests/check_fatal_report.cmake create mode 100644 tests/fatal_report_faults.c create mode 100644 tests/test_fatal_report.f90 diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 6d2c02d9..8af4f8b5 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -394,6 +394,8 @@ src/ cabledyn_blas.c BLAS thread policy; run-time OpenBLAS loading (Windows GNU) cabledyn_mutex.c C-ABI registry and initialisation locks cabledyn_crt_locale.c MinGW-w64 guard for libgfortran's locale restore + CableDyn_FatalReport.f90 abnormal-end report of the driver (time reached) + cabledyn_fatal.c interrupt, fault and stack-overflow handlers of the driver EI = 0 cable path: CableDyn_Mesh.f90 connectivity, property, and DOF-partition validation diff --git a/CHANGELOG.md b/CHANGELOG.md index 26b0e781..86222753 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,20 @@ All notable changes to CableDyn are recorded in this file. The format follows ### Fixed +- The standalone driver no longer ends without a word when its process is interrupted or hits a + fatal fault. Ctrl+C, Ctrl+Break, closing the console window, `SIGINT`, `SIGTERM` and `SIGHUP` + are reported as `CableDyn_driver: stopped by `, and an access violation, a stack + overflow or another fatal fault as `CableDyn_driver: fatal error: `. Both reports give + the simulated time of the last committed step, and the exit status is unchanged. Previously a + stack overflow in a Windows GNU build ended the process with no message. +- Every non-zero exit the driver makes itself now ends stderr with the closing line + `CableDyn_driver: ended with exit code `. A process ended from outside (`taskkill /F`, + *End task*, `kill -9`) runs none of its own code and cannot report anything; the missing + closing line now identifies it, since `taskkill /F` leaves exit code 1, the same code as a + refused input. `CableDynDriver.run`, `cabledyn-run` and `cabledyn-study` report such a run as + ended early, with the time its output reached, instead of relaying the start-up log as the + error. The documentation explains how to recognise each kind of early end and warns that + `taskkill /IM CableDyn_driver.exe /F` ends every CableDyn run on the computer. - The static solve of a taut, neutrally buoyant finite-EI line (mass per length equal to the displaced mass) no longer depends on the sign of the round-off weight: a line whose total weight is below 1e-12 of its axial stiffness is seeded as a straight, uniformly diff --git a/CMakeLists.txt b/CMakeLists.txt index c084f075..931f1b0b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -329,7 +329,9 @@ add_library(cabledyn_obj OBJECT src/cabledyn_blas.c src/cabledyn_crt_locale.c src/cabledyn_path.c + src/cabledyn_fatal.c src/CableDyn_Precision.f90 + src/CableDyn_FatalReport.f90 src/CableDyn_PathIO.f90 src/CableDyn_Banner.f90 src/CableDyn_Linalg.f90 @@ -1807,6 +1809,37 @@ add_executable(test_crt_locale_guard tests/test_crt_locale_guard.f90 tests/crt_l target_link_libraries(test_crt_locale_guard PRIVATE cabledyn_core) add_test(NAME crt_locale_guard COMMAND test_crt_locale_guard $,1,0>) set_tests_properties(crt_locale_guard PROPERTIES LABELS "fortran;unit") +# An abnormal end of the driver process is never silent: a stack overflow (on a 1 MiB stack +# in Windows GNU builds), an access violation and, off Windows, SIGTERM each end the process +# with a report on stderr of the cause and the simulated time reached; the march records the +# time of every committed step for that report. +add_executable(test_fatal_report tests/test_fatal_report.f90 tests/fatal_report_faults.c) +target_link_libraries(test_fatal_report PRIVATE cabledyn_core) +if(CABLEDYN_GNU_WINDOWS_STACK) + target_link_options(test_fatal_report PRIVATE "-Wl,--stack,${CABLEDYN_SMALL_STACK_RESERVE}") +endif() +file(MAKE_DIRECTORY "${CMAKE_BINARY_DIR}/fatal_report") +add_test(NAME fatal_report_march + COMMAND test_fatal_report march ${CMAKE_CURRENT_SOURCE_DIR}/examples/dynamic_chain_held.dat + ${CMAKE_BINARY_DIR}/fatal_report/march) +set_tests_properties(fatal_report_march PROPERTIES LABELS "fortran;unit;driver;dynamic") +# The cause names the fault as each platform delivers it (macOS may deliver SIGBUS). +foreach(_fatal IN ITEMS "overflow|during|(stack overflow|segmentation fault|bus error)" + "overflow|before|(stack overflow|segmentation fault|bus error)" + "null|during|(access violation|segmentation fault|bus error)" + "term|during|a termination request") + string(REPLACE "|" ";" _fatal_case "${_fatal}") + list(GET _fatal_case 0 _fatal_mode) + list(GET _fatal_case 1 _fatal_when) + list(SUBLIST _fatal_case 2 -1 _fatal_cause) + string(JOIN "|" _fatal_cause ${_fatal_cause}) + add_test(NAME fatal_report_${_fatal_mode}_${_fatal_when} + COMMAND ${CMAKE_COMMAND} -DEXE=$ -DMODE=${_fatal_mode} + -DWHEN=${_fatal_when} "-DCAUSE=${_fatal_cause}" + -P ${CMAKE_CURRENT_SOURCE_DIR}/tests/check_fatal_report.cmake) + set_tests_properties(fatal_report_${_fatal_mode}_${_fatal_when} PROPERTIES + LABELS "fortran;unit;driver;stack" TIMEOUT 120) +endforeach() add_test( NAME driver_repeat_exit COMMAND ${CMAKE_COMMAND} diff --git a/app/cabledyn.f90 b/app/cabledyn.f90 index dc3fe3bf..1b2f5324 100644 --- a/app/cabledyn.f90 +++ b/app/cabledyn.f90 @@ -14,7 +14,10 @@ PROGRAM cabledyn !! BODIES and RODS -- and, when dtM/TMax are present, marches the dynamics while !! writing the requested OUTPUTS at each row (doc/cli.rst). !! - !! Exit codes: 0 converged, 1 bad/unparseable deck, 2 solve failure. + !! Exit codes: 0 converged, 1 bad/unparseable deck, 2 solve failure. Every non-zero exit + !! the driver makes itself ends stderr with the closing line "CableDyn_driver: ended with + !! exit code " (stop_run); an interrupt or a fatal fault is reported by + !! CableDyn_FatalReport with the simulated time reached. USE CableDyn_Precision, ONLY: wp, CD_ZERO USE CableDyn_DeckDriver, ONLY: CD_Run_Deck_Driver, CD_Deck_Query_dtM, & CD_Classify_Dynamic_Completion, CD_DECKDRV_OK, & @@ -30,6 +33,7 @@ PROGRAM cabledyn CD_AGG_Range_Sample, CD_AGG_Range_Write, CD_AGG_OK, CD_AGG_BADINPUT USE CableDyn_Banner, ONLY: CD_Print_Banner USE CableDyn_Linalg, ONLY: CD_Blas_Runtime_Check, CD_LINALG_OK + USE CableDyn_FatalReport, ONLY: CD_Fatal_Report_Install, CD_Fatal_Report_Time USE, INTRINSIC :: ISO_FORTRAN_ENV, ONLY: error_unit, output_unit, int64 IMPLICIT NONE TYPE :: StandaloneProgress @@ -78,14 +82,14 @@ PROGRAM cabledyn IF (deck_path(1:1) == '-') THEN WRITE (error_unit, '(A)') PROG//': unknown option "'//TRIM(deck_path)//'"' CALL print_usage(error_unit) - STOP 1, QUIET = .TRUE. + CALL stop_run(1) END IF END DO IF (nargs /= 2) THEN CALL CD_Print_Banner(error_unit) CALL print_usage(error_unit) - STOP 1, QUIET = .TRUE. + CALL stop_run(1) END IF CALL get_argument(1, deck_path) CALL get_argument(2, out_root) @@ -101,8 +105,13 @@ PROGRAM cabledyn CALL CD_Blas_Runtime_Check(PROG, ErrStat, ErrMsg) IF (ErrStat /= CD_LINALG_OK) THEN WRITE (error_unit, '(A)') TRIM(ErrMsg) - STOP 2, QUIET = .TRUE. + CALL stop_run(2) END IF + ! From here on an interrupt or a fatal fault (an access violation, a stack overflow) is + ! reported on stderr with the simulated time reached, instead of ending the process + ! without a word. Installed after the LAPACK runtime has loaded, so its handlers are the + ! process's own. + CALL CD_Fatal_Report_Install() CALL CD_Deck_Query_dtM(TRIM(deck_path), dtM, has_dtm, ErrStat, ErrMsg, tmax=tmax, & has_tmax=has_tmax, n_ei0=n_ei0, n_finite_ei=n_finite_ei, & has_motion_file=has_motion_file, standalone_scan=.TRUE., & @@ -125,9 +134,9 @@ PROGRAM cabledyn IF (ErrStat /= CD_DECKDRV_OK) THEN WRITE (error_unit, '(A)') TRIM(ErrMsg) IF (ErrStat == CD_DECKDRV_BADINPUT) THEN - STOP 1, QUIET = .TRUE. + CALL stop_run(1) ELSE - STOP 2, QUIET = .TRUE. + CALL stop_run(2) END IF END IF ! Keep the process-level contract fail-closed even if a future driver route @@ -138,12 +147,23 @@ PROGRAM cabledyn ELSE WRITE (error_unit, '(A)') 'CableDyn_driver: run did not converge; output is for inspection only.' END IF - STOP 2, QUIET = .TRUE. + CALL stop_run(2) END IF WRITE (output_unit, '(A,A,A)') PROG//': converged run written to ', TRIM(out_root), '.out' CONTAINS + SUBROUTINE stop_run(code) + !! End the process with a non-zero exit code, after the diagnostic already written. + !! The closing line is the last stderr record of every exit the driver makes itself, so + !! a caller can tell such an exit from a process ended from outside (Task Manager, + !! `taskkill /F`, `kill -9`), which runs no code of the driver and leaves no closing line. + INTEGER, INTENT(IN) :: code + WRITE (error_unit, '(A,I0)') PROG//': ended with exit code ', code + FLUSH (error_unit) + STOP code, QUIET = .TRUE. + END SUBROUTINE stop_run + SUBROUTINE get_argument(index, value) !! Command-line argument that fails closed when it cannot be retrieved whole !! (a path longer than the buffer would otherwise be silently truncated). @@ -155,14 +175,14 @@ SUBROUTINE get_argument(index, value) IF (arg_stat == -1) THEN WRITE (error_unit, '(A,I0,A,I0,A)') PROG//': argument ', index, ' is longer than ', LEN(value), & ' characters' - STOP 1, QUIET = .TRUE. + CALL stop_run(1) ELSE IF (arg_stat /= 0) THEN WRITE (error_unit, '(A,I0)') PROG//': cannot read command-line argument ', index - STOP 1, QUIET = .TRUE. + CALL stop_run(1) END IF IF (arg_len < 1) THEN WRITE (error_unit, '(A,I0,A)') PROG//': command-line argument ', index, ' is empty' - STOP 1, QUIET = .TRUE. + CALL stop_run(1) END IF END SUBROUTINE get_argument @@ -180,12 +200,12 @@ SUBROUTINE prepare_paths(deck, root, deck_spelling, root_spelling) CALL CD_Native_Path(deck, deck_spelling, stat, why) IF (stat /= CD_PATH_OK) THEN WRITE (error_unit, '(A)') PROG//': cannot read deck "'//deck//'": '//TRIM(why) - STOP 1, QUIET = .TRUE. + CALL stop_run(1) END IF CALL CD_Native_Path(root, root_spelling, stat, why, for_output=.TRUE., reserve=MAX_OUTPUT_SUFFIX) IF (stat /= CD_PATH_OK) THEN WRITE (error_unit, '(A)') PROG//': cannot write output files at "'//root//'": '//TRIM(why) - STOP 1, QUIET = .TRUE. + CALL stop_run(1) END IF END SUBROUTINE prepare_paths @@ -206,7 +226,7 @@ SUBROUTINE check_output_root(deck, deck_spelling, root, root_spelling) IF (ios /= 0) THEN WRITE (error_unit, '(A)') PROG//': cannot write output files at "'//root// & '" (check that the directory exists and is writable)' - STOP 1, QUIET = .TRUE. + CALL stop_run(1) END IF IF (exists) THEN CLOSE (unit) @@ -265,7 +285,7 @@ SUBROUTINE check_input_not_output(input, root, root_spelling, what, input_spelli WRITE (error_unit, '(A)') PROG//': output root "'//root//'" would overwrite the '//what//' "'// & input//'" that the deck reads' END IF - STOP 1, QUIET = .TRUE. + CALL stop_run(1) END IF END SUBROUTINE check_input_not_output @@ -337,7 +357,7 @@ SUBROUTINE lock_output_root(root) CASE DEFAULT WRITE (error_unit, '(A)') PROG//': cannot create the output lock file "'//root//LOCK_SUFFIX//'"' END SELECT - STOP 1, QUIET = .TRUE. + CALL stop_run(1) END SUBROUTINE lock_output_root LOGICAL FUNCTION same_file(a, b) RESULT(same) @@ -773,6 +793,8 @@ SUBROUTINE update_standalone_progress(progress, step, simulated_time) INTEGER(int64) :: now REAL(wp) :: elapsed, remaining, percent CHARACTER(16) :: elapsed_text, remaining_text + ! Every committed step: the time an abnormal end of the process reports. + CALL CD_Fatal_Report_Time(simulated_time) IF (step < progress%nstep .AND. MOD(step, progress%stride) /= 0) RETURN CALL SYSTEM_CLOCK(now) elapsed = CD_ZERO diff --git a/doc/api_reference.md b/doc/api_reference.md index e12f92f6..2bfb0122 100644 --- a/doc/api_reference.md +++ b/doc/api_reference.md @@ -176,6 +176,8 @@ The exchange contract is in {doc}`coupling_boundary`. - modal analysis about the static equilibrium (`nModes`, `.modes.out`) * - `CableDyn_PathIO` - UTF-8 file names converted to the spelling the Fortran runtime opens exactly +* - `CableDyn_FatalReport` + - the driver's report of an interrupt or a fatal fault, with the simulated time of the last committed step ``` ## C helper sources @@ -194,4 +196,6 @@ The exchange contract is in {doc}`coupling_boundary`. - process-local locks of the C ABI (handle registry and deck initialisation) * - `cabledyn_crt_locale.c` - a guard around the numeric-locale handling of the MinGW-w64 Fortran runtime +* - `cabledyn_fatal.c` + - the driver's report of an interrupt or a fatal fault with the simulated time reached (the counterpart of `CableDyn_FatalReport`) ``` diff --git a/doc/cli.rst b/doc/cli.rst index 05f34807..1dd9f598 100644 --- a/doc/cli.rst +++ b/doc/cli.rst @@ -169,8 +169,11 @@ Exit codes message names the library, each location tried and the Windows load error. The release ``CableDyn_driver.exe`` is statically linked and has no such dependency -Every non-zero exit is accompanied by a message on stderr. See :doc:`troubleshooting` for the -messages and their remedies. +Every non-zero exit the driver makes itself writes its message on stderr and ends stderr with +the closing line ``CableDyn_driver: ended with exit code ``. An interrupt or a fatal fault is +reported with the simulated time reached; a process ended from outside (``taskkill /F``, +``kill -9``) cannot report anything. See :ref:`run-ended-early` for how to recognise each case, +and :doc:`troubleshooting` for the messages and their remedies. Examples ~~~~~~~~ diff --git a/doc/standalone_driver.rst b/doc/standalone_driver.rst index e057be96..fc7acc25 100644 --- a/doc/standalone_driver.rst +++ b/doc/standalone_driver.rst @@ -224,8 +224,46 @@ Exit status and automation result became non-finite; or, in a Windows GNU build, ``openblas.dll`` could not be loaded at start-up. The ``.out`` keeps the converged prefix for inspection only -The reason is always in the stderr message. See :doc:`troubleshooting` for the message-by-message -fixes. +The reason is always in the stderr message, and the last stderr line of every such exit is the +closing line ``CableDyn_driver: ended with exit code ``. See :doc:`troubleshooting` for the +message-by-message fixes. + +.. _run-ended-early: + +When a run ends early +~~~~~~~~~~~~~~~~~~~~~ + +A run can also end without the driver choosing to. The output files then hold every row written +before the end, and their last time is below ``TMax``. + +.. list-table:: + :header-rows: 1 + :widths: 24 76 + + * - How it ended + - What you see + * - interrupted: Ctrl+C or Ctrl+Break, closing the console window, logging off or shutting + down, ``SIGINT``, ``SIGTERM`` or ``SIGHUP`` + - ``CableDyn_driver: stopped by after the step at simulated time t = s`` on + stderr, followed by the Fortran runtime's own line where it has one (the release + Windows executable adds ``forrtl: error (200)`` and exits with ``1``; a GNU build exits + with ``0xC000013A`` on Windows and with the signal on Linux and macOS). Rows written while + the process was stopping may extend slightly past the reported time + * - a fatal fault, such as an access violation or a stack overflow + - ``CableDyn_driver: fatal error: (exception ) after the step at simulated + time t = s`` on stderr, possibly followed by the runtime's own report (``forrtl: + severe (157)`` or ``(170)`` in the release Windows executable, which then exits with that + number). Otherwise the exit status is the exception code (Windows) or the signal. Please + report it, with the deck (see SUPPORT.md) + * - ended from outside: *End task* in Task Manager, ``taskkill /F`` or ``kill -9`` + - nothing. These end a process without running any of its code, so no program can report + them. ``taskkill /F`` leaves exit code ``1``, the code of a refused input, but without the + closing line; ``kill -9`` leaves ``SIGKILL`` + +To tell a completed run from one that ended early, check the exit status, the closing line, and +that the last time in ``.out`` reaches ``TMax``; the Python wrapper below reports +such an end. ``taskkill /IM CableDyn_driver.exe /F`` ends *every* CableDyn run on the computer, not one; to +stop a single run, end it by its process id (``taskkill /PID /F``). Output streams ~~~~~~~~~~~~~~ @@ -238,7 +276,9 @@ Output streams - Content * - ``stderr`` - the identity banner of a normal run; every error message; the initialisation report of a - single-family (all ``EI = 0`` or all finite-EI) deck + single-family (all ``EI = 0`` or all finite-EI) deck; the report of an interrupt or a + fatal fault; and, last on a failed run, the closing line + ``CableDyn_driver: ended with exit code `` * - ``stdout`` - ``--version``/``--help`` output; the mixed-deck initialisation summary; the ``Dynamic simulation:`` header and ``Progress:`` records with elapsed time and ETA; @@ -260,7 +300,8 @@ Python automation The :class:`cabledyn.CableDynDriver` wrapper applies these rules automatically: it finds the executable, defaults the working directory to the deck directory, captures diagnostics, checks the exit code, rejects stale output unless overwrite is explicit, and validates every numeric -row. See :doc:`python`. +row. A run that ended early raises :class:`cabledyn.DriverExecutionError` with a message that +says so and gives the time the output reached. See :doc:`python`. Choosing the standalone driver or OpenFAST ------------------------------------------- diff --git a/doc/troubleshooting.rst b/doc/troubleshooting.rst index 18d5bf10..7858414c 100644 --- a/doc/troubleshooting.rst +++ b/doc/troubleshooting.rst @@ -43,6 +43,12 @@ Exit codes at a glance The complete stream and exit-code contract is in :doc:`standalone_driver`. Error messages are on **stderr**. If you redirected it away (``2>/dev/null``), rerun without the redirect. +A run that stops before ``TMax`` with no error message and without the closing line +``CableDyn_driver: ended with exit code `` was ended from outside the driver, for example by +``taskkill /F`` (which leaves exit code ``1``) or *End task*; ``taskkill /IM CableDyn_driver.exe`` +ends every CableDyn run on the computer at once. An interrupt or a fatal fault is reported on +stderr with the simulated time reached. See :ref:`run-ended-early`. + Reading an error message ------------------------ diff --git a/python/cabledyn/driver.py b/python/cabledyn/driver.py index a127c262..3b1f24a2 100644 --- a/python/cabledyn/driver.py +++ b/python/cabledyn/driver.py @@ -64,6 +64,59 @@ def _timeout_stream(value: str | bytes | None) -> str: return value +# The last stderr line of every non-zero exit the driver makes itself. +_CLOSING_LINE = re.compile(r"CableDyn_driver: ended with exit code -?\d+\s*\Z") +# The driver's report of an interrupt or a fatal fault (src/cabledyn_fatal.c). +_ABNORMAL_REPORT = re.compile(r"^CableDyn_driver: (?:fatal error:|stopped by) .*$", re.MULTILINE) + + +def _last_output_time(main: Path, window: int = 1 << 20) -> float | None: + """Time of the last complete row of a main output table, or ``None``.""" + try: + with main.open("rb") as handle: + size = handle.seek(0, os.SEEK_END) + start = max(0, size - window) + handle.seek(start) + tail = handle.read() + except OSError: + return None + lines = tail.splitlines(keepends=True) + # A row is trusted only whole: ended by a line end, and begun inside the window. + first_whole = 0 if start == 0 else 1 + for line in reversed(lines[first_whole:]): + fields = line.split() + if not line.endswith(b"\n") or not fields: + continue + try: + value = float(fields[0]) + except ValueError: + return None + return value if math.isfinite(value) else None + return None + + +def _failure_message(returncode: int, stdout: str, stderr: str, main: Path) -> str: + """Describe a non-zero exit, telling the driver's own failures from an abnormal end.""" + closing = _CLOSING_LINE.search(stderr) + if closing: + # The closing line repeats the exit code; the diagnostic is what precedes it. + diagnostic = stderr[: closing.start()].strip() or stdout.strip() or "no diagnostic" + return f"CableDyn driver failed with exit code {returncode}: {diagnostic}" + last = _last_output_time(main) + reached = "" if last is None else f"; its output ends at t = {last:g} s" + report = _ABNORMAL_REPORT.findall(stderr) + if report: + return f"CableDyn driver ended abnormally (exit code {returncode}){reached}: {report[-1]}" + # The conclusion goes last: summaries show the last line of a message. + diagnostic = stderr.strip() or stdout.strip() + tail = "".join(f"{line}\n" for line in diagnostic.splitlines()[-20:]) + return ( + f"{tail}CableDyn driver ended with exit code {returncode} without reporting a " + f"result{reached}: the process was ended from outside (for example by Task Manager, " + "taskkill /F or kill -9) or by a fault it could not report." + ) + + @dataclass(frozen=True) class DriverResult: """Files and captured streams from one completed driver run. @@ -511,9 +564,10 @@ def _execute( stderr=_timeout_stream(exc.stderr), ) from exc if completed.returncode != 0: - diagnostic = completed.stderr.strip() or completed.stdout.strip() or "no diagnostic" raise DriverExecutionError( - f"CableDyn driver failed with exit code {completed.returncode}: {diagnostic}", + _failure_message( + completed.returncode, completed.stdout, completed.stderr, Path(f"{root}.out") + ), returncode=completed.returncode, stdout=completed.stdout, stderr=completed.stderr, diff --git a/python/tests/conftest.py b/python/tests/conftest.py index d2e8640a..a989766f 100644 --- a/python/tests/conftest.py +++ b/python/tests/conftest.py @@ -40,6 +40,7 @@ with open(root + ".out", "w") as stream: stream.write("Time(s) FairTen1\n0.0 1.0\n") sys.stderr.write("CableDyn_driver: deck line 3: synthetic failure\n") + sys.stderr.write("CableDyn_driver: ended with exit code 3\n") sys.exit(3) if mode == "no-output": sys.exit(0) diff --git a/python/tests/test_cli.py b/python/tests/test_cli.py index 2c10a29d..8f868f18 100644 --- a/python/tests/test_cli.py +++ b/python/tests/test_cli.py @@ -321,7 +321,10 @@ def test_cabledyn_study_failed_case_exits_one(fake_driver, tmp_path, capsys, mon assert "only: failed: CableDyn driver failed with exit code 3" in out assert (tmp_path / "study-results" / "only.stderr.log").read_text( encoding="utf-8" - ).strip() == "CableDyn_driver: deck line 3: synthetic failure" + ).strip().splitlines() == [ + "CableDyn_driver: deck line 3: synthetic failure", + "CableDyn_driver: ended with exit code 3", + ] def test_cabledyn_study_preflight_errors_exit_two(fake_driver, tmp_path, capsys): diff --git a/python/tests/test_driver.py b/python/tests/test_driver.py index a6652ef2..a9f4e53f 100644 --- a/python/tests/test_driver.py +++ b/python/tests/test_driver.py @@ -241,6 +241,87 @@ def test_nonzero_exit_preserves_native_diagnostic(tmp_path, monkeypatch): assert "Newton failed" in str(caught.value) +_CLOSING = "CableDyn_driver: ended with exit code 2\n" +_PARTIAL_TABLE = "# CableDyn\nTime\tFairTen1\n0.0\t1.0\n7534.6\t2.0\n7534.65\t2." + + +@pytest.mark.parametrize( + ("returncode", "stderr", "table", "expected"), + [ + # An exit the driver chose: its closing line ends stderr. + ( + 2, + " init\nstep 12 did not converge\n" + _CLOSING, + None, + "failed with exit code 2: init\nstep 12 did not converge", + ), + # An interrupt or a fault the driver reported, with the time its output reached. + ( + 3221225725, + " init\n\nCableDyn_driver: fatal error: stack overflow (exception 0xC00000FD) after " + "the step at simulated time t = 7534.600 s. The run did not finish; its output " + "files end at the last step written.\n", + _PARTIAL_TABLE, + "ended abnormally (exit code 3221225725); its output ends at t = 7534.6 s: " + "CableDyn_driver: fatal error: stack overflow", + ), + # Ended from outside (taskkill /F): no closing line and no report. + ( + 1, + " CableDyn initialization completed.\n", + _PARTIAL_TABLE, + "initialization completed.\nCableDyn driver ended with exit code 1 without " + "reporting a result; its output ends at t = 7534.6 s: the process was ended from " + "outside", + ), + # The same with no output table yet. + (-9, "", None, "ended with exit code -9 without reporting a result: the process"), + ], +) +def test_nonzero_exit_tells_driver_failures_from_abnormal_ends( + tmp_path, monkeypatch, returncode, stderr, table, expected +): + executable = tmp_path / "driver" + executable.write_bytes(b"placeholder") + executable.chmod(0o755) + deck = tmp_path / "model.dat" + deck.write_text("deck", encoding="ascii") + + def fake_run(*args, **kwargs): + if table is not None: + (tmp_path / "run.out").write_bytes(table.encode("ascii")) + return subprocess.CompletedProcess(args[0], returncode, " Progress: 65.0%\n", stderr) + + monkeypatch.setattr(subprocess, "run", fake_run) + with pytest.raises(DriverExecutionError) as caught: + CableDynDriver(executable).run(deck, "run") + assert caught.value.returncode == returncode + assert expected in str(caught.value) + assert caught.value.stderr == stderr + + +@pytest.mark.parametrize( + ("content", "window", "expected"), + [ + (b"Time\ta\n0.0\t1\n0.5\t2\n", 1 << 20, 0.5), + (b"Time\ta\n0.0\t1\n0.5\t2\n1.0\t", 1 << 20, 0.5), # last row cut short + (b"Time\ta\n0.0\t1\nnan\t2\n", 1 << 20, None), + (b"Time\ta\n0.0\t1\nq\t2\n", 1 << 20, None), + (b"Time\ta\n\n", 1 << 20, None), + # The window starts inside a row: that row is not trusted, the next whole one is. + (b"Time\ta\n123456.0\t1\n2.0\t2\n", 10, 2.0), + (b"Time\ta\n123456.0\t1\n", 10, None), + ], +) +def test_last_output_time_reads_only_whole_rows(tmp_path, content, window, expected): + from cabledyn.driver import _last_output_time + + path = tmp_path / "run.out" + path.write_bytes(content) + assert _last_output_time(path, window) == expected + assert _last_output_time(tmp_path / "missing.out") is None + + def test_element_table_is_protected_reported_and_replaced(tmp_path, monkeypatch): executable = tmp_path / "driver" executable.write_bytes(b"placeholder") diff --git a/src/CableDyn_DeckDriver.f90 b/src/CableDyn_DeckDriver.f90 index 4dc02be3..ee0fb538 100644 --- a/src/CableDyn_DeckDriver.f90 +++ b/src/CableDyn_DeckDriver.f90 @@ -31,6 +31,7 @@ MODULE CableDyn_DeckDriver !! scope. Features outside it parse but FAIL CLOSED with a clear error naming the unsupported !! feature. USE CableDyn_Precision, ONLY: wp, CD_ZERO, CD_ONE + USE CableDyn_FatalReport, ONLY: CD_Fatal_Report_Time USE CableDyn_Loads, ONLY: CD_Submerged_Weight, CD_Equivalent_Buoyant_Section, CD_Assemble_Distributed_Load USE CableDyn_Assemble, ONLY: CD_Compute_Cable_Tension, CD_Cable_Free_Bandwidth, & CD_Assemble_Cable_Tangent_Force_Banded_Free @@ -25770,8 +25771,9 @@ SUBROUTINE start_driver_progress(progress, nstep, dt) END SUBROUTINE start_driver_progress SUBROUTINE update_driver_progress(progress, step, simulated_time) - !! Report only committed steps. ETA is based on average elapsed wall time per - !! committed step, so failed/retried nonlinear work is honestly included. + !! Called after every committed step; prints at 5% spacing. ETA is based on average + !! elapsed wall time per committed step, so failed/retried nonlinear work is honestly + !! included. TYPE(DriverProgress), INTENT(IN) :: progress INTEGER, INTENT(IN) :: step REAL(wp), INTENT(IN) :: simulated_time @@ -25780,6 +25782,8 @@ SUBROUTINE update_driver_progress(progress, step, simulated_time) REAL(wp) :: elapsed, remaining, percent CHARACTER(16) :: elapsed_text, remaining_text + ! Every committed step: the time an abnormal end of the process reports. + CALL CD_Fatal_Report_Time(simulated_time) IF (progress%nstep <= 0 .OR. step <= 0) RETURN IF (step < progress%nstep .AND. MOD(step, progress%report_stride) /= 0) RETURN CALL SYSTEM_CLOCK(now) diff --git a/src/CableDyn_FatalReport.f90 b/src/CableDyn_FatalReport.f90 new file mode 100644 index 00000000..3582eb67 --- /dev/null +++ b/src/CableDyn_FatalReport.f90 @@ -0,0 +1,42 @@ +! File: src/CableDyn_FatalReport.f90 +! SPDX-License-Identifier: Apache-2.0 +! Copyright (c) 2026 Jae Hoon Seo, SMI Lab, Inha University +MODULE CableDyn_FatalReport + !! The standalone driver's last report when the process ends abnormally (src/cabledyn_fatal.c). + !! + !! CD_Fatal_Report_Install installs, for the driver process only, the handlers that write + !! one line to stderr when the run is interrupted (Ctrl+C, a closed console, a termination + !! signal) or ends in a fatal fault (an access violation, a stack overflow), naming the cause + !! and the simulated time of the last committed step. The event then continues to the + !! handler that had it before, so the exit status is unchanged. CD_Fatal_Report_Time records + !! that time; the time marches call it after every committed step. It only stores a number, + !! so a host that loads the library without installing the handlers is not affected. + USE, INTRINSIC :: ISO_C_BINDING, ONLY: C_DOUBLE + USE CableDyn_Precision, ONLY: wp + IMPLICIT NONE + PRIVATE + PUBLIC :: CD_Fatal_Report_Install, CD_Fatal_Report_Time + + INTERFACE + SUBROUTINE c_fatal_report_install() BIND(C, name='cabledyn_fatal_report_install') + END SUBROUTINE c_fatal_report_install + SUBROUTINE c_fatal_report_time(simulated_time) BIND(C, name='cabledyn_fatal_report_time') + IMPORT :: C_DOUBLE + REAL(C_DOUBLE), VALUE :: simulated_time + END SUBROUTINE c_fatal_report_time + END INTERFACE + +CONTAINS + + SUBROUTINE CD_Fatal_Report_Install() + !! Install the abnormal-end report for this process (idempotent). Executables only. + CALL c_fatal_report_install() + END SUBROUTINE CD_Fatal_Report_Install + + SUBROUTINE CD_Fatal_Report_Time(simulated_time) + !! Record the simulated time of the last committed step for the abnormal-end report. + REAL(wp), INTENT(IN) :: simulated_time + CALL c_fatal_report_time(REAL(simulated_time, C_DOUBLE)) + END SUBROUTINE CD_Fatal_Report_Time + +END MODULE CableDyn_FatalReport diff --git a/src/cabledyn_fatal.c b/src/cabledyn_fatal.c new file mode 100644 index 00000000..730f795c --- /dev/null +++ b/src/cabledyn_fatal.c @@ -0,0 +1,396 @@ +/* File: src/cabledyn_fatal.c + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2026 Jae Hoon Seo, SMI Lab, Inha University + * + * A last report from the standalone driver when the process ends abnormally. + * + * The driver reports every failure it detects itself on stderr. A process can also end + * without the program choosing to: an interrupt (Ctrl+C, closing the console window, a + * termination signal) or a fatal fault (an access violation, a stack overflow). The runtime + * libraries print nothing for some of these -- a Windows stack overflow in a GNU build ends + * the process without a word -- and none of them knows how far the simulation got. + * + * cabledyn_fatal_report_install() installs handlers that write one line to stderr naming the + * cause and the simulated time of the last committed step, then let the event continue to + * whatever handled it before (the Fortran runtime's own report, the operating system's + * default action), so the exit status is the one the event would have produced anyway. + * cabledyn_fatal_report_time() records that time; the time march calls it after each + * committed step. Only the driver installs the handlers: the shared library never changes + * the signal or exception handling of the host process that loads it. + * + * A process that is ended from outside by TerminateProcess (Task Manager, `taskkill /F`) or + * by SIGKILL (`kill -9`) runs none of its own code, so no program can report that; the + * driver's closing status line (app/cabledyn.f90) lets a caller recognise such an exit. + */ +/* sigaction(), sigaltstack() and SA_ONSTACK are outside strict C11; request the platform + * declarations. */ +#define _DEFAULT_SOURCE +#define _DARWIN_C_SOURCE +#include +#include + +void cabledyn_fatal_report_install(void); +void cabledyn_fatal_report_time(double simulated_time); +double cabledyn_fatal_report_last_time(void); + +/* Simulated time of the last committed step; negative until the first one. An aligned + * double is written in one store on every supported target, and the handlers only read it. */ +static volatile double cabledyn_last_time = -1.0; + +void cabledyn_fatal_report_time(double simulated_time) +{ + if (simulated_time == simulated_time && simulated_time >= 0.0) { + cabledyn_last_time = simulated_time; + } +} + +/* The recorded time (negative before the first committed step); for the tests. */ +double cabledyn_fatal_report_last_time(void) +{ + return cabledyn_last_time; +} + +/* ---- message assembly: no allocation, no stdio, usable from a signal handler ---------- */ + +typedef struct { + char text[512]; + size_t len; +} cabledyn_line; + +static void line_add(cabledyn_line *line, const char *s) +{ + while (*s != '\0' && line->len + 1 < sizeof line->text) { + line->text[line->len++] = *s++; + } + line->text[line->len] = '\0'; +} + +static void line_add_unsigned(cabledyn_line *line, unsigned long long value, int min_digits) +{ + char digits[24]; + int n = 0; + do { + digits[n++] = (char)('0' + (int)(value % 10ULL)); + value /= 10ULL; + } while (value != 0ULL && n < (int)sizeof digits); + while (n < min_digits && n < (int)sizeof digits) { + digits[n++] = '0'; + } + while (n > 0 && line->len + 1 < sizeof line->text) { + line->text[line->len++] = digits[--n]; + } + line->text[line->len] = '\0'; +} + +static void line_add_hex(cabledyn_line *line, unsigned long value) +{ + static const char hex[] = "0123456789ABCDEF"; + char digits[2 * sizeof value]; + int n; + line_add(line, "0x"); + for (n = (int)(2 * sizeof value) - 1; n >= 0; --n) { + digits[n] = hex[value & 0xFUL]; + value >>= 4; + } + for (n = 0; n < (int)(2 * sizeof value) - 8 && digits[n] == '0'; ++n) { + } + for (; n < (int)(2 * sizeof value) && line->len + 1 < sizeof line->text; ++n) { + line->text[line->len++] = digits[n]; + } + line->text[line->len] = '\0'; +} + +/* "after the step at simulated time t = 7534.600 s" or the pre-march wording. */ +static void line_add_when(cabledyn_line *line) +{ + double t = cabledyn_last_time; + unsigned long long millis; + if (!(t >= 0.0) || t > 1.0e15) { + line_add(line, "before the first time step"); + return; + } + millis = (unsigned long long)(t * 1000.0 + 0.5); + line_add(line, "after the step at simulated time t = "); + line_add_unsigned(line, millis / 1000ULL, 1); + line_add(line, "."); + line_add_unsigned(line, millis % 1000ULL, 3); + line_add(line, " s"); +} + +static void line_finish(cabledyn_line *line) +{ + line_add(line, ". The run did not finish; its output files end at the last step written.\n"); +} + +#if defined(_WIN32) +#include + +static LPTOP_LEVEL_EXCEPTION_FILTER cabledyn_previous_filter = NULL; +/* Set by the first report, so one event that reaches two handlers is reported once. */ +static volatile LONG cabledyn_reported = 0; + +static void write_stderr(const cabledyn_line *line) +{ + HANDLE err = GetStdHandle(STD_ERROR_HANDLE); + DWORD written = 0; + if (err != NULL && err != INVALID_HANDLE_VALUE) { + WriteFile(err, line->text, (DWORD)line->len, &written, NULL); + } +} + +static const char *exception_name(DWORD code) +{ + switch (code) { + case EXCEPTION_STACK_OVERFLOW: + return "stack overflow"; + case EXCEPTION_ACCESS_VIOLATION: + return "access violation"; + case EXCEPTION_IN_PAGE_ERROR: + return "in-page error"; + case EXCEPTION_ILLEGAL_INSTRUCTION: + return "illegal instruction"; + case EXCEPTION_PRIV_INSTRUCTION: + return "privileged instruction"; + case EXCEPTION_INT_DIVIDE_BY_ZERO: + return "integer division by zero"; + case EXCEPTION_INT_OVERFLOW: + return "integer overflow"; + case EXCEPTION_FLT_DIVIDE_BY_ZERO: + return "floating-point division by zero"; + case EXCEPTION_FLT_INVALID_OPERATION: + return "invalid floating-point operation"; + case EXCEPTION_FLT_OVERFLOW: + return "floating-point overflow"; + case EXCEPTION_FLT_UNDERFLOW: + return "floating-point underflow"; + case EXCEPTION_FLT_INEXACT_RESULT: + return "inexact floating-point result"; + case EXCEPTION_FLT_DENORMAL_OPERAND: + return "denormal floating-point operand"; + case EXCEPTION_FLT_STACK_CHECK: + return "floating-point stack check"; + case EXCEPTION_DATATYPE_MISALIGNMENT: + return "misaligned data access"; + case EXCEPTION_ARRAY_BOUNDS_EXCEEDED: + return "array bounds exceeded"; + case EXCEPTION_NONCONTINUABLE_EXCEPTION: + return "non-continuable exception"; + case 0xC0000409UL: /* STATUS_STACK_BUFFER_OVERRUN: also raised by abort() / fast-fail */ + return "fast-fail abort"; + case 0xC0000374UL: /* STATUS_HEAP_CORRUPTION */ + return "heap corruption"; + default: + return "unhandled exception"; + } +} + +/* Faults no part of the driver recovers from. Floating-point exceptions are not trapped by + * default; when a build enables traps, the Fortran runtime reports them itself. */ +static int is_fatal_fault(DWORD code) +{ + switch (code) { + case EXCEPTION_STACK_OVERFLOW: + case EXCEPTION_ACCESS_VIOLATION: + case EXCEPTION_IN_PAGE_ERROR: + case EXCEPTION_ILLEGAL_INSTRUCTION: + case EXCEPTION_PRIV_INSTRUCTION: + case EXCEPTION_INT_DIVIDE_BY_ZERO: + case 0xC0000409UL: + case 0xC0000374UL: + return 1; + default: + return 0; + } +} + +static void report_exception(DWORD code) +{ + if (InterlockedExchange(&cabledyn_reported, 1) == 0) { + cabledyn_line line; + line.len = 0; + line.text[0] = '\0'; + line_add(&line, "\nCableDyn_driver: fatal error: "); + line_add(&line, exception_name(code)); + line_add(&line, " (exception "); + line_add_hex(&line, (unsigned long)code); + line_add(&line, ") "); + line_add_when(&line); + line_finish(&line); + write_stderr(&line); + } +} + +/* First in line for the fatal faults: the GNU runtime handles an access violation in a + * frame-based handler (its SIGSEGV backtrace) that ends the process before any unhandled- + * exception filter runs. The search continues afterwards, so the runtime still reports. */ +static LONG WINAPI cabledyn_vectored_handler(EXCEPTION_POINTERS *info) +{ + if (info != NULL && info->ExceptionRecord != NULL && + is_fatal_fault(info->ExceptionRecord->ExceptionCode)) { + report_exception(info->ExceptionRecord->ExceptionCode); + } + return EXCEPTION_CONTINUE_SEARCH; +} + +/* Any other exception nothing handled. */ +static LONG WINAPI cabledyn_exception_filter(EXCEPTION_POINTERS *info) +{ + if (info != NULL && info->ExceptionRecord != NULL) { + report_exception(info->ExceptionRecord->ExceptionCode); + } + if (cabledyn_previous_filter != NULL) { + return cabledyn_previous_filter(info); + } + return EXCEPTION_CONTINUE_SEARCH; +} + +static BOOL WINAPI cabledyn_console_handler(DWORD event) +{ + const char *cause; + switch (event) { + case CTRL_C_EVENT: + cause = "Ctrl+C"; + break; + case CTRL_BREAK_EVENT: + cause = "Ctrl+Break"; + break; + case CTRL_CLOSE_EVENT: + cause = "closing its console window"; + break; + case CTRL_LOGOFF_EVENT: + cause = "the user logging off"; + break; + case CTRL_SHUTDOWN_EVENT: + cause = "the system shutting down"; + break; + default: + return FALSE; + } + if (InterlockedExchange(&cabledyn_reported, 1) == 0) { + cabledyn_line line; + line.len = 0; + line.text[0] = '\0'; + line_add(&line, "\nCableDyn_driver: stopped by "); + line_add(&line, cause); + line_add(&line, " "); + line_add_when(&line); + line_finish(&line); + write_stderr(&line); + } + /* Not handled here: the next handler (the Fortran runtime's, or the default one) ends + * the process with its usual status. */ + return FALSE; +} + +void cabledyn_fatal_report_install(void) +{ + static volatile LONG installed = 0; + ULONG guarantee = 64UL * 1024UL; + if (InterlockedExchange(&installed, 1) != 0) { + return; + } + /* Keep room on the main thread's stack for the handlers to run after a stack overflow. */ + SetThreadStackGuarantee(&guarantee); + AddVectoredExceptionHandler(1, cabledyn_vectored_handler); + cabledyn_previous_filter = SetUnhandledExceptionFilter(cabledyn_exception_filter); + SetConsoleCtrlHandler(cabledyn_console_handler, TRUE); +} + +#else /* POSIX */ +#include +#include + +#define CABLEDYN_NSIG 8 +static const int cabledyn_signals[CABLEDYN_NSIG] = {SIGSEGV, SIGBUS, SIGFPE, SIGILL, + SIGABRT, SIGINT, SIGTERM, SIGHUP}; +static struct sigaction cabledyn_previous[CABLEDYN_NSIG]; +static volatile sig_atomic_t cabledyn_reported = 0; +/* The alternate stack the handlers run on, so a stack overflow can still be reported. */ +static char cabledyn_altstack[64 * 1024]; + +static const char *signal_name(int sig) +{ + switch (sig) { + case SIGSEGV: + return "segmentation fault (SIGSEGV; a stack overflow also ends this way)"; + case SIGBUS: + return "bus error (SIGBUS)"; + case SIGFPE: + return "floating-point exception (SIGFPE)"; + case SIGILL: + return "illegal instruction (SIGILL)"; + case SIGABRT: + return "abort (SIGABRT)"; + case SIGINT: + return "an interrupt (SIGINT, Ctrl+C)"; + case SIGTERM: + return "a termination request (SIGTERM)"; + case SIGHUP: + return "a hang-up (SIGHUP, terminal closed)"; + default: + return "a signal"; + } +} + +static void cabledyn_signal_handler(int sig) +{ + int k; + if (!cabledyn_reported) { + cabledyn_line line; + int fault = sig == SIGSEGV || sig == SIGBUS || sig == SIGFPE || sig == SIGILL || + sig == SIGABRT; + cabledyn_reported = 1; + line.len = 0; + line.text[0] = '\0'; + line_add(&line, fault ? "\nCableDyn_driver: fatal error: " : "\nCableDyn_driver: stopped by "); + line_add(&line, signal_name(sig)); + line_add(&line, " "); + line_add_when(&line); + line_finish(&line); + { + ssize_t ignored = write(STDERR_FILENO, line.text, line.len); + (void)ignored; + } + } + /* Hand the signal to whatever handled it before (the Fortran runtime's backtrace or the + * default action), so the exit status is unchanged. SA_NODEFER delivers it at once. */ + for (k = 0; k < CABLEDYN_NSIG; ++k) { + if (cabledyn_signals[k] == sig) { + sigaction(sig, &cabledyn_previous[k], NULL); + break; + } + } + raise(sig); +} + +void cabledyn_fatal_report_install(void) +{ + static int installed = 0; + stack_t alt; + struct sigaction action; + int k; + if (installed) { + return; + } + installed = 1; + memset(&alt, 0, sizeof alt); + alt.ss_sp = cabledyn_altstack; + alt.ss_size = sizeof cabledyn_altstack; + alt.ss_flags = 0; + sigaltstack(&alt, NULL); + memset(&action, 0, sizeof action); + action.sa_handler = cabledyn_signal_handler; + sigemptyset(&action.sa_mask); + action.sa_flags = SA_ONSTACK | SA_NODEFER; + for (k = 0; k < CABLEDYN_NSIG; ++k) { + if (sigaction(cabledyn_signals[k], NULL, &cabledyn_previous[k]) != 0) { + continue; + } + /* A signal the parent set to be ignored (nohup, background job) stays ignored. */ + if (cabledyn_previous[k].sa_handler == SIG_IGN) { + continue; + } + sigaction(cabledyn_signals[k], &action, NULL); + } +} +#endif diff --git a/tests/check_driver_nonconvergence.cmake b/tests/check_driver_nonconvergence.cmake index c78df2d2..8e00f23b 100644 --- a/tests/check_driver_nonconvergence.cmake +++ b/tests/check_driver_nonconvergence.cmake @@ -16,6 +16,10 @@ if(NOT driver_result EQUAL 2) "stdout:\n${driver_stdout}\nstderr:\n${driver_stderr}") endif() +if(NOT driver_stderr MATCHES "CableDyn_driver: ended with exit code 2[\r\n]*$") + message(FATAL_ERROR "stderr does not end with the closing line for exit code 2:\n${driver_stderr}") +endif() + set(driver_text "${driver_stdout}\n${driver_stderr}") if(NOT driver_text MATCHES "did not converge" OR NOT driver_text MATCHES "inspection only") diff --git a/tests/check_driver_output.cmake b/tests/check_driver_output.cmake index 81887a20..2c49c0fd 100644 --- a/tests/check_driver_output.cmake +++ b/tests/check_driver_output.cmake @@ -28,6 +28,13 @@ if(NOT driver_result EQUAL EXPECT_RC) message(FATAL_ERROR "driver returned ${driver_result}, expected ${EXPECT_RC}\n${driver_text}") endif() +# Every exit the driver makes itself with a non-zero code ends stderr with its closing line, +# which tells such an exit from a process ended from outside. +if(NOT EXPECT_RC EQUAL 0 AND + NOT driver_stderr MATCHES "CableDyn_driver: ended with exit code ${EXPECT_RC}[\r\n]*$") + message(FATAL_ERROR + "stderr does not end with the closing line for exit code ${EXPECT_RC}:\n${driver_stderr}") +endif() foreach(pattern IN LISTS MUST_MATCH) if(NOT driver_text MATCHES "${pattern}") message(FATAL_ERROR "driver output does not match \"${pattern}\":\n${driver_text}") diff --git a/tests/check_fatal_report.cmake b/tests/check_fatal_report.cmake new file mode 100644 index 00000000..fd5e7a93 --- /dev/null +++ b/tests/check_fatal_report.cmake @@ -0,0 +1,41 @@ +# SPDX-License-Identifier: Apache-2.0 +# An abnormal end of the driver process is never silent: run test_fatal_report with one +# fault and require a non-zero exit status together with the report line on stderr that +# names the cause and the simulated time reached. +# EXE the test_fatal_report executable +# MODE overflow | null | term +# WHEN before | during (whether a simulated time was recorded) +# CAUSE regular expression the cause must match +if(NOT DEFINED EXE OR NOT DEFINED MODE OR NOT DEFINED WHEN OR NOT DEFINED CAUSE) + message(FATAL_ERROR "EXE, MODE, WHEN and CAUSE are required") +endif() + +execute_process( + COMMAND "${EXE}" "${MODE}" "${WHEN}" + RESULT_VARIABLE result + OUTPUT_VARIABLE out + ERROR_VARIABLE err) + +if(out MATCHES "SKIP:") + message(STATUS "${out}") + return() +endif() +if("${result}" STREQUAL "0" OR err MATCHES "FAIL: test_fatal_report") + message(FATAL_ERROR "the ${MODE} fault did not end the process abnormally (status ${result}):\n" + "stdout:\n${out}\nstderr:\n${err}") +endif() +if(WHEN STREQUAL "during") + set(when_text "after the step at simulated time t = 7534\\.600 s") +else() + set(when_text "before the first time step") +endif() +set(expected "CableDyn_driver: (fatal error: |stopped by )${CAUSE}[^\n]* ${when_text}\\. The run did not finish") +if(NOT err MATCHES "${expected}") + message(FATAL_ERROR "status ${result}, but stderr lacks the report \"${expected}\":\n${err}") +endif() +string(REGEX MATCHALL "CableDyn_driver: (fatal error|stopped by)" reports "${err}") +list(LENGTH reports nreports) +if(NOT nreports EQUAL 1) + message(FATAL_ERROR "the event was reported ${nreports} times, expected once:\n${err}") +endif() +message(STATUS "status ${result}; stderr:\n${err}") diff --git a/tests/fatal_report_faults.c b/tests/fatal_report_faults.c new file mode 100644 index 00000000..4ff24705 --- /dev/null +++ b/tests/fatal_report_faults.c @@ -0,0 +1,29 @@ +/* File: tests/fatal_report_faults.c + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2026 Jae Hoon Seo, SMI Lab, Inha University + * + * Faults that tests/test_fatal_report.f90 provokes to check the abnormal-end report. + */ +#include + +void fatal_report_test_null_write(void); +int fatal_report_test_raise_term(void); + +static volatile int *volatile fatal_report_null_target = 0; + +/* A write through a null pointer: an access violation / SIGSEGV. */ +void fatal_report_test_null_write(void) +{ + *fatal_report_null_target = 1; +} + +/* SIGTERM to this process; returns 0 where the report does not handle it (Windows). */ +int fatal_report_test_raise_term(void) +{ +#if defined(_WIN32) + return 0; +#else + raise(SIGTERM); + return 1; +#endif +} diff --git a/tests/test_fatal_report.f90 b/tests/test_fatal_report.f90 new file mode 100644 index 00000000..badda91d --- /dev/null +++ b/tests/test_fatal_report.f90 @@ -0,0 +1,99 @@ +! File: tests/test_fatal_report.f90 +! SPDX-License-Identifier: Apache-2.0 +! Copyright (c) 2026 Jae Hoon Seo, SMI Lab, Inha University +PROGRAM test_fatal_report + !! The driver's abnormal-end report (src/cabledyn_fatal.c, CableDyn_FatalReport). + !! + !! test_fatal_report march + !! runs a dynamic deck and checks that the march recorded the time of its last + !! committed step (TMax) for the report; + !! test_fatal_report before|during + !! installs the report, optionally records a simulated time, and ends the process + !! with : overflow (unbounded recursion, a stack overflow), null (a write + !! through a null pointer) or term (SIGTERM; POSIX only). tests/check_fatal_report.cmake + !! checks the exit status and the stderr line. + USE, INTRINSIC :: ISO_C_BINDING, ONLY: C_DOUBLE, C_INT + USE, INTRINSIC :: ISO_FORTRAN_ENV, ONLY: error_unit + USE CableDyn_Precision, ONLY: wp + USE CableDyn_FatalReport, ONLY: CD_Fatal_Report_Install, CD_Fatal_Report_Time + USE CableDyn_DeckDriver, ONLY: CD_Run_Deck_Driver, CD_DECKDRV_OK + IMPLICIT NONE + + INTERFACE + REAL(C_DOUBLE) FUNCTION last_time() BIND(C, name='cabledyn_fatal_report_last_time') + IMPORT :: C_DOUBLE + END FUNCTION last_time + SUBROUTINE null_write() BIND(C, name='fatal_report_test_null_write') + END SUBROUTINE null_write + INTEGER(C_INT) FUNCTION raise_term() BIND(C, name='fatal_report_test_raise_term') + IMPORT :: C_INT + END FUNCTION raise_term + END INTERFACE + + CHARACTER(4096) :: mode, arg2, arg3 + CHARACTER(1024) :: msg + LOGICAL :: converged + INTEGER :: stat + + CALL GET_COMMAND_ARGUMENT(1, mode) + CALL GET_COMMAND_ARGUMENT(2, arg2) + CALL GET_COMMAND_ARGUMENT(3, arg3) + + IF (mode == 'march') THEN + IF (last_time() >= 0.0_C_DOUBLE) CALL fail('a time is recorded before any step') + ! Negative and NaN times are not committed steps and leave the record unchanged. + CALL CD_Fatal_Report_Time(-1.0_wp) + IF (last_time() >= 0.0_C_DOUBLE) CALL fail('a negative time was recorded') + CALL CD_Run_Deck_Driver(TRIM(arg2), TRIM(arg3), converged, stat, msg) + IF (stat /= CD_DECKDRV_OK .OR. .NOT. converged) CALL fail('the deck did not run: '//TRIM(msg)) + ! examples/dynamic_chain_held.dat: TMax 10 s. + IF (ABS(last_time() - 10.0_C_DOUBLE) > 1.0E-9_C_DOUBLE) THEN + WRITE (msg, '(A,ES23.15)') 'the march recorded t = ', last_time() + CALL fail(TRIM(msg)//', expected the last committed step at TMax = 10 s') + END IF + WRITE (*, '(A)') 'PASS: the march records the time of its last committed step' + STOP + END IF + + CALL CD_Fatal_Report_Install() + CALL CD_Fatal_Report_Install() ! idempotent + IF (arg2 == 'during') THEN + CALL CD_Fatal_Report_Time(0.05_wp) + CALL CD_Fatal_Report_Time(7534.6_wp) + END IF + SELECT CASE (TRIM(mode)) + CASE ('overflow') + WRITE (error_unit, '(A,I0)') 'unreachable: ', deep(1) + CASE ('null') + CALL null_write() + CASE ('term') + IF (raise_term() == 0_C_INT) THEN + WRITE (*, '(A)') 'SKIP: no SIGTERM on this platform' + STOP + END IF + CASE DEFAULT + CALL fail('unknown mode "'//TRIM(mode)//'"') + END SELECT + CALL fail('the '//TRIM(mode)//' fault did not end the process') + +CONTAINS + + RECURSIVE INTEGER FUNCTION deep(depth) RESULT(r) + !! 64 KiB of stack per level without bound: a stack overflow on any stack size. + INTEGER, INTENT(IN) :: depth + INTEGER :: frame(16384) + frame = depth + IF (depth < 0) THEN + r = 0 + RETURN + END IF + r = deep(depth + 1) + frame(MOD(depth, 16384) + 1) + END FUNCTION deep + + SUBROUTINE fail(text) + CHARACTER(*), INTENT(IN) :: text + WRITE (error_unit, '(A)') 'FAIL: test_fatal_report: '//text + ERROR STOP 3 + END SUBROUTINE fail + +END PROGRAM test_fatal_report From 367d7b42df86994181674b973a0f1027a3b8b7ed Mon Sep 17 00:00:00 2001 From: Jae Hoon Seo Date: Mon, 5 Oct 2026 11:11:58 +0900 Subject: [PATCH 2/6] Prepare worker threads and keep the fault context in the abnormal-end report Every OpenMP parallel region of the library now starts with CD_Fatal_Thread_Init, which gives the calling thread room to report its own stack overflow: a stack guarantee on Windows, an alternate signal stack (freed when the thread ends) on Linux and macOS. It does nothing until the driver has installed the report, and costs one thread-local test after a thread's first call. Previously only the installing thread had that room, so an overflow on a worker thread could still end the process without a word. On Linux and macOS the handlers are now three-argument (SA_SIGINFO) handlers. A hardware fault returns from the handler under the previous disposition, so the faulting instruction raises the same fault with its own address and context; a previous three-argument handler is called with the original siginfo and context. Only asynchronous signals and faults sent by kill or raise are sent again. Tests: a stack overflow on an OpenMP worker with a 1 MiB stack is reported (it ended silently without the per-thread setup); a forked child with an earlier SA_SIGINFO handler faults at a known address and that handler receives the kernel's own siginfo, the child ends by the fault's signal, and the report is written once; and every OpenMP parallel region in src/ and app/ is checked to begin with the per-thread call. --- CHANGELOG.md | 5 +- CMakeLists.txt | 22 ++++ src/CableDyn_Assemble.f90 | 3 + src/CableDyn_CosseratAssemble.f90 | 4 + src/CableDyn_FatalReport.f90 | 10 +- src/CableDyn_HermiteCableDynamic.f90 | 2 + src/CableDyn_HermiteCableStatic.f90 | 3 + src/CableDyn_OpenFAST_Aggregate.f90 | 3 + src/CableDyn_System.f90 | 3 + src/cabledyn_fatal.c | 155 +++++++++++++++++++++++---- tests/fatal_report_faults.c | 135 +++++++++++++++++++++++ tests/test_fatal_report.f90 | 48 ++++++++- tests/test_omp_fatal_init.py | 99 +++++++++++++++++ 13 files changed, 464 insertions(+), 28 deletions(-) create mode 100644 tests/test_omp_fatal_init.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 86222753..8ddab333 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,7 +25,10 @@ All notable changes to CableDyn are recorded in this file. The format follows are reported as `CableDyn_driver: stopped by `, and an access violation, a stack overflow or another fatal fault as `CableDyn_driver: fatal error: `. Both reports give the simulated time of the last committed step, and the exit status is unchanged. Previously a - stack overflow in a Windows GNU build ended the process with no message. + stack overflow in a Windows GNU build ended the process with no message. Every thread that + enters an OpenMP parallel region keeps room to report its own stack overflow, and on Linux + and macOS a hardware fault reaches the handler that was there before (the Fortran runtime's + backtrace, or the default action and its core dump) with its original address and context. - Every non-zero exit the driver makes itself now ends stderr with the closing line `CableDyn_driver: ended with exit code `. A process ended from outside (`taskkill /F`, *End task*, `kill -9`) runs none of its own code and cannot report anything; the missing diff --git a/CMakeLists.txt b/CMakeLists.txt index 931f1b0b..adcf6241 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1321,6 +1321,13 @@ if(Python3_Interpreter_FOUND) LABELS "python;documentation;examples" ENVIRONMENT "${_cabledyn_python_test_environment}") + # Every OpenMP parallel region prepares its threads for the abnormal-end report. + add_test(NAME omp_fatal_init + COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_omp_fatal_init.py) + set_tests_properties(omp_fatal_init PROPERTIES + LABELS "python;unit;stack" + ENVIRONMENT "${_cabledyn_python_test_environment}") + add_test(NAME documentation_sources COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tests/test_documentation.py) set_tests_properties(documentation_sources PROPERTIES @@ -1840,6 +1847,21 @@ foreach(_fatal IN ITEMS "overflow|during|(stack overflow|segmentation fault|bus set_tests_properties(fatal_report_${_fatal_mode}_${_fatal_when} PROPERTIES LABELS "fortran;unit;driver;stack" TIMEOUT 120) endforeach() +# A stack overflow on an OpenMP worker thread with a 1 MiB stack is reported too: every +# thread that enters a parallel region gets room for the handlers (tests/test_omp_fatal_init.py +# checks that every region of the library and the driver asks for it). +add_test(NAME fatal_report_omp_overflow_during + COMMAND ${CMAKE_COMMAND} -DEXE=$ -DMODE=omp_overflow + -DWHEN=during "-DCAUSE=(stack overflow|segmentation fault|bus error)" + -P ${CMAKE_CURRENT_SOURCE_DIR}/tests/check_fatal_report.cmake) +set_tests_properties(fatal_report_omp_overflow_during PROPERTIES + LABELS "fortran;unit;driver;stack" TIMEOUT 120 + ENVIRONMENT "OMP_NUM_THREADS=2;OMP_STACKSIZE=1M") +# POSIX: a hardware fault keeps its own siginfo and context through the report. +add_test(NAME fatal_report_fault_context COMMAND test_fatal_report context) +set_tests_properties(fatal_report_fault_context PROPERTIES + LABELS "fortran;unit;driver;stack" TIMEOUT 120 + FAIL_REGULAR_EXPRESSION "FAIL:") add_test( NAME driver_repeat_exit COMMAND ${CMAKE_COMMAND} diff --git a/src/CableDyn_Assemble.f90 b/src/CableDyn_Assemble.f90 index 1e15a9c7..0756e308 100644 --- a/src/CableDyn_Assemble.f90 +++ b/src/CableDyn_Assemble.f90 @@ -21,6 +21,7 @@ MODULE CableDyn_Assemble !! This module adds NO physics -- gravity, buoyancy, seabed, damping, drag, and !! the Newton solve live in later modules. USE CableDyn_Precision, ONLY: wp, CD_ZERO, CD_ONE, CD_All_Finite + USE CableDyn_FatalReport, ONLY: CD_Fatal_Thread_Init USE CableDyn_CableElem, ONLY: CD_Compute_Cable_Element USE CableDyn_Mesh, ONLY: CD_Validate_Connectivity, CD_Validate_Positive, CD_Validate_NonNegative !$ USE OMP_LIB, ONLY: omp_in_parallel, omp_get_max_threads @@ -380,6 +381,7 @@ SUBROUTINE CD_Assemble_Cable_Internal_Force(nodes, elem_conn, l0, ea, tension_on !$OMP SHARED(n_elem, elem_conn, nodes, ea, l0, tension_only, fe, elem_es) & !$OMP PRIVATE(e, a, b, nodes6, Kt6, Te, em) DO e = 1, n_elem + CALL CD_Fatal_Thread_Init() ! this thread can report its own stack overflow a = elem_conn(1, e) b = elem_conn(2, e) nodes6(1:3) = nodes(:, a) @@ -460,6 +462,7 @@ SUBROUTINE CD_Compute_Cable_Tension(nodes, elem_conn, l0, ea, tension_only, & !$OMP SHARED(n_elem, elem_conn, nodes, ea, l0, tension_only, te_buf, elem_es) & !$OMP PRIVATE(e, a, b, nodes6, Kt6, fint6, em) DO e = 1, n_elem + CALL CD_Fatal_Thread_Init() ! this thread can report its own stack overflow a = elem_conn(1, e) b = elem_conn(2, e) nodes6(1:3) = nodes(:, a) diff --git a/src/CableDyn_CosseratAssemble.f90 b/src/CableDyn_CosseratAssemble.f90 index 6d9a39c3..a54eea35 100644 --- a/src/CableDyn_CosseratAssemble.f90 +++ b/src/CableDyn_CosseratAssemble.f90 @@ -18,6 +18,7 @@ MODULE CableDyn_CosseratAssemble !! and scatter-adds Kt(12,12)/fint(12). The static solver can also assemble the !! free-DOF tangent block directly into LAPACK DGBSV band storage. USE CableDyn_Precision, ONLY: wp, CD_All_Finite, CD_Is_Finite + USE CableDyn_FatalReport, ONLY: CD_Fatal_Thread_Init USE CableDyn_Cosserat, ONLY: CD_Reference_Frame, CD_Cosserat_Internal_Force, CD_Cosserat_Force_Tangent USE CableDyn_Mesh, ONLY: CD_Validate_Connectivity USE, INTRINSIC :: ISO_FORTRAN_ENV, ONLY: INT64 @@ -116,6 +117,7 @@ SUBROUTINE CD_Assemble_Cosserat_Tangent_Force(nodes_ref, elem_conn, ea, gas, ei, !$OMP SHARED(n_elem, elem_conn, nodes_ref, q, ea, gas, ei, gj, reduced_shear, fe, Ke, elem_es, elem_em) & !$OMP PRIVATE(e, a, b, Lam0, L0, qe, finte, Kte, es, em) DO e = 1, n_elem + CALL CD_Fatal_Thread_Init() ! this thread can report its own stack overflow a = elem_conn(1, e) b = elem_conn(2, e) CALL CD_Reference_Frame(nodes_ref(:, a), nodes_ref(:, b), Lam0, L0) @@ -316,6 +318,7 @@ SUBROUTINE CD_Assemble_Cosserat_Tangent_Force_Banded_Free_Workspace(nodes_ref, e !$OMP SHARED(n_elem, elem_conn, nodes_ref, q, ea, gas, ei, gj, reduced_shear, work) & !$OMP PRIVATE(e, a, b, Lam0, L0, qe, finte, Kte, es, em) DO e = 1, n_elem + CALL CD_Fatal_Thread_Init() ! this thread can report its own stack overflow a = elem_conn(1, e) b = elem_conn(2, e) CALL CD_Reference_Frame(nodes_ref(:, a), nodes_ref(:, b), Lam0, L0) @@ -429,6 +432,7 @@ SUBROUTINE CD_Assemble_Cosserat_Internal_Force_Workspace(nodes_ref, elem_conn, e !$OMP SHARED(n_elem, elem_conn, nodes_ref, q, ea, gas, ei, gj, reduced_shear, work) & !$OMP PRIVATE(e, a, b, Lam0, L0, qe, finte, es, em) DO e = 1, n_elem + CALL CD_Fatal_Thread_Init() ! this thread can report its own stack overflow a = elem_conn(1, e) b = elem_conn(2, e) CALL CD_Reference_Frame(nodes_ref(:, a), nodes_ref(:, b), Lam0, L0) diff --git a/src/CableDyn_FatalReport.f90 b/src/CableDyn_FatalReport.f90 index 3582eb67..ff6cb78f 100644 --- a/src/CableDyn_FatalReport.f90 +++ b/src/CableDyn_FatalReport.f90 @@ -11,11 +11,12 @@ MODULE CableDyn_FatalReport !! handler that had it before, so the exit status is unchanged. CD_Fatal_Report_Time records !! that time; the time marches call it after every committed step. It only stores a number, !! so a host that loads the library without installing the handlers is not affected. + !! CD_Fatal_Thread_Init prepares each OpenMP worker thread in the same way. USE, INTRINSIC :: ISO_C_BINDING, ONLY: C_DOUBLE USE CableDyn_Precision, ONLY: wp IMPLICIT NONE PRIVATE - PUBLIC :: CD_Fatal_Report_Install, CD_Fatal_Report_Time + PUBLIC :: CD_Fatal_Report_Install, CD_Fatal_Report_Time, CD_Fatal_Thread_Init INTERFACE SUBROUTINE c_fatal_report_install() BIND(C, name='cabledyn_fatal_report_install') @@ -24,6 +25,13 @@ SUBROUTINE c_fatal_report_time(simulated_time) BIND(C, name='cabledyn_fatal_repo IMPORT :: C_DOUBLE REAL(C_DOUBLE), VALUE :: simulated_time END SUBROUTINE c_fatal_report_time + SUBROUTINE CD_Fatal_Thread_Init() BIND(C, name='cabledyn_fatal_thread_init') + !! The first statement of every OpenMP parallel region: gives the calling thread room + !! to report its own stack overflow (a Windows stack guarantee, a POSIX alternate + !! signal stack). Does nothing until the driver has installed the report, and costs one + !! thread-local test after a thread's first call. tests/check_omp_fatal_init.cmake + !! checks that no parallel region lacks it. + END SUBROUTINE CD_Fatal_Thread_Init END INTERFACE CONTAINS diff --git a/src/CableDyn_HermiteCableDynamic.f90 b/src/CableDyn_HermiteCableDynamic.f90 index 638a66ea..8a17b562 100644 --- a/src/CableDyn_HermiteCableDynamic.f90 +++ b/src/CableDyn_HermiteCableDynamic.f90 @@ -51,6 +51,7 @@ MODULE CableDyn_HermiteCableDynamic !! + (1-alpha_f) gamma/(beta dt) Kv(q_alpha, v_alpha), !! where Kv = dR/dv is the drag velocity Jacobian and gamma/(beta dt) = d v_{n+1}/d q_{n+1}. USE CableDyn_Precision, ONLY: wp, CD_ZERO, CD_ONE, CD_All_Finite, CD_Is_Finite + USE CableDyn_FatalReport, ONLY: CD_Fatal_Thread_Init USE CableDyn_EndConnection, ONLY: CD_EndConn_Spring, CD_EndConn_Energy, & CD_EndConn_Reaction_Moment, CD_EndConn_Basis, & CD_EndConn_Project, CD_ENDCONN_OK, & @@ -5256,6 +5257,7 @@ SUBROUTINE assemble_residual(model, q_cfg, v_cfg, t_eval, want_residual, want_ta IF (tangent_only) THEN !$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(qe) SCHEDULE(STATIC) NUM_THREADS(omp_threads) IF(ne >= 32) DO e = 1, ne + CALL CD_Fatal_Thread_Init() ! this thread can report its own stack overflow CALL evaluate_tangent_element(model, e, q_cfg, v_cfg, t_eval, model%ws_elem_K(:, :, e), & model%ws_elem_Kdrag(:, :, e), model%ws_elem_Kv(:, :, e), & model%ws_elem_Kwave(:, :, e), model%ws_elem_Kheld(:, :, e), & diff --git a/src/CableDyn_HermiteCableStatic.f90 b/src/CableDyn_HermiteCableStatic.f90 index ed10896c..d819115b 100644 --- a/src/CableDyn_HermiteCableStatic.f90 +++ b/src/CableDyn_HermiteCableStatic.f90 @@ -31,6 +31,7 @@ MODULE CableDyn_HermiteCableStatic !! chart nor suffers the rotational-vs-axial conditioning split; the bending !! stiffness enters as a well-scaled block of the position/tangent tangent. USE CableDyn_Precision, ONLY: wp, CD_ZERO, CD_ONE, CD_All_Finite, CD_Is_Finite + USE CableDyn_FatalReport, ONLY: CD_Fatal_Thread_Init USE CableDyn_HermiteCable, ONLY: CD_HermiteCable_Element, CD_HermiteCable_Curvature, & CD_HermiteCable_Peak_Curvature, & CD_HermiteCable_Axial_Resultant, & @@ -1012,6 +1013,7 @@ RECURSIVE SUBROUTINE newton_step(rnorm, es, em, do_update) ! then scatter in element order to retain deterministic residual/tangent sums. !$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(qe) SCHEDULE(STATIC) NUM_THREADS(omp_threads) IF(ne >= 32) DO e = 1, ne + CALL CD_Fatal_Thread_Init() ! this thread can report its own stack overflow qe(1:3) = q(6*(e - 1) + 1:6*(e - 1) + 3) qe(4:6) = q(6*(e - 1) + 4:6*(e - 1) + 6) qe(7:9) = q(6*e + 1:6*e + 3) @@ -2127,6 +2129,7 @@ SUBROUTINE add_current_drag(ecs, ecm) ! the sums are those of the serial loop. !$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(qe) SCHEDULE(STATIC) NUM_THREADS(omp_threads) IF(ne >= 32) DO e = 1, ne + CALL CD_Fatal_Thread_Init() ! this thread can report its own stack overflow qe(1:6) = q(6*(e - 1) + 1:6*e) qe(7:12) = q(6*e + 1:6*e + 6) CALL current_element_load(current, qe, l0(e), cur_d(e), cur_cn(e), cur_ct(e), want_jac, & diff --git a/src/CableDyn_OpenFAST_Aggregate.f90 b/src/CableDyn_OpenFAST_Aggregate.f90 index 3ec7744d..095b15c4 100644 --- a/src/CableDyn_OpenFAST_Aggregate.f90 +++ b/src/CableDyn_OpenFAST_Aggregate.f90 @@ -37,6 +37,7 @@ MODULE CableDyn_OpenFAST_Aggregate !! dt that differs from it (a mismatched size would silently desync the cables, whose !! generalised-alpha step carries the Init dt, from the mooring columns). USE CableDyn_Precision, ONLY: wp, CD_ZERO, CD_ONE, CD_All_Finite + USE CableDyn_FatalReport, ONLY: CD_Fatal_Thread_Init USE CableDyn_Linalg, ONLY: CD_Blas_Runtime_Check, CD_LINALG_OK USE CableDyn_OpenFAST_FMF, ONLY: CD_FMF_ModuleType, CD_FMF_Init_From_System, CD_FMF_NMovingPoints, & CD_FMF_GetPointMesh, CD_FMF_UpdateStates, & @@ -1102,6 +1103,7 @@ SUBROUTINE CD_AGG_Step_Moving(self, dt, position, velocity, acceleration, conver !$OMP PRIVATE(c, col, es, em) & !$OMP IF(self%ncable > 1 .AND. .NOT. CD_HermiteCable_Dyn_Profile_Enabled()) SCHEDULE(STATIC) DO c = 1, self%ncable + CALL CD_Fatal_Thread_Init() ! this thread can report its own stack overflow col = self%ncp_sys + c CALL CD_HFMF_UpdateStates(self%cables(c), position(:, col), velocity(:, col), acceleration(:, col), & es, em, orientation(:, :, col), self%mv_omega(:, col), self%mv_alpha(:, col)) @@ -1113,6 +1115,7 @@ SUBROUTINE CD_AGG_Step_Moving(self, dt, position, velocity, acceleration, conver !$OMP PARALLEL DO DEFAULT(NONE) SHARED(self, position, velocity, acceleration) PRIVATE(c, col, es, em) & !$OMP IF(self%ncable > 1 .AND. .NOT. CD_HermiteCable_Dyn_Profile_Enabled()) SCHEDULE(STATIC) DO c = 1, self%ncable + CALL CD_Fatal_Thread_Init() ! this thread can report its own stack overflow col = self%ncp_sys + c CALL CD_HFMF_UpdateStates(self%cables(c), position(:, col), velocity(:, col), acceleration(:, col), es, em, & u_angular_velocity=self%mv_omega(:, col), & diff --git a/src/CableDyn_System.f90 b/src/CableDyn_System.f90 index 70aa427d..fdfab1ba 100644 --- a/src/CableDyn_System.f90 +++ b/src/CableDyn_System.f90 @@ -9,6 +9,7 @@ MODULE CableDyn_System !! dynamic (free/connect) points are composed around them by this owner rather than !! bypassing it. Rigid6 bodies and rods are marched over a system by the deck driver. USE CableDyn_Precision, ONLY: wp, CD_ZERO, CD_All_Finite, CD_Is_Finite + USE CableDyn_FatalReport, ONLY: CD_Fatal_Thread_Init USE CableDyn_Model, ONLY: CD_ModelType, CD_Model_Is_Initialized, CD_End_Model, & CD_Model_NCoupledDOF, CD_Model_NDOF, CD_Model_NElem, & CD_Get_Model_CoupledDofs, CD_Get_Model_CoupledMotion, & @@ -1308,6 +1309,7 @@ RECURSIVE SUBROUTINE CD_Step_System(system, dt, q_coupled, v_coupled, a_coupled, n_team = line_team_size(system) !$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(i) SCHEDULE(STATIC) NUM_THREADS(n_team) IF(n_team > 1) DO i = 1, system%n_lines + CALL CD_Fatal_Thread_Init() ! this thread can report its own stack overflow CALL CD_Step_Model(system%lines(i), dt, system%line_converged(i), system%line_stalled(i), & system%line_iter(i), system%line_stat(i), system%line_msg(i), & prescribed_q=system%lines(i)%q_prescribed_work, & @@ -2082,6 +2084,7 @@ SUBROUTINE CD_Calc_System_CoupledLoads(system, coupled_loads, ErrStat, ErrMsg) n_team = line_team_size(system) !$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(i, lo, hi) SCHEDULE(STATIC) NUM_THREADS(n_team) IF(n_team > 1) DO i = 1, system%n_lines + CALL CD_Fatal_Thread_Init() ! this thread can report its own stack overflow lo = system%line_off(i) hi = system%line_off(i + 1) - 1 CALL CD_Calc_Model_CoupledLoads(system%lines(i), system%line_load_work(lo:hi), & diff --git a/src/cabledyn_fatal.c b/src/cabledyn_fatal.c index 730f795c..3e0d16ae 100644 --- a/src/cabledyn_fatal.c +++ b/src/cabledyn_fatal.c @@ -32,6 +32,19 @@ void cabledyn_fatal_report_install(void); void cabledyn_fatal_report_time(double simulated_time); double cabledyn_fatal_report_last_time(void); +void cabledyn_fatal_thread_init(void); + +/* Stack kept for the handlers on each thread: the Windows stack guarantee, the size of a + * POSIX alternate signal stack. */ +#define CABLEDYN_HANDLER_STACK (64 * 1024) + +#if defined(_MSC_VER) +#define CABLEDYN_THREAD_LOCAL __declspec(thread) +#else +#define CABLEDYN_THREAD_LOCAL _Thread_local +#endif +/* Whether this thread has its handler stack (or needs none). */ +static CABLEDYN_THREAD_LOCAL int cabledyn_thread_ready = 0; /* Simulated time of the last committed step; negative until the first one. An aligned * double is written in one store on every supported target, and the handlers only read it. */ @@ -128,6 +141,8 @@ static void line_finish(cabledyn_line *line) static LPTOP_LEVEL_EXCEPTION_FILTER cabledyn_previous_filter = NULL; /* Set by the first report, so one event that reaches two handlers is reported once. */ static volatile LONG cabledyn_reported = 0; +/* Set once the driver has installed the report. */ +static volatile LONG cabledyn_installed = 0; static void write_stderr(const cabledyn_line *line) { @@ -282,22 +297,29 @@ static BOOL WINAPI cabledyn_console_handler(DWORD event) return FALSE; } +/* Keep room on this thread's stack for the handlers to run after a stack overflow. */ +static void cabledyn_thread_setup(void) +{ + ULONG guarantee = CABLEDYN_HANDLER_STACK; + SetThreadStackGuarantee(&guarantee); +} + void cabledyn_fatal_report_install(void) { - static volatile LONG installed = 0; - ULONG guarantee = 64UL * 1024UL; - if (InterlockedExchange(&installed, 1) != 0) { + if (InterlockedExchange(&cabledyn_installed, 1) != 0) { return; } - /* Keep room on the main thread's stack for the handlers to run after a stack overflow. */ - SetThreadStackGuarantee(&guarantee); + cabledyn_thread_ready = 1; + cabledyn_thread_setup(); AddVectoredExceptionHandler(1, cabledyn_vectored_handler); cabledyn_previous_filter = SetUnhandledExceptionFilter(cabledyn_exception_filter); SetConsoleCtrlHandler(cabledyn_console_handler, TRUE); } #else /* POSIX */ +#include #include +#include #include #define CABLEDYN_NSIG 8 @@ -305,8 +327,14 @@ static const int cabledyn_signals[CABLEDYN_NSIG] = {SIGSEGV, SIGBUS, SIGFPE, SIG SIGABRT, SIGINT, SIGTERM, SIGHUP}; static struct sigaction cabledyn_previous[CABLEDYN_NSIG]; static volatile sig_atomic_t cabledyn_reported = 0; -/* The alternate stack the handlers run on, so a stack overflow can still be reported. */ -static char cabledyn_altstack[64 * 1024]; +/* Set once the driver has installed the report. */ +static volatile int cabledyn_installed = 0; +/* The main thread's alternate stack, so a stack overflow can still be reported. */ +static char cabledyn_altstack[CABLEDYN_HANDLER_STACK]; +/* Worker threads' alternate stacks, freed when each thread ends. */ +static pthread_key_t cabledyn_altstack_key; +static pthread_once_t cabledyn_altstack_once = PTHREAD_ONCE_INIT; +static int cabledyn_altstack_key_ok = 0; static const char *signal_name(int sig) { @@ -332,17 +360,31 @@ static const char *signal_name(int sig) } } -static void cabledyn_signal_handler(int sig) +static int signal_index(int sig) { int k; + for (k = 0; k < CABLEDYN_NSIG; ++k) { + if (cabledyn_signals[k] == sig) { + return k; + } + } + return -1; +} + +static void cabledyn_signal_handler(int sig, siginfo_t *info, void *context) +{ + int k = signal_index(sig); + int fault = sig == SIGSEGV || sig == SIGBUS || sig == SIGFPE || sig == SIGILL; + /* A fault the hardware raised re-executes when the handler returns; one sent by kill() + * or raise() (si_code <= 0) does not, and is re-sent below. */ + int synchronous = fault && info != NULL && info->si_code > 0; if (!cabledyn_reported) { cabledyn_line line; - int fault = sig == SIGSEGV || sig == SIGBUS || sig == SIGFPE || sig == SIGILL || - sig == SIGABRT; cabledyn_reported = 1; line.len = 0; line.text[0] = '\0'; - line_add(&line, fault ? "\nCableDyn_driver: fatal error: " : "\nCableDyn_driver: stopped by "); + line_add(&line, fault || sig == SIGABRT ? "\nCableDyn_driver: fatal error: " + : "\nCableDyn_driver: stopped by "); line_add(&line, signal_name(sig)); line_add(&line, " "); line_add_when(&line); @@ -352,45 +394,112 @@ static void cabledyn_signal_handler(int sig) (void)ignored; } } + if (k < 0) { + return; + } /* Hand the signal to whatever handled it before (the Fortran runtime's backtrace or the - * default action), so the exit status is unchanged. SA_NODEFER delivers it at once. */ - for (k = 0; k < CABLEDYN_NSIG; ++k) { - if (cabledyn_signals[k] == sig) { - sigaction(sig, &cabledyn_previous[k], NULL); - break; - } + * default action), so the exit status and the fault context are unchanged. */ + sigaction(sig, &cabledyn_previous[k], NULL); + if ((cabledyn_previous[k].sa_flags & SA_SIGINFO) != 0 && + cabledyn_previous[k].sa_sigaction != NULL) { + /* A previous three-argument handler receives the original siginfo and context. */ + cabledyn_previous[k].sa_sigaction(sig, info, context); + return; + } + if (synchronous) { + /* Returning re-executes the faulting instruction, which raises the same fault, with + * its own address and context, under the previous disposition. */ + return; } + /* An asynchronous signal (or a fault sent by kill/raise): send it again. SA_NODEFER + * delivers it at once. */ raise(sig); } +/* Free a worker thread's alternate stack when the thread ends. */ +static void cabledyn_altstack_release(void *stack) +{ + stack_t off; + memset(&off, 0, sizeof off); + off.ss_flags = SS_DISABLE; + sigaltstack(&off, NULL); + free(stack); +} + +static void cabledyn_altstack_key_create(void) +{ + cabledyn_altstack_key_ok = pthread_key_create(&cabledyn_altstack_key, cabledyn_altstack_release) == 0; +} + +/* An alternate signal stack for this thread, so a stack overflow can still be reported. A + * thread that already has one keeps it. */ +static void cabledyn_thread_setup(void) +{ + stack_t current, alt; + void *stack; + if (sigaltstack(NULL, ¤t) == 0 && (current.ss_flags & SS_DISABLE) == 0) { + return; + } + pthread_once(&cabledyn_altstack_once, cabledyn_altstack_key_create); + if (!cabledyn_altstack_key_ok) { + return; + } + stack = malloc(CABLEDYN_HANDLER_STACK); + if (stack == NULL) { + return; + } + memset(&alt, 0, sizeof alt); + alt.ss_sp = stack; + alt.ss_size = CABLEDYN_HANDLER_STACK; + alt.ss_flags = 0; + if (sigaltstack(&alt, NULL) != 0 || pthread_setspecific(cabledyn_altstack_key, stack) != 0) { + cabledyn_altstack_release(stack); + } +} + void cabledyn_fatal_report_install(void) { - static int installed = 0; stack_t alt; struct sigaction action; int k; - if (installed) { + if (cabledyn_installed) { return; } - installed = 1; + cabledyn_installed = 1; + cabledyn_thread_ready = 1; + /* The main thread uses a static alternate stack, which outlives every handler. */ memset(&alt, 0, sizeof alt); alt.ss_sp = cabledyn_altstack; alt.ss_size = sizeof cabledyn_altstack; alt.ss_flags = 0; sigaltstack(&alt, NULL); memset(&action, 0, sizeof action); - action.sa_handler = cabledyn_signal_handler; + action.sa_sigaction = cabledyn_signal_handler; sigemptyset(&action.sa_mask); - action.sa_flags = SA_ONSTACK | SA_NODEFER; + action.sa_flags = SA_SIGINFO | SA_ONSTACK | SA_NODEFER; for (k = 0; k < CABLEDYN_NSIG; ++k) { if (sigaction(cabledyn_signals[k], NULL, &cabledyn_previous[k]) != 0) { continue; } /* A signal the parent set to be ignored (nohup, background job) stays ignored. */ - if (cabledyn_previous[k].sa_handler == SIG_IGN) { + if ((cabledyn_previous[k].sa_flags & SA_SIGINFO) == 0 && + cabledyn_previous[k].sa_handler == SIG_IGN) { continue; } sigaction(cabledyn_signals[k], &action, NULL); } } #endif + +/* Prepare the calling thread for the report: called at the start of every OpenMP parallel + * region, it gives each worker thread room to report its own stack overflow. It does nothing + * until the driver has installed the report, so a host that loads the library is untouched, + * and after the first call on a thread it costs one thread-local test. */ +void cabledyn_fatal_thread_init(void) +{ + if (cabledyn_thread_ready || !cabledyn_installed) { + return; + } + cabledyn_thread_ready = 1; + cabledyn_thread_setup(); +} diff --git a/tests/fatal_report_faults.c b/tests/fatal_report_faults.c index 4ff24705..342cd528 100644 --- a/tests/fatal_report_faults.c +++ b/tests/fatal_report_faults.c @@ -4,10 +4,13 @@ * * Faults that tests/test_fatal_report.f90 provokes to check the abnormal-end report. */ +#define _DEFAULT_SOURCE +#define _DARWIN_C_SOURCE #include void fatal_report_test_null_write(void); int fatal_report_test_raise_term(void); +int fatal_report_test_fault_context(void); static volatile int *volatile fatal_report_null_target = 0; @@ -27,3 +30,135 @@ int fatal_report_test_raise_term(void) return 1; #endif } + +#if defined(_WIN32) +/* The fault-context check is POSIX-only: returns -1 (skipped). */ +int fatal_report_test_fault_context(void) +{ + return -1; +} +#else +#include +#include +#include +#include +#include +#include + +void cabledyn_fatal_report_install(void); +void cabledyn_fatal_report_time(double simulated_time); + +#define FAULT_ADDRESS ((uintptr_t)16) + +typedef struct { + int sig; + int code; + uintptr_t addr; +} fault_record; + +static int fault_pipe = -1; +/* Read through a volatile, so the compiler cannot treat the store as a known-invalid one. */ +static volatile uintptr_t fault_address = FAULT_ADDRESS; + +/* A handler installed before the report, as a runtime's would be: it records the siginfo it + * receives, restores the default action and returns, so the fault re-executes and ends the + * process. */ +static void prior_handler(int sig, siginfo_t *info, void *context) +{ + fault_record record; + ssize_t ignored; + (void)context; + record.sig = sig; + record.code = info != NULL ? info->si_code : -99; + record.addr = info != NULL ? (uintptr_t)info->si_addr : 0; + ignored = write(fault_pipe, &record, sizeof record); + (void)ignored; + signal(sig, SIG_DFL); +} + +static void read_all(int fd, char *buffer, size_t size) +{ + size_t used = 0; + ssize_t got; + while (used + 1 < size && (got = read(fd, buffer + used, size - 1 - used)) > 0) { + used += (size_t)got; + } + buffer[used] = '\0'; +} + +static int count(const char *text, const char *what) +{ + int n = 0; + const char *at = text; + while ((at = strstr(at, what)) != NULL) { + ++n; + at += strlen(what); + } + return n; +} + +/* A child process with a three-argument SIGSEGV/SIGBUS handler installs the report and writes + * to address 16. The previous handler must receive the hardware fault's own siginfo (a + * positive si_code and si_addr 16, not a re-sent signal), the child must end by that signal, + * and the report must be written once. Returns 0 when all hold, 1 otherwise. */ +int fatal_report_test_fault_context(void) +{ + int records[2], errors[2], status = 0, ok; + pid_t child; + fault_record record; + ssize_t got; + char text[8192]; + struct sigaction action; + + if (pipe(records) != 0 || pipe(errors) != 0) { + fprintf(stderr, "FAIL: fault context: pipe() failed\n"); + return 1; + } + fflush(NULL); + child = fork(); + if (child < 0) { + fprintf(stderr, "FAIL: fault context: fork() failed\n"); + return 1; + } + if (child == 0) { + close(records[0]); + close(errors[0]); + dup2(errors[1], STDERR_FILENO); + fault_pipe = records[1]; + memset(&action, 0, sizeof action); + action.sa_sigaction = prior_handler; + sigemptyset(&action.sa_mask); + action.sa_flags = SA_SIGINFO; + sigaction(SIGSEGV, &action, NULL); + sigaction(SIGBUS, &action, NULL); + cabledyn_fatal_report_install(); + cabledyn_fatal_report_time(7534.6); + *(volatile int *)fault_address = 1; + _exit(3); + } + close(records[1]); + close(errors[1]); + read_all(errors[0], text, sizeof text); + got = read(records[0], &record, sizeof record); + waitpid(child, &status, 0); + fprintf(stderr, "%s", text); + if (got != (ssize_t)sizeof record) { + fprintf(stderr, "FAIL: fault context: the previous handler was not called\n"); + return 1; + } + fprintf(stderr, + "fault context: previous handler got signal %d, si_code %d, si_addr %#lx; child " + "ended %s %d\n", + record.sig, record.code, (unsigned long)record.addr, + WIFSIGNALED(status) ? "by signal" : "with status", + WIFSIGNALED(status) ? WTERMSIG(status) : WEXITSTATUS(status)); + ok = (record.sig == SIGSEGV || record.sig == SIGBUS) && record.code > 0 && + record.addr == FAULT_ADDRESS && WIFSIGNALED(status) && WTERMSIG(status) == record.sig && + count(text, "CableDyn_driver: fatal error:") == 1 && count(text, "t = 7534.600 s") == 1; + if (!ok) { + fprintf(stderr, "FAIL: fault context: the fault did not keep its own context\n"); + return 1; + } + return 0; +} +#endif diff --git a/tests/test_fatal_report.f90 b/tests/test_fatal_report.f90 index badda91d..5f5c76ab 100644 --- a/tests/test_fatal_report.f90 +++ b/tests/test_fatal_report.f90 @@ -10,12 +10,17 @@ PROGRAM test_fatal_report !! test_fatal_report before|during !! installs the report, optionally records a simulated time, and ends the process !! with : overflow (unbounded recursion, a stack overflow), null (a write - !! through a null pointer) or term (SIGTERM; POSIX only). tests/check_fatal_report.cmake - !! checks the exit status and the stderr line. + !! through a null pointer), omp_overflow (a stack overflow on an OpenMP worker thread) + !! or term (SIGTERM; POSIX only). tests/check_fatal_report.cmake checks the exit status + !! and the stderr line; + !! test_fatal_report context + !! (POSIX) a child process with an earlier three-argument SIGSEGV handler faults at a + !! known address: that handler must receive the fault's own siginfo, the child must end + !! by the fault's signal, and the report must be written once. USE, INTRINSIC :: ISO_C_BINDING, ONLY: C_DOUBLE, C_INT USE, INTRINSIC :: ISO_FORTRAN_ENV, ONLY: error_unit USE CableDyn_Precision, ONLY: wp - USE CableDyn_FatalReport, ONLY: CD_Fatal_Report_Install, CD_Fatal_Report_Time + USE CableDyn_FatalReport, ONLY: CD_Fatal_Report_Install, CD_Fatal_Report_Time, CD_Fatal_Thread_Init USE CableDyn_DeckDriver, ONLY: CD_Run_Deck_Driver, CD_DECKDRV_OK IMPLICIT NONE @@ -28,6 +33,9 @@ END SUBROUTINE null_write INTEGER(C_INT) FUNCTION raise_term() BIND(C, name='fatal_report_test_raise_term') IMPORT :: C_INT END FUNCTION raise_term + INTEGER(C_INT) FUNCTION fault_context() BIND(C, name='fatal_report_test_fault_context') + IMPORT :: C_INT + END FUNCTION fault_context END INTERFACE CHARACTER(4096) :: mode, arg2, arg3 @@ -55,6 +63,18 @@ END FUNCTION raise_term STOP END IF + IF (mode == 'context') THEN + SELECT CASE (fault_context()) + CASE (-1) + WRITE (*, '(A)') 'SKIP: the fault-context check is POSIX-only' + CASE (0) + WRITE (*, '(A)') 'PASS: the fault reached the earlier handler with its own context' + CASE DEFAULT + CALL fail('the fault did not keep its own context') + END SELECT + STOP + END IF + CALL CD_Fatal_Report_Install() CALL CD_Fatal_Report_Install() ! idempotent IF (arg2 == 'during') THEN @@ -66,6 +86,8 @@ END FUNCTION raise_term WRITE (error_unit, '(A,I0)') 'unreachable: ', deep(1) CASE ('null') CALL null_write() + CASE ('omp_overflow') + CALL overflow_on_worker() CASE ('term') IF (raise_term() == 0_C_INT) THEN WRITE (*, '(A)') 'SKIP: no SIGTERM on this platform' @@ -90,6 +112,26 @@ RECURSIVE INTEGER FUNCTION deep(depth) RESULT(r) r = deep(depth + 1) + frame(MOD(depth, 16384) + 1) END FUNCTION deep + SUBROUTINE overflow_on_worker() + !! A stack overflow on OpenMP worker thread 1 (its stack set by OMP_STACKSIZE) while the + !! main thread waits at the region's barrier. +!$ USE omp_lib, ONLY: omp_get_num_threads, omp_get_thread_num + INTEGER :: r + r = 0 + !$OMP PARALLEL NUM_THREADS(2) DEFAULT(SHARED) FIRSTPRIVATE(r) + CALL CD_Fatal_Thread_Init() +!$ IF (omp_get_num_threads() >= 2) THEN +!$ IF (omp_get_thread_num() == 1) THEN +!$ r = deep(1) +!$ WRITE (error_unit, '(A,I0)') 'unreachable: ', r +!$ END IF +!$ END IF + !$OMP END PARALLEL +!$ CALL fail('the parallel region had no worker thread') + WRITE (*, '(A)') 'SKIP: this build has no OpenMP' + STOP + END SUBROUTINE overflow_on_worker + SUBROUTINE fail(text) CHARACTER(*), INTENT(IN) :: text WRITE (error_unit, '(A)') 'FAIL: test_fatal_report: '//text diff --git a/tests/test_omp_fatal_init.py b/tests/test_omp_fatal_init.py new file mode 100644 index 00000000..eefa77ba --- /dev/null +++ b/tests/test_omp_fatal_init.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 Jae Hoon Seo, SMI Lab, Inha University +"""Every OpenMP parallel region of the library and the driver prepares its threads for the +abnormal-end report: its first statement (the first statement of the loop body, for a +PARALLEL DO) is ``CALL CD_Fatal_Thread_Init()``, so a stack overflow on a worker thread is +reported like one on the main thread (src/cabledyn_fatal.c).""" + +from __future__ import annotations + +import re +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +DIRECTIVE = re.compile(r"^\s*!\$omp\s+parallel\b(?P.*)$", re.IGNORECASE) +DO_CLAUSE = re.compile(r"^\s*do\b", re.IGNORECASE) +DO_STATEMENT = re.compile(r"^\s*(?:\w+\s*:\s*)?do\b", re.IGNORECASE) +INIT = re.compile(r"^\s*call\s+cd_fatal_thread_init\s*(?:\(\s*\))?\s*(?:!.*)?$", re.IGNORECASE) + + +def sources() -> list[Path]: + """The Fortran sources of the core library and the driver (not the OpenFAST tree).""" + files = sorted((ROOT / "src").glob("*.f90")) + sorted((ROOT / "app").glob("*.f90")) + return [path for path in files if "openfast" not in path.parent.name.lower()] + + +def statements(lines: list[str], start: int): + """Yield (index, text) of the statements after ``start``, skipping comments and blanks.""" + for index in range(start, len(lines)): + text = lines[index].strip() + if not text or text.startswith("!"): + continue + yield index, lines[index] + + +def missing_inits(lines: list[str]) -> tuple[int, list[int]]: + """Count the parallel regions and list the 1-based lines of those without the call.""" + found, missing = 0, [] + index = 0 + while index < len(lines): + match = DIRECTIVE.match(lines[index]) + if match is None: + index += 1 + continue + found += 1 + first = index + loop = DO_CLAUSE.match(match.group("rest")) is not None + while lines[index].rstrip().endswith("&"): + index += 1 + following = statements(lines, index + 1) + nxt = next(following, None) + if loop: + if nxt is None or not DO_STATEMENT.match(nxt[1]): + missing.append(first + 1) + index += 1 + continue + nxt = next(following, None) + if nxt is None or not INIT.match(nxt[1]): + missing.append(first + 1) + index += 1 + return found, missing + + +class OmpFatalInitTest(unittest.TestCase): + def test_every_parallel_region_prepares_its_threads(self) -> None: + total = 0 + problems = [] + for path in sources(): + found, missing = missing_inits(path.read_text(encoding="utf-8").splitlines()) + total += found + problems += [f"{path.relative_to(ROOT)}:{line}" for line in missing] + self.assertGreater(total, 0, "no OpenMP parallel region found: the scan is broken") + self.assertEqual( + problems, + [], + "OpenMP parallel regions whose first statement is not CALL CD_Fatal_Thread_Init()", + ) + + def test_the_scan_catches_a_region_without_the_call(self) -> None: + good = [ + " !$OMP PARALLEL DO DEFAULT(SHARED) &", + " !$OMP PRIVATE(i)", + " DO i = 1, n", + " ! comment", + " CALL CD_Fatal_Thread_Init() ! note", + " END DO", + " !$omp parallel", + " call cd_fatal_thread_init()", + " !$omp end parallel", + ] + self.assertEqual(missing_inits(good), (2, [])) + bad = [" !$OMP PARALLEL DO", " DO i = 1, n", " x(i) = 0", " END DO", + " !$OMP PARALLEL", " x = 1"] + self.assertEqual(missing_inits(bad), (2, [1, 5])) + + +if __name__ == "__main__": + unittest.main() From b965006e3ea67f9a698aede89201086af8acbe6d Mon Sep 17 00:00:00 2001 From: Jae Hoon Seo Date: Mon, 5 Oct 2026 12:24:22 +0900 Subject: [PATCH 3/6] Keep ignored signals ignored when the report is installed The previous disposition of each signal is now classified by one helper that compares sa_handler with SIG_IGN, SIG_DFL and SIG_ERR whatever the SA_SIGINFO flag says (sa_handler and sa_sigaction share storage). The report is not installed on a signal that was ignored, with or without SA_SIGINFO, or whose disposition cannot be classified, and it chains only to a real handler function. Previously an inherited SIG_IGN stored with SA_SIGINFO set was taken for a handler: the report was installed, and the ignored signal was then passed to the SIG_IGN sentinel as a function. If restoring the previous disposition fails, the default action is set so the handler is never entered again. Test (POSIX, forked children): SIGTERM set to SIG_IGN with SA_SIGINFO, and to plain SIG_IGN, leaves the child running to a normal exit with no report; SIG_DFL is reported once and ends the child by SIGTERM; a one-argument handler is reported once and then runs. --- CHANGELOG.md | 1 + CMakeLists.txt | 6 +++ src/cabledyn_fatal.c | 38 ++++++++++++--- tests/fatal_report_faults.c | 97 +++++++++++++++++++++++++++++++++++++ tests/test_fatal_report.f90 | 20 +++++++- 5 files changed, 155 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ddab333..fecc9211 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ All notable changes to CableDyn are recorded in this file. The format follows enters an OpenMP parallel region keeps room to report its own stack overflow, and on Linux and macOS a hardware fault reaches the handler that was there before (the Fortran runtime's backtrace, or the default action and its core dump) with its original address and context. + A signal the driver was started with ignored (for example under `nohup`) stays ignored. - Every non-zero exit the driver makes itself now ends stderr with the closing line `CableDyn_driver: ended with exit code `. A process ended from outside (`taskkill /F`, *End task*, `kill -9`) runs none of its own code and cannot report anything; the missing diff --git a/CMakeLists.txt b/CMakeLists.txt index adcf6241..14e56099 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1862,6 +1862,12 @@ add_test(NAME fatal_report_fault_context COMMAND test_fatal_report context) set_tests_properties(fatal_report_fault_context PROPERTIES LABELS "fortran;unit;driver;stack" TIMEOUT 120 FAIL_REGULAR_EXPRESSION "FAIL:") +# POSIX: the report keeps every previous disposition -- an ignored signal (with or without +# SA_SIGINFO) stays ignored, a default one still ends the process, a handler still runs. +add_test(NAME fatal_report_dispositions COMMAND test_fatal_report dispositions) +set_tests_properties(fatal_report_dispositions PROPERTIES + LABELS "fortran;unit;driver" TIMEOUT 120 + FAIL_REGULAR_EXPRESSION "FAIL:") add_test( NAME driver_repeat_exit COMMAND ${CMAKE_COMMAND} diff --git a/src/cabledyn_fatal.c b/src/cabledyn_fatal.c index 3e0d16ae..3f4e0197 100644 --- a/src/cabledyn_fatal.c +++ b/src/cabledyn_fatal.c @@ -371,6 +371,26 @@ static int signal_index(int sig) return -1; } +/* What a signal's previous disposition is. sa_handler and sa_sigaction share storage, so the + * SIG_IGN / SIG_DFL / SIG_ERR sentinels are recognised whatever the SA_SIGINFO flag says, and + * only a real function is ever called. */ +enum { DISPOSITION_DEFAULT, DISPOSITION_IGNORED, DISPOSITION_UNKNOWN, DISPOSITION_HANDLER, + DISPOSITION_SIGINFO_HANDLER }; + +static int disposition(const struct sigaction *action) +{ + if (action->sa_handler == SIG_IGN) { + return DISPOSITION_IGNORED; + } + if (action->sa_handler == SIG_DFL) { + return DISPOSITION_DEFAULT; + } + if (action->sa_handler == SIG_ERR) { + return DISPOSITION_UNKNOWN; + } + return (action->sa_flags & SA_SIGINFO) != 0 ? DISPOSITION_SIGINFO_HANDLER : DISPOSITION_HANDLER; +} + static void cabledyn_signal_handler(int sig, siginfo_t *info, void *context) { int k = signal_index(sig); @@ -399,9 +419,11 @@ static void cabledyn_signal_handler(int sig, siginfo_t *info, void *context) } /* Hand the signal to whatever handled it before (the Fortran runtime's backtrace or the * default action), so the exit status and the fault context are unchanged. */ - sigaction(sig, &cabledyn_previous[k], NULL); - if ((cabledyn_previous[k].sa_flags & SA_SIGINFO) != 0 && - cabledyn_previous[k].sa_sigaction != NULL) { + if (sigaction(sig, &cabledyn_previous[k], NULL) != 0) { + /* Never leave this handler in place to be entered again. */ + signal(sig, SIG_DFL); + } + if (disposition(&cabledyn_previous[k]) == DISPOSITION_SIGINFO_HANDLER) { /* A previous three-argument handler receives the original siginfo and context. */ cabledyn_previous[k].sa_sigaction(sig, info, context); return; @@ -481,10 +503,14 @@ void cabledyn_fatal_report_install(void) if (sigaction(cabledyn_signals[k], NULL, &cabledyn_previous[k]) != 0) { continue; } - /* A signal the parent set to be ignored (nohup, background job) stays ignored. */ - if ((cabledyn_previous[k].sa_flags & SA_SIGINFO) == 0 && - cabledyn_previous[k].sa_handler == SIG_IGN) { + /* A signal the parent set to be ignored (nohup, background job) stays ignored, with + * or without SA_SIGINFO; a disposition that cannot be classified is left alone. */ + switch (disposition(&cabledyn_previous[k])) { + case DISPOSITION_IGNORED: + case DISPOSITION_UNKNOWN: continue; + default: + break; } sigaction(cabledyn_signals[k], &action, NULL); } diff --git a/tests/fatal_report_faults.c b/tests/fatal_report_faults.c index 342cd528..ba96d3a9 100644 --- a/tests/fatal_report_faults.c +++ b/tests/fatal_report_faults.c @@ -11,6 +11,7 @@ void fatal_report_test_null_write(void); int fatal_report_test_raise_term(void); int fatal_report_test_fault_context(void); +int fatal_report_test_dispositions(void); static volatile int *volatile fatal_report_null_target = 0; @@ -37,6 +38,12 @@ int fatal_report_test_fault_context(void) { return -1; } + +/* The disposition check is POSIX-only: returns -1 (skipped). */ +int fatal_report_test_dispositions(void) +{ + return -1; +} #else #include #include @@ -161,4 +168,94 @@ int fatal_report_test_fault_context(void) } return 0; } + +/* SIGTERM handler of the "handler" case: a real one-argument function. */ +static void prior_term_handler(int sig) +{ + static const char text[] = "prior SIGTERM handler ran\n"; + ssize_t ignored = write(STDERR_FILENO, text, sizeof text - 1); + (void)ignored; + (void)sig; + _exit(7); +} + +/* Run one child that sets SIGTERM to `setup`, installs the report and sends itself SIGTERM. + * setup 0: SIG_IGN with SA_SIGINFO; 1: SIG_IGN; 2: SIG_DFL; 3: a one-argument handler. */ +static int disposition_case(int setup, const char *name) +{ + int errors[2], status = 0, ok, reports, ran; + pid_t child; + char text[4096]; + struct sigaction action; + + if (pipe(errors) != 0) { + fprintf(stderr, "FAIL: dispositions: pipe() failed\n"); + return 1; + } + fflush(NULL); + child = fork(); + if (child < 0) { + fprintf(stderr, "FAIL: dispositions: fork() failed\n"); + return 1; + } + if (child == 0) { + close(errors[0]); + dup2(errors[1], STDERR_FILENO); + memset(&action, 0, sizeof action); + sigemptyset(&action.sa_mask); + if (setup == 0) { + action.sa_handler = SIG_IGN; /* the sentinel, stored with SA_SIGINFO set */ + action.sa_flags = SA_SIGINFO; + } else if (setup == 1) { + action.sa_handler = SIG_IGN; + } else if (setup == 2) { + action.sa_handler = SIG_DFL; + } else { + action.sa_handler = prior_term_handler; + } + sigaction(SIGTERM, &action, NULL); + cabledyn_fatal_report_install(); + cabledyn_fatal_report_time(7534.6); + kill(getpid(), SIGTERM); + /* Still running: an ignored SIGTERM must leave the process alone. */ + _exit(0); + } + close(errors[1]); + read_all(errors[0], text, sizeof text); + waitpid(child, &status, 0); + reports = count(text, "CableDyn_driver: stopped by a termination request"); + ran = count(text, "prior SIGTERM handler ran"); + if (setup <= 1) { + ok = WIFEXITED(status) && WEXITSTATUS(status) == 0 && reports == 0; + } else if (setup == 2) { + ok = WIFSIGNALED(status) && WTERMSIG(status) == SIGTERM && reports == 1; + } else { + ok = WIFEXITED(status) && WEXITSTATUS(status) == 7 && reports == 1 && ran == 1; + } + fprintf(stderr, "dispositions: %s: child ended %s %d, %d report(s)%s\n", name, + WIFSIGNALED(status) ? "by signal" : "with status", + WIFSIGNALED(status) ? WTERMSIG(status) : WEXITSTATUS(status), reports, + ok ? "" : " -- unexpected"); + if (!ok) { + fprintf(stderr, "%s", text); + } + return ok ? 0 : 1; +} + +/* An ignored SIGTERM (with or without SA_SIGINFO) stays ignored and is never reported or + * called; a default one is reported and ends the process by SIGTERM; a previous handler is + * reported and then runs. Returns 0 when all hold, 1 otherwise. */ +int fatal_report_test_dispositions(void) +{ + int failures = 0; + failures += disposition_case(0, "SIG_IGN with SA_SIGINFO"); + failures += disposition_case(1, "SIG_IGN"); + failures += disposition_case(2, "SIG_DFL"); + failures += disposition_case(3, "a one-argument handler"); + if (failures != 0) { + fprintf(stderr, "FAIL: dispositions: %d case(s) wrong\n", failures); + return 1; + } + return 0; +} #endif diff --git a/tests/test_fatal_report.f90 b/tests/test_fatal_report.f90 index 5f5c76ab..c5aaa6f2 100644 --- a/tests/test_fatal_report.f90 +++ b/tests/test_fatal_report.f90 @@ -16,7 +16,10 @@ PROGRAM test_fatal_report !! test_fatal_report context !! (POSIX) a child process with an earlier three-argument SIGSEGV handler faults at a !! known address: that handler must receive the fault's own siginfo, the child must end - !! by the fault's signal, and the report must be written once. + !! by the fault's signal, and the report must be written once; + !! test_fatal_report dispositions + !! (POSIX) an ignored SIGTERM, with or without SA_SIGINFO, stays ignored and unreported; + !! a default one is reported and ends the process; a previous handler still runs. USE, INTRINSIC :: ISO_C_BINDING, ONLY: C_DOUBLE, C_INT USE, INTRINSIC :: ISO_FORTRAN_ENV, ONLY: error_unit USE CableDyn_Precision, ONLY: wp @@ -36,6 +39,9 @@ END FUNCTION raise_term INTEGER(C_INT) FUNCTION fault_context() BIND(C, name='fatal_report_test_fault_context') IMPORT :: C_INT END FUNCTION fault_context + INTEGER(C_INT) FUNCTION dispositions() BIND(C, name='fatal_report_test_dispositions') + IMPORT :: C_INT + END FUNCTION dispositions END INTERFACE CHARACTER(4096) :: mode, arg2, arg3 @@ -75,6 +81,18 @@ END FUNCTION fault_context STOP END IF + IF (mode == 'dispositions') THEN + SELECT CASE (dispositions()) + CASE (-1) + WRITE (*, '(A)') 'SKIP: the disposition check is POSIX-only' + CASE (0) + WRITE (*, '(A)') 'PASS: ignored signals stay ignored; others are reported and passed on' + CASE DEFAULT + CALL fail('a previous signal disposition was not kept') + END SELECT + STOP + END IF + CALL CD_Fatal_Report_Install() CALL CD_Fatal_Report_Install() ! idempotent IF (arg2 == 'during') THEN From 9d7b17b9dc16cfcb2fec0a2b77d3a15462981e7c Mon Sep 17 00:00:00 2001 From: Jae Hoon Seo Date: Mon, 5 Oct 2026 14:27:14 +0900 Subject: [PATCH 4/6] Re-deliver chained signals through the kernel; make test_modal explain a crash The report no longer calls a previous signal handler itself. It re-installs the saved action exactly as it was (handler, flags and mask) and lets the kernel deliver the signal again: a hardware fault returns so the instruction re-executes, and an asynchronous signal is raised. The previous handler therefore runs under its own SA_RESETHAND, sa_mask and self-blocking, as it would have without the report. Tests (POSIX, forked children): a previous handler with SA_RESETHAND gets the first SIGTERM and a second SIGTERM ends the process; a previous handler with SIGUSR1 in its sa_mask runs with SIGUSR1 and SIGTERM blocked. The fault-context test still sees the kernel's own siginfo. test_modal and stack_modal ended once on a hosted Windows runner with "Exception: SegFault" and no output, and passed on a re-run of the same commit; the toolchain CI resolves passes here. They now install the abnormal-end report, so a recurrence names the fault (stack overflow or access violation), and print the OpenBLAS build and kernel the host selected before any case runs. --- CHANGELOG.md | 3 +- CMakeLists.txt | 4 +-- src/cabledyn_fatal.c | 19 +++++------- tests/blas_config_probe.c | 59 +++++++++++++++++++++++++++++++++++++ tests/fatal_report_faults.c | 58 ++++++++++++++++++++++++++++++++---- tests/test_modal.f90 | 37 +++++++++++++++++++++++ 6 files changed, 161 insertions(+), 19 deletions(-) create mode 100644 tests/blas_config_probe.c diff --git a/CHANGELOG.md b/CHANGELOG.md index fecc9211..13300527 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,7 +28,8 @@ All notable changes to CableDyn are recorded in this file. The format follows stack overflow in a Windows GNU build ended the process with no message. Every thread that enters an OpenMP parallel region keeps room to report its own stack overflow, and on Linux and macOS a hardware fault reaches the handler that was there before (the Fortran runtime's - backtrace, or the default action and its core dump) with its original address and context. + backtrace, or the default action and its core dump) with its original address and context, + and a previous handler runs under its own flags and signal mask. A signal the driver was started with ignored (for example under `nohup`) stays ignored. - Every non-zero exit the driver makes itself now ends stderr with the closing line `CableDyn_driver: ended with exit code `. A process ended from outside (`taskkill /F`, diff --git a/CMakeLists.txt b/CMakeLists.txt index 14e56099..de4b259b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1669,7 +1669,7 @@ target_link_libraries(test_stream_wave PRIVATE cabledyn_core) add_test(NAME stream_wave COMMAND test_stream_wave) set_tests_properties(stream_wave PROPERTIES LABELS "fortran;unit;driver;dynamic;validation;orcaflex") -add_executable(test_modal tests/test_modal.f90) +add_executable(test_modal tests/test_modal.f90 tests/blas_config_probe.c) target_link_libraries(test_modal PRIVATE cabledyn_core) add_test(NAME modal COMMAND test_modal) set_tests_properties(modal PROPERTIES LABELS "fortran;unit;driver;static;validation") @@ -1683,7 +1683,7 @@ set_tests_properties(modal PROPERTIES LABELS "fortran;unit;driver;static;validat # the library on every platform. if(CABLEDYN_GNU_WINDOWS_STACK) file(MAKE_DIRECTORY "${CMAKE_BINARY_DIR}/stack_small") - add_executable(test_modal_small_stack tests/test_modal.f90) + add_executable(test_modal_small_stack tests/test_modal.f90 tests/blas_config_probe.c) target_link_libraries(test_modal_small_stack PRIVATE cabledyn_core) add_executable(cabledyn_small_stack app/cabledyn.f90) target_link_libraries(cabledyn_small_stack PRIVATE cabledyn_core) diff --git a/src/cabledyn_fatal.c b/src/cabledyn_fatal.c index 3f4e0197..72317798 100644 --- a/src/cabledyn_fatal.c +++ b/src/cabledyn_fatal.c @@ -372,10 +372,8 @@ static int signal_index(int sig) } /* What a signal's previous disposition is. sa_handler and sa_sigaction share storage, so the - * SIG_IGN / SIG_DFL / SIG_ERR sentinels are recognised whatever the SA_SIGINFO flag says, and - * only a real function is ever called. */ -enum { DISPOSITION_DEFAULT, DISPOSITION_IGNORED, DISPOSITION_UNKNOWN, DISPOSITION_HANDLER, - DISPOSITION_SIGINFO_HANDLER }; + * SIG_IGN / SIG_DFL / SIG_ERR sentinels are recognised whatever the SA_SIGINFO flag says. */ +enum { DISPOSITION_DEFAULT, DISPOSITION_IGNORED, DISPOSITION_UNKNOWN, DISPOSITION_HANDLER }; static int disposition(const struct sigaction *action) { @@ -388,11 +386,12 @@ static int disposition(const struct sigaction *action) if (action->sa_handler == SIG_ERR) { return DISPOSITION_UNKNOWN; } - return (action->sa_flags & SA_SIGINFO) != 0 ? DISPOSITION_SIGINFO_HANDLER : DISPOSITION_HANDLER; + return DISPOSITION_HANDLER; } static void cabledyn_signal_handler(int sig, siginfo_t *info, void *context) { + (void)context; int k = signal_index(sig); int fault = sig == SIGSEGV || sig == SIGBUS || sig == SIGFPE || sig == SIGILL; /* A fault the hardware raised re-executes when the handler returns; one sent by kill() @@ -418,16 +417,14 @@ static void cabledyn_signal_handler(int sig, siginfo_t *info, void *context) return; } /* Hand the signal to whatever handled it before (the Fortran runtime's backtrace or the - * default action), so the exit status and the fault context are unchanged. */ + * default action). The saved action is re-installed exactly as it was -- handler, flags + * and mask -- and the signal is delivered again by the kernel, never by a direct call, so + * SA_RESETHAND, the saved sa_mask and the blocking of the signal during its own handler + * all apply as they would have without the report. */ if (sigaction(sig, &cabledyn_previous[k], NULL) != 0) { /* Never leave this handler in place to be entered again. */ signal(sig, SIG_DFL); } - if (disposition(&cabledyn_previous[k]) == DISPOSITION_SIGINFO_HANDLER) { - /* A previous three-argument handler receives the original siginfo and context. */ - cabledyn_previous[k].sa_sigaction(sig, info, context); - return; - } if (synchronous) { /* Returning re-executes the faulting instruction, which raises the same fault, with * its own address and context, under the previous disposition. */ diff --git a/tests/blas_config_probe.c b/tests/blas_config_probe.c new file mode 100644 index 00000000..e1b71968 --- /dev/null +++ b/tests/blas_config_probe.c @@ -0,0 +1,59 @@ +/* File: tests/blas_config_probe.c + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2026 Jae Hoon Seo, SMI Lab, Inha University + * + * The OpenBLAS build configuration and kernel (openblas_get_config) of the runtime this test + * process loaded, for its log: a failure that depends on the host CPU then names the kernel + * it ran. Windows GNU builds only (openblas.dll is loaded at run time); elsewhere, or when + * the library is not OpenBLAS, the text is empty. + */ +#include + +int cabledyn_test_blas_config(char *text, int capacity); + +#if defined(_WIN32) +#include + +typedef char *(__cdecl *config_fn)(void); + +int cabledyn_test_blas_config(char *text, int capacity) +{ + HMODULE blas = GetModuleHandleA("openblas.dll"); + FARPROC proc; + config_fn config; + const char *value; + size_t n; + if (text == NULL || capacity < 1) { + return 0; + } + text[0] = '\0'; + if (blas == NULL) { + return 0; + } + proc = GetProcAddress(blas, "openblas_get_config"); + if (proc == NULL) { + return 0; + } + /* A FARPROC converted without a function-pointer cast (as in src/cabledyn_blas.c). */ + memcpy(&config, &proc, sizeof config); + value = config(); + if (value == NULL) { + return 0; + } + n = strlen(value); + if (n > (size_t)(capacity - 1)) { + n = (size_t)(capacity - 1); + } + memcpy(text, value, n); + text[n] = '\0'; + return (int)n; +} +#else +int cabledyn_test_blas_config(char *text, int capacity) +{ + if (text != NULL && capacity > 0) { + text[0] = '\0'; + } + return 0; +} +#endif diff --git a/tests/fatal_report_faults.c b/tests/fatal_report_faults.c index ba96d3a9..204b7460 100644 --- a/tests/fatal_report_faults.c +++ b/tests/fatal_report_faults.c @@ -179,11 +179,40 @@ static void prior_term_handler(int sig) _exit(7); } +/* SIGTERM handler of the SA_RESETHAND case: notes the call and returns. */ +static void resethand_term_handler(int sig) +{ + static const char text[] = "reset-on-delivery handler ran\n"; + ssize_t ignored = write(STDERR_FILENO, text, sizeof text - 1); + (void)ignored; + (void)sig; +} + +/* SIGTERM handler of the sa_mask case: SIGUSR1 (its sa_mask) and SIGTERM itself (no + * SA_NODEFER) must be blocked while it runs. */ +static void masked_term_handler(int sig) +{ + static const char ok_text[] = "masked handler: mask as saved\n"; + static const char bad_text[] = "masked handler: mask NOT as saved\n"; + sigset_t now; + ssize_t ignored; + int ok; + (void)sig; + sigemptyset(&now); + sigprocmask(SIG_BLOCK, NULL, &now); + ok = sigismember(&now, SIGUSR1) == 1 && sigismember(&now, SIGTERM) == 1; + ignored = ok ? write(STDERR_FILENO, ok_text, sizeof ok_text - 1) + : write(STDERR_FILENO, bad_text, sizeof bad_text - 1); + (void)ignored; + _exit(5); +} + /* Run one child that sets SIGTERM to `setup`, installs the report and sends itself SIGTERM. - * setup 0: SIG_IGN with SA_SIGINFO; 1: SIG_IGN; 2: SIG_DFL; 3: a one-argument handler. */ + * setup 0: SIG_IGN with SA_SIGINFO; 1: SIG_IGN; 2: SIG_DFL; 3: a one-argument handler; + * 4: a handler with SA_RESETHAND (then a second SIGTERM); 5: a handler with sa_mask SIGUSR1. */ static int disposition_case(int setup, const char *name) { - int errors[2], status = 0, ok, reports, ran; + int errors[2], status = 0, ok, reports, ran, reset_ran; pid_t child; char text[4096]; struct sigaction action; @@ -210,13 +239,23 @@ static int disposition_case(int setup, const char *name) action.sa_handler = SIG_IGN; } else if (setup == 2) { action.sa_handler = SIG_DFL; - } else { + } else if (setup == 3) { action.sa_handler = prior_term_handler; + } else if (setup == 4) { + action.sa_handler = resethand_term_handler; + action.sa_flags = SA_RESETHAND; + } else { + action.sa_handler = masked_term_handler; + sigaddset(&action.sa_mask, SIGUSR1); } sigaction(SIGTERM, &action, NULL); cabledyn_fatal_report_install(); cabledyn_fatal_report_time(7534.6); kill(getpid(), SIGTERM); + if (setup == 4) { + /* The first SIGTERM reset the disposition: this one takes the default action. */ + kill(getpid(), SIGTERM); + } /* Still running: an ignored SIGTERM must leave the process alone. */ _exit(0); } @@ -225,12 +264,18 @@ static int disposition_case(int setup, const char *name) waitpid(child, &status, 0); reports = count(text, "CableDyn_driver: stopped by a termination request"); ran = count(text, "prior SIGTERM handler ran"); + reset_ran = count(text, "reset-on-delivery handler ran"); if (setup <= 1) { ok = WIFEXITED(status) && WEXITSTATUS(status) == 0 && reports == 0; } else if (setup == 2) { ok = WIFSIGNALED(status) && WTERMSIG(status) == SIGTERM && reports == 1; - } else { + } else if (setup == 3) { ok = WIFEXITED(status) && WEXITSTATUS(status) == 7 && reports == 1 && ran == 1; + } else if (setup == 4) { + ok = WIFSIGNALED(status) && WTERMSIG(status) == SIGTERM && reports == 1 && reset_ran == 1; + } else { + ok = WIFEXITED(status) && WEXITSTATUS(status) == 5 && reports == 1 && + count(text, "masked handler: mask as saved") == 1; } fprintf(stderr, "dispositions: %s: child ended %s %d, %d report(s)%s\n", name, WIFSIGNALED(status) ? "by signal" : "with status", @@ -244,7 +289,8 @@ static int disposition_case(int setup, const char *name) /* An ignored SIGTERM (with or without SA_SIGINFO) stays ignored and is never reported or * called; a default one is reported and ends the process by SIGTERM; a previous handler is - * reported and then runs. Returns 0 when all hold, 1 otherwise. */ + * reported and then runs under its own flags and mask (SA_RESETHAND resets it, its sa_mask + * and SIGTERM itself are blocked while it runs). Returns 0 when all hold, 1 otherwise. */ int fatal_report_test_dispositions(void) { int failures = 0; @@ -252,6 +298,8 @@ int fatal_report_test_dispositions(void) failures += disposition_case(1, "SIG_IGN"); failures += disposition_case(2, "SIG_DFL"); failures += disposition_case(3, "a one-argument handler"); + failures += disposition_case(4, "a handler with SA_RESETHAND"); + failures += disposition_case(5, "a handler with an sa_mask"); if (failures != 0) { fprintf(stderr, "FAIL: dispositions: %d case(s) wrong\n", failures); return 1; diff --git a/tests/test_modal.f90 b/tests/test_modal.f90 index 00790762..a6cdee8f 100644 --- a/tests/test_modal.f90 +++ b/tests/test_modal.f90 @@ -18,12 +18,25 @@ PROGRAM test_modal USE CableDyn_Precision, ONLY: wp USE CableDyn_Modal USE CableDyn_DeckDriver, ONLY: CD_Run_Deck_Driver, CD_DECKDRV_OK + USE CableDyn_FatalReport, ONLY: CD_Fatal_Report_Install + USE CableDyn_Linalg, ONLY: CD_Blas_Runtime_Check, CD_LINALG_OK + USE, INTRINSIC :: ISO_C_BINDING, ONLY: C_CHAR, C_INT, C_NULL_CHAR + USE, INTRINSIC :: ISO_FORTRAN_ENV, ONLY: output_unit IMPLICIT NONE + INTERFACE + INTEGER(C_INT) FUNCTION blas_config(text, capacity) BIND(C, name='cabledyn_test_blas_config') + IMPORT :: C_CHAR, C_INT + CHARACTER(KIND=C_CHAR), INTENT(OUT) :: text(*) + INTEGER(C_INT), VALUE :: capacity + END FUNCTION blas_config + END INTERFACE + REAL(wp), PARAMETER :: PI = 3.141592653589793238462643383279502884197_wp INTEGER :: nfail nfail = 0 + CALL report_environment() CALL case_taut_string() CALL case_hermite_beam() CALL case_hermite_long_beam() @@ -38,6 +51,30 @@ PROGRAM test_modal CONTAINS + SUBROUTINE report_environment() + !! A crash of this test on some CI hosts left no output. Report an abnormal end (the + !! fault, e.g. a stack overflow or an access violation) on stderr, and name the LAPACK + !! runtime and the OpenBLAS kernel this host selected, before any case runs. + CHARACTER(KIND=C_CHAR) :: config(512) + CHARACTER(512) :: text + CHARACTER(1024) :: em + INTEGER :: es, n, i + CALL CD_Fatal_Report_Install() + CALL CD_Blas_Runtime_Check('test_modal', es, em) + IF (es /= CD_LINALG_OK) THEN + WRITE (*, '(A)') 'test_modal: '//TRIM(em) + ERROR STOP 1 + END IF + n = blas_config(config, INT(SIZE(config), C_INT)) + text = '' + DO i = 1, MIN(n, LEN(text)) + IF (config(i) == C_NULL_CHAR) EXIT + text(i:i) = config(i) + END DO + IF (n > 0) WRITE (*, '(A)') 'test_modal: LAPACK runtime '//TRIM(text) + FLUSH (output_unit) + END SUBROUTINE report_environment + INCLUDE 'nan_max_abs.inc' SUBROUTINE require(cond, label) From 5aebdc19eb3087b0eae2349a4461167ec6d41bcd Mon Sep 17 00:00:00 2001 From: Jae Hoon Seo Date: Fri, 9 Oct 2026 14:11:48 +0900 Subject: [PATCH 5/6] State the exit contract at start; hoist per-thread setup out of loops The driver now writes, after its banner, one line stating that every failure ends stderr with "CableDyn_driver: ended with exit code ". The Python runner infers an outside end (no closing line) only when that line is present; with a driver that does not write it (0.1.0 and earlier), a failure is reported with the driver's own text and the time its output reached, as before. Every OpenMP region is now a PARALLEL region whose first statement is CD_Fatal_Thread_Init, followed by the worksharing DO, so a thread is prepared once per region instead of once per element. The source guard now refuses combined PARALLEL DO constructs and a per-iteration call. Hardening: a thread checks the process-wide "installed" flag before its thread-local one, so a host that never installs the report never touches it; the POSIX report flag is set with an atomic exchange, so two threads that fault together report once; alternate signal stacks are at least 64 KiB and at least the system's SIGSTKSZ / _SC_SIGSTKSZ (128 KiB on macOS; larger on CPUs with big vector state). Tests: an older driver's refusal is reported with its own text, a new driver without a closing line as ended from outside; the report followed by runtime lines (forrtl, a libgfortran backtrace) is read as an abnormal end; a library deck run leaves the host's signal dispositions and unhandled-exception filter unchanged; each prepared thread has one alternate stack of the system size; only app/cabledyn.f90 installs the handlers; the real driver writes the exact start-up line. --- CHANGELOG.md | 8 +- CMakeLists.txt | 4 + app/cabledyn.f90 | 5 + doc/standalone_driver.rst | 11 +- python/cabledyn/driver.py | 11 ++ python/tests/test_driver.py | 48 +++++++- src/CableDyn_Assemble.f90 | 16 ++- src/CableDyn_CosseratAssemble.f90 | 24 ++-- src/CableDyn_HermiteCableDynamic.f90 | 8 +- src/CableDyn_HermiteCableStatic.f90 | 16 ++- src/CableDyn_OpenFAST_Aggregate.f90 | 20 ++-- src/CableDyn_System.f90 | 16 ++- src/cabledyn_fatal.c | 52 ++++++--- tests/check_driver_nonconvergence.cmake | 8 ++ tests/fatal_report_faults.c | 136 ++++++++++++++++++++++ tests/test_fatal_report.f90 | 28 ++++- tests/test_omp_fatal_init.py | 148 +++++++++++++++--------- 17 files changed, 444 insertions(+), 115 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 13300527..886c3453 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,9 +35,11 @@ All notable changes to CableDyn are recorded in this file. The format follows `CableDyn_driver: ended with exit code `. A process ended from outside (`taskkill /F`, *End task*, `kill -9`) runs none of its own code and cannot report anything; the missing closing line now identifies it, since `taskkill /F` leaves exit code 1, the same code as a - refused input. `CableDynDriver.run`, `cabledyn-run` and `cabledyn-study` report such a run as - ended early, with the time its output reached, instead of relaying the start-up log as the - error. The documentation explains how to recognise each kind of early end and warns that + refused input. The driver states this at start, after the banner, in an `Exit status:` line. + `CableDynDriver.run`, `cabledyn-run` and `cabledyn-study` report such a run as ended early, + with the time its output reached, instead of relaying the start-up log as the error; with a + driver that does not write the `Exit status:` line (0.1.0 and earlier), a failure is + reported with the driver's own text as before. The documentation explains how to recognise each kind of early end and warns that `taskkill /IM CableDyn_driver.exe /F` ends every CableDyn run on the computer. - The static solve of a taut, neutrally buoyant finite-EI line (mass per length equal to the displaced mass) no longer depends on the sign of the round-off weight: a line whose diff --git a/CMakeLists.txt b/CMakeLists.txt index de4b259b..38b87a38 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1864,6 +1864,10 @@ set_tests_properties(fatal_report_fault_context PROPERTIES FAIL_REGULAR_EXPRESSION "FAIL:") # POSIX: the report keeps every previous disposition -- an ignored signal (with or without # SA_SIGINFO) stays ignored, a default one still ends the process, a handler still runs. +add_test(NAME fatal_report_altstack COMMAND test_fatal_report altstack) +set_tests_properties(fatal_report_altstack PROPERTIES + LABELS "fortran;unit;driver;stack" TIMEOUT 120 + FAIL_REGULAR_EXPRESSION "FAIL:") add_test(NAME fatal_report_dispositions COMMAND test_fatal_report dispositions) set_tests_properties(fatal_report_dispositions PROPERTIES LABELS "fortran;unit;driver" TIMEOUT 120 diff --git a/app/cabledyn.f90 b/app/cabledyn.f90 index 1b2f5324..9bdf3425 100644 --- a/app/cabledyn.f90 +++ b/app/cabledyn.f90 @@ -100,6 +100,11 @@ PROGRAM cabledyn ! remain visible on stdout; automation must use the exit status rather than assuming ! stdout contains only one machine-readable record. CALL CD_Print_Banner(error_unit) + ! The exit contract, stated once at start: a caller that finds this line and later no + ! closing line knows the process was ended from outside (a driver older than this line + ! writes neither, so its failures must not be read that way). + WRITE (error_unit, '(A)') ' Exit status: every failure ends stderr with "'//PROG// & + ': ended with exit code ".' ! The LAPACK runtime is loaded on first use in the Windows GNU build; a missing or ! incompatible library stops here with its diagnostic instead of as a solver failure. CALL CD_Blas_Runtime_Check(PROG, ErrStat, ErrMsg) diff --git a/doc/standalone_driver.rst b/doc/standalone_driver.rst index fc7acc25..db755c66 100644 --- a/doc/standalone_driver.rst +++ b/doc/standalone_driver.rst @@ -262,8 +262,12 @@ before the end, and their last time is below ``TMax``. To tell a completed run from one that ended early, check the exit status, the closing line, and that the last time in ``.out`` reaches ``TMax``; the Python wrapper below reports -such an end. ``taskkill /IM CableDyn_driver.exe /F`` ends *every* CableDyn run on the computer, not one; to -stop a single run, end it by its process id (``taskkill /PID /F``). +such an end. The driver states this contract at start, after the banner, with the line +``Exit status: every failure ends stderr with "CableDyn_driver: ended with exit code ".``; +a missing closing line means an outside end only when that line is present, since drivers of +version 0.1.0 and earlier write neither. ``taskkill /IM CableDyn_driver.exe /F`` ends *every* +CableDyn run on the computer, not one; to stop a single run, end it by its process id +(``taskkill /PID /F``). Output streams ~~~~~~~~~~~~~~ @@ -275,7 +279,8 @@ Output streams * - Stream - Content * - ``stderr`` - - the identity banner of a normal run; every error message; the initialisation report of a + - the identity banner of a normal run and the ``Exit status:`` line after it; every error + message; the initialisation report of a single-family (all ``EI = 0`` or all finite-EI) deck; the report of an interrupt or a fatal fault; and, last on a failed run, the closing line ``CableDyn_driver: ended with exit code `` diff --git a/python/cabledyn/driver.py b/python/cabledyn/driver.py index 3b1f24a2..98ce8672 100644 --- a/python/cabledyn/driver.py +++ b/python/cabledyn/driver.py @@ -66,6 +66,12 @@ def _timeout_stream(value: str | bytes | None) -> str: # The last stderr line of every non-zero exit the driver makes itself. _CLOSING_LINE = re.compile(r"CableDyn_driver: ended with exit code -?\d+\s*\Z") +# The start-up line of a driver that writes the closing line on every failure (app/cabledyn.f90). +_EXIT_CONTRACT = re.compile( + r"^\s*Exit status: every failure ends stderr with " + r"\"CableDyn_driver: ended with exit code \"", + re.MULTILINE, +) # The driver's report of an interrupt or a fatal fault (src/cabledyn_fatal.c). _ABNORMAL_REPORT = re.compile(r"^CableDyn_driver: (?:fatal error:|stopped by) .*$", re.MULTILINE) @@ -107,6 +113,11 @@ def _failure_message(returncode: int, stdout: str, stderr: str, main: Path) -> s report = _ABNORMAL_REPORT.findall(stderr) if report: return f"CableDyn driver ended abnormally (exit code {returncode}){reached}: {report[-1]}" + if not _EXIT_CONTRACT.search(stderr): + # A driver that does not state the exit contract (0.1.0 and older) writes no closing + # line on any failure, so its own refusals look just like this: report them as such. + diagnostic = stderr.strip() or stdout.strip() or "no diagnostic" + return f"CableDyn driver failed with exit code {returncode}{reached}: {diagnostic}" # The conclusion goes last: summaries show the last line of a message. diagnostic = stderr.strip() or stdout.strip() tail = "".join(f"{line}\n" for line in diagnostic.splitlines()[-20:]) diff --git a/python/tests/test_driver.py b/python/tests/test_driver.py index a9f4e53f..8705d584 100644 --- a/python/tests/test_driver.py +++ b/python/tests/test_driver.py @@ -242,6 +242,10 @@ def test_nonzero_exit_preserves_native_diagnostic(tmp_path, monkeypatch): _CLOSING = "CableDyn_driver: ended with exit code 2\n" +# The start-up line of a driver that writes the closing line on every failure. +_CONTRACT = ( + ' Exit status: every failure ends stderr with "CableDyn_driver: ended with exit code ".\n' +) _PARTIAL_TABLE = "# CableDyn\nTime\tFairTen1\n0.0\t1.0\n7534.6\t2.0\n7534.65\t2." @@ -265,17 +269,54 @@ def test_nonzero_exit_preserves_native_diagnostic(tmp_path, monkeypatch): "ended abnormally (exit code 3221225725); its output ends at t = 7534.6 s: " "CableDyn_driver: fatal error: stack overflow", ), - # Ended from outside (taskkill /F): no closing line and no report. + # The report followed by the runtime's own lines: the release executable's Ctrl+Break + # (forrtl, exit 1) and a GNU access violation (the libgfortran backtrace). + ( + 1, + _CONTRACT + "\nCableDyn_driver: stopped by Ctrl+Break after the step at simulated " + "time t = 97.400 s. The run did not finish; its output files end at the last step " + "written.\nforrtl: error (200): program aborting due to control-BREAK event\n" + "Image PC Routine Line Source\n", + _PARTIAL_TABLE, + "ended abnormally (exit code 1); its output ends at t = 7534.6 s: " + "CableDyn_driver: stopped by Ctrl+Break after the step at simulated time t = 97.400 s", + ), + ( + 3, + _CONTRACT + "\nCableDyn_driver: fatal error: access violation (exception 0xC0000005) " + "after the step at simulated time t = 12.000 s. The run did not finish; its output " + "files end at the last step written.\n\nProgram received signal SIGSEGV: " + "Segmentation fault - invalid memory reference.\n\nBacktrace for this error:\n", + None, + "ended abnormally (exit code 3): CableDyn_driver: fatal error: access violation", + ), + # Ended from outside (taskkill /F): the driver stated the exit contract, then wrote + # no closing line and no report. ( 1, - " CableDyn initialization completed.\n", + _CONTRACT + " CableDyn initialization completed.\n", _PARTIAL_TABLE, "initialization completed.\nCableDyn driver ended with exit code 1 without " "reporting a result; its output ends at t = 7534.6 s: the process was ended from " "outside", ), # The same with no output table yet. - (-9, "", None, "ended with exit code -9 without reporting a result: the process"), + ( + -9, + _CONTRACT, + None, + "ended with exit code -9 without reporting a result: the process", + ), + # A driver older than the exit contract (0.1.0) writes no closing line on its own + # refusals: they are reported with their own text, never as ended from outside. + ( + 1, + " CableDyn v0.1.0\nCableDyn_DeckDriver: deck line 42: bad value\n", + _PARTIAL_TABLE, + "failed with exit code 1; its output ends at t = 7534.6 s: CableDyn v0.1.0\n" + "CableDyn_DeckDriver: deck line 42: bad value", + ), + (2, "", None, "failed with exit code 2: Progress: 65.0%"), ], ) def test_nonzero_exit_tells_driver_failures_from_abnormal_ends( @@ -298,6 +339,7 @@ def fake_run(*args, **kwargs): assert caught.value.returncode == returncode assert expected in str(caught.value) assert caught.value.stderr == stderr + assert ("ended from outside" in str(caught.value)) == ("without reporting" in expected) @pytest.mark.parametrize( diff --git a/src/CableDyn_Assemble.f90 b/src/CableDyn_Assemble.f90 index 0756e308..9e2e4f07 100644 --- a/src/CableDyn_Assemble.f90 +++ b/src/CableDyn_Assemble.f90 @@ -377,18 +377,20 @@ SUBROUTINE CD_Assemble_Cable_Internal_Force(nodes, elem_conn, l0, ea, tension_on ALLOCATE (fe(6, n_elem), elem_es(n_elem)) n_team = axial_team_size() - !$OMP PARALLEL DO DEFAULT(NONE) SCHEDULE(static) NUM_THREADS(n_team) & + !$OMP PARALLEL DEFAULT(NONE) NUM_THREADS(n_team) & !$OMP SHARED(n_elem, elem_conn, nodes, ea, l0, tension_only, fe, elem_es) & !$OMP PRIVATE(e, a, b, nodes6, Kt6, Te, em) + CALL CD_Fatal_Thread_Init() ! this thread can report its own stack overflow + !$OMP DO SCHEDULE(static) DO e = 1, n_elem - CALL CD_Fatal_Thread_Init() ! this thread can report its own stack overflow a = elem_conn(1, e) b = elem_conn(2, e) nodes6(1:3) = nodes(:, a) nodes6(4:6) = nodes(:, b) CALL CD_Compute_Cable_Element(nodes6, ea(e), l0(e), tension_only, Kt6, fe(:, e), Te, elem_es(e), em) END DO - !$OMP END PARALLEL DO + !$OMP END DO + !$OMP END PARALLEL DO e = 1, n_elem IF (elem_es(e) /= 0) THEN ! Re-evaluate the first failing element serially for its message. @@ -458,18 +460,20 @@ SUBROUTINE CD_Compute_Cable_Tension(nodes, elem_conn, l0, ea, tension_only, & ALLOCATE (te_buf(n_elem), elem_es(n_elem)) n_team = axial_team_size() - !$OMP PARALLEL DO DEFAULT(NONE) SCHEDULE(static) NUM_THREADS(n_team) & + !$OMP PARALLEL DEFAULT(NONE) NUM_THREADS(n_team) & !$OMP SHARED(n_elem, elem_conn, nodes, ea, l0, tension_only, te_buf, elem_es) & !$OMP PRIVATE(e, a, b, nodes6, Kt6, fint6, em) + CALL CD_Fatal_Thread_Init() ! this thread can report its own stack overflow + !$OMP DO SCHEDULE(static) DO e = 1, n_elem - CALL CD_Fatal_Thread_Init() ! this thread can report its own stack overflow a = elem_conn(1, e) b = elem_conn(2, e) nodes6(1:3) = nodes(:, a) nodes6(4:6) = nodes(:, b) CALL CD_Compute_Cable_Element(nodes6, ea(e), l0(e), tension_only, Kt6, fint6, te_buf(e), elem_es(e), em) END DO - !$OMP END PARALLEL DO + !$OMP END DO + !$OMP END PARALLEL DO e = 1, n_elem IF (elem_es(e) /= 0) THEN ! Re-evaluate the first failing element serially for its message. diff --git a/src/CableDyn_CosseratAssemble.f90 b/src/CableDyn_CosseratAssemble.f90 index a54eea35..f6472df8 100644 --- a/src/CableDyn_CosseratAssemble.f90 +++ b/src/CableDyn_CosseratAssemble.f90 @@ -113,11 +113,12 @@ SUBROUTINE CD_Assemble_Cosserat_Tangent_Force(nodes_ref, elem_conn, ea, gas, ei, Ke = 0.0_wp elem_es = 0 elem_em = '' - !$OMP PARALLEL DO DEFAULT(NONE) SCHEDULE(static) & + !$OMP PARALLEL DEFAULT(NONE) & !$OMP SHARED(n_elem, elem_conn, nodes_ref, q, ea, gas, ei, gj, reduced_shear, fe, Ke, elem_es, elem_em) & !$OMP PRIVATE(e, a, b, Lam0, L0, qe, finte, Kte, es, em) + CALL CD_Fatal_Thread_Init() ! this thread can report its own stack overflow + !$OMP DO SCHEDULE(static) DO e = 1, n_elem - CALL CD_Fatal_Thread_Init() ! this thread can report its own stack overflow a = elem_conn(1, e) b = elem_conn(2, e) CALL CD_Reference_Frame(nodes_ref(:, a), nodes_ref(:, b), Lam0, L0) @@ -133,7 +134,8 @@ SUBROUTINE CD_Assemble_Cosserat_Tangent_Force(nodes_ref, elem_conn, ea, gas, ei, Ke(:, :, e) = Kte END IF END DO - !$OMP END PARALLEL DO + !$OMP END DO + !$OMP END PARALLEL DO e = 1, n_elem IF (elem_es(e) /= 0) THEN Kt = 0.0_wp; fint = 0.0_wp @@ -314,11 +316,12 @@ SUBROUTINE CD_Assemble_Cosserat_Tangent_Force_Banded_Free_Workspace(nodes_ref, e work%Ke(:, :, 1:n_elem) = 0.0_wp work%elem_es(1:n_elem) = 0 work%elem_em(1:n_elem) = '' - !$OMP PARALLEL DO DEFAULT(NONE) SCHEDULE(static) & + !$OMP PARALLEL DEFAULT(NONE) & !$OMP SHARED(n_elem, elem_conn, nodes_ref, q, ea, gas, ei, gj, reduced_shear, work) & !$OMP PRIVATE(e, a, b, Lam0, L0, qe, finte, Kte, es, em) + CALL CD_Fatal_Thread_Init() ! this thread can report its own stack overflow + !$OMP DO SCHEDULE(static) DO e = 1, n_elem - CALL CD_Fatal_Thread_Init() ! this thread can report its own stack overflow a = elem_conn(1, e) b = elem_conn(2, e) CALL CD_Reference_Frame(nodes_ref(:, a), nodes_ref(:, b), Lam0, L0) @@ -334,7 +337,8 @@ SUBROUTINE CD_Assemble_Cosserat_Tangent_Force_Banded_Free_Workspace(nodes_ref, e work%Ke(:, :, e) = Kte END IF END DO - !$OMP END PARALLEL DO + !$OMP END DO + !$OMP END PARALLEL DO e = 1, n_elem IF (work%elem_es(e) /= 0) THEN Kb = 0.0_wp; fint = 0.0_wp @@ -428,11 +432,12 @@ SUBROUTINE CD_Assemble_Cosserat_Internal_Force_Workspace(nodes_ref, elem_conn, e work%fe(:, 1:n_elem) = 0.0_wp work%elem_es(1:n_elem) = 0 work%elem_em(1:n_elem) = '' - !$OMP PARALLEL DO DEFAULT(NONE) SCHEDULE(static) & + !$OMP PARALLEL DEFAULT(NONE) & !$OMP SHARED(n_elem, elem_conn, nodes_ref, q, ea, gas, ei, gj, reduced_shear, work) & !$OMP PRIVATE(e, a, b, Lam0, L0, qe, finte, es, em) + CALL CD_Fatal_Thread_Init() ! this thread can report its own stack overflow + !$OMP DO SCHEDULE(static) DO e = 1, n_elem - CALL CD_Fatal_Thread_Init() ! this thread can report its own stack overflow a = elem_conn(1, e) b = elem_conn(2, e) CALL CD_Reference_Frame(nodes_ref(:, a), nodes_ref(:, b), Lam0, L0) @@ -447,7 +452,8 @@ SUBROUTINE CD_Assemble_Cosserat_Internal_Force_Workspace(nodes_ref, elem_conn, e work%fe(:, e) = finte END IF END DO - !$OMP END PARALLEL DO + !$OMP END DO + !$OMP END PARALLEL DO e = 1, n_elem IF (work%elem_es(e) /= 0) THEN fint = 0.0_wp diff --git a/src/CableDyn_HermiteCableDynamic.f90 b/src/CableDyn_HermiteCableDynamic.f90 index 8a17b562..6e73a63b 100644 --- a/src/CableDyn_HermiteCableDynamic.f90 +++ b/src/CableDyn_HermiteCableDynamic.f90 @@ -5255,15 +5255,17 @@ SUBROUTINE assemble_residual(model, q_cfg, v_cfg, t_eval, want_residual, want_ta ! combined-output requests retain the lower-overhead serial path. A four-thread cap avoids ! tiny-kernel oversubscription while respecting a smaller host OpenMP thread limit. IF (tangent_only) THEN - !$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(qe) SCHEDULE(STATIC) NUM_THREADS(omp_threads) IF(ne >= 32) + !$OMP PARALLEL DEFAULT(SHARED) PRIVATE(qe) NUM_THREADS(omp_threads) IF(ne >= 32) + CALL CD_Fatal_Thread_Init() ! this thread can report its own stack overflow + !$OMP DO SCHEDULE(STATIC) DO e = 1, ne - CALL CD_Fatal_Thread_Init() ! this thread can report its own stack overflow CALL evaluate_tangent_element(model, e, q_cfg, v_cfg, t_eval, model%ws_elem_K(:, :, e), & model%ws_elem_Kdrag(:, :, e), model%ws_elem_Kv(:, :, e), & model%ws_elem_Kwave(:, :, e), model%ws_elem_Kheld(:, :, e), & model%ws_elem_es(e), model%ws_elem_em(e)) END DO - !$OMP END PARALLEL DO + !$OMP END DO + !$OMP END PARALLEL DO e = 1, ne IF (model%ws_elem_es(e) /= CD_HCABLE_OK) THEN ErrStat = CD_HCDYN_NOCONVERGE; ErrMsg = 'element: '//TRIM(model%ws_elem_em(e)); RETURN diff --git a/src/CableDyn_HermiteCableStatic.f90 b/src/CableDyn_HermiteCableStatic.f90 index d819115b..54f5021e 100644 --- a/src/CableDyn_HermiteCableStatic.f90 +++ b/src/CableDyn_HermiteCableStatic.f90 @@ -1011,9 +1011,10 @@ RECURSIVE SUBROUTINE newton_step(rnorm, es, em, do_update) ! Evaluate independent element kernels concurrently into solve-lifetime buffers, ! then scatter in element order to retain deterministic residual/tangent sums. - !$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(qe) SCHEDULE(STATIC) NUM_THREADS(omp_threads) IF(ne >= 32) + !$OMP PARALLEL DEFAULT(SHARED) PRIVATE(qe) NUM_THREADS(omp_threads) IF(ne >= 32) + CALL CD_Fatal_Thread_Init() ! this thread can report its own stack overflow + !$OMP DO SCHEDULE(STATIC) DO e = 1, ne - CALL CD_Fatal_Thread_Init() ! this thread can report its own stack overflow qe(1:3) = q(6*(e - 1) + 1:6*(e - 1) + 3) qe(4:6) = q(6*(e - 1) + 4:6*(e - 1) + 6) qe(7:9) = q(6*e + 1:6*e + 3) @@ -1030,7 +1031,8 @@ RECURSIVE SUBROUTINE newton_step(rnorm, es, em, do_update) bending_quadrature_order=bending_order) END IF END DO - !$OMP END PARALLEL DO + !$OMP END DO + !$OMP END PARALLEL pi_total = SUM(elem_E) pi_abs = SUM(ABS(elem_E)) DO e = 1, ne @@ -2127,15 +2129,17 @@ SUBROUTINE add_current_drag(ecs, ecm) ! The element loads are independent: evaluated concurrently into the element buffers ! (free once the internal forces are scattered), then scattered in element order, so ! the sums are those of the serial loop. - !$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(qe) SCHEDULE(STATIC) NUM_THREADS(omp_threads) IF(ne >= 32) + !$OMP PARALLEL DEFAULT(SHARED) PRIVATE(qe) NUM_THREADS(omp_threads) IF(ne >= 32) + CALL CD_Fatal_Thread_Init() ! this thread can report its own stack overflow + !$OMP DO SCHEDULE(STATIC) DO e = 1, ne - CALL CD_Fatal_Thread_Init() ! this thread can report its own stack overflow qe(1:6) = q(6*(e - 1) + 1:6*e) qe(7:12) = q(6*e + 1:6*e + 6) CALL current_element_load(current, qe, l0(e), cur_d(e), cur_cn(e), cur_ct(e), want_jac, & elem_f(:, e), elem_K(:, :, e), elem_es(e), elem_em(e)) END DO - !$OMP END PARALLEL DO + !$OMP END DO + !$OMP END PARALLEL DO e = 1, ne IF (elem_es(e) /= CD_HCSTAT_OK) THEN ecs = elem_es(e) diff --git a/src/CableDyn_OpenFAST_Aggregate.f90 b/src/CableDyn_OpenFAST_Aggregate.f90 index 095b15c4..33c24088 100644 --- a/src/CableDyn_OpenFAST_Aggregate.f90 +++ b/src/CableDyn_OpenFAST_Aggregate.f90 @@ -1099,23 +1099,26 @@ SUBROUTINE CD_AGG_Step_Moving(self, dt, position, velocity, acceleration, conver n_iter = MAX(n_iter, nsys) END IF IF (PRESENT(orientation)) THEN - !$OMP PARALLEL DO DEFAULT(NONE) SHARED(self, position, velocity, acceleration, orientation) & + !$OMP PARALLEL DEFAULT(NONE) SHARED(self, position, velocity, acceleration, orientation) & !$OMP PRIVATE(c, col, es, em) & - !$OMP IF(self%ncable > 1 .AND. .NOT. CD_HermiteCable_Dyn_Profile_Enabled()) SCHEDULE(STATIC) + !$OMP IF(self%ncable > 1 .AND. .NOT. CD_HermiteCable_Dyn_Profile_Enabled()) + CALL CD_Fatal_Thread_Init() ! this thread can report its own stack overflow + !$OMP DO SCHEDULE(STATIC) DO c = 1, self%ncable - CALL CD_Fatal_Thread_Init() ! this thread can report its own stack overflow col = self%ncp_sys + c CALL CD_HFMF_UpdateStates(self%cables(c), position(:, col), velocity(:, col), acceleration(:, col), & es, em, orientation(:, :, col), self%mv_omega(:, col), self%mv_alpha(:, col)) self%cable_stat(c) = es self%cable_msg(c) = em END DO - !$OMP END PARALLEL DO + !$OMP END DO + !$OMP END PARALLEL ELSE - !$OMP PARALLEL DO DEFAULT(NONE) SHARED(self, position, velocity, acceleration) PRIVATE(c, col, es, em) & - !$OMP IF(self%ncable > 1 .AND. .NOT. CD_HermiteCable_Dyn_Profile_Enabled()) SCHEDULE(STATIC) + !$OMP PARALLEL DEFAULT(NONE) SHARED(self, position, velocity, acceleration) PRIVATE(c, col, es, em) & + !$OMP IF(self%ncable > 1 .AND. .NOT. CD_HermiteCable_Dyn_Profile_Enabled()) + CALL CD_Fatal_Thread_Init() ! this thread can report its own stack overflow + !$OMP DO SCHEDULE(STATIC) DO c = 1, self%ncable - CALL CD_Fatal_Thread_Init() ! this thread can report its own stack overflow col = self%ncp_sys + c CALL CD_HFMF_UpdateStates(self%cables(c), position(:, col), velocity(:, col), acceleration(:, col), es, em, & u_angular_velocity=self%mv_omega(:, col), & @@ -1123,7 +1126,8 @@ SUBROUTINE CD_AGG_Step_Moving(self, dt, position, velocity, acceleration, conver self%cable_stat(c) = es self%cable_msg(c) = em END DO - !$OMP END PARALLEL DO + !$OMP END DO + !$OMP END PARALLEL END IF DO c = 1, self%ncable IF (self%cable_stat(c) /= CD_HFMF_OK) THEN diff --git a/src/CableDyn_System.f90 b/src/CableDyn_System.f90 index fdfab1ba..65c2c140 100644 --- a/src/CableDyn_System.f90 +++ b/src/CableDyn_System.f90 @@ -1307,16 +1307,18 @@ RECURSIVE SUBROUTINE CD_Step_System(system, dt, q_coupled, v_coupled, a_coupled, ! Line models own disjoint state and Newton workspaces. Workers stage results; ! the following serial pass selects the first error and reduces flags in deck order. n_team = line_team_size(system) - !$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(i) SCHEDULE(STATIC) NUM_THREADS(n_team) IF(n_team > 1) + !$OMP PARALLEL DEFAULT(SHARED) PRIVATE(i) NUM_THREADS(n_team) IF(n_team > 1) + CALL CD_Fatal_Thread_Init() ! this thread can report its own stack overflow + !$OMP DO SCHEDULE(STATIC) DO i = 1, system%n_lines - CALL CD_Fatal_Thread_Init() ! this thread can report its own stack overflow CALL CD_Step_Model(system%lines(i), dt, system%line_converged(i), system%line_stalled(i), & system%line_iter(i), system%line_stat(i), system%line_msg(i), & prescribed_q=system%lines(i)%q_prescribed_work, & prescribed_v=system%lines(i)%v_prescribed_work, & prescribed_a=system%lines(i)%a_prescribed_work) END DO - !$OMP END PARALLEL DO + !$OMP END DO + !$OMP END PARALLEL DO i = 1, system%n_lines IF (system%line_stat(i) /= CD_MODEL_OK) THEN CALL restore_failed_step(system, system%line_stat(i), 'line step failed: '//TRIM(system%line_msg(i)), & @@ -2082,15 +2084,17 @@ SUBROUTINE CD_Calc_System_CoupledLoads(system, coupled_loads, ErrStat, ErrMsg) RETURN END IF n_team = line_team_size(system) - !$OMP PARALLEL DO DEFAULT(SHARED) PRIVATE(i, lo, hi) SCHEDULE(STATIC) NUM_THREADS(n_team) IF(n_team > 1) + !$OMP PARALLEL DEFAULT(SHARED) PRIVATE(i, lo, hi) NUM_THREADS(n_team) IF(n_team > 1) + CALL CD_Fatal_Thread_Init() ! this thread can report its own stack overflow + !$OMP DO SCHEDULE(STATIC) DO i = 1, system%n_lines - CALL CD_Fatal_Thread_Init() ! this thread can report its own stack overflow lo = system%line_off(i) hi = system%line_off(i + 1) - 1 CALL CD_Calc_Model_CoupledLoads(system%lines(i), system%line_load_work(lo:hi), & system%line_stat(i), system%line_msg(i)) END DO - !$OMP END PARALLEL DO + !$OMP END DO + !$OMP END PARALLEL DO i = 1, system%n_lines IF (system%line_stat(i) /= CD_MODEL_OK) THEN ErrStat = CD_SYSTEM_SOLVEFAIL diff --git a/src/cabledyn_fatal.c b/src/cabledyn_fatal.c index 72317798..7b7f84b6 100644 --- a/src/cabledyn_fatal.c +++ b/src/cabledyn_fatal.c @@ -326,11 +326,11 @@ void cabledyn_fatal_report_install(void) static const int cabledyn_signals[CABLEDYN_NSIG] = {SIGSEGV, SIGBUS, SIGFPE, SIGILL, SIGABRT, SIGINT, SIGTERM, SIGHUP}; static struct sigaction cabledyn_previous[CABLEDYN_NSIG]; -static volatile sig_atomic_t cabledyn_reported = 0; +/* Set by the first report, with an atomic exchange (lock-free, so usable in a handler): two + * threads that fault together report once. */ +static volatile int cabledyn_reported = 0; /* Set once the driver has installed the report. */ static volatile int cabledyn_installed = 0; -/* The main thread's alternate stack, so a stack overflow can still be reported. */ -static char cabledyn_altstack[CABLEDYN_HANDLER_STACK]; /* Worker threads' alternate stacks, freed when each thread ends. */ static pthread_key_t cabledyn_altstack_key; static pthread_once_t cabledyn_altstack_once = PTHREAD_ONCE_INIT; @@ -397,9 +397,8 @@ static void cabledyn_signal_handler(int sig, siginfo_t *info, void *context) /* A fault the hardware raised re-executes when the handler returns; one sent by kill() * or raise() (si_code <= 0) does not, and is re-sent below. */ int synchronous = fault && info != NULL && info->si_code > 0; - if (!cabledyn_reported) { + if (__sync_lock_test_and_set(&cabledyn_reported, 1) == 0) { cabledyn_line line; - cabledyn_reported = 1; line.len = 0; line.text[0] = '\0'; line_add(&line, fault || sig == SIGABRT ? "\nCableDyn_driver: fatal error: " @@ -435,6 +434,31 @@ static void cabledyn_signal_handler(int sig, siginfo_t *info, void *context) raise(sig); } +/* The size of an alternate signal stack: at least CABLEDYN_HANDLER_STACK and at least what the + * system asks for (SIGSTKSZ is 128 KiB on macOS; on Linux it grows with the CPU's vector + * state, e.g. AVX-512 or AMX). */ +size_t cabledyn_fatal_altstack_size(void); +size_t cabledyn_fatal_altstack_size(void) +{ + size_t size = CABLEDYN_HANDLER_STACK; +#if defined(_SC_SIGSTKSZ) + long system_size = sysconf(_SC_SIGSTKSZ); + if (system_size > 0 && (size_t)system_size > size) { + size = (size_t)system_size; + } +#endif +#if defined(SIGSTKSZ) + { + /* SIGSTKSZ is a sysconf() call in recent glibc, which may return -1. */ + long default_size = (long)SIGSTKSZ; + if (default_size > 0 && (size_t)default_size > size) { + size = (size_t)default_size; + } + } +#endif + return size; +} + /* Free a worker thread's alternate stack when the thread ends. */ static void cabledyn_altstack_release(void *stack) { @@ -456,6 +480,7 @@ static void cabledyn_thread_setup(void) { stack_t current, alt; void *stack; + size_t size; if (sigaltstack(NULL, ¤t) == 0 && (current.ss_flags & SS_DISABLE) == 0) { return; } @@ -463,13 +488,14 @@ static void cabledyn_thread_setup(void) if (!cabledyn_altstack_key_ok) { return; } - stack = malloc(CABLEDYN_HANDLER_STACK); + size = cabledyn_fatal_altstack_size(); + stack = malloc(size); if (stack == NULL) { return; } memset(&alt, 0, sizeof alt); alt.ss_sp = stack; - alt.ss_size = CABLEDYN_HANDLER_STACK; + alt.ss_size = size; alt.ss_flags = 0; if (sigaltstack(&alt, NULL) != 0 || pthread_setspecific(cabledyn_altstack_key, stack) != 0) { cabledyn_altstack_release(stack); @@ -478,7 +504,6 @@ static void cabledyn_thread_setup(void) void cabledyn_fatal_report_install(void) { - stack_t alt; struct sigaction action; int k; if (cabledyn_installed) { @@ -486,12 +511,7 @@ void cabledyn_fatal_report_install(void) } cabledyn_installed = 1; cabledyn_thread_ready = 1; - /* The main thread uses a static alternate stack, which outlives every handler. */ - memset(&alt, 0, sizeof alt); - alt.ss_sp = cabledyn_altstack; - alt.ss_size = sizeof cabledyn_altstack; - alt.ss_flags = 0; - sigaltstack(&alt, NULL); + cabledyn_thread_setup(); memset(&action, 0, sizeof action); action.sa_sigaction = cabledyn_signal_handler; sigemptyset(&action.sa_mask); @@ -520,7 +540,9 @@ void cabledyn_fatal_report_install(void) * and after the first call on a thread it costs one thread-local test. */ void cabledyn_fatal_thread_init(void) { - if (cabledyn_thread_ready || !cabledyn_installed) { + /* The process-wide flag first: in a host that never installs the report, the thread-local + * flag is never touched (an emulated-TLS build would allocate it per thread). */ + if (!cabledyn_installed || cabledyn_thread_ready) { return; } cabledyn_thread_ready = 1; diff --git a/tests/check_driver_nonconvergence.cmake b/tests/check_driver_nonconvergence.cmake index 8e00f23b..469205d4 100644 --- a/tests/check_driver_nonconvergence.cmake +++ b/tests/check_driver_nonconvergence.cmake @@ -19,6 +19,14 @@ endif() if(NOT driver_stderr MATCHES "CableDyn_driver: ended with exit code 2[\r\n]*$") message(FATAL_ERROR "stderr does not end with the closing line for exit code 2:\n${driver_stderr}") endif() +# The start-up statement of that contract, exactly as python/cabledyn/driver.py recognises it: +# without it a caller cannot tell an outside kill from an older driver's refusal. +string(FIND "${driver_stderr}" + " Exit status: every failure ends stderr with \"CableDyn_driver: ended with exit code \"." + contract_at) +if(contract_at LESS 0) + message(FATAL_ERROR "stderr lacks the exit-status statement:\n${driver_stderr}") +endif() set(driver_text "${driver_stdout}\n${driver_stderr}") if(NOT driver_text MATCHES "did not converge" OR diff --git a/tests/fatal_report_faults.c b/tests/fatal_report_faults.c index 204b7460..ab64de80 100644 --- a/tests/fatal_report_faults.c +++ b/tests/fatal_report_faults.c @@ -12,6 +12,8 @@ void fatal_report_test_null_write(void); int fatal_report_test_raise_term(void); int fatal_report_test_fault_context(void); int fatal_report_test_dispositions(void); +int fatal_report_test_altstack(void); +int fatal_report_test_handlers_unchanged(int phase); static volatile int *volatile fatal_report_null_target = 0; @@ -44,6 +46,34 @@ int fatal_report_test_dispositions(void) { return -1; } + +/* The alternate-stack check is POSIX-only: returns -1 (skipped). */ +int fatal_report_test_altstack(void) +{ + return -1; +} + +#include +#include + +/* The unhandled-exception filter a host process set is kept by every library call when the + * driver has not installed the report. phase 0 records, phase 1 compares. */ +static LPTOP_LEVEL_EXCEPTION_FILTER filter_before = NULL; + +int fatal_report_test_handlers_unchanged(int phase) +{ + LPTOP_LEVEL_EXCEPTION_FILTER now = SetUnhandledExceptionFilter(NULL); + SetUnhandledExceptionFilter(now); + if (phase == 0) { + filter_before = now; + return 0; + } + if (now != filter_before) { + fprintf(stderr, "FAIL: host: the unhandled-exception filter changed\n"); + return 1; + } + return 0; +} #else #include #include @@ -306,4 +336,110 @@ int fatal_report_test_dispositions(void) } return 0; } + +#include + +void cabledyn_fatal_thread_init(void); +size_t cabledyn_fatal_altstack_size(void); + +static int altstack_ok(const char *who, size_t need, void **base) +{ + stack_t current; + if (sigaltstack(NULL, ¤t) != 0 || (current.ss_flags & SS_DISABLE) != 0) { + fprintf(stderr, "FAIL: altstack: %s has no alternate signal stack\n", who); + return 0; + } + if (current.ss_size < need) { + fprintf(stderr, "FAIL: altstack: %s stack is %lu bytes, below %lu\n", who, + (unsigned long)current.ss_size, (unsigned long)need); + return 0; + } + if (base != NULL) { + *base = current.ss_sp; + } + return 1; +} + +static void *prepared_worker(void *arg) +{ + void *first = NULL, *second = NULL; + int *ok = (int *)arg; + cabledyn_fatal_thread_init(); + *ok = altstack_ok("a prepared worker", cabledyn_fatal_altstack_size(), &first); + cabledyn_fatal_thread_init(); /* once per thread: no second stack */ + *ok = *ok && altstack_ok("a prepared worker", cabledyn_fatal_altstack_size(), &second) && + first == second; + return NULL; +} + +static void *plain_worker(void *arg) +{ + stack_t current; + int *ok = (int *)arg; + *ok = sigaltstack(NULL, ¤t) == 0 && (current.ss_flags & SS_DISABLE) != 0; + return NULL; +} + +/* After the report is installed, the main thread and every worker that calls + * cabledyn_fatal_thread_init have an alternate signal stack of at least 64 KiB and at least the + * system's SIGSTKSZ / _SC_SIGSTKSZ; a worker that does not call it gets none. */ +int fatal_report_test_altstack(void) +{ + pthread_t thread; + int prepared = 0, plain = 0; + size_t need = cabledyn_fatal_altstack_size(); + if (need < 64 * 1024) { + fprintf(stderr, "FAIL: altstack: size %lu is below 64 KiB\n", (unsigned long)need); + return 1; + } +#if defined(_SC_SIGSTKSZ) + if (sysconf(_SC_SIGSTKSZ) > 0 && need < (size_t)sysconf(_SC_SIGSTKSZ)) { + fprintf(stderr, "FAIL: altstack: size %lu is below _SC_SIGSTKSZ\n", (unsigned long)need); + return 1; + } +#endif + cabledyn_fatal_report_install(); + if (!altstack_ok("the main thread", need, NULL)) { + return 1; + } + if (pthread_create(&thread, NULL, prepared_worker, &prepared) != 0 || + pthread_join(thread, NULL) != 0 || !prepared) { + fprintf(stderr, "FAIL: altstack: a prepared worker thread\n"); + return 1; + } + if (pthread_create(&thread, NULL, plain_worker, &plain) != 0 || + pthread_join(thread, NULL) != 0 || !plain) { + fprintf(stderr, "FAIL: altstack: a worker that did not ask has an alternate stack\n"); + return 1; + } + fprintf(stderr, "altstack: %lu bytes on the main thread and each prepared worker\n", + (unsigned long)need); + return 0; +} + +/* The signal dispositions a host process set (here: the defaults) are kept by every library + * call when the driver has not installed the report. phase 0 records, phase 1 compares. */ +static struct sigaction handlers_before[8]; +static const int handler_signals[8] = {SIGSEGV, SIGBUS, SIGFPE, SIGILL, + SIGABRT, SIGINT, SIGTERM, SIGHUP}; + +int fatal_report_test_handlers_unchanged(int phase) +{ + int k; + struct sigaction now; + for (k = 0; k < 8; ++k) { + if (phase == 0) { + sigaction(handler_signals[k], NULL, &handlers_before[k]); + continue; + } + sigaction(handler_signals[k], NULL, &now); + if (now.sa_handler != handlers_before[k].sa_handler || + now.sa_flags != handlers_before[k].sa_flags) { + fprintf(stderr, "FAIL: host: the disposition of signal %d changed\n", + handler_signals[k]); + return 1; + } + } + return 0; +} #endif diff --git a/tests/test_fatal_report.f90 b/tests/test_fatal_report.f90 index c5aaa6f2..0d308106 100644 --- a/tests/test_fatal_report.f90 +++ b/tests/test_fatal_report.f90 @@ -6,7 +6,12 @@ PROGRAM test_fatal_report !! !! test_fatal_report march !! runs a dynamic deck and checks that the march recorded the time of its last - !! committed step (TMax) for the report; + !! committed step (TMax) for the report, and that the library, as a host such as + !! OpenFAST uses it (the report not installed), left the process's signal and + !! exception handlers as they were; + !! test_fatal_report altstack + !! (POSIX) the main thread and each worker that calls CD_Fatal_Thread_Init get one + !! alternate signal stack of at least 64 KiB and the system's SIGSTKSZ; !! test_fatal_report before|during !! installs the report, optionally records a simulated time, and ends the process !! with : overflow (unbounded recursion, a stack overflow), null (a write @@ -42,6 +47,13 @@ END FUNCTION fault_context INTEGER(C_INT) FUNCTION dispositions() BIND(C, name='fatal_report_test_dispositions') IMPORT :: C_INT END FUNCTION dispositions + INTEGER(C_INT) FUNCTION altstack() BIND(C, name='fatal_report_test_altstack') + IMPORT :: C_INT + END FUNCTION altstack + INTEGER(C_INT) FUNCTION handlers_unchanged(phase) BIND(C, name='fatal_report_test_handlers_unchanged') + IMPORT :: C_INT + INTEGER(C_INT), VALUE :: phase + END FUNCTION handlers_unchanged END INTERFACE CHARACTER(4096) :: mode, arg2, arg3 @@ -58,7 +70,9 @@ END FUNCTION dispositions ! Negative and NaN times are not committed steps and leave the record unchanged. CALL CD_Fatal_Report_Time(-1.0_wp) IF (last_time() >= 0.0_C_DOUBLE) CALL fail('a negative time was recorded') + IF (handlers_unchanged(0_C_INT) /= 0_C_INT) CALL fail('cannot record the handlers') CALL CD_Run_Deck_Driver(TRIM(arg2), TRIM(arg3), converged, stat, msg) + IF (handlers_unchanged(1_C_INT) /= 0_C_INT) CALL fail('the library changed the host handlers') IF (stat /= CD_DECKDRV_OK .OR. .NOT. converged) CALL fail('the deck did not run: '//TRIM(msg)) ! examples/dynamic_chain_held.dat: TMax 10 s. IF (ABS(last_time() - 10.0_C_DOUBLE) > 1.0E-9_C_DOUBLE) THEN @@ -81,6 +95,18 @@ END FUNCTION dispositions STOP END IF + IF (mode == 'altstack') THEN + SELECT CASE (altstack()) + CASE (-1) + WRITE (*, '(A)') 'SKIP: the alternate-stack check is POSIX-only' + CASE (0) + WRITE (*, '(A)') 'PASS: each prepared thread has one alternate signal stack of the system size' + CASE DEFAULT + CALL fail('an alternate signal stack is missing or too small') + END SELECT + STOP + END IF + IF (mode == 'dispositions') THEN SELECT CASE (dispositions()) CASE (-1) diff --git a/tests/test_omp_fatal_init.py b/tests/test_omp_fatal_init.py index eefa77ba..3694685c 100644 --- a/tests/test_omp_fatal_init.py +++ b/tests/test_omp_fatal_init.py @@ -2,9 +2,11 @@ # SPDX-License-Identifier: Apache-2.0 # Copyright (c) 2026 Jae Hoon Seo, SMI Lab, Inha University """Every OpenMP parallel region of the library and the driver prepares its threads for the -abnormal-end report: its first statement (the first statement of the loop body, for a -PARALLEL DO) is ``CALL CD_Fatal_Thread_Init()``, so a stack overflow on a worker thread is -reported like one on the main thread (src/cabledyn_fatal.c).""" +abnormal-end report once per thread: its first statement is ``CALL CD_Fatal_Thread_Init()``, +before any worksharing loop, so a stack overflow on a worker thread is reported like one on +the main thread (src/cabledyn_fatal.c). A combined ``PARALLEL DO`` (or ``SECTIONS``, +``WORKSHARE``, ``LOOP``) construct has no place for that statement and is refused, as is the +call made inside a loop body, where it would run once per iteration.""" from __future__ import annotations @@ -14,9 +16,17 @@ ROOT = Path(__file__).resolve().parents[1] DIRECTIVE = re.compile(r"^\s*!\$omp\s+parallel\b(?P.*)$", re.IGNORECASE) -DO_CLAUSE = re.compile(r"^\s*do\b", re.IGNORECASE) +COMBINED = re.compile(r"^\s*(?:do|sections|workshare|loop)\b", re.IGNORECASE) DO_STATEMENT = re.compile(r"^\s*(?:\w+\s*:\s*)?do\b", re.IGNORECASE) -INIT = re.compile(r"^\s*call\s+cd_fatal_thread_init\s*(?:\(\s*\))?\s*(?:!.*)?$", re.IGNORECASE) +# A call that installs the abnormal-end handlers, from Fortran or C (not a declaration). +INSTALL = re.compile( + r"^\s*call\s+cd_fatal_report_install\b" + r"|^\s*(?!void\b)[^/*]*\bcabledyn_fatal_report_install\s*\(\s*\)\s*;", + re.IGNORECASE, +) +INIT = re.compile( + r"^\s*call\s+cd_fatal_thread_init\s*(?:\(\s*\))?\s*(?:!.*)?$", re.IGNORECASE +) def sources() -> list[Path]: @@ -25,74 +35,108 @@ def sources() -> list[Path]: return [path for path in files if "openfast" not in path.parent.name.lower()] -def statements(lines: list[str], start: int): - """Yield (index, text) of the statements after ``start``, skipping comments and blanks.""" - for index in range(start, len(lines)): - text = lines[index].strip() - if not text or text.startswith("!"): - continue - yield index, lines[index] +def is_statement(text: str) -> bool: + """A line that is neither blank nor a comment (OpenMP directives count as statements).""" + stripped = text.strip() + return bool(stripped) and (not stripped.startswith("!") or stripped[:2] == "!$") -def missing_inits(lines: list[str]) -> tuple[int, list[int]]: - """Count the parallel regions and list the 1-based lines of those without the call.""" - found, missing = 0, [] +def check(lines: list[str]) -> tuple[int, list[str]]: + """Count the parallel regions and describe each problem as ': '.""" + found, problems = 0, [] + previous = "" index = 0 while index < len(lines): - match = DIRECTIVE.match(lines[index]) - if match is None: - index += 1 - continue - found += 1 - first = index - loop = DO_CLAUSE.match(match.group("rest")) is not None - while lines[index].rstrip().endswith("&"): - index += 1 - following = statements(lines, index + 1) - nxt = next(following, None) - if loop: - if nxt is None or not DO_STATEMENT.match(nxt[1]): - missing.append(first + 1) + line = lines[index] + if INIT.match(line) and DO_STATEMENT.match(previous): + problems.append(f"{index + 1}: the call runs once per loop iteration") + match = DIRECTIVE.match(line) + if match is not None: + found += 1 + first = index + while lines[index].rstrip().endswith("&"): index += 1 - continue - nxt = next(following, None) - if nxt is None or not INIT.match(nxt[1]): - missing.append(first + 1) + if COMBINED.match(match.group("rest")): + problems.append( + f"{first + 1}: a combined construct; split it into PARALLEL + DO" + ) + else: + following = ( + lines[k] + for k in range(index + 1, len(lines)) + if is_statement(lines[k]) + ) + if not INIT.match(next(following, "")): + problems.append(f"{first + 1}: the first statement is not the call") + if is_statement(lines[index]): + previous = lines[index] index += 1 - return found, missing + return found, problems + + +def install_calls() -> list[str]: + """Every place in the library, the driver and the OpenFAST integration that installs the + abnormal-end handlers, as ':'.""" + found = [] + for folder in ("src", "app", "integration"): + for path in sorted((ROOT / folder).rglob("*")): + if path.suffix.lower() not in {".f90", ".c", ".h"} or not path.is_file(): + continue + for number, line in enumerate(path.read_text(encoding="utf-8", errors="replace").splitlines(), 1): + if INSTALL.search(line): + found.append(f"{path.relative_to(ROOT).as_posix()}:{number}") + return found class OmpFatalInitTest(unittest.TestCase): - def test_every_parallel_region_prepares_its_threads(self) -> None: + def test_only_the_driver_installs_the_handlers(self) -> None: + # A host that loads the library (OpenFAST, Python) keeps its own signal and exception + # handling: only the standalone driver program installs the report. + calls = install_calls() + self.assertEqual([c.split(":")[0] for c in calls], ["app/cabledyn.f90"], calls) + + def test_every_parallel_region_prepares_its_threads_once(self) -> None: total = 0 problems = [] for path in sources(): - found, missing = missing_inits(path.read_text(encoding="utf-8").splitlines()) + found, issues = check(path.read_text(encoding="utf-8").splitlines()) total += found - problems += [f"{path.relative_to(ROOT)}:{line}" for line in missing] - self.assertGreater(total, 0, "no OpenMP parallel region found: the scan is broken") + problems += [f"{path.relative_to(ROOT)}:{issue}" for issue in issues] + self.assertGreater( + total, 0, "no OpenMP parallel region found: the scan is broken" + ) self.assertEqual( - problems, - [], - "OpenMP parallel regions whose first statement is not CALL CD_Fatal_Thread_Init()", + problems, [], "OpenMP regions that do not prepare their threads once" ) - def test_the_scan_catches_a_region_without_the_call(self) -> None: + def test_the_scan_accepts_the_split_form(self) -> None: good = [ - " !$OMP PARALLEL DO DEFAULT(SHARED) &", + " !$OMP PARALLEL DEFAULT(SHARED) &", " !$OMP PRIVATE(i)", + " ! comment", + " CALL CD_Fatal_Thread_Init() ! note", + " !$OMP DO SCHEDULE(STATIC)", " DO i = 1, n", - " ! comment", - " CALL CD_Fatal_Thread_Init() ! note", + " x(i) = 0", " END DO", - " !$omp parallel", - " call cd_fatal_thread_init()", - " !$omp end parallel", + " !$OMP END DO", + " !$OMP END PARALLEL", + ] + self.assertEqual(check(good), (1, [])) + + def test_the_scan_refuses_missing_combined_and_per_iteration_calls(self) -> None: + bad = [ + " !$OMP PARALLEL DO", + " DO i = 1, n", + " CALL CD_Fatal_Thread_Init()", + " END DO", + " !$OMP PARALLEL", + " x = 1", + " !$OMP END PARALLEL", ] - self.assertEqual(missing_inits(good), (2, [])) - bad = [" !$OMP PARALLEL DO", " DO i = 1, n", " x(i) = 0", " END DO", - " !$OMP PARALLEL", " x = 1"] - self.assertEqual(missing_inits(bad), (2, [1, 5])) + found, problems = check(bad) + self.assertEqual(found, 2) + self.assertEqual([p.split(":")[0] for p in problems], ["1", "3", "5"], problems) if __name__ == "__main__": From 782e63e1c59747e685ecb6f9d0fcc018188aaeea Mon Sep 17 00:00:00 2001 From: Jae Hoon Seo Date: Fri, 9 Oct 2026 15:34:49 +0900 Subject: [PATCH 6/6] Tell sent faults apart portably; locate Windows faults; guard-page LAPACK test A fault signal sent by a process is now recognised by its sender code (SI_USER, SI_QUEUE, SI_TKILL, SI_TIMER, SI_ASYNCIO, SI_MESGQ where defined, or any si_code <= 0), not by sign: on macOS SI_USER is positive, so a SIGSEGV sent with kill() was taken for a hardware fault and the handler returned instead of sending it again. Test (POSIX, forked children): SIGSEGV and SIGFPE sent with kill() end the child by that signal with one report. On Windows a fault report now names the module and offset of the fault and, for an access violation, the address read or written, so a fault inside a library (such as the intermittent one seen in the modal tests on some hosted runners) identifies itself. New test lapack_guard_pages: DSYGV and DSBGVX through CableDyn's LAPACK layer, with every argument array against a no-access page at its end and at its start (n <= 96, half band widths 0..11, OPENBLAS_CORETYPE=Haswell), with the argument sizes the modal code uses. It passes with OpenBLAS 0.3.33 and 0.3.34: the modal code's arguments and workspaces are within bounds. --- CHANGELOG.md | 7 +- CMakeLists.txt | 16 +++ src/cabledyn_fatal.c | 100 +++++++++++++++- tests/check_fatal_report.cmake | 5 + tests/fatal_report_faults.c | 62 ++++++++++ tests/test_fatal_report.f90 | 18 +++ tests/test_lapack_guard_pages.c | 194 ++++++++++++++++++++++++++++++++ 7 files changed, 394 insertions(+), 8 deletions(-) create mode 100644 tests/test_lapack_guard_pages.c diff --git a/CHANGELOG.md b/CHANGELOG.md index 886c3453..ef9de9a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,12 +24,15 @@ All notable changes to CableDyn are recorded in this file. The format follows fatal fault. Ctrl+C, Ctrl+Break, closing the console window, `SIGINT`, `SIGTERM` and `SIGHUP` are reported as `CableDyn_driver: stopped by `, and an access violation, a stack overflow or another fatal fault as `CableDyn_driver: fatal error: `. Both reports give - the simulated time of the last committed step, and the exit status is unchanged. Previously a + the simulated time of the last committed step, and the exit status is unchanged. On Windows a + fault report also names the module and offset of the fault and, for an access violation, the + address read or written. Previously a stack overflow in a Windows GNU build ended the process with no message. Every thread that enters an OpenMP parallel region keeps room to report its own stack overflow, and on Linux and macOS a hardware fault reaches the handler that was there before (the Fortran runtime's backtrace, or the default action and its core dump) with its original address and context, - and a previous handler runs under its own flags and signal mask. + and a previous handler runs under its own flags and signal mask; a fault signal sent by + another process (`kill`) is passed on as sent. A signal the driver was started with ignored (for example under `nohup`) stays ignored. - Every non-zero exit the driver makes itself now ends stderr with the closing line `CableDyn_driver: ended with exit code `. A process ended from outside (`taskkill /F`, diff --git a/CMakeLists.txt b/CMakeLists.txt index 38b87a38..55cffa4a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1673,6 +1673,18 @@ add_executable(test_modal tests/test_modal.f90 tests/blas_config_probe.c) target_link_libraries(test_modal PRIVATE cabledyn_core) add_test(NAME modal COMMAND test_modal) set_tests_properties(modal PROPERTIES LABELS "fortran;unit;driver;static;validation") +# The modal eigensolves through CableDyn's LAPACK layer with every argument array against a +# no-access page: no read or write outside the arrays the modal code passes. +if(NOT CABLEDYN_STATIC_WINDOWS_RELEASE) + add_executable(test_lapack_guard_pages tests/test_lapack_guard_pages.c) + target_link_libraries(test_lapack_guard_pages PRIVATE cabledyn_core) + set_target_properties(test_lapack_guard_pages PROPERTIES LINKER_LANGUAGE Fortran) + add_test(NAME lapack_guard_pages COMMAND test_lapack_guard_pages) + set_tests_properties(lapack_guard_pages PROPERTIES + LABELS "c;unit;static" TIMEOUT 300 + ENVIRONMENT "OPENBLAS_CORETYPE=Haswell" + FAIL_REGULAR_EXPRESSION "FAIL:") +endif() # Stack bound (GNU Windows builds): the modal test and the driver relinked with a 1 MiB stack # reserve, half the MinGW default, instead of the 64 MiB reserve of the other executables. A @@ -1864,6 +1876,10 @@ set_tests_properties(fatal_report_fault_context PROPERTIES FAIL_REGULAR_EXPRESSION "FAIL:") # POSIX: the report keeps every previous disposition -- an ignored signal (with or without # SA_SIGINFO) stays ignored, a default one still ends the process, a handler still runs. +add_test(NAME fatal_report_sent_faults COMMAND test_fatal_report sent_faults) +set_tests_properties(fatal_report_sent_faults PROPERTIES + LABELS "fortran;unit;driver" TIMEOUT 120 + FAIL_REGULAR_EXPRESSION "FAIL:") add_test(NAME fatal_report_altstack COMMAND test_fatal_report altstack) set_tests_properties(fatal_report_altstack PROPERTIES LABELS "fortran;unit;driver;stack" TIMEOUT 120 diff --git a/src/cabledyn_fatal.c b/src/cabledyn_fatal.c index 7b7f84b6..2dd40d75 100644 --- a/src/cabledyn_fatal.c +++ b/src/cabledyn_fatal.c @@ -113,6 +113,23 @@ static void line_add_hex(cabledyn_line *line, unsigned long value) line->text[line->len] = '\0'; } +/* A 64-bit value as 0x and 16 hex digits. */ +static void line_add_hex64(cabledyn_line *line, unsigned long long value) +{ + static const char hex[] = "0123456789ABCDEF"; + char digits[16]; + int n; + for (n = 15; n >= 0; --n) { + digits[n] = hex[value & 0xFULL]; + value >>= 4; + } + line_add(line, "0x"); + for (n = 0; n < 16 && line->len + 1 < sizeof line->text; ++n) { + line->text[line->len++] = digits[n]; + } + line->text[line->len] = '\0'; +} + /* "after the step at simulated time t = 7534.600 s" or the pre-march wording. */ static void line_add_when(cabledyn_line *line) { @@ -218,16 +235,54 @@ static int is_fatal_fault(DWORD code) } } -static void report_exception(DWORD code) +/* " in openblas.dll+0x...": the module that holds a code address, and the offset in it. */ +static void line_add_location(cabledyn_line *line, const void *address) +{ + HMODULE module = NULL; + char path[MAX_PATH]; + DWORD len, i; + const char *name; + if (address == NULL || + !GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, + (LPCSTR)address, &module)) { + return; + } + len = GetModuleFileNameA(module, path, (DWORD)sizeof path); + if (len == 0 || len >= sizeof path) { + return; + } + name = path; + for (i = 0; i < len; ++i) { + if (path[i] == '\\' || path[i] == '/') { + name = path + i + 1; + } + } + line_add(line, " in "); + line_add(line, name); + line_add(line, "+"); + line_add_hex64(line, (unsigned long long)((const char *)address - (const char *)module)); +} + +static void report_exception(const EXCEPTION_RECORD *record) { if (InterlockedExchange(&cabledyn_reported, 1) == 0) { cabledyn_line line; + DWORD code = record->ExceptionCode; line.len = 0; line.text[0] = '\0'; line_add(&line, "\nCableDyn_driver: fatal error: "); line_add(&line, exception_name(code)); line_add(&line, " (exception "); line_add_hex(&line, (unsigned long)code); + /* Where it happened, and for an access violation what it touched: the evidence a + * report needs when the fault is inside a library. */ + line_add_location(&line, record->ExceptionAddress); + if ((code == EXCEPTION_ACCESS_VIOLATION || code == EXCEPTION_IN_PAGE_ERROR) && + record->NumberParameters >= 2) { + ULONG_PTR kind = record->ExceptionInformation[0]; + line_add(&line, kind == 1 ? ", writing " : kind == 8 ? ", executing " : ", reading "); + line_add_hex64(&line, (unsigned long long)record->ExceptionInformation[1]); + } line_add(&line, ") "); line_add_when(&line); line_finish(&line); @@ -242,7 +297,7 @@ static LONG WINAPI cabledyn_vectored_handler(EXCEPTION_POINTERS *info) { if (info != NULL && info->ExceptionRecord != NULL && is_fatal_fault(info->ExceptionRecord->ExceptionCode)) { - report_exception(info->ExceptionRecord->ExceptionCode); + report_exception(info->ExceptionRecord); } return EXCEPTION_CONTINUE_SEARCH; } @@ -251,7 +306,7 @@ static LONG WINAPI cabledyn_vectored_handler(EXCEPTION_POINTERS *info) static LONG WINAPI cabledyn_exception_filter(EXCEPTION_POINTERS *info) { if (info != NULL && info->ExceptionRecord != NULL) { - report_exception(info->ExceptionRecord->ExceptionCode); + report_exception(info->ExceptionRecord); } if (cabledyn_previous_filter != NULL) { return cabledyn_previous_filter(info); @@ -360,6 +415,39 @@ static const char *signal_name(int sig) } } +/* Whether a signal was sent by a process (kill, raise, sigqueue, a timer or an I/O or message + * completion) rather than raised by the hardware. The sender codes are compared by name: + * they are zero or negative on Linux, but SI_USER is 0x10001 on macOS. */ +static int sent_by_process(const siginfo_t *info) +{ + if (info == NULL || info->si_code <= 0) { + return 1; + } + switch (info->si_code) { +#if defined(SI_USER) + case SI_USER: +#endif +#if defined(SI_QUEUE) + case SI_QUEUE: +#endif +#if defined(SI_TKILL) + case SI_TKILL: +#endif +#if defined(SI_TIMER) + case SI_TIMER: +#endif +#if defined(SI_ASYNCIO) + case SI_ASYNCIO: +#endif +#if defined(SI_MESGQ) + case SI_MESGQ: +#endif + return 1; + default: + return 0; + } +} + static int signal_index(int sig) { int k; @@ -394,9 +482,9 @@ static void cabledyn_signal_handler(int sig, siginfo_t *info, void *context) (void)context; int k = signal_index(sig); int fault = sig == SIGSEGV || sig == SIGBUS || sig == SIGFPE || sig == SIGILL; - /* A fault the hardware raised re-executes when the handler returns; one sent by kill() - * or raise() (si_code <= 0) does not, and is re-sent below. */ - int synchronous = fault && info != NULL && info->si_code > 0; + /* A fault the hardware raised re-executes when the handler returns; one sent by a process + * (kill, raise) does not, and is re-sent below. */ + int synchronous = fault && !sent_by_process(info); if (__sync_lock_test_and_set(&cabledyn_reported, 1) == 0) { cabledyn_line line; line.len = 0; diff --git a/tests/check_fatal_report.cmake b/tests/check_fatal_report.cmake index fd5e7a93..46d24fe8 100644 --- a/tests/check_fatal_report.cmake +++ b/tests/check_fatal_report.cmake @@ -33,6 +33,11 @@ set(expected "CableDyn_driver: (fatal error: |stopped by )${CAUSE}[^\n]* ${when_ if(NOT err MATCHES "${expected}") message(FATAL_ERROR "status ${result}, but stderr lacks the report \"${expected}\":\n${err}") endif() +# On Windows a fault names the module and offset, and an access violation what it touched. +if(CMAKE_HOST_WIN32 AND MODE STREQUAL "null" AND + NOT err MATCHES "exception 0xC0000005 in [^ ]+[.]exe[+]0x[0-9A-F]+, writing 0x0+[)]") + message(FATAL_ERROR "the access violation report lacks its location and address:\n${err}") +endif() string(REGEX MATCHALL "CableDyn_driver: (fatal error|stopped by)" reports "${err}") list(LENGTH reports nreports) if(NOT nreports EQUAL 1) diff --git a/tests/fatal_report_faults.c b/tests/fatal_report_faults.c index ab64de80..e36d83a6 100644 --- a/tests/fatal_report_faults.c +++ b/tests/fatal_report_faults.c @@ -13,6 +13,7 @@ int fatal_report_test_raise_term(void); int fatal_report_test_fault_context(void); int fatal_report_test_dispositions(void); int fatal_report_test_altstack(void); +int fatal_report_test_sent_faults(void); int fatal_report_test_handlers_unchanged(int phase); static volatile int *volatile fatal_report_null_target = 0; @@ -53,6 +54,12 @@ int fatal_report_test_altstack(void) return -1; } +/* The sent-fault check is POSIX-only: returns -1 (skipped). */ +int fatal_report_test_sent_faults(void) +{ + return -1; +} + #include #include @@ -442,4 +449,59 @@ int fatal_report_test_handlers_unchanged(int phase) } return 0; } + +/* One child sends itself `sig` with kill() after installing the report. A fault signal sent by + * a process does not re-execute anything, so the report must send it again: the child ends by + * `sig` with one report, and never runs on to its normal exit. */ +static int sent_fault_case(int sig, const char *name) +{ + int errors[2], status = 0, ok, reports; + pid_t child; + char text[4096]; + if (pipe(errors) != 0) { + fprintf(stderr, "FAIL: sent faults: pipe() failed\n"); + return 1; + } + fflush(NULL); + child = fork(); + if (child < 0) { + fprintf(stderr, "FAIL: sent faults: fork() failed\n"); + return 1; + } + if (child == 0) { + close(errors[0]); + dup2(errors[1], STDERR_FILENO); + signal(sig, SIG_DFL); + cabledyn_fatal_report_install(); + cabledyn_fatal_report_time(7534.6); + kill(getpid(), sig); + _exit(0); /* reached only if the sent signal was swallowed */ + } + close(errors[1]); + read_all(errors[0], text, sizeof text); + waitpid(child, &status, 0); + reports = count(text, "CableDyn_driver: fatal error:"); + ok = WIFSIGNALED(status) && WTERMSIG(status) == sig && reports == 1; + fprintf(stderr, "sent faults: %s: child ended %s %d, %d report(s)%s\n", name, + WIFSIGNALED(status) ? "by signal" : "with status", + WIFSIGNALED(status) ? WTERMSIG(status) : WEXITSTATUS(status), reports, + ok ? "" : " -- unexpected"); + if (!ok) { + fprintf(stderr, "%s", text); + } + return ok ? 0 : 1; +} + +/* SIGSEGV and SIGFPE sent with kill() are reported and re-sent (on macOS SI_USER is positive, + * so the sender is told apart by name, not by sign). Returns 0 when both hold. */ +int fatal_report_test_sent_faults(void) +{ + int failures = sent_fault_case(SIGSEGV, "SIGSEGV from kill()") + + sent_fault_case(SIGFPE, "SIGFPE from kill()"); + if (failures != 0) { + fprintf(stderr, "FAIL: sent faults: %d case(s) wrong\n", failures); + return 1; + } + return 0; +} #endif diff --git a/tests/test_fatal_report.f90 b/tests/test_fatal_report.f90 index 0d308106..3c68afc1 100644 --- a/tests/test_fatal_report.f90 +++ b/tests/test_fatal_report.f90 @@ -9,6 +9,9 @@ PROGRAM test_fatal_report !! committed step (TMax) for the report, and that the library, as a host such as !! OpenFAST uses it (the report not installed), left the process's signal and !! exception handlers as they were; + !! test_fatal_report sent_faults + !! (POSIX) SIGSEGV and SIGFPE sent with kill() are reported and sent again (the child + !! ends by the signal), never treated as hardware faults that re-execute; !! test_fatal_report altstack !! (POSIX) the main thread and each worker that calls CD_Fatal_Thread_Init get one !! alternate signal stack of at least 64 KiB and the system's SIGSTKSZ; @@ -50,6 +53,9 @@ END FUNCTION dispositions INTEGER(C_INT) FUNCTION altstack() BIND(C, name='fatal_report_test_altstack') IMPORT :: C_INT END FUNCTION altstack + INTEGER(C_INT) FUNCTION sent_faults() BIND(C, name='fatal_report_test_sent_faults') + IMPORT :: C_INT + END FUNCTION sent_faults INTEGER(C_INT) FUNCTION handlers_unchanged(phase) BIND(C, name='fatal_report_test_handlers_unchanged') IMPORT :: C_INT INTEGER(C_INT), VALUE :: phase @@ -95,6 +101,18 @@ END FUNCTION handlers_unchanged STOP END IF + IF (mode == 'sent_faults') THEN + SELECT CASE (sent_faults()) + CASE (-1) + WRITE (*, '(A)') 'SKIP: the sent-fault check is POSIX-only' + CASE (0) + WRITE (*, '(A)') 'PASS: faults sent with kill() are reported and sent again' + CASE DEFAULT + CALL fail('a fault sent with kill() was not sent again') + END SELECT + STOP + END IF + IF (mode == 'altstack') THEN SELECT CASE (altstack()) CASE (-1) diff --git a/tests/test_lapack_guard_pages.c b/tests/test_lapack_guard_pages.c new file mode 100644 index 00000000..b55d69c7 --- /dev/null +++ b/tests/test_lapack_guard_pages.c @@ -0,0 +1,194 @@ +/* File: tests/test_lapack_guard_pages.c + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2026 Jae Hoon Seo, SMI Lab, Inha University + * + * The modal eigensolves (DSYGV, the dense reference; DSBGVX, the banded production solver) + * called through CableDyn's LAPACK layer -- in Windows GNU builds the run-time forwarders to + * openblas.dll -- with every argument array placed against a no-access page, at its end and, + * in a second pass, at its start. Any read or write outside the arrays the modal code passes + * faults at once and names the call. Sizes cover the test_modal cases and more (n <= 96, half + * band widths 0..11), with the argument sizes CableDyn_Modal uses (WORK 7n and IWORK 5n for + * DSBGVX, the queried LWORK for DSYGV). CTest runs it with OPENBLAS_CORETYPE=Haswell, the + * kernel selected on the hosts where test_modal was seen to fault. + */ +#if !defined(_WIN32) +#define _DEFAULT_SOURCE +#define _DARWIN_C_SOURCE +#endif +#include +#include +#include +#include + +void dsygv_(const int *itype, const char *jobz, const char *uplo, const int *n, double *a, const int *lda, + double *b, const int *ldb, double *w, double *work, const int *lwork, int *info, size_t jobz_len, + size_t uplo_len); +void dsbgvx_(const char *jobz, const char *range, const char *uplo, const int *n, const int *ka, const int *kb, + double *ab, const int *ldab, double *bb, const int *ldbb, double *q, const int *ldq, const double *vl, + const double *vu, const int *il, const int *iu, const double *abstol, int *m, double *w, double *z, + const int *ldz, double *work, int *iwork, int *ifail, int *info, size_t jobz_len, size_t range_len, + size_t uplo_len); + +static char where[96] = "start"; + +#if defined(_WIN32) +#include + +static LONG WINAPI on_fault(EXCEPTION_POINTERS *ep) +{ + if (ep->ExceptionRecord->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) { + fprintf(stderr, "FAIL: an access outside the arrays during %s\n", where); + fflush(stderr); + ExitProcess(3); + } + return EXCEPTION_CONTINUE_SEARCH; +} + +static void install_fault_report(void) +{ + AddVectoredExceptionHandler(1, on_fault); +} + +static size_t page_size(void) +{ + SYSTEM_INFO si; + GetSystemInfo(&si); + return si.dwPageSize; +} + +static char *reserve(size_t bytes) +{ + return (char *)VirtualAlloc(NULL, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); +} + +static void no_access(char *at, size_t bytes) +{ + DWORD old; + VirtualProtect(at, bytes, PAGE_NOACCESS, &old); +} +#else +#include +#include +#include + +static void on_fault(int sig) +{ + static const char head[] = "FAIL: an access outside the arrays during "; + ssize_t ignored = write(STDERR_FILENO, head, sizeof head - 1); + ignored = write(STDERR_FILENO, where, strlen(where)); + ignored = write(STDERR_FILENO, "\n", 1); + (void)ignored; + (void)sig; + _exit(3); +} + +static void install_fault_report(void) +{ + signal(SIGSEGV, on_fault); + signal(SIGBUS, on_fault); +} + +static size_t page_size(void) +{ + return (size_t)sysconf(_SC_PAGESIZE); +} + +static char *reserve(size_t bytes) +{ + void *p = mmap(NULL, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); + return p == MAP_FAILED ? NULL : (char *)p; +} + +static void no_access(char *at, size_t bytes) +{ + mprotect(at, bytes, PROT_NONE); +} +#endif + +/* `bytes` usable bytes between two no-access pages: flush against the page after them + * (front = 0) or the page before them (front = 1). Never freed: a few MB for the run. */ +static void *guarded(size_t bytes, int front) +{ + size_t page = page_size(); + size_t span = (bytes + page - 1) / page * page; + char *base = reserve(span + 2 * page); + if (base == NULL) { + fprintf(stderr, "FAIL: cannot reserve guarded memory\n"); + exit(2); + } + no_access(base, page); + no_access(base + page + span, page); + return front ? (void *)(base + page) : (void *)(base + page + span - bytes); +} + +static double next_random(unsigned *state) +{ + *state = *state * 1664525u + 1013904223u; + return (double)(*state >> 8) / 16777216.0; +} + +int main(void) +{ + unsigned state = 2026u; + int front, n, failures = 0; + install_fault_report(); + for (front = 0; front <= 1; ++front) { + for (n = 1; n <= 96; ++n) { + int i, j, info = 0, itype = 1, lwork = -1, kd; + double query = 0.0; + double *a = guarded(sizeof(double) * (size_t)n * (size_t)n, front); + double *b = guarded(sizeof(double) * (size_t)n * (size_t)n, front); + double *w = guarded(sizeof(double) * (size_t)n, front); + double *work; + for (j = 0; j < n; ++j) { + for (i = 0; i <= j; ++i) { + double v = next_random(&state) - 0.5; + a[i + j * n] = a[j + i * n] = v + (i == j ? 2.0 * n : 0.0); + b[i + j * n] = b[j + i * n] = i == j ? 1.0 + next_random(&state) : 0.01 * v; + } + } + snprintf(where, sizeof where, "DSYGV workspace query, n = %d", n); + dsygv_(&itype, "V", "U", &n, a, &n, b, &n, w, &query, &lwork, &info, 1, 1); + lwork = (int)query; + if (lwork < 3 * n) { + lwork = 3 * n; + } + work = guarded(sizeof(double) * (size_t)lwork, front); + snprintf(where, sizeof where, "DSYGV, n = %d, %s guard", n, front ? "front" : "end"); + dsygv_(&itype, "V", "U", &n, a, &n, b, &n, w, work, &lwork, &info, 1, 1); + if (info != 0) { + fprintf(stderr, "FAIL: DSYGV n = %d returned INFO = %d\n", n, info); + ++failures; + } + for (kd = 0; kd <= 11 && kd < n; ++kd) { + int ld = kd + 1, il = 1, iu = n < 6 ? n : 6, m = 0, one = 1; + double vl = 0.0, vu = 0.0, tol = 4.4501477170144028e-308, q = 0.0, z = 0.0; + double *ab = guarded(sizeof(double) * (size_t)ld * (size_t)n, front); + double *bb = guarded(sizeof(double) * (size_t)ld * (size_t)n, front); + double *ww = guarded(sizeof(double) * (size_t)n, front); + double *bwork = guarded(sizeof(double) * 7 * (size_t)n, front); + int *iwork = guarded(sizeof(int) * 5 * (size_t)n, front); + int *ifail = guarded(sizeof(int) * (size_t)n, front); + for (j = 0; j < n; ++j) { + for (i = j - kd > 0 ? j - kd : 0; i <= j; ++i) { + ab[(kd + i - j) + j * ld] = i == j ? 2.0 * n : next_random(&state) - 0.5; + bb[(kd + i - j) + j * ld] = i == j ? 1.0 + next_random(&state) : 0.0; + } + } + snprintf(where, sizeof where, "DSBGVX, n = %d, kd = %d, %s guard", n, kd, + front ? "front" : "end"); + dsbgvx_("N", "I", "U", &n, &kd, &kd, ab, &ld, bb, &ld, &q, &one, &vl, &vu, &il, &iu, &tol, &m, + ww, &z, &one, bwork, iwork, ifail, &info, 1, 1, 1); + if (info != 0 || m != iu) { + fprintf(stderr, "FAIL: DSBGVX n = %d kd = %d returned INFO = %d, M = %d\n", n, kd, info, m); + ++failures; + } + } + } + } + if (failures != 0) { + return 1; + } + printf("PASS: DSYGV and DSBGVX stay inside the arrays the modal code passes (n <= 96)\n"); + return 0; +}