diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index aa83ab50..6daf4978 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -23,6 +23,12 @@ jobs: - name: Show installed packages run: uv tree + # The math-snippet guard validates docs snippets with the MathJax + # engine (utils/check_math_snippets_node.cjs); Node is preinstalled + # on the runner. + - name: Install MathJax with npm + run: npm ci + # Per-metric gate rationale: see "Test pipeline" in docs/developer.rst. # term-missing keeps the uncovered lines in the CI log. - name: Test FLPQ_Data with coverage (line and branch >= 95%) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 6ae3087d..8ad8ff76 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -20,6 +20,12 @@ jobs: - name: Install dependencies with uv run: uv sync --frozen --all-groups + # The pre-commit math-snippet hook (check-math-snippets) runs the + # MathJax Node oracle (utils/check_math_snippets_node.cjs); Node is + # preinstalled on the runner. + - name: Install MathJax with npm + run: npm ci + - name: Run pre-commit run: | uv run pre-commit run --all-files --color always --verbose --show-diff-on-failure diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d4492e60..04d08844 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -28,5 +28,11 @@ jobs: - name: Show installed packages run: uv tree + # The math-snippet guard validates docs snippets with the MathJax + # engine (utils/check_math_snippets_node.cjs); Node is preinstalled + # on all runner OSes. + - name: Install MathJax with npm + run: npm ci + - name: Test FLPQ_Data run: uv run pytest diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a592c88b..8ab28508 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,3 +31,9 @@ repos: language: system pass_filenames: false files: \.py$ + - id: check-math-snippets + name: Check math snippets with the MathJax engine + entry: python3 utils/check_math_snippets.py + language: system + pass_filenames: false + always_run: true diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f877a4b..e3b5cdb4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,36 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [6.1.2] - 2026-10-28 + +### Fixed + +- The reachable-pair count columns of the graph tables were inconsistently + named (e.g. `grammar` on the biological_uniprot page, bare grammar names + on the rdf page); they are now uniformly `Reachable pairs`, with the rdf + page's per-grammar columns prefixed `Reachable pairs / ` (#169). +- The field-sensitive-alias pages rendered the edge label `f_r_i` as a TeX + double subscript, which MathJax rejects with "Double subscripts: use + braces to clarify" (#173); the label is now `f_{r\_i}` in the ten + per-graph Edges Statistics tables and the canonical-grammar block. +- Seven grammar pages used `&` alignment tabs in plain display math, which + MathJax rejects with "Misplaced &" (#174); the blocks are now wrapped in + `\begin{aligned} ... \end{aligned}`. +- The eclass graph page's Source link (and the historical old_graphs + page's Origin link) pointed to a URL the hosting university removed from + its site; they now point to the archived original (Wayback Machine + snapshot of the eClassOWL 5.1.4 ontology file the original page + redirected to). + +### Changed + +- The math-snippet guard now validates every docs-page and package-docstring + math snippet with the MathJax engine itself (mathjax 4.1.3 on Node, + `utils/check_math_snippets_node.cjs`) instead of hand-crafted patterns, + reporting the engine's own error message; the browser bundle is pinned to + the same version (`mathjax_js` in `docs/conf.py`), and the guard also runs + as a pre-commit hook. + ## [6.1.1] - 2026-10-15 ### Changed diff --git a/docs/conf.py b/docs/conf.py index 2d856b41..2d6b4da4 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -70,6 +70,12 @@ "texext", ] +# Pin the browser-side MathJax to the exact version the math-snippet guard +# validates with (utils/check_math_snippets_node.cjs, version in +# package.json). The guard and the renderer must stay on the same engine +# version or they can disagree about what renders. +mathjax_js = "https://cdn.jsdelivr.net/npm/mathjax@4.1.3/tex-mml-chtml.js" + # generate autosummary pages autosummary_generate = True diff --git a/docs/developer.rst b/docs/developer.rst index eaff6a5d..642076a2 100644 --- a/docs/developer.rst +++ b/docs/developer.rst @@ -55,6 +55,16 @@ then from the repository root:: - The committed :file:`uv.lock` pins every dependency; CI passes ``--frozen`` to fail on a lockfile that is out of sync with ``pyproject.toml``. +The math-snippet guard (see "Test pipeline") validates docs math with the +MathJax engine itself, which runs on Node.js (>= 20). Install Node.js once, +then from the repository root:: + + npm ci + +The committed :file:`package-lock.json` pins the engine version; it must +match the ``mathjax_js`` URL in :file:`docs/conf.py`, so the guard and the +browser render with the same MathJax version. + .. _developer-quality: Quality checks @@ -64,9 +74,10 @@ Linting, formatting, and type checking are managed by uv and run through ``pre-commit``. The hook list — hygiene checks, the official ``uv-lock`` and ruff hooks (check with autofix plus format), a version-sync check that fails when :file:`flpq_data/config.py` and :file:`pyproject.toml` declare different -versions, and a local ty type check — is defined in -:file:`.pre-commit-config.yaml`, which is the source of truth; do not -maintain a copy of it elsewhere. +versions, a local ty type check, and a math-snippet check that validates the +docs and docstring math with the MathJax engine (see "Test pipeline") — is +defined in :file:`.pre-commit-config.yaml`, which is the source of truth; do +not maintain a copy of it elsewhere. The ``ruff-pre-commit`` hook revision must match the ruff version locked in :file:`uv.lock`; bump both together when upgrading ruff. @@ -112,13 +123,16 @@ coverage gate:: uv run pytest tests/graphs/utils/test_add_reverse_edges.py -The suite also validates the math snippets of the docs pages: Sphinx passes -math verbatim to MathJax, which renders it in the browser, so a broken -snippet (e.g. an unescaped underscore inside a ``\textit{...}`` group) sails -through the Sphinx build and shows up only as an error on the deployed site. -``tests/utils/test_check_math_snippets.py`` runs -``utils/check_math_snippets.py`` over every page and fails the suite on any -broken snippet. +The suite also validates the math snippets of the docs pages and of the +package docstrings (rendered on the reference pages): Sphinx passes math +verbatim to MathJax, which renders it in the browser, so a broken snippet +sails through the Sphinx build and shows up only as an error on the +deployed site. The guard therefore validates every snippet with the MathJax +engine itself — ``utils/check_math_snippets.py`` extracts the ``:math:`` +roles and ``.. math::`` bodies and compiles them with +``utils/check_math_snippets_node.cjs`` (the same engine version the browser +loads, see "Development setup") — and fails the suite on any snippet the +engine rejects, reporting the engine's own error message. - Doctest discovery and test paths are configured in ``pyproject.toml`` (``[tool.pytest.ini_options]``). diff --git a/docs/graphs/biological_uniprot.rst b/docs/graphs/biological_uniprot.rst index e9678bc4..66490832 100644 --- a/docs/graphs/biological_uniprot.rst +++ b/docs/graphs/biological_uniprot.rst @@ -28,12 +28,11 @@ as in `"Subgraph queries by context-free grammars" `_ * - Source - - `link `_ + - `link `_ Graph Statistics diff --git a/docs/graphs/data/imagick_field_sensitive_alias.rst b/docs/graphs/data/imagick_field_sensitive_alias.rst index f76adaa1..da27ce06 100644 --- a/docs/graphs/data/imagick_field_sensitive_alias.rst +++ b/docs/graphs/data/imagick_field_sensitive_alias.rst @@ -49,5 +49,5 @@ Edges Statistics - 4202 * - :math:`f_i` - 26813 - * - :math:`f_r_i` + * - :math:`f_{r\_i}` - 26813 diff --git a/docs/graphs/data/leela_field_sensitive_alias.rst b/docs/graphs/data/leela_field_sensitive_alias.rst index 8af18a06..5674c4fa 100644 --- a/docs/graphs/data/leela_field_sensitive_alias.rst +++ b/docs/graphs/data/leela_field_sensitive_alias.rst @@ -49,5 +49,5 @@ Edges Statistics - 2306 * - :math:`f_i` - 3856 - * - :math:`f_r_i` + * - :math:`f_{r\_i}` - 3856 diff --git a/docs/graphs/data/nab_field_sensitive_alias.rst b/docs/graphs/data/nab_field_sensitive_alias.rst index 996877c6..b07ef377 100644 --- a/docs/graphs/data/nab_field_sensitive_alias.rst +++ b/docs/graphs/data/nab_field_sensitive_alias.rst @@ -49,5 +49,5 @@ Edges Statistics - 836 * - :math:`f_i` - 1192 - * - :math:`f_r_i` + * - :math:`f_{r\_i}` - 1192 diff --git a/docs/graphs/data/omnetpp_field_sensitive_alias.rst b/docs/graphs/data/omnetpp_field_sensitive_alias.rst index 338c6390..9d3dc268 100644 --- a/docs/graphs/data/omnetpp_field_sensitive_alias.rst +++ b/docs/graphs/data/omnetpp_field_sensitive_alias.rst @@ -49,5 +49,5 @@ Edges Statistics - 12579 * - :math:`f_i` - 16822 - * - :math:`f_r_i` + * - :math:`f_{r\_i}` - 16822 diff --git a/docs/graphs/data/parest_field_sensitive_alias.rst b/docs/graphs/data/parest_field_sensitive_alias.rst index b718c20f..9d42a650 100644 --- a/docs/graphs/data/parest_field_sensitive_alias.rst +++ b/docs/graphs/data/parest_field_sensitive_alias.rst @@ -49,5 +49,5 @@ Edges Statistics - 7610 * - :math:`f_i` - 8983 - * - :math:`f_r_i` + * - :math:`f_{r\_i}` - 8983 diff --git a/docs/graphs/data/perlbench_field_sensitive_alias.rst b/docs/graphs/data/perlbench_field_sensitive_alias.rst index b37d12dd..90c0a114 100644 --- a/docs/graphs/data/perlbench_field_sensitive_alias.rst +++ b/docs/graphs/data/perlbench_field_sensitive_alias.rst @@ -49,5 +49,5 @@ Edges Statistics - 8520 * - :math:`f_i` - 16381 - * - :math:`f_r_i` + * - :math:`f_{r\_i}` - 16381 diff --git a/docs/graphs/data/povray_field_sensitive_alias.rst b/docs/graphs/data/povray_field_sensitive_alias.rst index 23294548..709c0881 100644 --- a/docs/graphs/data/povray_field_sensitive_alias.rst +++ b/docs/graphs/data/povray_field_sensitive_alias.rst @@ -49,5 +49,5 @@ Edges Statistics - 3436 * - :math:`f_i` - 6951 - * - :math:`f_r_i` + * - :math:`f_{r\_i}` - 6951 diff --git a/docs/graphs/data/x264_field_sensitive_alias.rst b/docs/graphs/data/x264_field_sensitive_alias.rst index 74b70e41..78fb7213 100644 --- a/docs/graphs/data/x264_field_sensitive_alias.rst +++ b/docs/graphs/data/x264_field_sensitive_alias.rst @@ -49,5 +49,5 @@ Edges Statistics - 4747 * - :math:`f_i` - 9532 - * - :math:`f_r_i` + * - :math:`f_{r\_i}` - 9532 diff --git a/docs/graphs/data/xz_field_sensitive_alias.rst b/docs/graphs/data/xz_field_sensitive_alias.rst index aeee8292..884f12ee 100644 --- a/docs/graphs/data/xz_field_sensitive_alias.rst +++ b/docs/graphs/data/xz_field_sensitive_alias.rst @@ -49,5 +49,5 @@ Edges Statistics - 588 * - :math:`f_i` - 1193 - * - :math:`f_r_i` + * - :math:`f_{r\_i}` - 1193 diff --git a/docs/graphs/data_provenance.rst b/docs/graphs/data_provenance.rst index e52445b6..21765a21 100644 --- a/docs/graphs/data_provenance.rst +++ b/docs/graphs/data_provenance.rst @@ -35,12 +35,11 @@ Python projects, following the W3C PROV model (``Entity``, ``Activity``, data/provenance_airflow .. list-table:: - :header-rows: 1 * - Graph - Num Nodes - Num Edges - - prov_derivation + - Reachable pairs - Size (MB) - Download * - :ref:`provenance_sampleproject` @@ -159,6 +158,7 @@ The grammar file is attached to the archive. .. math:: + \begin{aligned} S &\to Entity \quad Core \quad Entity \\ Core &\to wasDerivedFrom \\ &\mid wasDerivedFrom_{r} \\ @@ -174,3 +174,4 @@ The grammar file is attached to the archive. &\quad Entity \quad wasDerivedFrom_{r} \\ &\mid specializationOf \quad Entity \quad Core \\ &\quad Entity \quad specializationOf_{r} + \end{aligned} diff --git a/docs/graphs/field_sensitive_alias.rst b/docs/graphs/field_sensitive_alias.rst index 26137f33..91171afc 100644 --- a/docs/graphs/field_sensitive_alias.rst +++ b/docs/graphs/field_sensitive_alias.rst @@ -27,12 +27,11 @@ field (the ``f_i`` family), as in `"Taming Transitive Redundancy for Context-Fre data/omnetpp_field_sensitive_alias .. list-table:: - :header-rows: 1 * - Graph - Num Nodes - Num Edges - - vf + - Reachable pairs - Size (MB) - Download * - :ref:`xz_field_sensitive_alias` @@ -106,7 +105,7 @@ Reversed edges (``a_r``, ``d_r``, ``f_r_i``) are auto-generated from forward edg .. math:: M \, \rightarrow \, d_r \, V \, d \, \\ - V \, \rightarrow \, A_r \, V \mid V \, A \mid f_r_i \, V \, f_i \mid M \mid \varepsilon \, \\ + V \, \rightarrow \, A_r \, V \mid V \, A \mid f_{r\_i} \, V \, f_i \mid M \mid \varepsilon \, \\ A_r \, \rightarrow \, M \, a_r \mid a_r \mid \varepsilon \, \\ A \, \rightarrow \, a \, M \mid a \mid \varepsilon \, \\ diff --git a/docs/graphs/index.rst b/docs/graphs/index.rst index 332f4882..d6fe4ae1 100644 --- a/docs/graphs/index.rst +++ b/docs/graphs/index.rst @@ -110,16 +110,18 @@ is dropped. Contents -------- -The per-category tables include one column per grammar language -applicable to that category. Each cell shows the number of vertex pairs -reachable with respect to the respective language (i.e. the number of -pairs ``(u, v)`` for which some string in the language labels a path from -:math:`u` to :math:`v`). The count depends on the language, not on a -particular grammar: several grammars may generate the same language and -yield the same count. A cell reading "not available" means the value has -not been computed yet; an empty cell means the grammar does not apply to -that graph. Each table also has a ``Size (MB)`` column: the download size -of the graph's archive, in megabytes. +The per-category tables include one count column per grammar language +applicable to that category, each headed ``Reachable pairs`` (or +``Reachable pairs / `` in categories with several languages, such +as RDF). Each cell shows the number of vertex pairs reachable with respect +to the respective language (i.e. the number of pairs ``(u, v)`` for which +some string in the language labels a path from :math:`u` to :math:`v`). The +count depends on the language, not on a particular grammar: several grammars +may generate the same language and yield the same count. A cell reading +"not available" means the value has not been computed yet; an empty cell +means the grammar does not apply to that graph. Each table also has a +``Size (MB)`` column: the download size of the graph's archive, in +megabytes. The canonical grammar behind each column is documented once on the respective category page, in its "Canonical grammars" section — a grammar diff --git a/docs/graphs/java_points_to.rst b/docs/graphs/java_points_to.rst index ea88abcb..351a2166 100644 --- a/docs/graphs/java_points_to.rst +++ b/docs/graphs/java_points_to.rst @@ -40,12 +40,11 @@ are not stored (see :ref:`graphs`). data/tradesoap .. list-table:: - :header-rows: 1 * - Graph - Num Nodes - Num Edges - - java_points_to + - Reachable pairs - Size (MB) - Download * - :ref:`gson` diff --git a/docs/graphs/name_resolution.rst b/docs/graphs/name_resolution.rst index bb2f1f7e..037d14dd 100644 --- a/docs/graphs/name_resolution.rst +++ b/docs/graphs/name_resolution.rst @@ -22,12 +22,11 @@ scopes. data/name_resolution_libgdx .. list-table:: - :header-rows: 1 * - Graph - Num Nodes - Num Edges - - name_resolution + - Reachable pairs - Size (MB) - Download * - :ref:`name_resolution_jiaozi` @@ -62,6 +61,7 @@ The grammar file is attached to the archive. .. math:: + \begin{aligned} S &\to \varepsilon \\ &\mid eps \quad S \\ &\mid Q \quad S \\ @@ -70,3 +70,4 @@ The grammar file is attached to the archive. S\#vpsh_{i} &\to vpsh_{i} \quad S \\ V &\to S\#vpsh_{i} \quad vpp_{i} \\ Q &\to S\#psh_{i} \quad pp_{i} + \end{aligned} diff --git a/docs/graphs/rdf.rst b/docs/graphs/rdf.rst index 41d27926..549959ce 100644 --- a/docs/graphs/rdf.rst +++ b/docs/graphs/rdf.rst @@ -37,15 +37,14 @@ queries over RDF as in `"Context-Free Path Queries on RDF Graphs" `_ * - Origin - - `link `_ + - `link `_ CSV File Structure diff --git a/docs/queries/cfpq/data/c_alias.rst b/docs/queries/cfpq/data/c_alias.rst index 9172f57a..e200f794 100644 --- a/docs/queries/cfpq/data/c_alias.rst +++ b/docs/queries/cfpq/data/c_alias.rst @@ -42,8 +42,10 @@ Grammar Template .. math:: + \begin{aligned} S \, &\rightarrow \, d_r \, V \, d \, \\ V \, &\rightarrow \, ((S \, \mid \, \varepsilon) \, a_r)^{*} \, (S \, \mid \, \varepsilon) \, (a \, (S \, \mid \, \varepsilon))^{*} \, \\ + \end{aligned} Description diff --git a/docs/queries/cfpq/data/dyck.rst b/docs/queries/cfpq/data/dyck.rst index 2dd62b69..19e20cda 100644 --- a/docs/queries/cfpq/data/dyck.rst +++ b/docs/queries/cfpq/data/dyck.rst @@ -42,10 +42,12 @@ Grammar Template .. math:: + \begin{aligned} S \, &\rightarrow \, \varepsilon \, \qquad \qquad &\textit{if } \textit{eps} = \textit{True} \, \\ S \, &\rightarrow \, \textit{op}_i \, \textit{cp}_i \qquad \qquad &\textit{if } \textit{eps} = \textit{False} \, \\ S \, &\rightarrow \, \textit{op}_i \, S \, \textit{cp}_i \, S \, &\\ &\forall \, (\textit{op}_i, \textit{cp}_i) \, \in \, \textit{types} \, &\\ + \end{aligned} Description diff --git a/docs/queries/cfpq/data/java_points_to.rst b/docs/queries/cfpq/data/java_points_to.rst index e992e106..88da0575 100644 --- a/docs/queries/cfpq/data/java_points_to.rst +++ b/docs/queries/cfpq/data/java_points_to.rst @@ -39,10 +39,12 @@ Grammar Template ---------------- .. math:: + \begin{aligned} \textit{PointsTo} \, &\rightarrow \, (\textit{assign} \, \mid \, \textit{load}_f \, \textit{Alias} \, \textit{store}_f)^{*} \, \textit{alloc} \, \\ \textit{Alias} \, &\rightarrow \, \textit{PointsTo} \, \textit{FlowsTo} \, \\ \textit{FlowsTo} \, &\rightarrow \, \textit{alloc}_r \, (\textit{assign}_r \, \mid \, \textit{store}_{r\_f} \, \textit{Alias} \, \textit{load}_{r\_f})^* \, \\ &\forall \, f \, \in \, \textit{fields} \, \\ + \end{aligned} Description diff --git a/docs/queries/cfpq/data/nested_parentheses.rst b/docs/queries/cfpq/data/nested_parentheses.rst index aec2a0c3..55fac11e 100644 --- a/docs/queries/cfpq/data/nested_parentheses.rst +++ b/docs/queries/cfpq/data/nested_parentheses.rst @@ -42,10 +42,12 @@ Grammar Template .. math:: + \begin{aligned} S \, &\rightarrow \, \varepsilon \, \qquad \qquad &\textit{if } \textit{eps} = \textit{True} \, \\ S \, &\rightarrow \, \textit{op}_i \, \textit{cp}_i \qquad \qquad &\textit{if } \textit{eps} = \textit{False} \, \\ S \, &\rightarrow \, \textit{op}_i \, S \, \textit{cp}_i \, &\\ &\forall \, (\textit{op}_i, \textit{cp}_i) \, \in \, \textit{types} \, &\\ + \end{aligned} Grammar Description diff --git a/flpq_data/config.py b/flpq_data/config.py index 853d51de..092f65c1 100644 --- a/flpq_data/config.py +++ b/flpq_data/config.py @@ -6,7 +6,7 @@ "ROOT", ] -VERSION = "6.1.1" +VERSION = "6.1.2" #: The dataset version served on object storage: the package major version #: with zeroed minor and patch (a minor/patch release does not move the diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 00000000..5db63ee1 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,27 @@ +{ + "name": "FLPQ_Data", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "dependencies": { + "mathjax": "4.1.3" + } + }, + "node_modules/@mathjax/mathjax-newcm-font": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/@mathjax/mathjax-newcm-font/-/mathjax-newcm-font-4.1.3.tgz", + "integrity": "sha512-gzAB3dFHilHX1l5x2xUqRL+1jDQt3Fyza1DkEMVXWC4E8SvsGdlgEza47HYi2WhVcgfkvf4zgUGzuhbq3Pjlew==", + "license": "Apache-2.0" + }, + "node_modules/mathjax": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/mathjax/-/mathjax-4.1.3.tgz", + "integrity": "sha512-BN/8Pkgn7G1pIDYJqd9md+JHsE/jydSYbyOZnSdSA0WziuVO8mRxdYiWFumkVVly/8U+hm9DpIIoWuvySverzw==", + "license": "Apache-2.0", + "dependencies": { + "@mathjax/mathjax-newcm-font": "^4.1.3" + } + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 00000000..0aabbd19 --- /dev/null +++ b/package.json @@ -0,0 +1,7 @@ +{ + "private": true, + "description": "MathJax version pin for the math-snippet guard (utils/check_math_snippets_node.cjs); must match the mathjax_js URL pinned in docs/conf.py.", + "dependencies": { + "mathjax": "4.1.3" + } +} diff --git a/pyproject.toml b/pyproject.toml index 0af95ecc..55ee0651 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "flpq-data" -version = "6.1.1" +version = "6.1.2" description = "Python package containing Graphs and Grammars for experimental analysis of Formal Language constrained Path Querying algorithms" readme = "README.rst" license = "Apache-2.0 AND CC-BY-4.0" diff --git a/tasks/detailed_plan.md b/tasks/detailed_plan.md index 823e2b3a..ca66c5be 100644 --- a/tasks/detailed_plan.md +++ b/tasks/detailed_plan.md @@ -1,154 +1,207 @@ -# Detailed Plan: Task 167 — Single contribution page under the Dataset root - -Issue #167. The data-contribution process is documented in four scattered -places: the "Contributing data" section of the Graphs page -(`docs/graphs/index.rst`), a stranded "How to add a new grammar?" section on -the Dataset root (`docs/dataset.rst`), a stale pointer in the developer guide -(`docs/developer.rst`, referencing a README section that does not exist under -that name), and the README. Consolidate it into one page under the Dataset -root, and elevate the "File structure" archive-layout reference (currently on -the Graphs page) to its own dataset-level page. - -## Confirmed decisions (user guidance) - -- Placement: **flat file + toctree** — new pages are siblings of - `dataset.rst`, listed in its toctree, so they appear as subpages under - Dataset in the navigation. No existing public URL changes (no directory - restructure). -- File structure: **own dataset-level page** — it describes the full archive - layout and is a neutral format reference used by contributors *and* end - users (tutorial, issue template); it does not live inside the contribution - page. -- No-duplication: every fact keeps exactly one home; all other mentions become - cross-references re-pointed to the new locations. - -## Reuse analysis - -- No package code changes (docs-only task). -- All new-page content is **moved verbatim** from existing sections — nothing - is re-described: "Contributing data" (`docs/graphs/index.rst:12-63`), - "How to add a new grammar?" (`docs/dataset.rst:26-29`), "File structure" - (`docs/graphs/index.rst:65-152`). -- The issue template (`.github/ISSUE_TEMPLATE/new-data-template.md`) remains - the single source of truth for required fields; the pages keep pointing at - it instead of restating its content. -- Page header pattern (`.. only:: html` + `:Release:` / `:Date:`) reused from - existing pages; label `contributing_data` reused on the new page; - `graph_file_structure` renamed to `file_structure` (all four `:ref:` uses - updated in the same commit). - ---- - -### S1: Move "File structure" to its own dataset-level page [done] 202952d - -**Code:** none (docs/templates/skills only). - -**Tests:** fresh docs build under the no-warnings policy (orphan-document and -duplicate-label checks included). - -**Docs:** -- New `docs/file_structure.rst` — title "File structure", label - `.. _file_structure:` (renamed from `graph_file_structure`), standard page - header; content moved verbatim from `docs/graphs/index.rst:65-152` (archive - layout, partial archives, MTX format, "Indexed labels" and "Reversed edges" - subsections demoted one level). -- `docs/dataset.rst`: add `file_structure` to the toctree (same commit — an - unwired document is an orphan warning under the no-warnings policy). -- `docs/graphs/index.rst`: remove the "File structure" section. -- Re-point every reference to the old location: - - `:ref:`graph_file_structure`` → `:ref:`file_structure``: - `docs/tutorial.rst` (2×), `docs/queries/rpq/index.rst`. - - Prose "the 'File structure' section of the :ref:`graphs` page" → the new - page: `docs/flpq.rst` (2×), `docs/queries/rpq/data/reachability.rst`, - `docs/queries/rpq/data/label_star.rst`, - `docs/reference/tools/index.rst`. - - `.github/ISSUE_TEMPLATE/new-data-template.md`: the "Data format" link - `.../graphs/index.html` → `.../file_structure.html`. - - `.opencode/skills/add-graph/SKILL.md`: "File structure section of - `docs/graphs/index.rst`" → `docs/file_structure.rst`. - - Discovered during implementation (different phrasing, caught by the - build + grep sweep): `docs/flpq.rst` RSM "Placement" paragraph and the - `flpq_data/tools/check_archive_structure.py` module docstring — both - re-pointed to the new page (docstring-only change; the module's 80 tests - pass). +# Detailed Plan: Task 173 — Fix broken TeX in the docs and harden the math guard + +Task: #173 (bug: "Double subscripts: use braces to clarify" on the +field-sensitive-alias pages) +Linked: #174 (bug: "Misplaced &" on seven grammar pages — discovered by the +new guard during this task) +Branch: `feature/173-mathjax-guard` (from `dev`) + +## Background + +- The deployed site renders math with MathJax 4.1.3: `sphinx.ext.mathjax` + (Sphinx 9.0.4) loads + `https://cdn.jsdelivr.net/npm/mathjax@4/tex-mml-chtml.js`. A snippet the + engine cannot parse shows up as MathJax error text on the deployed page + while the Sphinx build stays green. +- The existing guard (`utils/check_math_snippets.py`, added for #127) + detects only one hand-crafted pattern (unescaped `_` inside text-mode + groups) and missed the double-subscript class of #173. +- Decision (user guidance): no hand-crafted TeX parser — validate with the + standard tooling, i.e. the MathJax engine itself, run on Node through the + officially supported `mathjax` npm package entry point + (`require('mathjax').init(...)` + `tex2mml`). Both sides are pinned to the + same version (4.1.3) so the checker and the browser cannot drift apart. + +## Findings (verified with mathjax 4.1.3 on Node 22) + +- 11 snippets with `f_r_i` (double subscript): + `docs/graphs/data/{cactus,imagick,leela,nab,omnetpp,parest,perlbench,povray,x264,xz}_field_sensitive_alias.rst:52` + and `docs/graphs/field_sensitive_alias.rst:108`. +- 7 `.. math::` blocks using `&` alignment tabs in plain display math + ("Misplaced &"): `docs/graphs/biological_uniprot.rst`, + `docs/graphs/data_provenance.rst`, `docs/graphs/name_resolution.rst`, + `docs/queries/cfpq/data/{c_alias,dyck,java_points_to,nested_parentheses}.rst`. +- All other 556 snippets of the docs tree compile cleanly. +- No broken math in `flpq_data` docstrings (only `mcfg.py` carries + `:math:` roles; all valid). + +## Subtasks + +### S1: Fix the 18 broken math snippets in the docs [done] 823be4b + +**Code:** docs only: +- 11× `f_r_i` → `f_{r\_i}` (braced subscript with escaped inner underscore, + matching the existing `\textit{store}_{r\_f}` convention on the + java_points_to pages): + - `docs/graphs/field_sensitive_alias.rst` (canonical-grammar math block) + - `docs/graphs/data/{cactus,imagick,leela,nab,omnetpp,parest,perlbench,povray,x264,xz}_field_sensitive_alias.rst` + (Edges Statistics table row) +- 7× wrap the `&`-aligned math blocks in `\begin{aligned} ... \end{aligned}`: + - `docs/graphs/biological_uniprot.rst` + - `docs/graphs/data_provenance.rst` + - `docs/graphs/name_resolution.rst` + - `docs/queries/cfpq/data/c_alias.rst` + - `docs/queries/cfpq/data/dyck.rst` + - `docs/queries/cfpq/data/java_points_to.rst` + - `docs/queries/cfpq/data/nested_parentheses.rst` + +**Tests:** the existing suite stays green (the old guard passes; the new +guard lands in S2/S3). Every fixed snippet is verified to compile cleanly +in the MathJax 4.1.3 oracle during implementation. + +**Docs:** the change is the docs themselves. **Spec:** -- The moved text is verbatim except heading levels; no sentence is rewritten - or dropped. -- After the commit, no file in the repo references `graph_file_structure` or - describes the archive layout outside `docs/file_structure.rst`. -- The Graphs page keeps "Format conversion", "Contents", and the category - tables untouched. - -### S2: Create the "Contributing data" page; remove the old sections [done] 46dabfd - -**Code:** none (docs only). +- `f_{r\_i}` renders f with subscript "r_i" (literal underscore), the + faithful rendering of the edge-label string `f_r_i`. +- The `aligned` wrapper keeps the visual layout: `&` becomes the alignment + tab of the environment, `\\` line breaks are unchanged. +- No other content on the affected pages changes. + +### S2: Add the MathJax Node oracle [done] 9295c6c + +**Code:** +- `package.json` + `package-lock.json` at the repo root: exact pin + `"mathjax": "4.1.3"` (the version the browser bundle serves). +- `utils/check_math_snippets_node.cjs`: reads a JSON array of + `{file, line, display, snippet}` from stdin; compiles each with + `tex2mml(snippet, {display})`; prints one line per failure + `file:line: :: snippet`; exit 0 clean, 1 on failures, + 2 on oracle malfunction. +- `docs/conf.py`: `mathjax_js = + 'https://cdn.jsdelivr.net/npm/mathjax@4.1.3/tex-mml-chtml.js'` — pin the + browser to the checker's version. +- `.github/workflows/tests.yml` + `.github/workflows/coverage.yml`: an + `npm ci` step before the test step (Node is preinstalled on all runner + OSes). + +**Tests:** `tests/utils/test_check_math_snippets_node.py` — runs the `.cjs` +script with synthetic snippets: broken (`f_r_i`, `\textit{a_b}`, `a_{b`, +`x_`, `a^b^c`) and valid (`f_{r\_i}`, `a_{b_c}`, `a_b^c`, +`\textit{store}_{r\_f}`, an `aligned` block); asserts exit codes and the +reported messages. + +**Docs:** `docs/developer.rst` "Development setup": Node.js ≥ 20 and +`npm ci` as prerequisites of the math guard. -**Tests:** fresh docs build under the no-warnings policy. - -**Docs:** -- New `docs/contributing_data.rst` — title "Contributing data", label - `.. _contributing_data:` (moved from the Graphs page), standard page header. - Sections, in order: - - Intro (from `docs/graphs/index.rst:17-23`): all new data enters through - the single CONTRIBUTE NEW DATA issue; prepare → validate → publish → open - issue; maintainers upload and merge; no PR needed. Plus one sentence - pointing to :ref:`file_structure` for the archive layout every - contribution must follow. - - "Contribution cases" (from `docs/graphs/index.rst:25-33`). - - "Validation" (from `docs/graphs/index.rst:35-45`). - - "Consistency invariants" (from `docs/graphs/index.rst:47-55`). - - "Not accepted via this flow" (from `docs/graphs/index.rst:57-63`). - - "Grammars" (from `docs/dataset.rst:26-29`, condensed to avoid repeating - the intro's issue-creation sentence): grammars arrive with the data they - apply to (template's "Queries and grammars" section); accepted only if - instantiated as at least one query with reference results in a - contributed archive. -- `docs/dataset.rst`: remove the "How to add a new grammar?" section; add - `contributing_data` to the toctree (before `file_structure`); extend the - intro by one sentence naming the two new subpages. -- `docs/graphs/index.rst`: remove the "Contributing data" section (the page - becomes a pure catalog: intro, Format conversion, Contents, tables). The - page previously had no intro paragraph (it started with the moved - section), so one sentence is added naming what the catalog page holds. +**Spec:** +- The oracle is the only TeX validator: no hand-crafted TeX pattern code. +- The node script is a pure function of its stdin: no file access, no + network, deterministic. +- merror messages are reported verbatim (whitespace-collapsed). +- `display` flag: `true` for `.. math::` block bodies, `false` for + `:math:` inline roles — the engine must compile in the same mode the + browser renders. + +### S3: Rewire the checker to the oracle; scan docstrings [done] c92eacd + +**Code:** `utils/check_math_snippets.py`: +- `extract_math_snippets`: keep the RST extraction; return + `(line, snippet, display)` triples (`display=True` for `.. math::` + bodies, `False` for `:math:` roles). +- Delete the hand-crafted TeX pattern code: `_TEXT_MODE_RE`, + `_text_mode_groups`, `_has_unescaped_underscore`, `find_broken_snippets`. +- New `check_snippets(snippets)`: one Node subprocess per call (batched + JSON on stdin, bounded timeout); returns one message per broken snippet. +- New `check_package(package_dir)`: ast-walks `*.py` docstrings, extracts + math snippets with `extract_math_snippets`, validates them through the + same oracle. +- `check_docs`: same signature, now oracle-backed. +- `main`: `--dir` (default `docs`) and `--package` (default + `flpq_data`); runs both scans. +- Fails loudly (clear message, non-zero exit) when `node` or + `node_modules` is missing — never silently skips. + +**Tests:** `tests/utils/test_check_math_snippets.py`: +- keep the pure-Python extraction tests (updated to the triple form); +- oracle-backed broken/valid tests through `check_snippets` (synthetic); +- docstring tests: a tmp package tree with a broken `:math:` role in a + docstring is flagged, a clean one passes; +- `test_real_docs_tree_is_clean` (now oracle-backed) + new + `test_real_package_is_clean`; +- missing-node behavior (monkeypatched `shutil.which`) raises the clear + error. + +**Docs:** `docs/developer.rst` "Test pipeline": the guard validates every +docs-page and package-docstring math snippet with the same MathJax engine +the browser uses. **Spec:** -- The issue-template link appears exactly once on the new page (in the - intro); the grammar section references the template's section name without - repeating the full link. -- After the commit, "Contributing data" content exists only in - `docs/contributing_data.rst`; the label `contributing_data` resolves to the - new page. +- Message format: `path:line: :: snippet`, sorted by + file. +- Docstring line numbers: docstring start line + snippet offset within the + docstring (1-based). +- One Node process per `check_docs`/`check_package` call, not per snippet. +- The hand-crafted pattern code is deleted, not kept behind a flag. -### S3: Re-point external pointers; changelog [done] deef55a +### S4: Pre-commit hook, docs, changelog [done] 0578e61 -**Code:** none (README/docs/changelog only). +**Code:** `.pre-commit-config.yaml`: local hook `check-math-snippets` +(entry `python3 utils/check_math_snippets.py`, `language: system`, +`pass_filenames: false`, `always_run: true`). -**Tests:** fresh docs build under the no-warnings policy. +**Tests:** none new (the hook config is exercised by pre-commit itself; the +driver is covered by the S3 tests). **Docs:** -- `README.rst` "How to contribute new data?": "the full contribution guide is - on the Graphs page" → the Contributing data page - (`.../FLPQ_Data/contributing_data.html`). -- `docs/developer.rst` "Adding graphs and grammars": replace the stale - pointer (it references a README section titled "How to add a new graph?" - that does not exist) with :doc:`/contributing_data`. -- `CHANGELOG.md` Unreleased → Changed: contribution documentation - consolidated under the Dataset section — new "Contributing data" and - "File structure" pages; both sections removed from the Graphs page and the - Dataset root. +- `docs/developer.rst` "Quality checks": add the math guard to the + enumerated hook list (the config file stays the source of truth). +- `CHANGELOG.md` `[Unreleased]`: + - Fixed: the 11 double-subscript snippets (#173) and the 7 misplaced-& + grammar blocks (#174). + - Changed: the math-snippet guard validates with the MathJax engine + itself (mathjax 4.1.3 on Node) instead of hand-crafted patterns; the + browser bundle is pinned to the same version; the guard also covers + package docstrings; pre-commit hook. **Spec:** -- No file references the old locations of either section (verified by grep - for "How to add a new graph", "How to add a new grammar", and the two old - section headings). -- The last subtask's commit carries `Closes #167`. +- The hook runs the full driver (docs + package); ~2–3 s. +- The last commit of the task carries `Fixes #173` and `Fixes #174` as + standalone lines. + +### S5: Repoint the dead eclass Source links to the archived ontology [done] bdfdd03 + +Discovered during the quality gate: the full link check (the CI "Check +links" step, run locally per the no-push rule) reports +`http://www.ebusiness-unibw.org/ontologies/eclass/5.1.4` as broken. It is +the Source link of `docs/graphs/data/eclass.rst` and the Origin link of the +historical `docs/old_graphs/data/eclass.rst`. The university removed the +`/ontologies/` subtree from its site (its redirector mangles the path and +the target 404s; verified directly, not a datacenter-IP block). + +**Code:** both eclass pages: the link now points to the Wayback Machine +snapshot of the original artifact, +`https://web.archive.org/web/20160309143952/http://ebusiness-unibw.org/ontologies/eclass/5.1.4/eclass_514en.owl`. +The original landing page was a 303 redirect to exactly this file, so the +link serves the same content as before; the snapshot is verified to be the +eClassOWL 5.1.4 ontology (40 MB RDF/XML, "eClassOWL Version 5.1.4"). + +**Tests:** the full link check runs to completion; the eclass links pass. +The only failure is `https://junit.org/junit5/` (pre-existing, on dev +since May 2026), unreachable from this machine's network (IPv6 unrouted, +IPv4 silently dropped) but green in CI on the base commit — ruled +environmental by the user, not a broken link. + +**Docs:** the change is the docs themselves; `CHANGELOG.md` `[Unreleased]` +Fixed gains the link repair. ---- +**Spec:** +- The link resolves (HTTP 200) and points to the actual source artifact. +- No `linkcheck_ignore` entry: the ignore list is for rate-limited sites, + not dead URLs; a dead link must not stay visible on the deployed page. -## Execution order +## Out of scope -S1 → S2 → S3. One commit per subtask (`docs(167-SN)`); docs-only task — code -gates (tests, lint, format) are skipped per the workflow rules, but each -subtask must leave a green fresh docs build; code review + quality gate -(docs build, CI linkcheck) before merge to `dev`. +- `$...$` spans via the `texext` extension: zero occurrences in the docs; + mirroring texext's matching rules is a follow-up if/when used. +- `README.rst`: contains no math. diff --git a/tests/utils/test_archive_sizes.py b/tests/utils/test_archive_sizes.py index a61219a3..7205cedb 100644 --- a/tests/utils/test_archive_sizes.py +++ b/tests/utils/test_archive_sizes.py @@ -5,6 +5,7 @@ from archive_sizes import ( extract_url, fetch_sizes, + header_columns, iter_graph_tables, update_table, ) @@ -76,6 +77,114 @@ def test_iter_graph_tables_keeps_empty_cells_aligned(): ] +def test_iter_graph_tables_parses_multi_row_header(): + # A two-row header: a bare ``* -`` row start (row 2) plus the option line. + # docutils merges the group positionally, so ``header_columns`` joins the + # non-empty header cells of each column with " / " in row order. + text = ( + ".. list-table::\n" + " :header-rows: 2\n" + " :align: left\n" + "\n" + " * - Graph\n" + " - Num Nodes\n" + " - Reachable pairs\n" + " -\n" + " - Size (MB)\n" + " - Download\n" + " * -\n" + " -\n" + " - subClassOf\n" + " - type\n" + " -\n" + " -\n" + f" * - :ref:`skos`\n" + " - 144\n" + " - 1\n" + " - 2\n" + " - 0.002\n" + f" - `skos.tar.gz <{WC_URL}>`_ 📥\n" + ) + + table = iter_graph_tables(text)[0] + + assert table.header_rows == 2 + assert len(table.rows) == 3 + # docutils merges the group positionally: the row-2 subheaders land in the + # columns after the group label, so header_columns joins them per column. + assert header_columns(table) == [ + "Graph", + "Num Nodes / subClassOf", + "Reachable pairs / type", + "", + "Size (MB)", + "Download", + ] + # Data rows start after the two header rows and keep all their cells. + assert [cell for _, cell in table.rows[2]] == [ + ":ref:`skos`", + "144", + "1", + "2", + "0.002", + f"`skos.tar.gz <{WC_URL}>`_ 📥", + ] + + +def test_header_columns_single_row_is_flat(): + table = iter_graph_tables(TABLE_WITHOUT_SIZE)[0] + assert table.header_rows == 1 + assert header_columns(table) == ["Graph", "Num Nodes", "Download"] + + +def test_update_table_fills_size_with_multi_row_header(): + text = ( + ".. list-table::\n" + " :header-rows: 2\n" + " :align: left\n" + "\n" + " * - Graph\n" + " - Num Nodes\n" + " - Reachable pairs\n" + " -\n" + " - Size (MB)\n" + " - Download\n" + " * -\n" + " -\n" + " - subClassOf\n" + " - type\n" + " -\n" + " -\n" + f" * - :ref:`skos`\n" + " - 144\n" + " - 1\n" + " - 2\n" + " - 9.99\n" + f" - `skos.tar.gz <{WC_URL}>`_ 📥\n" + ) + + updated, changed = update_table(text, SIZES) + + # Size (MB) is located via header_columns (column index 4); the size cell + # is the one before Download (index 5 -> data index 4). + assert changed == 1 + table = iter_graph_tables(updated)[0] + assert header_columns(table)[4] == "Size (MB)" + assert header_columns(table)[5] == "Download" + assert [cell for _, cell in table.rows[2]] == [ + ":ref:`skos`", + "144", + "1", + "2", + "0.002", + f"`skos.tar.gz <{WC_URL}>`_ 📥", + ] + # Idempotent: the size column is now correct, nothing else to change. + again, changed_again = update_table(updated, SIZES) + assert changed_again == 0 + assert again == updated + + def test_iter_graph_tables_skips_tables_without_download_column(): text = ( ".. list-table::\n" diff --git a/tests/utils/test_check_math_snippets.py b/tests/utils/test_check_math_snippets.py index 1d60ced5..4e9e177e 100644 --- a/tests/utils/test_check_math_snippets.py +++ b/tests/utils/test_check_math_snippets.py @@ -1,40 +1,43 @@ from pathlib import Path +import pytest from check_math_snippets import ( check_docs, + check_package, + check_snippets, extract_math_snippets, - find_broken_snippets, main, ) DOCS_DIR = Path(__file__).resolve().parents[2] / "docs" +PACKAGE_DIR = Path(__file__).resolve().parents[2] / "flpq_data" def test_extract_inline_role(): - assert extract_math_snippets("x :math:`a_1` y") == [(1, "a_1")] + assert extract_math_snippets("x :math:`a_1` y") == [(1, "a_1", False)] def test_extract_multiple_inline_roles_on_one_line(): assert extract_math_snippets(":math:`a` and :math:`b_2`") == [ - (1, "a"), - (1, "b_2"), + (1, "a", False), + (1, "b_2", False), ] def test_extract_multiline_inline_role(): text = "for :math:`\\{w \\mid\nw \\in L\\}` end" - assert extract_math_snippets(text) == [(1, "\\{w \\mid w \\in L\\}")] + assert extract_math_snippets(text) == [(1, "\\{w \\mid w \\in L\\}", False)] def test_extract_math_block(): text = ".. math::\n\n S \\rightarrow a_1 \\\\\n V \\rightarrow b_2 \\\\\n" assert extract_math_snippets(text) == [ - (1, "S \\rightarrow a_1 \\\\ V \\rightarrow b_2 \\\\") + (1, "S \\rightarrow a_1 \\\\ V \\rightarrow b_2 \\\\", True) ] def test_extract_indented_math_block(): - assert extract_math_snippets(" .. math::\n\n x_1\n") == [(1, "x_1")] + assert extract_math_snippets(" .. math::\n\n x_1\n") == [(1, "x_1", True)] def test_extract_math_block_without_body(): @@ -43,74 +46,84 @@ def test_extract_math_block_without_body(): def test_extract_skips_code_blocks(): text = ".. code-block:: latex\n\n :math:`\\textit{a_b}`\n\nreal :math:`c_d` end\n" - assert extract_math_snippets(text) == [(5, "c_d")] + assert extract_math_snippets(text) == [(5, "c_d", False)] def test_extract_code_block_without_body(): text = ".. code-block:: python\nplain text :math:`a_b`\n" - assert extract_math_snippets(text) == [(2, "a_b")] + assert extract_math_snippets(text) == [(2, "a_b", False)] def test_extract_inline_role_inside_math_block_is_not_double_counted(): text = ".. math::\n\n :math:`a_b`\n" - assert extract_math_snippets(text) == [(1, ":math:`a_b`")] + assert extract_math_snippets(text) == [(1, ":math:`a_b`", True)] def test_extract_no_math(): assert extract_math_snippets("plain text\nno math here\n") == [] -def test_broken_underscore_in_textit_group(): - snippet = r"\textit{assigment_labels}" - assert find_broken_snippets([(1, snippet)]) == [(1, snippet)] - - -def test_valid_subscript_outside_group(): - assert find_broken_snippets([(1, r"\textit{load}_f")]) == [] - - -def test_escaped_underscore_in_group_is_valid(): - assert find_broken_snippets([(1, r"\textit{a\_b}")]) == [] - - -def test_double_backslash_before_underscore_is_broken(): - # A literal backslash (\\) followed by a raw underscore is not escaped. - snippet = r"\textit{a\\_b}" - assert find_broken_snippets([(1, snippet)]) == [(1, snippet)] - - -def test_nested_text_mode_group_is_checked(): - snippet = r"\textit{\mbox{a_b}}" - assert find_broken_snippets([(1, snippet)]) == [(1, snippet)] +def test_check_snippets_reports_broken_items(): + messages = check_snippets( + [ + ("a.rst", 1, "f_r_i", False), + ("a.rst", 2, r"\textit{a_b}", False), + ] + ) + assert len(messages) == 2 + assert messages[0].startswith("a.rst:1:") + assert "Double subscripts" in messages[0] + assert "f_r_i" in messages[0] + assert messages[1].startswith("a.rst:2:") + assert "'_' allowed only in math mode" in messages[1] + + +def test_check_snippets_is_clean_for_valid_items(): + assert ( + check_snippets( + [ + ("a.rst", 1, r"f_{r\_i}", False), + ("a.rst", 2, "a_{b_c}", False), + ("b.rst", 3, r"\begin{aligned} S &\to a \end{aligned}", True), + ] + ) + == [] + ) -def test_plain_subscript_without_command_is_valid(): - assert find_broken_snippets([(1, r"V_1 \rightarrow a_r")]) == [] +def test_check_snippets_empty_input_runs_no_oracle(monkeypatch): + def fail(*args, **kwargs): + raise AssertionError("oracle must not run for empty input") + monkeypatch.setattr("check_math_snippets.subprocess.run", fail) + assert check_snippets([]) == [] -def test_all_text_mode_commands_are_checked(): - for cmd in ("text", "textit", "textrm", "textbf", "mbox"): - snippet = rf"\{cmd}{{a_b}}" - assert find_broken_snippets([(1, snippet)]) == [(1, snippet)] +def test_check_snippets_missing_node_fails_loudly(monkeypatch): + monkeypatch.setattr("check_math_snippets.shutil.which", lambda _: None) + with pytest.raises(RuntimeError, match="node is not on PATH"): + check_snippets([("a.rst", 1, "x_1", False)]) -def test_unbalanced_group_still_checked(): - snippet = r"\textit{a_b" - assert find_broken_snippets([(1, snippet)]) == [(1, snippet)] +def test_check_snippets_missing_node_modules_fails_loudly(monkeypatch, tmp_path): + import check_math_snippets as cms -def test_no_snippets_no_violations(): - assert find_broken_snippets([]) == [] + # Point the repo-root lookup at an empty tmp dir via the script location. + script = tmp_path / "utils" / "check_math_snippets_node.cjs" + script.parent.mkdir() + script.write_text("", encoding="utf-8") + monkeypatch.setattr(cms, "_oracle_script", lambda: script) + with pytest.raises(RuntimeError, match="npm ci"): + check_snippets([("a.rst", 1, "x_1", False)]) def test_check_docs_reports_violations(tmp_path): - (tmp_path / "page.rst").write_text( - "T\n=\n\n:math:`\\textit{a_b}`\n", encoding="utf-8" - ) + (tmp_path / "page.rst").write_text("T\n=\n\n:math:`f_r_i`\n", encoding="utf-8") messages = check_docs(tmp_path) assert len(messages) == 1 assert "page.rst:4" in messages[0] - assert r"\textit{a_b}" in messages[0] + assert "Double subscripts" in messages[0] + assert "f_r_i" in messages[0] def test_check_docs_is_clean_for_valid_pages(tmp_path): @@ -123,7 +136,7 @@ def test_check_docs_is_clean_for_valid_pages(tmp_path): def test_check_docs_skips_build_artifacts(tmp_path): build = tmp_path / "_build" build.mkdir() - (build / "page.rst").write_text(":math:`\\textit{a_b}`\n", encoding="utf-8") + (build / "page.rst").write_text(":math:`f_r_i`\n", encoding="utf-8") assert check_docs(tmp_path) == [] @@ -131,18 +144,61 @@ def test_check_docs_empty_dir(tmp_path): assert check_docs(tmp_path) == [] +def test_check_package_reports_broken_docstring(tmp_path): + (tmp_path / "mod.py").write_text( + '"""Module with :math:`f_r_i` in its docstring."""\n\n\n' + "def f():\n" + ' """Uses :math:`a_{b_c}`."""\n' + " return 1\n", + encoding="utf-8", + ) + messages = check_package(tmp_path) + assert len(messages) == 1 + assert "mod.py:1" in messages[0] + assert "Double subscripts" in messages[0] + + +def test_check_package_is_clean(tmp_path): + (tmp_path / "mod.py").write_text( + 'r"""Module with :math:`f_{r\\_i}` in its docstring."""\n', encoding="utf-8" + ) + assert check_package(tmp_path) == [] + + +def test_check_package_skips_unparseable_modules(tmp_path): + (tmp_path / "broken.py").write_text("def f(:\n", encoding="utf-8") + assert check_package(tmp_path) == [] + + +def test_check_package_empty_dir(tmp_path): + assert check_package(tmp_path) == [] + + def test_main_returns_zero_when_clean(tmp_path, capsys): - (tmp_path / "page.rst").write_text("no math\n", encoding="utf-8") - assert main(["--dir", str(tmp_path)]) == 0 + docs = tmp_path / "docs" + docs.mkdir() + (docs / "page.rst").write_text("no math\n", encoding="utf-8") + package = tmp_path / "pkg" + package.mkdir() + (package / "mod.py").write_text('"""Clean :math:`x_1`."""\n', encoding="utf-8") + assert main(["--dir", str(docs), "--package", str(package)]) == 0 assert "valid" in capsys.readouterr().out def test_main_returns_one_on_violation(tmp_path, capsys): - (tmp_path / "page.rst").write_text(":math:`\\textit{a_b}`\n", encoding="utf-8") - assert main(["--dir", str(tmp_path)]) == 1 + docs = tmp_path / "docs" + docs.mkdir() + (docs / "page.rst").write_text(":math:`f_r_i`\n", encoding="utf-8") + package = tmp_path / "pkg" + package.mkdir() + assert main(["--dir", str(docs), "--package", str(package)]) == 1 out = capsys.readouterr().out assert "page.rst:1" in out def test_real_docs_tree_is_clean(): assert check_docs(DOCS_DIR) == [] + + +def test_real_package_is_clean(): + assert check_package(PACKAGE_DIR) == [] diff --git a/tests/utils/test_check_math_snippets_node.py b/tests/utils/test_check_math_snippets_node.py new file mode 100644 index 00000000..5ba4367b --- /dev/null +++ b/tests/utils/test_check_math_snippets_node.py @@ -0,0 +1,119 @@ +"""Tests for the MathJax oracle script (utils/check_math_snippets_node.cjs). + +The oracle is the only TeX validator of the math-snippet guard: it compiles +snippets with the same MathJax engine and version the deployed docs load in +the browser, so these tests pin the observable contract (exit codes, message +format) of that engine integration. +""" + +import json +import pathlib +import shutil +import subprocess + +SCRIPT = ( + pathlib.Path(__file__).resolve().parents[2] + / "utils" + / "check_math_snippets_node.cjs" +) +TIMEOUT = 60 + + +def _node() -> str: + node = shutil.which("node") + if node is None: + raise AssertionError( + "node is not on PATH; install Node.js (>= 20) and run `npm ci` " + "at the repository root (see docs/developer.rst, Development setup)" + ) + return node + + +def run_oracle(payload) -> subprocess.CompletedProcess: + data = payload if isinstance(payload, str) else json.dumps(payload) + return subprocess.run( + [_node(), str(SCRIPT)], + input=data, + capture_output=True, + text=True, + timeout=TIMEOUT, + ) + + +def test_oracle_flags_broken_snippets(): + result = run_oracle( + [ + {"file": "a.rst", "line": 1, "display": False, "snippet": "f_r_i"}, + {"file": "a.rst", "line": 2, "display": False, "snippet": r"\textit{a_b}"}, + {"file": "b.rst", "line": 3, "display": False, "snippet": "a_{b"}, + {"file": "b.rst", "line": 4, "display": False, "snippet": "x_"}, + {"file": "c.rst", "line": 5, "display": False, "snippet": "a^b^c"}, + ] + ) + assert result.returncode == 1 + lines = result.stdout.splitlines() + assert len(lines) == 5 + assert lines[0].startswith("a.rst:1:") + assert "Double subscripts" in lines[0] + assert "f_r_i" in lines[0] + assert "'_' allowed only in math mode" in lines[1] + assert "brace" in lines[2] + assert "Missing superscript or subscript argument" in lines[3] + assert "Double exponent" in lines[4] + + +def test_oracle_accepts_valid_snippets(): + result = run_oracle( + [ + {"file": "a.rst", "line": 1, "display": False, "snippet": r"f_{r\_i}"}, + {"file": "a.rst", "line": 2, "display": False, "snippet": "a_{b_c}"}, + {"file": "a.rst", "line": 3, "display": False, "snippet": "a_b^c"}, + { + "file": "b.rst", + "line": 4, + "display": False, + "snippet": r"\textit{store}_{r\_f}", + }, + { + "file": "b.rst", + "line": 5, + "display": True, + "snippet": r"\begin{aligned} S &\to a \\ T &\to b \end{aligned}", + }, + ] + ) + assert result.returncode == 0 + assert result.stdout == "" + + +def test_alignment_tab_requires_an_alignment_environment(): + broken = run_oracle( + [ + { + "file": "a.rst", + "line": 1, + "display": True, + "snippet": "S &\\to a \\\\ T &\\to b", + } + ] + ) + assert broken.returncode == 1 + assert "Misplaced &" in broken.stdout + + fixed = run_oracle( + [ + { + "file": "a.rst", + "line": 1, + "display": True, + "snippet": r"\begin{aligned} S &\to a \\ T &\to b \end{aligned}", + } + ] + ) + assert fixed.returncode == 0 + + +def test_bad_input_exits_two(): + result = run_oracle("not json") + assert result.returncode == 2 + assert "bad input" in result.stderr diff --git a/tests/utils/test_reachable_pairs_tables.py b/tests/utils/test_reachable_pairs_tables.py index e7e74722..785b56a8 100644 --- a/tests/utils/test_reachable_pairs_tables.py +++ b/tests/utils/test_reachable_pairs_tables.py @@ -1,4 +1,5 @@ import pathlib +from typing import Optional import pytest import reachable_pairs_tables @@ -258,6 +259,101 @@ def test_update_category_columns_empty_cell_for_missing_grammar(): assert any("beta/b.cnf" in p for p in problems) +def test_update_category_columns_multi_row_header(): + # A two-row header: the count-column cells carry the flat key + # ("Reachable pairs / ") and data rows start after the two header + # rows. docutils merges the group positionally, so the subheader under a + # count column lands in the column after the group label. + text = ( + ".. list-table::\n" + " :header-rows: 2\n" + " :align: left\n" + "\n" + " * - Graph\n" + " - Num Nodes\n" + " - Reachable pairs\n" + " -\n" + " -\n" + " -\n" + " - Size (MB)\n" + " - Download\n" + " * -\n" + " -\n" + " - subClassOf\n" + " - subClassOf_type\n" + " - type\n" + " - broaderTransitive\n" + " -\n" + " -\n" + " * - :ref:`g1`\n" + " - 10\n" + " - 3\n" + " -\n" + " -\n" + " -\n" + " - 0.003\n" + f" - `alpha.tar.gz <{ALPHA_URL}>`_ 📥\n" + " * - :ref:`g2`\n" + " - 20\n" + " - 7\n" + " - 8\n" + " - 9\n" + " - 10\n" + " - 0.004\n" + f" - `beta.tar.gz <{BETA_URL}>`_ 📥\n" + ) + # Flat keys produced by header_columns for this positional parse (col 2 = + # "Reachable pairs / subClassOf_type", col 3 = "type"). + columns: list[tuple[str, Optional[str]]] = [ + ("Reachable pairs / subClassOf_type", "a.cnf"), + ("type", "b.cnf"), + ] + # g1/a.cnf is 3 (count col 2 already), g1/b.cnf is missing -> col 3 empty; + # g2/a.cnf is 5 (was 7), g2/b.cnf is 9 (was 9). + updated, _ = update_category_columns(text, ROWS, "cat_a", PAGES, columns=columns) + + lines = updated.splitlines() + # g1 (alpha) count cells: col2 (a.cnf) stays 3, col3 (b.cnf) -> "not available". + assert lines[22] == " - 3" + assert lines[23] == " - not available" + # g2 (beta) count cells: col2 (a.cnf) becomes 5, col3 (b.cnf) becomes 9. + assert lines[30] == " - 5" + assert lines[31] == " - 9" + # Idempotent once in sync. + again, problems_again = update_category_columns( + updated, ROWS, "cat_a", PAGES, columns=columns + ) + assert again == updated + assert problems_again == [] + + +def test_update_category_columns_missing_flat_key_is_a_problem(): + # The table's count column is named "subClassOf"; a wanted flat key that + # does not match it ("subClassOf / type") is reported as missing. + text = ( + ".. list-table::\n" + " :header-rows: 1\n" + "\n" + " * - Graph\n" + " - subClassOf\n" + " - Download\n" + f" * - :ref:`g1`\n" + " - 3\n" + f" - `alpha.tar.gz <{ALPHA_URL}>`_ 📥\n" + ) + # "subClassOf" matches the table's column (selects it); "subClassOf / type" + # does not, so it is reported as missing from the header. + columns: list[tuple[str, Optional[str]]] = [ + ("subClassOf", "a.cnf"), + ("subClassOf / type", "b.cnf"), + ] + _, problems = update_category_columns(text, ROWS, "cat_a", PAGES, columns=columns) + assert any( + "column subClassOf / type is missing from the table header" in p + for p in problems + ) + + def test_update_category_columns_short_row_is_a_problem(): # The row has cells for Graph and a_col only. text = ( diff --git a/utils/archive_sizes.py b/utils/archive_sizes.py index e4ab467b..4acd030d 100644 --- a/utils/archive_sizes.py +++ b/utils/archive_sizes.py @@ -1,8 +1,8 @@ """Keep the Size (MB) column of the docs graph tables in sync with S3. -Every per-category graph table (and the old-graphs table) has a ``Download`` -column linking to the archive on Yandex Object Storage. This tool keeps the -``Size (MB)`` column next to it accurate: +Every per-category graph table has a ``Download`` column linking to the +archive on Yandex Object Storage. This tool keeps the ``Size (MB)`` column +next to it accurate: - check mode (default): HEAD every referenced archive and report the rows whose size cell disagrees with the stored object; @@ -29,9 +29,11 @@ __all__ = [ "SIZE_COLUMN", + "DOWNLOAD_COLUMN", "GRAPH_TABLE_FILES", "GraphTable", "iter_graph_tables", + "header_columns", "extract_url", "fetch_sizes", "update_table", @@ -45,10 +47,9 @@ DOWNLOAD_COLUMN = "Download" #: The docs files holding graph tables, as glob patterns relative to docs/. -GRAPH_TABLE_FILES: tuple[str, ...] = ( - "graphs/*.rst", - "old_graphs/index.rst", -) +#: ``old_graphs`` is historical data and is intentionally not maintained by +#: this tool, so its table is not scanned. +GRAPH_TABLE_FILES: tuple[str, ...] = ("graphs/*.rst",) _ROW_RE = re.compile(r"^(\s+)\* - (.*)$") # The space after the dash is optional: empty cells are bare ``-`` lines. @@ -64,20 +65,27 @@ class GraphTable: ---------- start_line : int The 0-based index of the ``.. list-table::`` line in the file text. + header_rows : int + The number of header rows (the ``:header-rows:`` option; 1 when the + table declares none). ``rows[:header_rows]`` are the header rows and + ``rows[header_rows:]`` are the data rows. rows : list[list[tuple[int, str]]] - The table rows in order; ``rows[0]`` is the header. Each row is a - list of ``(line_index, cell_text)`` pairs in column order, where - ``line_index`` is the 0-based index of the cell's line. + The table rows in order. Each row is a list of ``(line_index, + cell_text)`` pairs in column order, where ``line_index`` is the + 0-based index of the cell's line. """ start_line: int + header_rows: int rows: list[list[tuple[int, str]]] def iter_graph_tables(text: str) -> list[GraphTable]: """Returns the graph tables (list-tables with a Download column) in text. - Tables without a ``Download`` header cell are skipped, as are files' + A table is a graph table when any of its header rows (the + ``:header-rows:`` count, ``1`` when the table declares none) contains a + ``Download`` cell. Tables without one are skipped, as are files' non-table content. Only the flat one-cell-per-line layout used by the docs graph tables is understood. @@ -89,9 +97,9 @@ def iter_graph_tables(text: str) -> list[GraphTable]: Examples -------- >>> text = ( - ... ".. list-table::\n :header-rows: 1\n\n" - ... " * - Graph\n - Download\n" - ... " * - wc\n - `wc.tar.gz `_ 📥\n" + ... ".. list-table::\\n :header-rows: 1\\n\\n" + ... " * - Graph\\n - Download\\n" + ... " * - wc\\n - `wc.tar.gz `_ \\u2605\\n" ... ) >>> tables = iter_graph_tables(text) >>> len(tables), [cell for _, cell in tables[0].rows[0]] @@ -120,6 +128,26 @@ def iter_graph_tables(text: str) -> list[GraphTable]: return tables +def _header_rows_of(lines: list[str], start: int, end: int) -> int: + """Returns the ``:header-rows:`` count of the directive at ``lines[start]``. + + Scans the directive's option lines (the ``:name: value`` lines before any + table row) and returns the value of ``:header-rows:``; ``1`` when the + table declares none. + """ + for idx in range(start + 1, end): + line = lines[idx].strip() + if not line: + continue + if line.startswith("*"): + break + if line.startswith(":header-rows:"): + value = line.split(":", 2)[2].strip() + if value.isdigit(): + return max(int(value), 1) + return 1 + + def _indent_of(lines: list[str], line_index: int) -> str: """Returns the leading whitespace of a line.""" line = lines[line_index] @@ -127,7 +155,12 @@ def _indent_of(lines: list[str], line_index: int) -> str: def _parse_table(lines: list[str], start: int, end: int) -> Optional[GraphTable]: - """Parses the list-table at ``lines[start]`` or returns None.""" + """Parses the list-table at ``lines[start]`` or returns None. + + A row starts at a ``* - `` line or at a bare ``* -`` line (a + row whose first cell is empty, as in the leading cell of a multi-row + header group); its remaining cells follow as indented ``- `` lines. + """ rows: list[list[tuple[int, str]]] = [] current: Optional[list[tuple[int, str]]] = None for idx in range(start + 1, end): @@ -136,12 +169,60 @@ def _parse_table(lines: list[str], start: int, end: int) -> Optional[GraphTable] current = [(idx, row_match.group(2))] rows.append(current) continue + if current is not None and re.match(r"^\s+\* ?-?$", lines[idx]): + current = [] + rows.append(current) + continue cell_match = _CELL_RE.match(lines[idx]) if cell_match is not None and current is not None: current.append((idx, cell_match.group(2))) - if not rows or DOWNLOAD_COLUMN not in [cell for _, cell in rows[0]]: + if not rows: + return None + header_rows = _header_rows_of(lines, start, end) + if not any( + DOWNLOAD_COLUMN in [cell for _, cell in row] for row in rows[:header_rows] + ): return None - return GraphTable(start_line=start, rows=rows) + return GraphTable(start_line=start, header_rows=header_rows, rows=rows) + + +def header_columns(table: GraphTable) -> list[str]: + """Returns the flat per-column header keys of a table. + + The non-empty header cells of each column (across the ``header_rows`` + header rows) are joined with ``" / "`` in row order. A column that is empty + in a header row contributes nothing to that row, so a merged group renders + as a single key per column (e.g. ``"Reachable pairs / subClassOf"``). A + table whose every column is empty yields one empty key. + + Parameters + ---------- + table : GraphTable + A parsed graph table. + + Examples + -------- + >>> from archive_sizes import GraphTable + >>> table = GraphTable( + ... start_line=0, header_rows=1, + ... rows=[[(0, "Graph"), (1, "Size (MB)"), (2, "Download")], + ... [(3, "wc"), (4, "0.002"), (5, "link")]], + ... ) + >>> header_columns(table) + ['Graph', 'Size (MB)', 'Download'] + + Returns + ------- + columns : list[str] + One flat key per column, in column order. + """ + header = table.rows[: table.header_rows] + width = max((len(row) for row in header), default=0) + columns: list[str] = [] + for col in range(width): + parts = [row[col][1] for row in header if col < len(row) and row[col][1]] + columns.append(" / ".join(parts)) + return columns def extract_url(cell: str) -> Optional[str]: @@ -277,16 +358,17 @@ def update_table(text: str, sizes: dict[str, int]) -> tuple[str, int]: replacements: list[tuple[int, str]] = [] changed = 0 for table in iter_graph_tables(text): - header = table.rows[0] + columns = header_columns(table) download_index = next( - i for i, (_, cell) in enumerate(header) if cell == DOWNLOAD_COLUMN + i for i, cell in enumerate(columns) if cell == DOWNLOAD_COLUMN ) - has_size = any(cell == SIZE_COLUMN for _, cell in header) + has_size = SIZE_COLUMN in columns if not has_size: - line_index, _ = header[download_index] + last_header = table.rows[table.header_rows - 1] + line_index, _ = last_header[download_index] indent = _indent_of(lines, line_index) insertions.append((line_index, f"{indent}- {SIZE_COLUMN}")) - for row in table.rows[1:]: + for row in table.rows[table.header_rows :]: if len(row) <= download_index: continue url = extract_url(row[download_index][1]) @@ -323,10 +405,11 @@ def _table_urls(text: str) -> list[str]: """Returns the archive URLs referenced by the graph tables of text.""" urls: list[str] = [] for table in iter_graph_tables(text): + columns = header_columns(table) download_index = next( - i for i, (_, cell) in enumerate(table.rows[0]) if cell == DOWNLOAD_COLUMN + i for i, cell in enumerate(columns) if cell == DOWNLOAD_COLUMN ) - for row in table.rows[1:]: + for row in table.rows[table.header_rows :]: if len(row) <= download_index: continue url = extract_url(row[download_index][1]) @@ -391,14 +474,14 @@ def main(argv: Optional[Sequence[str]] = None) -> int: rel_path = path.relative_to(MAIN_FOLDER) text = texts[path] for table in iter_graph_tables(text): - header = [cell for _, cell in table.rows[0]] - download_index = header.index(DOWNLOAD_COLUMN) - has_size = SIZE_COLUMN in header + columns = header_columns(table) + download_index = columns.index(DOWNLOAD_COLUMN) + has_size = SIZE_COLUMN in columns if not has_size: problems.append( f"{rel_path}:{table.start_line + 1}: no {SIZE_COLUMN} column" ) - for row in table.rows[1:]: + for row in table.rows[table.header_rows :]: rows_checked += 1 name = row[0][1] if len(row) <= download_index: diff --git a/utils/check_math_snippets.py b/utils/check_math_snippets.py index a2dae15e..8332d4e8 100644 --- a/utils/check_math_snippets.py +++ b/utils/check_math_snippets.py @@ -1,36 +1,41 @@ #!/usr/bin/env python3 -"""Validate the math snippets of the docs pages. +"""Validate the math snippets of the docs pages and package docstrings. Sphinx passes math content verbatim to MathJax, which renders it in the -browser; a snippet that MathJax cannot parse shows up as an error message on -the deployed page while the Sphinx build stays green (issue #127). This tool -scans every ``.rst`` file under ``docs/`` and rejects the known-broken -pattern: an unescaped underscore inside a text-mode command group (``\\text``, -``\\textit``, ``\\textrm``, ``\\textbf``, ``\\mbox``). Inside such a group the -content is typeset in text mode, where ``_`` is illegal ("'_' allowed only in -math mode"); a subscript outside the group (``\\textit{load}_f``) is valid. +browser; a snippet the engine cannot parse shows up as an error message on +the deployed page while the Sphinx build stays green (issues #127, #173). +This tool therefore validates every snippet with the MathJax engine itself +(``utils/check_math_snippets_node.cjs``, the same version the browser loads +— see ``mathjax_js`` in ``docs/conf.py``): it scans every ``.rst`` file +under ``docs`` and every docstring under the package, extracts the +``:math:`` roles and ``.. math::`` bodies, and reports each snippet the +engine rejects with the engine's own error message. Usage:: - python utils/check_math_snippets.py [--dir docs] + python utils/check_math_snippets.py [--dir docs] [--package flpq_data] """ import argparse +import ast +import json import pathlib import re +import shutil +import subprocess import sys from typing import Optional, Sequence __all__ = [ "check_docs", + "check_package", + "check_snippets", "extract_math_snippets", - "find_broken_snippets", "main", ] -#: Commands whose argument is typeset in text mode inside a math environment, -#: where an underscore is illegal. -_TEXT_MODE_RE = re.compile(r"\\(?:text|textit|textrm|textbf|mbox)\s*\{") +#: Upper bound for one oracle run (the engine starts up once per run). +_ORACLE_TIMEOUT = 120 _INLINE_MATH_RE = re.compile(r":math:`([^`]+)`", re.DOTALL) _MATH_BLOCK_RE = re.compile(r"^\s*\.\.\s+math::\s*$") @@ -93,13 +98,13 @@ def _mask_code_blocks(text: str) -> str: return "\n".join(lines) -def extract_math_snippets(text: str) -> list[tuple[int, str]]: - """Returns the (start line, snippet) pairs of every math snippet in an RST text. +def extract_math_snippets(text: str) -> list[tuple[int, str, bool]]: + """Returns the (start line, snippet, display) triples of every math snippet. - Collects inline ``:math:`` roles (which may span several lines) and the - bodies of ``.. math::`` directives; code-block directives are ignored. - Line numbers are 1-based and whitespace inside a snippet is collapsed to - single spaces. + Collects inline ``:math:`` roles (``display=False``, which may span + several lines) and the bodies of ``.. math::`` directives + (``display=True``); code-block directives are ignored. Line numbers are + 1-based and whitespace inside a snippet is collapsed to single spaces. Parameters ---------- @@ -109,17 +114,17 @@ def extract_math_snippets(text: str) -> list[tuple[int, str]]: Examples -------- >>> extract_math_snippets("a :math:`x_1` b\\n\\n.. math::\\n\\n y_2\\n") - [(1, 'x_1'), (3, 'y_2')] + [(1, 'x_1', False), (3, 'y_2', True)] Returns ------- - snippets : list[tuple[int, str]] - The (start line, snippet) pairs, in document order. + snippets : list[tuple[int, str, bool]] + The (start line, snippet, display) triples, in document order. """ masked = _mask_code_blocks(text) lines = masked.splitlines() - snippets: list[tuple[int, str]] = [] + snippets: list[tuple[int, str, bool]] = [] block_lines: set[int] = set() for i, line in enumerate(lines): if not _MATH_BLOCK_RE.match(line): @@ -127,73 +132,76 @@ def extract_math_snippets(text: str) -> list[tuple[int, str]]: j, body = _directive_body(lines, i) block_lines.update(range(i, j)) if body: - snippets.append((i + 1, " ".join(body))) + snippets.append((i + 1, " ".join(body), True)) for match in _INLINE_MATH_RE.finditer(masked): line_no = masked.count("\n", 0, match.start()) + 1 if line_no - 1 in block_lines: continue snippet = " ".join(match.group(1).split()) - snippets.append((line_no, snippet)) + snippets.append((line_no, snippet, False)) snippets.sort() return snippets -def _has_unescaped_underscore(text: str) -> bool: - """Returns True if text holds an underscore not escaped by a backslash.""" - for match in re.finditer(r"_", text): - i = match.start() - 1 - backslashes = 0 - while i >= 0 and text[i] == "\\": - backslashes += 1 - i -= 1 - if backslashes % 2 == 0: - return True - return False +def _oracle_script() -> pathlib.Path: + """Returns the path of the MathJax oracle script.""" + return pathlib.Path(__file__).resolve().parent / "check_math_snippets_node.cjs" -def _text_mode_groups(snippet: str) -> list[str]: - """Returns the brace-balanced arguments of the text-mode commands in a snippet. +def _require_oracle() -> str: + """Returns the node executable, failing loudly when the oracle is unusable.""" + node = shutil.which("node") + if node is None: + raise RuntimeError( + "node is not on PATH; install Node.js (>= 20) to run the " + "math-snippet guard (see docs/developer.rst, Development setup)" + ) + repo_root = _oracle_script().resolve().parents[1] + if not (repo_root / "node_modules" / "mathjax").is_dir(): + raise RuntimeError( + "node_modules/mathjax is missing; run `npm ci` at the " + "repository root to run the math-snippet guard" + ) + return node - An unbalanced group extends to the end of the snippet so its content is - still checked. - """ - groups: list[str] = [] - for match in _TEXT_MODE_RE.finditer(snippet): - depth = 1 - j = match.end() - while j < len(snippet) and depth > 0: - if snippet[j] == "{": - depth += 1 - elif snippet[j] == "}": - depth -= 1 - j += 1 - end = j - 1 if depth == 0 else len(snippet) - groups.append(snippet[match.end() : end]) - return groups +def check_snippets(items: list[tuple[str, int, str, bool]]) -> list[str]: + """Validates (file, line, snippet, display) items with the MathJax oracle. -def find_broken_snippets(snippets: list[tuple[int, str]]) -> list[tuple[int, str]]: - """Returns the (line, snippet) pairs with a raw underscore in a text-mode group. + Runs the oracle once for all items and returns one message per broken + snippet, ``file:line: :: snippet``, in input order. Parameters ---------- - snippets : list[tuple[int, str]] - The (start line, snippet) pairs as returned by ``extract_math_snippets``. + items : list[tuple[str, int, str, bool]] + The (file, line, snippet, display) tuples to validate. Returns ------- - broken : list[tuple[int, str]] - The violating pairs, in the order of the input. + messages : list[str] + One message per broken snippet, in input order. """ - broken: list[tuple[int, str]] = [] - for line, snippet in snippets: - if any( - _has_unescaped_underscore(group) for group in _text_mode_groups(snippet) - ): - broken.append((line, snippet)) - return broken + if not items: + return [] + node = _require_oracle() + payload = json.dumps( + [ + {"file": file, "line": line, "display": display, "snippet": snippet} + for file, line, snippet, display in items + ] + ) + proc = subprocess.run( + [node, str(_oracle_script())], + input=payload, + capture_output=True, + text=True, + timeout=_ORACLE_TIMEOUT, + ) + if proc.returncode == 2: + raise RuntimeError(f"math oracle failed: {proc.stderr.strip()}") + return proc.stdout.splitlines() def check_docs(docs_dir: pathlib.Path) -> list[str]: @@ -208,31 +216,88 @@ def check_docs(docs_dir: pathlib.Path) -> list[str]: Returns ------- messages : list[str] - One ``path:line: reason`` message per violation, sorted by file. + One ``file:line: :: snippet`` message per + violation, sorted by file and line. """ - messages: list[str] = [] + items: list[tuple[str, int, str, bool]] = [] for path in sorted(pathlib.Path(docs_dir).rglob("*.rst")): if "_build" in path.parts: continue text = path.read_text(encoding="utf-8") - for line, snippet in find_broken_snippets(extract_math_snippets(text)): - messages.append( - f"{path}:{line}: unescaped '_' inside a text-mode group: {snippet}" - ) - return messages + for line, snippet, display in extract_math_snippets(text): + items.append((str(path), line, snippet, display)) + items.sort(key=lambda item: (item[0], item[1])) + return check_snippets(items) + + +def _docstrings(source: str) -> list[tuple[int, str]]: + """Returns the (start line, docstring) pairs of every docstring in a module source. + + The start line is the 1-based line of the docstring's opening quotes. + """ + tree = ast.parse(source) + result: list[tuple[int, str]] = [] + for node in ast.walk(tree): + if not isinstance( + node, (ast.Module, ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef) + ): + continue + docstring = ast.get_docstring(node, clean=False) + if docstring is None: + continue + result.append((node.body[0].lineno, docstring)) + return result + + +def check_package(package_dir: pathlib.Path) -> list[str]: + """Returns one message per broken math snippet in the docstrings under package_dir. + + Docstrings are rendered on the autodoc reference pages, so broken math + there fails the same way as broken math in the docs pages. + + Parameters + ---------- + package_dir : pathlib.Path + The package directory to scan recursively for ``.py`` modules. + + Returns + ------- + messages : list[str] + One ``file:line: :: snippet`` message per + violation, sorted by file and line. + """ + items: list[tuple[str, int, str, bool]] = [] + for path in sorted(pathlib.Path(package_dir).rglob("*.py")): + source = path.read_text(encoding="utf-8") + try: + docstring_items = _docstrings(source) + except SyntaxError: + continue + for start, docstring in docstring_items: + for line, snippet, display in extract_math_snippets(docstring): + items.append((str(path), start + line - 1, snippet, display)) + items.sort(key=lambda item: (item[0], item[1])) + return check_snippets(items) def main(argv: Optional[Sequence[str]] = None) -> int: - """Checks all docs pages; returns 0 if clean, 1 on any violation.""" + """Checks docs pages and package docstrings; returns 0 if clean, 1 otherwise.""" parser = argparse.ArgumentParser( - description="Validate the math snippets of the docs pages." + description="Validate the math snippets of the docs and package docstrings." ) parser.add_argument( "--dir", default="docs", help="directory to scan for .rst pages" ) + parser.add_argument( + "--package", + default="flpq_data", + help="package directory to scan for docstrings", + ) args = parser.parse_args(argv) - messages = check_docs(pathlib.Path(args.dir)) + messages = check_docs(pathlib.Path(args.dir)) + check_package( + pathlib.Path(args.package) + ) if messages: for message in messages: print(message) diff --git a/utils/check_math_snippets_node.cjs b/utils/check_math_snippets_node.cjs new file mode 100644 index 00000000..77bc8d20 --- /dev/null +++ b/utils/check_math_snippets_node.cjs @@ -0,0 +1,78 @@ +#!/usr/bin/env node +/** + * MathJax oracle for the math-snippet guard. + * + * Compiles every snippet with the same MathJax engine (and version) the + * deployed docs load in the browser: docs/conf.py pins mathjax_js to the + * version this script requires from package.json. A snippet the engine + * cannot parse yields nodes carrying the exact message shown on + * the deployed page, so the guard and the renderer cannot disagree. + * + * Input: a JSON array of {file, line, display, snippet} objects on stdin. + * Output: one line per broken snippet, `file:line: :: snippet`. + * Exit codes: 0 clean, 1 broken snippets found, 2 oracle malfunction. + */ +'use strict'; + +const path = require('node:path'); +const { pathToFileURL } = require('node:url'); + +const MERROR_RE = /]*>([\s\S]*?)<\/merror>/g; + +function collapse(s) { + return s.replace(/\s+/g, ' ').trim(); +} + +function main() { + let data = ''; + process.stdin.on('data', (chunk) => { + data += chunk; + }); + process.stdin.on('end', () => { + let items; + try { + items = JSON.parse(data); + if (!Array.isArray(items)) { + throw new Error('expected a JSON array'); + } + } catch (err) { + console.error(`check_math_snippets_node: bad input: ${err.message}`); + process.exit(2); + } + require('mathjax') + .init({ + loader: { + load: ['input/tex'], + // MathJax's node entry resolves components to plain absolute + // paths and loads them with import(); Node's ESM loader rejects + // absolute paths without a URL scheme on Windows ("Received + // protocol 'd:'"), so convert them to file:// URLs. + require: (file) => + import(path.isAbsolute(file) ? pathToFileURL(file).href : file), + }, + startup: { typeset: false }, + }) + .then((MathJax) => { + let broken = 0; + for (const item of items) { + const out = MathJax.tex2mml(item.snippet, { display: !!item.display }); + const messages = [...out.matchAll(MERROR_RE)].map((m) => + collapse(m[1].replace(/<[^>]+>/g, '')) + ); + if (messages.length) { + broken += 1; + console.log( + `${item.file}:${item.line}: ${messages.join('; ')} :: ${item.snippet}` + ); + } + } + process.exit(broken ? 1 : 0); + }) + .catch((err) => { + console.error(`check_math_snippets_node: oracle failure: ${err.message}`); + process.exit(2); + }); + }); +} + +main(); diff --git a/utils/reachable_pairs_tables.py b/utils/reachable_pairs_tables.py index 5d35559c..c05d401c 100644 --- a/utils/reachable_pairs_tables.py +++ b/utils/reachable_pairs_tables.py @@ -35,7 +35,7 @@ import re from typing import Optional, Sequence -from archive_sizes import iter_graph_tables +from archive_sizes import header_columns, iter_graph_tables from config import MAIN_FOLDER from flpq_data.tools.check_archive_structure import QUERY_CLASSES @@ -75,18 +75,21 @@ #: count columns of its graph table; a None grammar file means the per-graph #: ``.cnf``. GRAMMAR_COLUMNS: dict[str, list[tuple[str, Optional[str]]]] = { - "biological_uniprot": [("grammar", None)], - "c_alias_analysis": [("c_alias", "c_alias.cnf")], - "context_sensitive_data_flow": [("aa", "aa.cnf")], - "data_provenance": [("prov_derivation", "prov_derivation.cnf")], - "field_sensitive_alias": [("vf", "vf.cnf")], - "java_points_to": [("java_points_to", "java_points_to.cnf")], - "name_resolution": [("name_resolution", "name_resolution.cnf")], + "biological_uniprot": [("Reachable pairs", None)], + "c_alias_analysis": [("Reachable pairs", "c_alias.cnf")], + "context_sensitive_data_flow": [("Reachable pairs", "aa.cnf")], + "data_provenance": [("Reachable pairs", "prov_derivation.cnf")], + "field_sensitive_alias": [("Reachable pairs", "vf.cnf")], + "java_points_to": [("Reachable pairs", "java_points_to.cnf")], + "name_resolution": [("Reachable pairs", "name_resolution.cnf")], "rdf": [ - ("subClassOf", "nested_parentheses_subClassOf.cnf"), - ("subClassOf_type", "nested_parentheses_subClassOf_type.cnf"), - ("type", "nested_parentheses_type.cnf"), - ("broaderTransitive", "nested_parentheses_broaderTransitive.cnf"), + ("Reachable pairs / subClassOf", "nested_parentheses_subClassOf.cnf"), + ("Reachable pairs / subClassOf_type", "nested_parentheses_subClassOf_type.cnf"), + ("Reachable pairs / type", "nested_parentheses_type.cnf"), + ( + "Reachable pairs / broaderTransitive", + "nested_parentheses_broaderTransitive.cnf", + ), ], } @@ -449,16 +452,14 @@ def update_category_columns( "add them to GRAMMAR_COLUMNS" ) from err wanted = {header for header, _ in columns} - tables = [ - t for t in iter_graph_tables(text) if wanted & {cell for _, cell in t.rows[0]} - ] + tables = [t for t in iter_graph_tables(text) if wanted & set(header_columns(t))] if len(tables) != 1: raise ValueError( f"expected exactly one table with a count column from {sorted(wanted)}, " f"found {len(tables)}" ) table = tables[0] - header = [cell for _, cell in table.rows[0]] + header = header_columns(table) col_index = {name: i for i, name in enumerate(header)} by_pair: dict[tuple[str, str], Optional[int]] = { @@ -473,7 +474,7 @@ def update_category_columns( for col_header, _ in columns: if col_header not in col_index: problems.append(f"column {col_header} is missing from the table header") - for row in table.rows[1:]: + for row in table.rows[table.header_rows :]: ref_match = _REF_RE.match(row[0][1]) if ref_match is None: continue diff --git a/uv.lock b/uv.lock index f19a9210..544aa366 100644 --- a/uv.lock +++ b/uv.lock @@ -538,7 +538,7 @@ wheels = [ [[package]] name = "flpq-data" -version = "6.1.1" +version = "6.1.2" source = { editable = "." } dependencies = [ { name = "lark" },