Merge pull request #39 from HolobiomicsLab/proposals/wave-2026-08-20 #94
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # validate.yml -- runs on every PR | |
| # Enforces CI gates per section 14 of ASB-Skills Release Design Doc v2. | |
| # | |
| # Gates implemented here: | |
| # 1. LinkML schema validation (collection.yaml, tools/*.yaml) | |
| # 2. No orphan skills (DOI resolution sample) | |
| # 5. Description discipline lint (leading phrase, length, no marketing) | |
| # 6. EDAM IRI resolution | |
| # 8. RO-Crate validation (Workflow Run Profile 0.5) | |
| # 9. Indicium round-trip (verify-claims) — WARN-ONLY / inert: the CLI ships in | |
| # indicium-adapters, not yet on PyPI, so this gate is advertised but does not | |
| # run in public CI (emits a ::warning:: annotation; never blocks). See gate9-inert. | |
| # 10. Plugin manifest validation | |
| name: Validate | |
| on: | |
| pull_request: | |
| branches: [main] | |
| push: | |
| branches: [main] | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| jobs: | |
| validate: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Python 3.12 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Install dependencies | |
| run: | | |
| pip install --upgrade pip | |
| # numpy backs scripts/skill_map.py; without it that suite silently skips. | |
| pip install pyyaml jsonschema requests pytest numpy | |
| pip install linkml linkml-runtime || echo "WARNING: linkml install failed" | |
| pip install rocrate || echo "WARNING: rocrate install failed" | |
| # indicium-adapters provides the verify-claims console script (Plan 1c) | |
| pip install indicium-adapters || echo "WARNING: indicium-adapters not yet available; gate 9 will warn-only" | |
| # -- Gate 0: Unit tests ------------------------------------------------- | |
| # Until now the suite ran only inside release.yml, so a test could sit broken | |
| # on main for weeks and nothing would say so. The suite is hermetic (pyyaml | |
| # + pytest, no network). | |
| - name: Run test suite | |
| run: python -m pytest tests/ -q | |
| # -- Gate 0b: Index coverage ------------------------------------------- | |
| # A skill absent from skills_index.json / kb_bundle.json ships on disk but | |
| # cannot be found by search, the MCP server, or the docs site. Nothing else | |
| # notices: check_license_tiers.py only inspects skills the index already | |
| # knows about, so the orphan stays invisible and CI stays green. | |
| - name: Every skill is in every index its collection publishes | |
| run: python -m scripts.skill_index | |
| # -- Gate 10: Plugin manifest validates -------------------------------- | |
| - name: Validate .claude-plugin/marketplace.json | |
| run: | | |
| python - <<'EOF' | |
| import json, pathlib, sys | |
| p = pathlib.Path(".claude-plugin/marketplace.json") | |
| if not p.exists(): | |
| print("SKIP: no marketplace.json found"); sys.exit(0) | |
| data = json.loads(p.read_text()) | |
| required = ["schema_version", "plugins"] | |
| missing = [k for k in required if k not in data] | |
| if missing: | |
| print(f"FAIL: marketplace.json missing keys: {missing}"); sys.exit(1) | |
| if not isinstance(data["plugins"], list): | |
| print("FAIL: plugins must be a list"); sys.exit(1) | |
| print(f"PASS: marketplace.json valid ({len(data['plugins'])} plugins)") | |
| EOF | |
| # -- Gate 5: Description discipline lint -------------------------------- | |
| # The rules used to live here as a heredoc, so the only way to run them was | |
| # to push. They are `scripts/lint_skill_descriptions.py` now, covered by | |
| # tests/test_lint_skill_descriptions.py and runnable before the push. | |
| - name: Lint skill descriptions | |
| run: python -m scripts.lint_skill_descriptions | |
| # -- Gate 2: No orphan skills (DOI resolution sample) ------------------ | |
| - name: Check derived_from DOIs resolve (sample) | |
| run: | | |
| python - <<'EOF' | |
| import sys, pathlib, yaml, urllib.request, urllib.error | |
| failures = [] | |
| unverified = [] | |
| checked = 0 | |
| skill_files = list(pathlib.Path("collections").rglob("SKILL.md")) | |
| skill_files += list(pathlib.Path("staged-collections").rglob("SKILL.md")) | |
| for skill_md in skill_files[:10]: | |
| text = skill_md.read_text() | |
| if not text.startswith("---"): | |
| continue | |
| try: | |
| fm = yaml.safe_load(text.split("---", 2)[1]) | |
| except Exception: | |
| continue | |
| derived = fm.get("derived_from") or [] | |
| if not derived: | |
| failures.append(f"{skill_md}: no derived_from DOIs") | |
| continue | |
| # Sample: check first DOI only to keep CI fast | |
| entry = derived[0] | |
| doi = entry.get("doi") if isinstance(entry, dict) else entry | |
| url = f"https://doi.org/{doi}" | |
| try: | |
| req = urllib.request.Request( | |
| url, method="HEAD", | |
| headers={"User-Agent": "asb-skill-collections/0.1"} | |
| ) | |
| with urllib.request.urlopen(req, timeout=10): | |
| checked += 1 | |
| except urllib.error.HTTPError as e: | |
| # 404/410 = the DOI genuinely does not resolve (orphan). | |
| # 403/429/401/405/5xx = the resolver answered but the | |
| # publisher bot-blocked HEAD; the DOI exists, not an orphan. | |
| if e.code in (404, 410): | |
| failures.append(f"{skill_md}: DOI {doi} does not resolve (HTTP {e.code})") | |
| else: | |
| unverified.append(f"{skill_md}: DOI {doi} reachable, target blocked HEAD (HTTP {e.code})") | |
| except Exception as e: | |
| # Network/timeout flakiness is inconclusive, not a missing DOI. | |
| unverified.append(f"{skill_md}: DOI {doi} check inconclusive ({e})") | |
| if unverified: | |
| print(f"NOTE: {len(unverified)} DOI(s) reachable but not target-verified (not orphans):") | |
| for u in unverified: | |
| print(f" - {u}") | |
| if failures: | |
| print("FAIL: orphan skill / DOI resolution failures:") | |
| for f in failures: | |
| print(f" - {f}") | |
| sys.exit(1) | |
| print(f"PASS: DOI resolution OK ({checked} verified, {len(unverified)} unverified)") | |
| EOF | |
| # -- Gate 6: EDAM IRI resolution ---------------------------------------- | |
| - name: Check EDAM IRIs | |
| run: | | |
| python - <<'EOF' | |
| import sys, pathlib, yaml | |
| EDAM_BASE = "http://edamontology.org/" | |
| failures = [] | |
| checked = set() | |
| skill_files = list(pathlib.Path("collections").rglob("SKILL.md")) | |
| skill_files += list(pathlib.Path("staged-collections").rglob("SKILL.md")) | |
| for skill_md in skill_files: | |
| text = skill_md.read_text() | |
| if not text.startswith("---"): | |
| continue | |
| try: | |
| fm = yaml.safe_load(text.split("---", 2)[1]) | |
| except Exception: | |
| continue | |
| meta = fm.get("metadata") or {} | |
| iris = [] | |
| if meta.get("edam_operation"): | |
| iris.append(meta["edam_operation"]) | |
| iris.extend(meta.get("edam_topics") or []) | |
| for iri in iris: | |
| if iri in checked: | |
| continue | |
| checked.add(iri) | |
| if not iri.startswith(EDAM_BASE): | |
| failures.append( | |
| f"{skill_md}: EDAM IRI {iri} does not start with {EDAM_BASE}" | |
| ) | |
| if failures: | |
| print("FAIL: EDAM IRI violations:") | |
| for f in failures: | |
| print(f" - {f}") | |
| sys.exit(1) | |
| print(f"PASS: EDAM IRIs OK ({len(checked)} unique IRIs validated)") | |
| EOF | |
| # -- Gate 8: RO-Crate validation ---------------------------------------- | |
| - name: Validate RO-Crate metadata | |
| # warn-only: depends on the `rocrate` package + crate files that may not be | |
| # present in CI; surfaced as a warning, does not block the Validate job. | |
| continue-on-error: true | |
| run: | | |
| python - <<'EOF' | |
| import sys, json, pathlib | |
| crate_files = list(pathlib.Path("collections").rglob("ro-crate-metadata.json")) | |
| crate_files += list(pathlib.Path("staged-collections").rglob("ro-crate-metadata.json")) | |
| failures = [] | |
| for crate_file in crate_files: | |
| try: | |
| data = json.loads(crate_file.read_text()) | |
| if "@context" not in data: | |
| failures.append(f"{crate_file}: missing @context") | |
| if "@graph" not in data: | |
| failures.append(f"{crate_file}: missing @graph") | |
| graph = data.get("@graph", []) | |
| root_ids = {"./", "."} | |
| root_entities = [e for e in graph if e.get("@id") in root_ids] | |
| if not root_entities: | |
| failures.append( | |
| f"{crate_file}: no root dataset entity (id ./ or .)" | |
| ) | |
| except json.JSONDecodeError as e: | |
| failures.append(f"{crate_file}: JSON parse error: {e}") | |
| if failures: | |
| print("FAIL: RO-Crate validation failures:") | |
| for f in failures: | |
| print(f" - {f}") | |
| sys.exit(1) | |
| print(f"PASS: RO-Crate validation OK ({len(crate_files)} crates checked)") | |
| EOF | |
| # -- Gate 9: indicium round-trip (verify-claims) — WARN-ONLY ------------- | |
| # This gate is ADVERTISED but INERT in public CI: the verify-claims CLI ships | |
| # in indicium-adapters, which is not yet on PyPI, so `pip install` above is a | |
| # no-op and the command is always absent. When that happens the step emits a | |
| # GitHub ::warning:: annotation (visible in the PR checks UI) and exits 0 — it | |
| # must never read as an enforced gate that passed. See the `gate9-inert` item | |
| # in agenticsciencebuilder_dev/docs/asbb/HUMAN_REVIEW_GATE.md. | |
| - name: "Gate 9: indicium round-trip (WARN-ONLY until indicium-adapters is published)" | |
| run: | | |
| if ! command -v verify-claims &> /dev/null; then | |
| echo "::warning title=Gate 9 inert (not enforced)::verify-claims CLI is absent (indicium-adapters is not on PyPI), so the indicium round-trip gate did NOT run. This is advisory only — do not read a green Validate as gate 9 passing. Tracked as gate9-inert." | |
| echo "==============================================================" | |
| echo " GATE 9 DID NOT RUN — verify-claims CLI not found." | |
| echo " indicium-adapters is not yet published to PyPI, so this gate" | |
| echo " is advertised but INERT. It is NOT enforced. Non-blocking." | |
| echo "==============================================================" | |
| exit 0 | |
| fi | |
| EXIT=0 | |
| for collection_dir in collections/*/v*; do | |
| [ -d "$collection_dir" ] || continue | |
| echo "Running verify-claims on $collection_dir ..." | |
| verify-claims --collection "$collection_dir" --format json || EXIT=$? | |
| done | |
| for collection_dir in staged-collections/*/v*; do | |
| [ -d "$collection_dir" ] || continue | |
| echo "Running verify-claims on $collection_dir ..." | |
| verify-claims --collection "$collection_dir" --format json || EXIT=$? | |
| done | |
| exit $EXIT | |
| # -- Gate 1: LinkML schema validation ------------------------------------ | |
| - name: LinkML schema validation | |
| # warn-only: requires the `asb-schema` package (sibling repo, not yet on | |
| # PyPI) for asb_skill_bundle.yaml; surfaced as a warning until published. | |
| continue-on-error: true | |
| run: | | |
| pip install asb-schema || echo "INFO: asb-schema not yet on PyPI; skipping LinkML gate" | |
| python - <<'EOF' | |
| import sys, pathlib, subprocess | |
| try: | |
| import linkml_runtime # noqa: F401 | |
| except ImportError: | |
| print("SKIP: linkml_runtime not available") | |
| sys.exit(0) | |
| collection_files = list(pathlib.Path("collections").rglob("collection.yaml")) | |
| collection_files += list( | |
| pathlib.Path("staged-collections").rglob("collection.yaml") | |
| ) | |
| if not collection_files: | |
| print("SKIP: no collection.yaml files found") | |
| sys.exit(0) | |
| failures = [] | |
| for cf in collection_files: | |
| result = subprocess.run( | |
| ["linkml-validate", "--schema", "asb_skill_bundle.yaml", str(cf)], | |
| capture_output=True, text=True | |
| ) | |
| if result.returncode != 0: | |
| failures.append(f"{cf}: {result.stderr.strip()}") | |
| if failures: | |
| print("FAIL: LinkML validation failures:") | |
| for f in failures: | |
| print(f" - {f}") | |
| sys.exit(1) | |
| print(f"PASS: LinkML validation OK ({len(collection_files)} files)") | |
| EOF | |
| # -- Gate: License-tier enforcement ------ | |
| - name: License-tier gate | |
| run: python -m scripts.check_license_tiers collections/metabolomics/v2 | |
| # Every published skill must be able to show where its content came from, | |
| # and the SKILL.md must agree with the index. Ported from the dev line and | |
| # wired here once its producer had been run over the corpus. | |
| - name: Provenance-tier gate | |
| run: python -m scripts.check_provenance_tiers collections/metabolomics/v2 | |
| # The catalogue's own licence fields must agree with tools_index.json, and | |
| # every tool a skill claims to use must exist. Wired once the enrichers had | |
| # been run over the corpus; the last of the gates ported from the dev line. | |
| - name: Tool-catalogue gate | |
| run: python -m scripts.check_tools_index collections/metabolomics/v2 |