Merge #26: consolidate v0 release-prep (OA/licence stack, PII hardeni… #76
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # validate.yml -- runs on every PR | |
| # Enforces CI gates per section 14 of ASB-Skills Release Design Doc v2. | |
| # | |
| # Gates implemented here: | |
| # 1. LinkML schema validation (collection.yaml, tools/*.yaml) | |
| # 2. No orphan skills (DOI resolution sample) | |
| # 5. Description discipline lint (leading phrase, length, no marketing) | |
| # 6. EDAM IRI resolution | |
| # 8. RO-Crate validation (Workflow Run Profile 0.5) | |
| # 9. Indicium round-trip (verify-claims) — WARN-ONLY / inert: the CLI ships in | |
| # indicium-adapters, not yet on PyPI, so this gate is advertised but does not | |
| # run in public CI (emits a ::warning:: annotation; never blocks). See gate9-inert. | |
| # 10. Plugin manifest validation | |
| name: Validate | |
| on: | |
| pull_request: | |
| branches: [main] | |
| push: | |
| branches: [main] | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| jobs: | |
| validate: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Python 3.12 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Install dependencies | |
| run: | | |
| pip install --upgrade pip | |
| pip install pyyaml jsonschema requests pytest | |
| pip install linkml linkml-runtime || echo "WARNING: linkml install failed" | |
| pip install rocrate || echo "WARNING: rocrate install failed" | |
| # indicium-adapters provides the verify-claims console script (Plan 1c) | |
| pip install indicium-adapters || echo "WARNING: indicium-adapters not yet available; gate 9 will warn-only" | |
| # -- Gate 0: Unit tests ------------------------------------------------- | |
| # Until now the suite ran only inside release.yml, so a test could sit broken | |
| # on main for weeks and nothing would say so. The suite is hermetic (pyyaml | |
| # + pytest, no network). | |
| - name: Run test suite | |
| run: python -m pytest tests/ -q | |
| # -- Gate 0b: Index coverage ------------------------------------------- | |
| # A skill absent from skills_index.json / kb_bundle.json ships on disk but | |
| # cannot be found by search, the MCP server, or the docs site. Nothing else | |
| # notices: check_license_tiers.py only inspects skills the index already | |
| # knows about, so the orphan stays invisible and CI stays green. | |
| - name: Every skill is in every index its collection publishes | |
| run: python -m scripts.skill_index | |
| # -- Gate 10: Plugin manifest validates -------------------------------- | |
| - name: Validate .claude-plugin/marketplace.json | |
| run: | | |
| python - <<'EOF' | |
| import json, pathlib, sys | |
| p = pathlib.Path(".claude-plugin/marketplace.json") | |
| if not p.exists(): | |
| print("SKIP: no marketplace.json found"); sys.exit(0) | |
| data = json.loads(p.read_text()) | |
| required = ["schema_version", "plugins"] | |
| missing = [k for k in required if k not in data] | |
| if missing: | |
| print(f"FAIL: marketplace.json missing keys: {missing}"); sys.exit(1) | |
| if not isinstance(data["plugins"], list): | |
| print("FAIL: plugins must be a list"); sys.exit(1) | |
| print(f"PASS: marketplace.json valid ({len(data['plugins'])} plugins)") | |
| EOF | |
| # -- Gate 5: Description discipline lint -------------------------------- | |
| - name: Lint skill descriptions | |
| run: | | |
| python - <<'EOF' | |
| import sys, pathlib, yaml | |
| APPROVED_PREFIXES = ( | |
| "Use when", "Reference for", "Explains", "Decision support for" | |
| ) | |
| MIN_LEN = 50 | |
| MAX_LEN = 300 | |
| MARKETING_TERMS = ["best", "state-of-the-art", "revolutionary", "leading", "superior"] | |
| # Workflow super-skills and `_`-prefixed scaffolds (e.g. `_router`) are | |
| # composites, gated separately by check_workflows — the release authority | |
| # release_gate.py::_iter_skill_md exempts them from the leaf description | |
| # lint, so this gate must too (they legitimately carry longer, composite | |
| # descriptions). Kept consistent by tests/test_ci_gate_consistency.py. | |
| def _is_leaf(p): | |
| return "workflows" not in p.parts and not any(s.startswith("_") for s in p.parts[:-1]) | |
| failures = [] | |
| skill_files = list(pathlib.Path("collections").rglob("SKILL.md")) | |
| skill_files += list(pathlib.Path("staged-collections").rglob("SKILL.md")) | |
| skill_files = [p for p in skill_files if _is_leaf(p)] | |
| for skill_md in skill_files: | |
| text = skill_md.read_text() | |
| if not text.startswith("---"): | |
| continue | |
| try: | |
| parts = text.split("---", 2) | |
| fm = yaml.safe_load(parts[1]) | |
| except Exception: | |
| continue | |
| desc = (fm.get("description") or "").strip() | |
| if not desc: | |
| failures.append(f"{skill_md}: missing description") | |
| continue | |
| if not any(desc.startswith(p) for p in APPROVED_PREFIXES): | |
| failures.append( | |
| f"{skill_md}: description must start with one of {APPROVED_PREFIXES}" | |
| ) | |
| if len(desc) < MIN_LEN: | |
| failures.append( | |
| f"{skill_md}: description too short ({len(desc)} < {MIN_LEN})" | |
| ) | |
| if len(desc) > MAX_LEN: | |
| failures.append( | |
| f"{skill_md}: description too long ({len(desc)} > {MAX_LEN})" | |
| ) | |
| for term in MARKETING_TERMS: | |
| if term.lower() in desc.lower(): | |
| failures.append( | |
| f"{skill_md}: marketing term '{term}' in description" | |
| ) | |
| if failures: | |
| print("FAIL: description discipline violations:") | |
| for f in failures: | |
| print(f" - {f}") | |
| sys.exit(1) | |
| print(f"PASS: description discipline OK ({len(skill_files)} skill files checked)") | |
| EOF | |
| # -- Gate 2: No orphan skills (DOI resolution sample) ------------------ | |
| - name: Check derived_from DOIs resolve (sample) | |
| run: | | |
| python - <<'EOF' | |
| import sys, pathlib, yaml, urllib.request, urllib.error | |
| failures = [] | |
| unverified = [] | |
| checked = 0 | |
| skill_files = list(pathlib.Path("collections").rglob("SKILL.md")) | |
| skill_files += list(pathlib.Path("staged-collections").rglob("SKILL.md")) | |
| for skill_md in skill_files[:10]: | |
| text = skill_md.read_text() | |
| if not text.startswith("---"): | |
| continue | |
| try: | |
| fm = yaml.safe_load(text.split("---", 2)[1]) | |
| except Exception: | |
| continue | |
| derived = fm.get("derived_from") or [] | |
| if not derived: | |
| failures.append(f"{skill_md}: no derived_from DOIs") | |
| continue | |
| # Sample: check first DOI only to keep CI fast | |
| entry = derived[0] | |
| doi = entry.get("doi") if isinstance(entry, dict) else entry | |
| url = f"https://doi.org/{doi}" | |
| try: | |
| req = urllib.request.Request( | |
| url, method="HEAD", | |
| headers={"User-Agent": "asb-skill-collections/0.1"} | |
| ) | |
| with urllib.request.urlopen(req, timeout=10): | |
| checked += 1 | |
| except urllib.error.HTTPError as e: | |
| # 404/410 = the DOI genuinely does not resolve (orphan). | |
| # 403/429/401/405/5xx = the resolver answered but the | |
| # publisher bot-blocked HEAD; the DOI exists, not an orphan. | |
| if e.code in (404, 410): | |
| failures.append(f"{skill_md}: DOI {doi} does not resolve (HTTP {e.code})") | |
| else: | |
| unverified.append(f"{skill_md}: DOI {doi} reachable, target blocked HEAD (HTTP {e.code})") | |
| except Exception as e: | |
| # Network/timeout flakiness is inconclusive, not a missing DOI. | |
| unverified.append(f"{skill_md}: DOI {doi} check inconclusive ({e})") | |
| if unverified: | |
| print(f"NOTE: {len(unverified)} DOI(s) reachable but not target-verified (not orphans):") | |
| for u in unverified: | |
| print(f" - {u}") | |
| if failures: | |
| print("FAIL: orphan skill / DOI resolution failures:") | |
| for f in failures: | |
| print(f" - {f}") | |
| sys.exit(1) | |
| print(f"PASS: DOI resolution OK ({checked} verified, {len(unverified)} unverified)") | |
| EOF | |
| # -- Gate 6: EDAM IRI resolution ---------------------------------------- | |
| - name: Check EDAM IRIs | |
| run: | | |
| python - <<'EOF' | |
| import sys, pathlib, yaml | |
| EDAM_BASE = "http://edamontology.org/" | |
| failures = [] | |
| checked = set() | |
| skill_files = list(pathlib.Path("collections").rglob("SKILL.md")) | |
| skill_files += list(pathlib.Path("staged-collections").rglob("SKILL.md")) | |
| for skill_md in skill_files: | |
| text = skill_md.read_text() | |
| if not text.startswith("---"): | |
| continue | |
| try: | |
| fm = yaml.safe_load(text.split("---", 2)[1]) | |
| except Exception: | |
| continue | |
| meta = fm.get("metadata") or {} | |
| iris = [] | |
| if meta.get("edam_operation"): | |
| iris.append(meta["edam_operation"]) | |
| iris.extend(meta.get("edam_topics") or []) | |
| for iri in iris: | |
| if iri in checked: | |
| continue | |
| checked.add(iri) | |
| if not iri.startswith(EDAM_BASE): | |
| failures.append( | |
| f"{skill_md}: EDAM IRI {iri} does not start with {EDAM_BASE}" | |
| ) | |
| if failures: | |
| print("FAIL: EDAM IRI violations:") | |
| for f in failures: | |
| print(f" - {f}") | |
| sys.exit(1) | |
| print(f"PASS: EDAM IRIs OK ({len(checked)} unique IRIs validated)") | |
| EOF | |
| # -- Gate 8: RO-Crate validation ---------------------------------------- | |
| - name: Validate RO-Crate metadata | |
| # warn-only: depends on the `rocrate` package + crate files that may not be | |
| # present in CI; surfaced as a warning, does not block the Validate job. | |
| continue-on-error: true | |
| run: | | |
| python - <<'EOF' | |
| import sys, json, pathlib | |
| crate_files = list(pathlib.Path("collections").rglob("ro-crate-metadata.json")) | |
| crate_files += list(pathlib.Path("staged-collections").rglob("ro-crate-metadata.json")) | |
| failures = [] | |
| for crate_file in crate_files: | |
| try: | |
| data = json.loads(crate_file.read_text()) | |
| if "@context" not in data: | |
| failures.append(f"{crate_file}: missing @context") | |
| if "@graph" not in data: | |
| failures.append(f"{crate_file}: missing @graph") | |
| graph = data.get("@graph", []) | |
| root_ids = {"./", "."} | |
| root_entities = [e for e in graph if e.get("@id") in root_ids] | |
| if not root_entities: | |
| failures.append( | |
| f"{crate_file}: no root dataset entity (id ./ or .)" | |
| ) | |
| except json.JSONDecodeError as e: | |
| failures.append(f"{crate_file}: JSON parse error: {e}") | |
| if failures: | |
| print("FAIL: RO-Crate validation failures:") | |
| for f in failures: | |
| print(f" - {f}") | |
| sys.exit(1) | |
| print(f"PASS: RO-Crate validation OK ({len(crate_files)} crates checked)") | |
| EOF | |
| # -- Gate 9: indicium round-trip (verify-claims) — WARN-ONLY ------------- | |
| # This gate is ADVERTISED but INERT in public CI: the verify-claims CLI ships | |
| # in indicium-adapters, which is not yet on PyPI, so `pip install` above is a | |
| # no-op and the command is always absent. When that happens the step emits a | |
| # GitHub ::warning:: annotation (visible in the PR checks UI) and exits 0 — it | |
| # must never read as an enforced gate that passed. See the `gate9-inert` item | |
| # in agenticsciencebuilder_dev/docs/asbb/HUMAN_REVIEW_GATE.md. | |
| - name: "Gate 9: indicium round-trip (WARN-ONLY until indicium-adapters is published)" | |
| run: | | |
| if ! command -v verify-claims &> /dev/null; then | |
| echo "::warning title=Gate 9 inert (not enforced)::verify-claims CLI is absent (indicium-adapters is not on PyPI), so the indicium round-trip gate did NOT run. This is advisory only — do not read a green Validate as gate 9 passing. Tracked as gate9-inert." | |
| echo "==============================================================" | |
| echo " GATE 9 DID NOT RUN — verify-claims CLI not found." | |
| echo " indicium-adapters is not yet published to PyPI, so this gate" | |
| echo " is advertised but INERT. It is NOT enforced. Non-blocking." | |
| echo "==============================================================" | |
| exit 0 | |
| fi | |
| EXIT=0 | |
| for collection_dir in collections/*/v*; do | |
| [ -d "$collection_dir" ] || continue | |
| echo "Running verify-claims on $collection_dir ..." | |
| verify-claims --collection "$collection_dir" --format json || EXIT=$? | |
| done | |
| for collection_dir in staged-collections/*/v*; do | |
| [ -d "$collection_dir" ] || continue | |
| echo "Running verify-claims on $collection_dir ..." | |
| verify-claims --collection "$collection_dir" --format json || EXIT=$? | |
| done | |
| exit $EXIT | |
| # -- Gate 1: LinkML schema validation ------------------------------------ | |
| - name: LinkML schema validation | |
| # warn-only: requires the `asb-schema` package (sibling repo, not yet on | |
| # PyPI) for asb_skill_bundle.yaml; surfaced as a warning until published. | |
| continue-on-error: true | |
| run: | | |
| pip install asb-schema || echo "INFO: asb-schema not yet on PyPI; skipping LinkML gate" | |
| python - <<'EOF' | |
| import sys, pathlib, subprocess | |
| try: | |
| import linkml_runtime # noqa: F401 | |
| except ImportError: | |
| print("SKIP: linkml_runtime not available") | |
| sys.exit(0) | |
| collection_files = list(pathlib.Path("collections").rglob("collection.yaml")) | |
| collection_files += list( | |
| pathlib.Path("staged-collections").rglob("collection.yaml") | |
| ) | |
| if not collection_files: | |
| print("SKIP: no collection.yaml files found") | |
| sys.exit(0) | |
| failures = [] | |
| for cf in collection_files: | |
| result = subprocess.run( | |
| ["linkml-validate", "--schema", "asb_skill_bundle.yaml", str(cf)], | |
| capture_output=True, text=True | |
| ) | |
| if result.returncode != 0: | |
| failures.append(f"{cf}: {result.stderr.strip()}") | |
| if failures: | |
| print("FAIL: LinkML validation failures:") | |
| for f in failures: | |
| print(f" - {f}") | |
| sys.exit(1) | |
| print(f"PASS: LinkML validation OK ({len(collection_files)} files)") | |
| EOF | |
| # -- Gate: License-tier enforcement ------ | |
| - name: License-tier gate | |
| run: python -m scripts.check_license_tiers collections/metabolomics/v2 |