Add search-xrefs command with OLS4 and MyChem.info providers

CLAUDE.md

@@ -41,6 +41,10 @@ uv run babel-explorer ids MONDO:0004979
 # Test concordance changes with NodeNorm
 uv run babel-explorer test-concord MONDO:0004979 HP:0000001
 
+# Search external providers (OLS4, MyChem.info) for candidate xrefs
+# and diff against Babel's Concord — finds xrefs worth importing into Babel.
+uv run babel-explorer search-xrefs CHEBI:31941 --ignore-known
+
 # Use custom Babel server or local directory
 uv run babel-explorer xrefs MONDO:0004979 --local-dir data/2025nov19 --babel-url https://stars.renci.org:443/var/babel/2025nov19/
 ```
@@ -91,7 +95,18 @@ uv run ruff format
 
 4. **CLI** (`src/babel_explorer/cli.py`):
    - Click-based command-line interface
-   - Three main commands: `xrefs`, `ids`, `test-concord`
+   - Four main commands: `xrefs`, `ids`, `test-concord`, `search-xrefs`
+   - Shared `@format_option` decorator adds `--format [console|json|tsv|csv]` and `--json-indent` to every command
+
+5. **XRef Providers** (`src/babel_explorer/core/providers/`):
+   - Pluggable external mapping sources used by `search-xrefs` (currently OLS4 and MyChem.info)
+   - `XRefProvider` `typing.Protocol` + module-level `PROVIDERS` registry dict (see `providers/__init__.py`)
+   - Each provider mirrors the `NodeNorm` pattern: `requests`, `@functools.lru_cache(maxsize=None)` on `fetch()`, empty-URL skip, frozen-dataclass results
+   - **Adding a new provider**: write a class with `name: str` and `fetch(curie) -> list[CandidateXRef]`, then append a factory to `PROVIDERS` in `providers/__init__.py`. No CLI changes needed — the registry drives `--providers` selection.
+
+6. **curie_utils** (`src/babel_explorer/core/curie_utils.py`):
+   - Shared CURIE↔IRI helpers (`split_curie`, `to_iri`, `from_iri`) and a `DEFAULT_PREFIX_MAP` of common Translator prefixes
+   - Used by providers that query external services by IRI (e.g. OLS4)
 
 ### Data Flow
 
@@ -123,6 +138,10 @@ Tests live in `tests/` and are split into fast **unit tests** (mocked, no networ
 | `tests/test_babel_xrefs.py` | 23 | 20 | 3 | 46 |
 | `tests/test_nodenorm.py` | 20 | 13 | 0 | 33 |
 | `tests/test_cli.py` | 24 | 0 | 0 | 24 |
+| `tests/test_curie_utils.py` | 26 | 0 | 0 | 26 |
+| `tests/test_providers_ols.py` | 17 | 2 | 0 | 19 |
+| `tests/test_providers_mychem.py` | 21 | 3 | 0 | 24 |
+| `tests/test_search_xrefs_cli.py` | 15 | 0 | 0 | 15 |
 
 ### Test Infrastructure
 
@@ -136,6 +155,7 @@ Tests live in `tests/` and are split into fast **unit tests** (mocked, no networ
 - **`CrossReference`** — Frozen dataclass for Concord.parquet rows (filename, subj, pred, obj)
 - **`LabeledCrossReference`** — Extends CrossReference with labels and biolink types from NodeNorm
 - **`IdentifierRecord`** — Frozen dataclass for Identifiers.parquet rows (curie + dynamic extra fields). Returned by `BabelXRefs.get_curie_ids()`.
+- **`CandidateXRef`** — Frozen dataclass for an xref candidate from an external provider (query_curie, target_curie, provider, predicate, confidence, evidence, in_babel, target_label, target_biolink_type). Returned by `XRefProvider.fetch()`.
 
 ## Important Notes
 

src/babel_explorer/cli.py

@@ -1,11 +1,18 @@
 # Command line interface for babel-explorer
+import dataclasses
 import click
 import logging
 from babel_explorer.core.downloader import BabelDownloader
 from babel_explorer.core.babel_xrefs import BabelXRefs
 from babel_explorer.core.nodenorm import NodeNorm
 from babel_explorer.core.babel_xrefs import LabeledCrossReference
-from babel_explorer.formatting import write_records, _record_to_dict, make_console, hl_curie
+from babel_explorer.core.providers import PROVIDERS
+from babel_explorer.formatting import (
+    write_records,
+    _record_to_dict,
+    make_console,
+    hl_curie,
+)
 from rich.markup import escape
 
 
@@ -181,7 +188,14 @@ def xrefs(
     "'never' disables re-checking and always uses cached files; '0' forces a re-check every time.",
 )
 @format_option
-def ids(curies: list[str], babel_url: str, local_dir: str, check_download: str, fmt: str, json_indent: int):
+def ids(
+    curies: list[str],
+    babel_url: str,
+    local_dir: str,
+    check_download: str,
+    fmt: str,
+    json_indent: int,
+):
     """
     Fetches and prints the ID records for the given CURIEs, along with Biolink type if provided.
 
@@ -248,5 +262,163 @@ def test_concord(curies, nodenorm_url, fmt, json_indent):
         write_records(rows, fmt=fmt, indent=json_indent)
 
 
+@cli.command("search-xrefs")
+@click.argument("curies", type=str, required=True, nargs=-1)
+@click.option(
+    "--providers",
+    "providers_arg",
+    type=str,
+    default="",
+    help="Comma-separated list of provider names to query "
+    "(default: all registered, e.g. 'ols,mychem').",
+)
+@click.option(
+    "--ols-url",
+    type=str,
+    default="https://www.ebi.ac.uk/ols4",
+    show_default=True,
+    help="Base URL of the OLS4 server.",
+)
+@click.option(
+    "--mychem-url",
+    type=str,
+    default="https://mychem.info/v1",
+    show_default=True,
+    help="Base URL of the MyChem.info API.",
+)
+@click.option(
+    "--external-timeout",
+    type=int,
+    default=30,
+    show_default=True,
+    help="HTTP request timeout (seconds) for external providers.",
+)
+@click.option(
+    "--ignore-known",
+    is_flag=True,
+    help="Drop candidates that already exist as a Concord edge in Babel.",
+)
+@click.option("--labels", is_flag=True, help="Resolve target labels via NodeNorm.")
+@click.option(
+    "--local-dir",
+    type=str,
+    default="data/2025nov19",
+    help="Local location to save Babel download files to",
+)
+@click.option(
+    "--babel-url",
+    type=str,
+    default="https://stars.renci.org:443/var/babel/2025nov19/",
+    help="Base URL of the Babel server",
+)
+@click.option(
+    "--nodenorm-url",
+    type=str,
+    default="https://nodenormalization-sri.renci.org/",
+    help="NodeNorm base URL (used for --labels and for MyChem InChIKey resolution).",
+)
+@click.option(
+    "--check-download",
+    type=str,
+    default="3h",
+    show_default=True,
+    help="How often to re-check downloads (e.g. '3h', '30m', '1d', '0', 'never').",
+)
+@format_option
+def search_xrefs(
+    curies: tuple[str, ...],
+    providers_arg: str,
+    ols_url: str,
+    mychem_url: str,
+    external_timeout: int,
+    ignore_known: bool,
+    labels: bool,
+    local_dir: str,
+    babel_url: str,
+    nodenorm_url: str,
+    check_download: str,
+    fmt: str,
+    json_indent: int,
+):
+    """Search external mapping providers for cross-references and diff against Babel.
+
+    For each CURIE, query the selected providers (OLS4, MyChem.info, ...), then
+    annotate each candidate with whether it already exists as an edge in Babel's
+    local Concord.parquet. Use ``--ignore-known`` to filter the output to only
+    candidates Babel does not yet know about.
+    """
+    logging.basicConfig(level=logging.INFO)
+
+    selected = [n.strip() for n in providers_arg.split(",") if n.strip()] or list(
+        PROVIDERS.keys()
+    )
+    for name in selected:
+        if name not in PROVIDERS:
+            raise click.BadParameter(
+                f"Unknown provider {name!r}. Known: {', '.join(PROVIDERS)}."
+            )
+
+    freshness = parse_duration(check_download)
+    nodenorm = NodeNorm(nodenorm_url)
+    bxref = BabelXRefs(
+        BabelDownloader(babel_url, local_path=local_dir, freshness_seconds=freshness),
+        nodenorm,
+    )
+
+    provider_kwargs = {
+        "ols_url": ols_url,
+        "mychem_url": mychem_url,
+        "nodenorm": nodenorm,
+        "timeout": external_timeout,
+    }
+    providers = [PROVIDERS[name](**provider_kwargs) for name in selected]
+
+    candidates = []
+    for curie in curies:
+        known_pairs = {x.curies for x in bxref.get_curie_xref(curie)}
+        for provider in providers:
+            for cand in provider.fetch(curie):
+                in_babel = (
+                    frozenset({cand.query_curie, cand.target_curie}) in known_pairs
+                )
+                if ignore_known and in_babel:
+                    continue
+                target_label = ""
+                target_biolink_type: tuple[str, ...] = ()
+                if labels:
+                    ident = nodenorm.get_identifier(cand.target_curie)
+                    target_label = ident.label
+                    target_biolink_type = ident.biolink_type
+                candidates.append(
+                    dataclasses.replace(
+                        cand,
+                        in_babel=in_babel,
+                        target_label=target_label,
+                        target_biolink_type=target_biolink_type,
+                    )
+                )
+
+    candidates.sort()
+
+    if fmt == "console":
+        console = make_console()
+        query_set = set(curies)
+        for c in candidates:
+            query_str = hl_curie(c.query_curie, c.query_curie in query_set)
+            target_str = hl_curie(c.target_curie, c.target_curie in query_set)
+            if c.target_label:
+                target_str += f" ({escape(c.target_label)})"
+            marker = (
+                "[bold green]NEW[/bold green]" if not c.in_babel else "[dim]known[/dim]"
+            )
+            console.print(
+                f"{query_str}  [dim]→[/dim]  {target_str}  "
+                f"[dim]\\[{escape(c.provider)}][/dim]  "
+                f"[dim]{escape(c.predicate)}[/dim]  {marker}"
+            )
+    else:
+        write_records(candidates, fmt=fmt, indent=json_indent)
+
+
 if __name__ == "__main__":
     cli()

src/babel_explorer/core/curie_utils.py

@@ -0,0 +1,68 @@
+"""CURIE/IRI helpers shared by external xref providers.
+
+A *CURIE* is a compact identifier of the form ``PREFIX:LOCAL_ID`` (e.g.
+``CHEBI:31941``). An *IRI* is the expanded form
+(``http://purl.obolibrary.org/obo/CHEBI_31941``). Providers like OLS4 query by
+IRI, so each provider client uses the helpers here to translate.
+
+The default prefix map covers the subset of identifier types currently used
+by babel-explorer. Extend it as new providers come online.
+"""
+
+from typing import Mapping
+
+
+DEFAULT_PREFIX_MAP: dict[str, str] = {
+    "CHEBI": "http://purl.obolibrary.org/obo/CHEBI_",
+    "MONDO": "http://purl.obolibrary.org/obo/MONDO_",
+    "HP": "http://purl.obolibrary.org/obo/HP_",
+    "PUBCHEM.COMPOUND": "http://identifiers.org/pubchem.compound/",
+    "UMLS": "http://linkedlifedata.com/resource/umls/id/",
+    "CHEMBL.COMPOUND": "http://identifiers.org/chembl.compound/",
+    "DRUGBANK": "http://identifiers.org/drugbank/",
+    "KEGG.COMPOUND": "http://identifiers.org/kegg.compound/",
+    "UNII": "http://fdasis.nlm.nih.gov/srs/unii/",
+    "INCHIKEY": "http://identifiers.org/inchikey/",
+}
+
+
+def split_curie(curie: str) -> tuple[str, str]:
+    """Split ``curie`` into ``(prefix, local_id)`` on the first colon.
+
+    :raises ValueError: If ``curie`` is empty or has no colon.
+    """
+    if not curie or ":" not in curie:
+        raise ValueError(f"Not a valid CURIE: {curie!r}")
+    prefix, local_id = curie.split(":", 1)
+    if not prefix or not local_id:
+        raise ValueError(f"Not a valid CURIE: {curie!r}")
+    return prefix, local_id
+
+
+def to_iri(curie: str, prefix_map: Mapping[str, str] = DEFAULT_PREFIX_MAP) -> str:
+    """Expand ``curie`` to an IRI using ``prefix_map``.
+
+    :raises KeyError: If the CURIE prefix is not in ``prefix_map``.
+    """
+    prefix, local_id = split_curie(curie)
+    try:
+        iri_prefix = prefix_map[prefix]
+    except KeyError:
+        raise KeyError(f"Unknown CURIE prefix {prefix!r} (not in prefix map)")
+    return iri_prefix + local_id
+
+
+def from_iri(iri: str, prefix_map: Mapping[str, str] = DEFAULT_PREFIX_MAP) -> str:
+    """Contract ``iri`` back to a CURIE using ``prefix_map``.
+
+    Tries the longest matching prefix first so e.g. ``CHEMBL.COMPOUND`` wins
+    over a hypothetical bare ``CHEMBL`` entry.
+
+    :raises ValueError: If no prefix in ``prefix_map`` matches the IRI.
+    """
+    for prefix, iri_prefix in sorted(
+        prefix_map.items(), key=lambda kv: len(kv[1]), reverse=True
+    ):
+        if iri.startswith(iri_prefix):
+            return f"{prefix}:{iri[len(iri_prefix) :]}"
+    raise ValueError(f"No CURIE prefix in map matches IRI {iri!r}")

src/babel_explorer/core/providers/__init__.py

@@ -0,0 +1,75 @@
+"""Pluggable cross-reference providers for `babel-explorer search-xrefs`.
+
+Each provider queries an external mapping source (OLS4, MyChem.info, ...) for
+candidate cross-references and returns a list of ``CandidateXRef`` records that
+the CLI then diffs against Babel's local ``Concord.parquet``.
+
+Adding a new provider:
+    1. Create a class in this package whose interface matches ``XRefProvider``
+       (a ``name`` attribute and a ``fetch(curie) -> list[CandidateXRef]`` method).
+    2. Register a factory in ``PROVIDERS`` below.
+"""
+
+import dataclasses
+from typing import Callable, Protocol, runtime_checkable
+
+
+@dataclasses.dataclass(frozen=True)
+class CandidateXRef:
+    """A candidate cross-reference proposed by an external mapping source."""
+
+    query_curie: str
+    target_curie: str
+    provider: str
+    predicate: str
+    confidence: float | None
+    evidence: str
+    in_babel: bool
+    target_label: str = ""
+    target_biolink_type: tuple[str, ...] = ()
+
+    def __lt__(self, other):
+        return (self.query_curie, self.provider, self.target_curie) < (
+            other.query_curie,
+            other.provider,
+            other.target_curie,
+        )
+
+
+@runtime_checkable
+class XRefProvider(Protocol):
+    """Interface that every cross-reference provider implements."""
+
+    name: str
+
+    def fetch(self, curie: str) -> list[CandidateXRef]: ...
+
+
+# Provider registry — populated after provider classes are imported.
+PROVIDERS: dict[str, Callable[..., XRefProvider]] = {}
+
+
+# Imports placed after type definitions so provider modules can safely
+# `from babel_explorer.core.providers import CandidateXRef`.
+from babel_explorer.core.providers.ols import OLS4Provider  # noqa: E402
+
+
+def _build_ols(**kw) -> "OLS4Provider":
+    return OLS4Provider(kw.get("ols_url", ""), timeout=kw.get("timeout", 30))
+
+
+PROVIDERS["ols"] = _build_ols
+
+
+from babel_explorer.core.providers.mychem import MyChemProvider  # noqa: E402
+
+
+def _build_mychem(**kw) -> "MyChemProvider":
+    return MyChemProvider(
+        kw.get("mychem_url", ""),
+        nodenorm=kw.get("nodenorm"),
+        timeout=kw.get("timeout", 30),
+    )
+
+
+PROVIDERS["mychem"] = _build_mychem