From 198c1406dc1c7ff837e1cbe85dd55158b3879b0e Mon Sep 17 00:00:00 2001 From: basil-chen <192173459+rastagan-git@users.noreply.github.com> Date: Sun, 19 Jul 2026 22:54:28 +0800 Subject: [PATCH] Make PubChem CAS resolution conservative --- .agents/skills/curate-aroma-data/SKILL.md | 7 +- .../references/output-schema.md | 22 +++- README-CN.md | 5 +- README.md | 5 +- aromanexus/__init__.py | 2 +- aromanexus/cli.py | 14 ++ aromanexus/workflows.py | 85 +++++++++++- pyproject.toml | 2 +- tests/test_cli.py | 36 +++++ tests/test_workflows.py | 124 ++++++++++++++++++ 10 files changed, 295 insertions(+), 7 deletions(-) diff --git a/.agents/skills/curate-aroma-data/SKILL.md b/.agents/skills/curate-aroma-data/SKILL.md index f5fd298..99650f1 100644 --- a/.agents/skills/curate-aroma-data/SKILL.md +++ b/.agents/skills/curate-aroma-data/SKILL.md @@ -12,6 +12,7 @@ Build traceable compound tables through the repository's `aromanexus` CLI. Keep 1. Inspect the input without modifying it. - Run `python .agents/skills/curate-aroma-data/scripts/inspect_workbook.py INPUT` from the repository root. - Confirm the row count, exact column names, identifier quality, duplicates, and formula-like cells. + - Identify section labels, headers, totals, and other structural rows before provider calls. Define an explicit dataset-specific skip rule; do not assume that text such as `C6` is globally non-chemical. 2. Choose the smallest provider set that supplies the requested fields. - Read [references/provider-matrix.md](references/provider-matrix.md) before any network or browser operation. - Prefer PubChem for canonical identity and sourced odor annotations. @@ -19,10 +20,11 @@ Build traceable compound tables through the repository's `aromanexus` CLI. Keep - Use Pyrfume only for explicitly selected archives after reviewing each manifest note. - Use M2OR only when receptor bioassay evidence is relevant; label species and assay scope. 3. Preview the operation. - - State the input, new output path, selected provider, expected columns, approximate request count, cache behavior, and material access caveats. + - State the input, new output path, selected provider, expected columns, skip patterns, approximate request count, cache behavior, and material access caveats. - Write a sibling output by default. Do not pass `--force` or overwrite the input unless the user explicitly requests that exact replacement. 4. Run one focused command. - Identity and odor metadata: `aromanexus pubchem INPUT --identifier-column "CAS Number"` + - Name lookup with dataset-specific structural rows: `aromanexus pubchem INPUT --identifier-column "Name" --skip-pattern '^C\d+$'` - Retention indices: `aromanexus nist-ri INPUT --cas-column "CAS Number" --calculated-ri-column "Calculated RI"` - Names to CAS: `aromanexus resolve-cas INPUT --name-column "Name"` - Curated descriptors: `aromanexus pyrfume INPUT --archives aromadb,superscent` @@ -31,6 +33,8 @@ Build traceable compound tables through the repository's `aromanexus` CLI. Keep 5. Verify the result. - Re-run the inspection script on the output. - Confirm identical row order and row count, expected new fields, typed status counts, source URL, retrieval time, version, and license/access fields. + - Treat `PubChem Status` as provider execution state, not proof of a uniquely resolved CAS. Check `PubChem CAS Resolution`, candidate count, and `Resolved CAS` separately. + - Leave `multiple` and `missing` CAS resolutions unresolved; retain all candidates and route only the affected rows to a targeted fallback source or manual review. - Treat `http_error`, `network_error`, `parse_error`, `missing_data`, `data_error`, `partial`, `blocked`, and `skipped` separately from `not_found`. - Consult [references/output-schema.md](references/output-schema.md) when reconciling columns or statuses. 6. Report the output path, provider versions, status counts, partial failures, and any source terms the user must still review. @@ -43,3 +47,4 @@ Build traceable compound tables through the repository's `aromanexus` CLI. Keep - Do not describe the toolkit as AI-powered. Say that structured exports can support downstream statistics, cheminformatics, or machine-learning experiments. - Do not bundle or republish downloaded Pyrfume or M2OR data in the repository. - Preserve remote text as literal spreadsheet cells to prevent formula execution. +- Never select the first PubChem CAS candidate merely because the provider status is `ok`. diff --git a/.agents/skills/curate-aroma-data/references/output-schema.md b/.agents/skills/curate-aroma-data/references/output-schema.md index 84b83cf..133b7f0 100644 --- a/.agents/skills/curate-aroma-data/references/output-schema.md +++ b/.agents/skills/curate-aroma-data/references/output-schema.md @@ -7,6 +7,25 @@ - Keep CAS as an external identifier and validate its checksum before exact-match providers. - Never silently pick an ambiguous name result. +## PubChem CAS resolution + +Keep provider execution and CAS curation in separate columns: + +- `PubChem Status`: whether the provider request and record parsing succeeded. +- `PubChem CAS Numbers`: every checksum-valid candidate retained for review. +- `PubChem CAS Candidate Count`: the number of distinct valid candidates. +- `PubChem CAS Resolution`: the conservative resolution decision. +- `Resolved CAS`: populate only for `query_confirmed` or `unique`. + +Interpret `PubChem CAS Resolution` as follows: + +- `query_confirmed`: the input was a checksum-valid CAS and PubChem resolved its record. +- `unique`: a successful name lookup returned exactly one valid CAS candidate. +- `multiple`: more than one valid candidate remains; keep `Resolved CAS` empty. +- `missing`: a successful record returned no valid CAS candidate; keep `Resolved CAS` empty. +- `not_evaluated`: provider failure prevented a defensible CAS decision. +- `skipped`: an explicit user-supplied rule excluded the row before lookup. + ## Status values - `ok`: a provider returned a parsed record. @@ -20,7 +39,7 @@ - `data_error`: every selected archive failed to load; inspect the per-archive diagnostic message. - `partial`: at least one selected source succeeded and at least one failed, or a provider returned usable data with warnings. - `blocked`: access policy or missing permission prevented the request. -- `skipped`: a user deliberately skipped an interactive record. +- `skipped`: a user deliberately excluded a record through an interactive choice or explicit skip rule. Do not merge an access, transport, HTTP, snapshot, parse, or partial failure into `not_found`. @@ -38,3 +57,4 @@ After every run, confirm: 4. Every processed row has a typed status. 5. Remote strings beginning with `=`, `+`, `-`, or `@` are stored as literal text. 6. Partial outputs are reported separately if a run is interrupted. +7. `multiple`, `missing`, `not_evaluated`, and `skipped` PubChem CAS resolutions never contain an automatic `Resolved CAS`. diff --git a/README-CN.md b/README-CN.md index cd4e1e4..2e3020b 100644 --- a/README-CN.md +++ b/README-CN.md @@ -66,6 +66,9 @@ CLI 支持 `.xlsx`、`.csv` 和 `.tsv`。列名均可修改；下面使用默认 # 规范化身份、理化性质、同义词、CAS 标识符与带来源的气味文本 aromanexus pubchem compounds.xlsx --identifier-column "CAS Number" +# 在名称查询前跳过当前数据集中的结构标签 +aromanexus pubchem compounds.xlsx --identifier-column "Name" --skip-pattern '^C\d+$' + # 在 NIST 中寻找最接近实验计算值的保留指数 aromanexus nist-ri data.xlsx \ --cas-column "CAS Number" \ @@ -112,7 +115,7 @@ aromanexus --cache-dir .cache/aromanexus --timeout 30 pubchem compounds.xlsx 所有表格命令默认在输入文件旁生成新文件，保留原有行序和列，再添加数据源字段。例如，PubChem 会将 `compounds.xlsx` 输出为 `compounds_pubchem.xlsx`。 -默认来源记录包括数据源状态、来源 URL、获取时间、是否命中缓存、固定版本、许可 URL 与诊断信息。只有在确实需要旧版形状时才使用 `--no-provenance`。 +默认来源记录包括数据源状态、来源 URL、获取时间、是否命中缓存、固定版本、许可 URL 与诊断信息。PubChem 会单独报告 CAS 解析状态；仅当输入 CAS 得到确认，或只剩一个校验有效的候选时，才填入 `Resolved CAS`。多个或缺失候选会保持未解析。只有在确实需要旧版形状时才使用 `--no-provenance`。 ```bash # 明确指定输出位置 diff --git a/README.md b/README.md index 1f4f7a7..7ab462b 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,9 @@ The CLI accepts `.xlsx`, `.csv`, and `.tsv` tables. Column names are configurabl # Canonical identity, properties, synonyms, CAS identifiers, and sourced odor text aromanexus pubchem compounds.xlsx --identifier-column "CAS Number" +# Skip dataset-specific structural labels before a name lookup +aromanexus pubchem compounds.xlsx --identifier-column "Name" --skip-pattern '^C\d+$' + # Closest NIST RI to an experimentally calculated RI aromanexus nist-ri data.xlsx \ --cas-column "CAS Number" \ @@ -112,7 +115,7 @@ aromanexus --cache-dir .cache/aromanexus --timeout 30 pubchem compounds.xlsx Every table command writes a sibling file by default, keeps the original row order and columns, and adds provider fields. For example, `compounds.xlsx` becomes `compounds_pubchem.xlsx` after a PubChem run. -By default, provenance columns include provider status, source URL, retrieval timestamp, cache hit, pinned version, license URL, and a diagnostic message. Use `--no-provenance` only for legacy-shaped output. +By default, provenance columns include provider status, source URL, retrieval timestamp, cache hit, pinned version, license URL, and a diagnostic message. PubChem reports CAS resolution separately and populates `Resolved CAS` only when the input is a confirmed CAS or exactly one checksum-valid candidate remains. Multiple or missing candidates stay unresolved. Use `--no-provenance` only for legacy-shaped output. ```bash # Choose an output explicitly diff --git a/aromanexus/__init__.py b/aromanexus/__init__.py index daaed9b..abd02c2 100644 --- a/aromanexus/__init__.py +++ b/aromanexus/__init__.py @@ -3,4 +3,4 @@ from aromanexus.models import LookupResult __all__ = ["LookupResult"] -__version__ = "0.3.0" +__version__ = "0.3.1" diff --git a/aromanexus/cli.py b/aromanexus/cli.py index 52566ff..90208b4 100644 --- a/aromanexus/cli.py +++ b/aromanexus/cli.py @@ -99,6 +99,18 @@ def build_parser() -> argparse.ArgumentParser: ) _add_table_arguments(pubchem) pubchem.add_argument("--identifier-column", default="CAS Number") + pubchem.add_argument( + "--skip-pattern", + action="append", + default=[], + metavar="REGEX", + help="Skip identifiers matching this regex before lookup; repeat for multiple patterns", + ) + pubchem.add_argument( + "--resolved-cas-column", + default="Resolved CAS", + help="Output column populated only for query-confirmed or unique CAS resolution", + ) pubchem.add_argument("--no-odor", action="store_true", help="Skip PUG-View odor annotations") pubchem.set_defaults(handler=_handle_pubchem) @@ -199,6 +211,8 @@ def _handle_pubchem(args: argparse.Namespace) -> RunSummary: args.input, client, identifier_column=args.identifier_column, + resolved_cas_column=args.resolved_cas_column, + skip_patterns=args.skip_pattern, include_odor=not args.no_odor, **_common_kwargs(args), ) diff --git a/aromanexus/workflows.py b/aromanexus/workflows.py index 3f51e6f..e559440 100644 --- a/aromanexus/workflows.py +++ b/aromanexus/workflows.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import re import time from collections import Counter from collections.abc import Callable, Iterable @@ -18,6 +19,7 @@ require_columns, write_table, ) +from aromanexus.identifiers import clean_text, is_valid_cas, normalize_cas from aromanexus.models import LookupResult from aromanexus.sources.chemicalbook import ManualVerificationRequired @@ -217,25 +219,99 @@ def run_resolve_cas( } +def _compile_skip_patterns(patterns: Iterable[str] | str | None) -> tuple[re.Pattern[str], ...]: + raw_patterns = (patterns,) if isinstance(patterns, str) else patterns or () + compiled: list[re.Pattern[str]] = [] + for pattern in raw_patterns: + try: + compiled.append(re.compile(pattern)) + except re.error as exc: + raise ValueError(f"Invalid --skip-pattern regex {pattern!r}: {exc}") from exc + return tuple(compiled) + + +def _valid_cas_candidates(value: Any) -> list[str]: + if isinstance(value, (list, tuple, set)): + raw_candidates = value + elif value is None: + raw_candidates = () + else: + raw_candidates = (value,) + + candidates: list[str] = [] + for candidate in raw_candidates: + normalized = normalize_cas(candidate) + if is_valid_cas(normalized) and normalized not in candidates: + candidates.append(normalized) + return candidates + + +def _apply_pubchem_resolution( + frame: pd.DataFrame, + index: Any, + identifier: Any, + result: LookupResult, + *, + resolved_cas_column: str, +) -> None: + candidates = _valid_cas_candidates(result.values.get("cas_numbers")) + resolved_cas = "" + if result.status == "skipped": + resolution = "skipped" + elif result.status != "ok": + resolution = "not_evaluated" + else: + query_cas = normalize_cas(identifier) + if is_valid_cas(query_cas): + resolution = "query_confirmed" + resolved_cas = query_cas + elif len(candidates) == 1: + resolution = "unique" + resolved_cas = candidates[0] + elif candidates: + resolution = "multiple" + else: + resolution = "missing" + + _set_cell(frame, index, "PubChem CAS Resolution", resolution) + _set_cell(frame, index, "PubChem CAS Candidate Count", len(candidates)) + _set_cell(frame, index, resolved_cas_column, resolved_cas) + + def run_pubchem( input_path: str | Path, client: Any, *, output_path: str | Path | None = None, identifier_column: str = "CAS Number", + resolved_cas_column: str = "Resolved CAS", + skip_patterns: Iterable[str] | str | None = None, include_odor: bool = True, include_provenance: bool = True, checkpoint_every: int = 25, force: bool = False, progress: ProgressCallback = console_progress, ) -> RunSummary: + compiled_skip_patterns = _compile_skip_patterns(skip_patterns) frame, destination = _prepare_run(input_path, output_path, suffix="_pubchem", force=force) require_columns(frame, identifier_column) statuses: Counter[str] = Counter() total = len(frame) for current, index, row in _indices(frame): identifier = row[identifier_column] - result = client.lookup(identifier, include_odor=include_odor) + identifier_text = clean_text(identifier) + matching_pattern = next( + (pattern for pattern in compiled_skip_patterns if pattern.search(identifier_text)), + None, + ) + if matching_pattern is not None: + result = LookupResult.failure( + "PubChem", + status="skipped", + message=f"Identifier matched --skip-pattern {matching_pattern.pattern!r}", + ) + else: + result = client.lookup(identifier, include_odor=include_odor) _apply_result( frame, index, @@ -244,6 +320,13 @@ def run_pubchem( prefix="PubChem", key_map=PUBCHEM_COLUMN_MAP, ) + _apply_pubchem_resolution( + frame, + index, + identifier, + result, + resolved_cas_column=resolved_cas_column, + ) statuses[result.status] += 1 progress(current, total, f"PubChem: {identifier} -> {result.status}") _checkpoint(frame, destination, current, checkpoint_every) diff --git a/pyproject.toml b/pyproject.toml index 35356be..e9f6dd1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "aromanexus" -version = "0.3.0" +version = "0.3.1" description = "Provenance-aware integration for odorant, flavor-molecule, and olfactory-receptor data" readme = "README.md" license = "MIT" diff --git a/tests/test_cli.py b/tests/test_cli.py index 9b2d4d5..e77aa85 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -13,3 +13,39 @@ def test_missing_input_returns_user_error(capsys, tmp_path): code = main(["pubchem", str(tmp_path / "missing.xlsx")]) assert code == 2 assert "does not exist" in capsys.readouterr().err + + +def test_pubchem_cli_forwards_repeatable_skip_patterns_and_resolved_column(monkeypatch, tmp_path): + input_path = tmp_path / "input.csv" + input_path.write_text("Name\nC6\n", encoding="utf-8") + captured = {} + + monkeypatch.setattr("aromanexus.sources.pubchem.PubChemClient", lambda **_kwargs: object()) + + def fake_run_pubchem(input_file, client, **kwargs): + captured["input"] = input_file + captured["client"] = client + captured["kwargs"] = kwargs + return 0 + + monkeypatch.setattr("aromanexus.cli.run_pubchem", fake_run_pubchem) + + code = main( + [ + "pubchem", + str(input_path), + "--identifier-column", + "Name", + "--skip-pattern", + r"^C\d+$", + "--skip-pattern", + "^Total$", + "--resolved-cas-column", + "Curated CAS", + ] + ) + + assert code == 0 + assert captured["input"] == input_path + assert captured["kwargs"]["skip_patterns"] == [r"^C\d+$", "^Total$"] + assert captured["kwargs"]["resolved_cas_column"] == "Curated CAS" diff --git a/tests/test_workflows.py b/tests/test_workflows.py index b35e2ee..c78a5b9 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -39,6 +39,16 @@ def lookup(self, identifier, include_odor=True): ) +class ScriptedPubChem: + def __init__(self, results): + self.results = results + self.calls = [] + + def lookup(self, identifier, include_odor=True): + self.calls.append((identifier, include_odor)) + return self.results[identifier] + + def _silent(*_): return None @@ -83,6 +93,120 @@ def test_pubchem_maps_fields_and_sanitizes_remote_text(tmp_path: Path): assert output.loc[0, "PubChem Source URL"].endswith("/240") +def test_pubchem_skip_pattern_excludes_structural_rows_before_lookup(tmp_path: Path): + source = tmp_path / "input.xlsx" + destination = tmp_path / "pubchem.xlsx" + pd.DataFrame({"Name": ["C6", "n-Hexane"]}).to_excel(source, index=False) + client = ScriptedPubChem( + { + "n-Hexane": LookupResult( + provider="PubChem", + values={"cid": 8058, "cas_numbers": ["110-54-3"]}, + ) + } + ) + + run_pubchem( + source, + client, + output_path=destination, + identifier_column="Name", + skip_patterns=[r"^C\d+$"], + resolved_cas_column="Resolved CAS", + checkpoint_every=0, + progress=_silent, + ) + + output = read_table(destination) + assert client.calls == [("n-Hexane", True)] + assert output.loc[0, "PubChem Status"] == "skipped" + assert output.loc[0, "PubChem CAS Resolution"] == "skipped" + assert output.loc[0, "PubChem CAS Candidate Count"] == 0 + assert pd.isna(output.loc[0, "Resolved CAS"]) + assert output.loc[1, "Resolved CAS"] == "110-54-3" + + +def test_pubchem_separates_lookup_status_from_cas_resolution(tmp_path: Path): + source = tmp_path / "input.xlsx" + destination = tmp_path / "pubchem.xlsx" + identifiers = [ + "100-52-7", + "benzaldehyde", + "n-Hexane", + "Matched without CAS", + "Unavailable", + ] + pd.DataFrame({"Name": identifiers}).to_excel(source, index=False) + client = ScriptedPubChem( + { + "100-52-7": LookupResult( + provider="PubChem", + values={"cas_numbers": ["64-17-5", "100-52-7"]}, + ), + "benzaldehyde": LookupResult( + provider="PubChem", + values={"cas_numbers": ["100-52-7"]}, + ), + "n-Hexane": LookupResult( + provider="PubChem", + values={"cas_numbers": ["110-54-3", "64-17-5"]}, + ), + "Matched without CAS": LookupResult(provider="PubChem", values={"cid": 999}), + "Unavailable": LookupResult.failure( + "PubChem", status="not_found", message="No matching record" + ), + } + ) + + run_pubchem( + source, + client, + output_path=destination, + identifier_column="Name", + resolved_cas_column="Resolved CAS", + checkpoint_every=0, + progress=_silent, + ) + + output = read_table(destination) + assert output["PubChem CAS Resolution"].tolist() == [ + "query_confirmed", + "unique", + "multiple", + "missing", + "not_evaluated", + ] + assert output["PubChem CAS Candidate Count"].tolist() == [2, 1, 2, 0, 0] + assert output.loc[0, "Resolved CAS"] == "100-52-7" + assert output.loc[1, "Resolved CAS"] == "100-52-7" + assert pd.isna(output.loc[2, "Resolved CAS"]) + assert pd.isna(output.loc[3, "Resolved CAS"]) + assert pd.isna(output.loc[4, "Resolved CAS"]) + assert output.loc[2, "PubChem CAS Numbers"] == "110-54-3; 64-17-5" + assert output.loc[4, "PubChem Status"] == "not_found" + + +def test_invalid_pubchem_skip_pattern_fails_before_provider_calls(tmp_path: Path): + source = tmp_path / "input.xlsx" + destination = tmp_path / "pubchem.xlsx" + pd.DataFrame({"Name": ["benzaldehyde"]}).to_excel(source, index=False) + client = ScriptedPubChem({}) + + with pytest.raises(ValueError): + run_pubchem( + source, + client, + output_path=destination, + identifier_column="Name", + skip_patterns=["["], + checkpoint_every=0, + progress=_silent, + ) + + assert client.calls == [] + assert not destination.exists() + + def test_existing_output_is_rejected_before_provider_calls(tmp_path: Path): source = tmp_path / "input.xlsx" destination = tmp_path / "output.xlsx"