From 2672ab100e5c1dc7743fc435b7652931817f90ad Mon Sep 17 00:00:00 2001 From: basil-chen <192173459+rastagan-git@users.noreply.github.com> Date: Sun, 19 Jul 2026 20:15:19 +0800 Subject: [PATCH] Package crawler as multi-source aroma data skill --- .agents/skills/curate-flavor-data/SKILL.md | 45 ++ .../curate-flavor-data/agents/openai.yaml | 4 + .../references/output-schema.md | 40 ++ .../references/provider-matrix.md | 14 + .../scripts/inspect_workbook.py | 70 +++ .github/workflows/ci.yml | 42 ++ .gitignore | 13 + README-CN.md | 195 ++++++++ README.md | 228 ++++++--- cb_spider.py | 210 ++------- flavor_data_crawler/__init__.py | 6 + flavor_data_crawler/cli.py | 288 ++++++++++++ flavor_data_crawler/excel_io.py | 96 ++++ flavor_data_crawler/http.py | 362 +++++++++++++++ flavor_data_crawler/identifiers.py | 63 +++ flavor_data_crawler/models.py | 58 +++ flavor_data_crawler/sources/__init__.py | 1 + flavor_data_crawler/sources/chemicalbook.py | 127 ++++++ flavor_data_crawler/sources/m2or.py | 258 +++++++++++ flavor_data_crawler/sources/mffi.py | 108 +++++ flavor_data_crawler/sources/nist.py | 314 +++++++++++++ flavor_data_crawler/sources/pubchem.py | 428 +++++++++++++++++ flavor_data_crawler/sources/pyrfume.py | 327 +++++++++++++ flavor_data_crawler/workflows.py | 431 ++++++++++++++++++ mffi_spider.py | 175 ++----- name_to_cas.py | 169 ++----- nist_excel_tool.py | 189 ++------ pyproject.toml | 52 +++ requirements-dev.txt | 1 + requirements.txt | 7 +- start1.bat | 42 +- start2.bat | 57 +-- start3.bat | 49 +- start4.bat | 49 +- tests/fixtures/M2OR.synthetic.csv | 7 + tests/test_browser_sources.py | 27 ++ tests/test_cli.py | 15 + tests/test_excel_io.py | 40 ++ tests/test_http.py | 184 ++++++++ tests/test_identifiers.py | 25 + tests/test_m2or.py | 88 ++++ tests/test_nist.py | 140 ++++++ tests/test_pubchem.py | 230 ++++++++++ tests/test_pyrfume.py | 121 +++++ tests/test_skill.py | 36 ++ tests/test_workflows.py | 94 ++++ 46 files changed, 4745 insertions(+), 780 deletions(-) create mode 100644 .agents/skills/curate-flavor-data/SKILL.md create mode 100644 .agents/skills/curate-flavor-data/agents/openai.yaml create mode 100644 .agents/skills/curate-flavor-data/references/output-schema.md create mode 100644 .agents/skills/curate-flavor-data/references/provider-matrix.md create mode 100644 .agents/skills/curate-flavor-data/scripts/inspect_workbook.py create mode 100644 .github/workflows/ci.yml create mode 100644 .gitignore create mode 100644 README-CN.md create mode 100644 flavor_data_crawler/__init__.py create mode 100644 flavor_data_crawler/cli.py create mode 100644 flavor_data_crawler/excel_io.py create mode 100644 flavor_data_crawler/http.py create mode 100644 flavor_data_crawler/identifiers.py create mode 100644 flavor_data_crawler/models.py create mode 100644 flavor_data_crawler/sources/__init__.py create mode 100644 flavor_data_crawler/sources/chemicalbook.py create mode 100644 flavor_data_crawler/sources/m2or.py create mode 100644 flavor_data_crawler/sources/mffi.py create mode 100644 flavor_data_crawler/sources/nist.py create mode 100644 flavor_data_crawler/sources/pubchem.py create mode 100644 flavor_data_crawler/sources/pyrfume.py create mode 100644 flavor_data_crawler/workflows.py create mode 100644 pyproject.toml create mode 100644 requirements-dev.txt create mode 100644 tests/fixtures/M2OR.synthetic.csv create mode 100644 tests/test_browser_sources.py create mode 100644 tests/test_cli.py create mode 100644 tests/test_excel_io.py create mode 100644 tests/test_http.py create mode 100644 tests/test_identifiers.py create mode 100644 tests/test_m2or.py create mode 100644 tests/test_nist.py create mode 100644 tests/test_pubchem.py create mode 100644 tests/test_pyrfume.py create mode 100644 tests/test_skill.py create mode 100644 tests/test_workflows.py diff --git a/.agents/skills/curate-flavor-data/SKILL.md b/.agents/skills/curate-flavor-data/SKILL.md new file mode 100644 index 0000000..e6e6839 --- /dev/null +++ b/.agents/skills/curate-flavor-data/SKILL.md @@ -0,0 +1,45 @@ +--- +name: curate-flavor-data +description: Validate, normalize, enrich, and export flavor, odor, aroma, and chemical-sensory datasets with source-level provenance. Use for XLSX, CSV, or TSV workbooks containing CAS numbers, compound names, calculated retention indices, sensory descriptors, odor thresholds, or olfactory-receptor evidence; for selecting among NIST, PubChem, Pyrfume, M2OR, MFFI, and the permission-gated ChemicalBook compatibility connector; and for verifying analysis-ready outputs. Do not use to bypass access controls, relicense source data, or make biomedical or machine-learning claims unsupported by the exported evidence. +--- + +# Curate Flavor Data + +Build traceable compound tables through the repository's `flavor-data` CLI. Keep the skill as an orchestration layer; modify provider behavior in the Python package, not here. + +## Workflow + +1. Inspect the input without modifying it. + - Run `python .agents/skills/curate-flavor-data/scripts/inspect_workbook.py INPUT` from the repository root. + - Confirm the row count, exact column names, identifier quality, duplicates, and formula-like cells. +2. Choose the smallest provider set that supplies the requested fields. + - Read [references/provider-matrix.md](references/provider-matrix.md) before any network or browser operation. + - Prefer PubChem for canonical identity and sourced odor annotations. + - Use NIST only for the existing retention-index or name-resolution workflows. + - Use Pyrfume only for explicitly selected archives after reviewing each manifest note. + - Use M2OR only when receptor bioassay evidence is relevant; label species and assay scope. +3. Preview the operation. + - State the input, new output path, selected provider, expected columns, approximate request count, cache behavior, and material access caveats. + - Write a sibling output by default. Do not pass `--force` or overwrite the input unless the user explicitly requests that exact replacement. +4. Run one focused command. + - Identity and odor metadata: `flavor-data pubchem INPUT --identifier-column "CAS Number"` + - Retention indices: `flavor-data nist-ri INPUT --cas-column "CAS Number" --calculated-ri-column "Calculated RI"` + - Names to CAS: `flavor-data resolve-cas INPUT --name-column "Name"` + - Curated descriptors: `flavor-data pyrfume INPUT --archives aromadb,superscent` + - Receptor evidence: `flavor-data m2or INPUT --cas-column "CAS Number"` + - Source inventory: `flavor-data sources` +5. Verify the result. + - Re-run the inspection script on the output. + - Confirm identical row order and row count, expected new fields, typed status counts, source URL, retrieval time, version, and license/access fields. + - Treat `http_error`, `network_error`, `parse_error`, `missing_data`, `data_error`, `partial`, `blocked`, and `skipped` separately from `not_found`. + - Consult [references/output-schema.md](references/output-schema.md) when reconciling columns or statuses. +6. Report the output path, provider versions, status counts, partial failures, and any source terms the user must still review. + +## Guardrails + +- Keep NIST's interval at five seconds or slower and retain caching. +- Never automate CAPTCHA solving. Keep browser sources visible when user intervention may be required. +- Do not run ChemicalBook automation unless the user confirms documented permission; its current robots policy excludes the legacy routes. +- Do not describe the toolkit as AI-powered. Say that structured exports can support downstream statistics, cheminformatics, or machine-learning experiments. +- Do not bundle or republish downloaded Pyrfume or M2OR data in the repository. +- Preserve remote text as literal spreadsheet cells to prevent formula execution. diff --git a/.agents/skills/curate-flavor-data/agents/openai.yaml b/.agents/skills/curate-flavor-data/agents/openai.yaml new file mode 100644 index 0000000..9835ee4 --- /dev/null +++ b/.agents/skills/curate-flavor-data/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Curate Flavor Data" + short_description: "Curate traceable odorant and flavor datasets" + default_prompt: "Use $curate-flavor-data to inspect and enrich this odorant workbook with source provenance." diff --git a/.agents/skills/curate-flavor-data/references/output-schema.md b/.agents/skills/curate-flavor-data/references/output-schema.md new file mode 100644 index 0000000..84b83cf --- /dev/null +++ b/.agents/skills/curate-flavor-data/references/output-schema.md @@ -0,0 +1,40 @@ +# Output and validation contract + +## Identity + +- Preserve the original input identifier. +- Use PubChem CID and InChIKey as canonical joins when available. +- Keep CAS as an external identifier and validate its checksum before exact-match providers. +- Never silently pick an ambiguous name result. + +## Status values + +- `ok`: a provider returned a parsed record. +- `not_found`: the provider responded successfully but no exact record matched. +- `ambiguous`: more than one defensible identity matched. +- `invalid_input`: the identifier is empty, malformed, or fails validation. +- `network_error`: transport or remote-service failure; safe to retry later. +- `http_error`: the provider returned a non-success HTTP response that was not a clean not-found result. +- `parse_error`: the response arrived but its structure could not be interpreted; investigate a selector/schema change. +- `missing_data`: a required local or cached snapshot is unavailable and downloading is disabled or failed before parsing. +- `data_error`: every selected archive failed to load; inspect the per-archive diagnostic message. +- `partial`: at least one selected source succeeded and at least one failed, or a provider returned usable data with warnings. +- `blocked`: access policy or missing permission prevented the request. +- `skipped`: a user deliberately skipped an interactive record. + +Do not merge an access, transport, HTTP, snapshot, parse, or partial failure into `not_found`. + +## Provenance + +Keep provider status, source URL, retrieval time, cache-hit flag, pinned version or snapshot, license URL, and message columns. For PubChem odor text, also keep contributor source names, URLs, and license URLs. + +## Workbook QA + +After every run, confirm: + +1. Output row count and row order equal the input. +2. Original columns remain present. +3. Requested output columns exist. +4. Every processed row has a typed status. +5. Remote strings beginning with `=`, `+`, `-`, or `@` are stored as literal text. +6. Partial outputs are reported separately if a run is interrupted. diff --git a/.agents/skills/curate-flavor-data/references/provider-matrix.md b/.agents/skills/curate-flavor-data/references/provider-matrix.md new file mode 100644 index 0000000..3c56e16 --- /dev/null +++ b/.agents/skills/curate-flavor-data/references/provider-matrix.md @@ -0,0 +1,14 @@ +# Provider matrix + +Review these notes before choosing or running a provider. Access policies can change; re-check the linked official page when a live run or redistribution decision matters. + +| Provider | Best use | Access behavior | Rights and scientific caveat | +| --- | --- | --- | --- | +| [PubChem PUG REST](https://pubchem.ncbi.nlm.nih.gov/docs/pug-rest) and [PUG-View](https://pubchem.ncbi.nlm.nih.gov/docs/pug-view) | CID, InChIKey, structures, formula, properties, sourced odor annotations | Live API, cached locally, below the official five-requests-per-second ceiling, bounded retries | PubChem aggregates contributor records. Preserve contributor source, URL, and license URL for annotation text. | +| [NIST Chemistry WebBook](https://webbook.nist.gov/chemistry/) | Existing GC retention-index and name-to-CAS workflows | Cached HTML, at least five seconds between uncached requests per [robots.txt](https://webbook.nist.gov/robots.txt) | NIST SRD compilation rights apply. Fetch on demand and cite; do not redistribute a bulk scrape. | +| [Pyrfume Public Data Archive](https://github.com/pyrfume/pyrfume-data) | Curated odor descriptors or collection membership keyed by PubChem CID | Pinned GitHub snapshot; explicit archive allowlist; files cached locally | The repository code is MIT, but manifests record upstream rights and sometimes copyright caveats. Do not treat every archive as MIT-licensed data. | +| [M2OR](https://github.com/chemosim-lab/M2OR) | Molecule-olfactory-receptor pairs, species, responsive assays, and study DOI | Optional pinned CSV download (about 43 MB), cached locally | Dataset repository is Apache-2.0. Results are assay evidence, not human odor perception or clinical prediction. | +| [MFFI](https://mffi.sjtu.edu.cn/database/search) | Chinese/English names, sensory characteristics, and water thresholds | Interactive Selenium browser, conservative pacing | No public API, rate policy, or reuse license was found. Robots allowance is not a data license; use conservatively and cite. | +| [ChemicalBook](https://www.chemicalbook.com/) | Original odor/threshold/type compatibility path only | Disabled until explicit permission confirmation; visible manual browser | Current [robots.txt](https://www.chemicalbook.com/robots.txt) excludes the search and product-property routes. Never bypass CAPTCHA or imply permission. | + +Avoid automated extraction from FlavorDB2, Flavornet, Good Scents, VCF, or other attractive-but-undocumented sites unless an official API, bulk download, or written permission clearly covers the intended use. diff --git a/.agents/skills/curate-flavor-data/scripts/inspect_workbook.py b/.agents/skills/curate-flavor-data/scripts/inspect_workbook.py new file mode 100644 index 0000000..6a7b314 --- /dev/null +++ b/.agents/skills/curate-flavor-data/scripts/inspect_workbook.py @@ -0,0 +1,70 @@ +"""Inspect a flavor-data workbook without modifying it.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[4] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from flavor_data_crawler.excel_io import FORMULA_PREFIXES, read_table # noqa: E402 +from flavor_data_crawler.identifiers import is_valid_cas, normalize_cas # noqa: E402 + + +def inspect(path: Path, cas_column: str | None = None) -> dict[str, object]: + frame = read_table(path) + detected_cas = cas_column or next( + (name for name in ("CAS Number", "CAS", "cas", "cas_number") if name in frame.columns), + None, + ) + formula_like = 0 + for column in frame.columns: + formula_like += int( + frame[column] + .map(lambda value: isinstance(value, str) and value.startswith(FORMULA_PREFIXES)) + .sum() + ) + report: dict[str, object] = { + "path": str(path.resolve()), + "rows": len(frame), + "columns": [str(column) for column in frame.columns], + "duplicate_rows": int(frame.duplicated().sum()), + "formula_like_cells": formula_like, + "missing_by_column": { + str(column): int(frame[column].isna().sum() + frame[column].eq("").sum()) + for column in frame.columns + }, + } + if detected_cas: + normalized = frame[detected_cas].map(normalize_cas) + valid_mask = normalized.map(is_valid_cas) + nonempty_mask = normalized.ne("") + invalid = normalized[nonempty_mask & ~valid_mask] + report["cas"] = { + "column": detected_cas, + "nonempty": int(nonempty_mask.sum()), + "valid": int(valid_mask.sum()), + "invalid": int((nonempty_mask & ~valid_mask).sum()), + "invalid_examples": list(dict.fromkeys(invalid.astype(str)))[:10], + "duplicates": int(normalized[nonempty_mask].duplicated().sum()), + } + else: + report["cas"] = {"column": None, "note": "No common CAS column name detected"} + return report + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("input", type=Path) + parser.add_argument("--cas-column") + args = parser.parse_args() + print(json.dumps(inspect(args.input, args.cas_column), ensure_ascii=False, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..97ad26a --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,42 @@ +name: CI + +on: + push: + pull_request: + +permissions: + contents: read + +jobs: + test: + name: ${{ matrix.os }} / Python ${{ matrix.python-version }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + python-version: ["3.11", "3.13"] + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: pip + + - name: Install package and development dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -e ".[dev]" + + - name: Lint + run: python -m ruff check . + + - name: Check formatting + run: python -m ruff format --check . + + - name: Test + run: python -m pytest diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8b02ec0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,13 @@ +.venv/ +venv/ +myenv/ +__pycache__/ +*.py[cod] +*.egg-info/ +.pytest_cache/ +.ruff_cache/ +.coverage +htmlcov/ +build/ +dist/ +.flavor-data-cache/ diff --git a/README-CN.md b/README-CN.md new file mode 100644 index 0000000..81cd99b --- /dev/null +++ b/README-CN.md @@ -0,0 +1,195 @@ +# Flavor Data Crawler + +[![CI](https://github.com/rastagan-git/Flavor-Data-Crawler/actions/workflows/ci.yml/badge.svg)](https://github.com/rastagan-git/Flavor-Data-Crawler/actions/workflows/ci.yml) +[![Python 3.11+](https://img.shields.io/badge/Python-3.11%2B-3776AB?logo=python&logoColor=white)](https://www.python.org/downloads/) + +[English](README.md) · **简体中文** + +一套重视数据来源追踪的化学—感官数据整理工具：把化合物工作簿扩充为可核查、可继续分析的数据表。 + +Flavor Data Crawler 将化学身份、气相色谱保留指数、气味描述、阈值，以及可选的嗅觉受体实验结果串联起来。它会保留原始表格，规范化不同来源的结果，并记录每项扩充数据来自哪里。输出可作为后续统计分析、化学信息学与边界清晰的机器学习实验输入。 + +本项目**不声称自己是 AI 系统**，也不会推断生物学效应或给出临床预测。M2OR 数据代表受体实验结果，不等同于人的气味感知。 + +```text +XLSX / CSV / TSV + │ + ▼ +验证标识符 ──► 带缓存的数据源适配器 ──► 规范字段 + 来源记录 + │ + ▼ + 新的、便于分析的数据表 +``` + +## 这次升级带来了什么 + +原有四套工作簿脚本仍然保留，同时新增了统一 CLI： + +- 严格验证 CAS，并明确标记名称匹配歧义； +- 默认记录状态、来源 URL、获取时间、缓存、版本、许可链接和诊断信息； +- 使用保守的访问间隔、有限重试与持久缓存； +- 原子写入、定期生成恢复检查点，默认不覆盖已有文件； +- 在原有 NIST、MFFI、ChemicalBook 流程之外，增加 PubChem、Pyrfume 与 M2OR 扩充。 + +## 安装 + +需要 Python 3.11 或更高版本。 + +### Windows PowerShell + +```powershell +python -m venv .venv +.\.venv\Scripts\Activate.ps1 +python -m pip install --upgrade pip +python -m pip install -e . +``` + +### macOS 或 Linux + +```bash +python3 -m venv .venv +source .venv/bin/activate +python -m pip install --upgrade pip +python -m pip install -e . +``` + +安装后先确认版本并查看所有数据源的访问模式： + +```bash +flavor-data --version +flavor-data sources +``` + +## 快速开始 + +CLI 支持 `.xlsx`、`.csv` 和 `.tsv`。列名均可修改；下面使用默认列名。 + +```bash +# 规范化身份、理化性质、同义词、CAS 标识符与带来源的气味文本 +flavor-data pubchem compounds.xlsx --identifier-column "CAS Number" + +# 在 NIST 中寻找最接近实验计算值的保留指数 +flavor-data nist-ri data.xlsx \ + --cas-column "CAS Number" \ + --calculated-ri-column "Calculated RI" + +# 通过 NIST WebBook 将化合物名称解析为 CAS +flavor-data resolve-cas names.xlsx --name-column "Name" + +# 查询选定的 Pyrfume 集合；缺少 CID 时会通过 PubChem 解析 +flavor-data pyrfume compounds.xlsx --archives aromadb,superscent + +# 可选的分子—嗅觉受体实验依据 +flavor-data m2or compounds.xlsx --cas-column "CAS Number" + +# 需要交互浏览器的兼容数据源 +flavor-data mffi compounds.xlsx --cas-column "CAS Number" + +# 设有许可门槛的旧版数据源；命令会要求明确确认 +flavor-data chemicalbook-legacy compounds.xlsx --cas-column "CAS Number" +``` + +在 PowerShell 中，请把多行命令写成一行，或将 Bash 的 `\` 续行符换成 PowerShell 的反引号。 + +## 命令一览 + +| 命令 | 默认输入列 | 用途 | 默认输出后缀 | +| --- | --- | --- | --- | +| `flavor-data sources` | 无 | 列出数据源、用途和访问方式；`providers` 是别名。 | 无 | +| `flavor-data nist-ri INPUT` | `CAS Number`、`Calculated RI` | 在原流程指定的 NIST 非极性柱、自定义升温 RI 表中匹配最接近值。 | `_nist_result` | +| `flavor-data resolve-cas INPUT` | `Name` | 通过 NIST 将无歧义的化合物名称解析为 CAS Registry Number。 | `_with_cas` | +| `flavor-data pubchem INPUT` | `CAS Number` | 添加 CID、名称、结构标识符、选定性质、同义词、CAS 标识符和带来源的气味注释。 | `_pubchem` | +| `flavor-data pyrfume INPUT` | `PubChem CID`；若需解析 CID，则用 `CAS Number` | 匹配白名单内的固定版本档案：`aromadb`、`flavornet`、`superscent`；默认 `aromadb,superscent`。 | `_pyrfume` | +| `flavor-data m2or INPUT` | `CAS Number` | 汇总分子—受体配对、响应配对、物种、人类响应受体和研究 DOI。 | `_m2or` | +| `flavor-data mffi INPUT` | `CAS Number` | 通过可见 Chrome 获取中英文名、感官特征和水中阈值；确认不需要交互时才使用 `--headless`。 | `_mffi_result` | +| `flavor-data chemicalbook-legacy INPUT` | `CAS Number` | 保留原有气味、阈值和香型交互流程；在确认有书面许可前禁用。 | `_cb_result` | + +使用 `flavor-data COMMAND --help` 查看列名及数据源专用选项。全局参数必须写在子命令之前： + +```bash +flavor-data --cache-dir .cache/flavor-data --timeout 30 pubchem compounds.xlsx +``` + +### 输出、检查点与覆盖保护 + +所有表格命令默认在输入文件旁生成新文件，保留原有行序和列，再添加数据源字段。例如，PubChem 会将 `compounds.xlsx` 输出为 `compounds_pubchem.xlsx`。 + +默认来源记录包括数据源状态、来源 URL、获取时间、是否命中缓存、固定版本、许可 URL 与诊断信息。只有在确实需要旧版形状时才使用 `--no-provenance`。 + +```bash +# 明确指定输出位置 +flavor-data pubchem compounds.xlsx --output results/compounds_enriched.xlsx + +# 每处理 10 行保存一次恢复检查点；设为 0 可关闭 +flavor-data pubchem compounds.xlsx --checkpoint-every 10 + +# 明确覆盖一个已存在的目标文件 +flavor-data pubchem compounds.xlsx --output compounds_pubchem.xlsx --force +``` + +检查点形如 `compounds_pubchem.partial.xlsx`：运行期间定期刷新，中断后保留，最终文件写入成功后删除。若目标文件已存在，命令会停止，除非显式传入 `--force`。建议输出到新文件，不要直接覆盖输入。 + +成功的 HTTP 响应与下载快照默认缓存到 `~/.cache/flavor-data-crawler`。如需更改位置，请在子命令之前传入 `--cache-dir`。 + +## 数据来源、访问方式与权利边界 + +访问规则与数据条款可能变化。正式抓取、发表或再分发前，请重新检查下列官方页面。本仓库不会替第三方数据授予使用权。 + +| 数据源 | 本工具使用的数据 | 访问与缓存方式 | 权利与科学边界 | +| --- | --- | --- | --- | +| [PubChem PUG REST](https://pubchem.ncbi.nlm.nih.gov/docs/pug-rest) + [PUG-View](https://pubchem.ncbi.nlm.nih.gov/docs/pug-view) | 化合物身份、选定性质、同义词/CAS 标识符，以及带贡献者来源的气味注释 | 调用 NCBI 在线 API 并持久缓存响应。未缓存请求间隔为 0.25 秒，即每秒 4 次，低于 PubChem 的每秒 5 次上限；瞬时失败只进行有限重试。 | PubChem 汇集不同贡献者记录。输出保留注释来源名、来源 URL 和许可 URL；请同时检查 [NCBI 政策](https://www.ncbi.nlm.nih.gov/home/about/policies/)及各贡献者条款。 | +| [NIST Chemistry WebBook，SRD 69](https://webbook.nist.gov/chemistry/) | 保留指数查询、名称解析 CAS | HTML 持久缓存；未缓存请求之间至少间隔 5 秒，遵守公开的 [robots.txt](https://webbook.nist.gov/robots.txt)。 | 适用 [NIST Standard Reference Data 权利规定](https://www.nist.gov/srd/public-law)。应按需获取并引用，不要把该服务当作可自由再分发的批量数据集。 | +| [Pyrfume Public Data Archive](https://github.com/pyrfume/pyrfume-data) | 以 PubChem CID 为键、固定版本的 `aromadb`、`flavornet` 与 `superscent` 文件 | 仅允许显式列出的档案；从固定提交下载选定文件并在本地缓存。 | 权利以**每个 manifest 和上游集合**为准。仓库代码的许可证不会自动覆盖每一份数据；输出会保留 manifest 的来源、备注与许可说明。 | +| [M2OR](https://github.com/chemosim-lab/M2OR) | 分子—嗅觉受体配对、响应标签、物种、受体和 DOI | 可选的固定版本 CSV 快照，约 43 MB；首次使用时下载并缓存，本仓库不捆绑该文件。 | 上游快照采用 Apache-2.0。这些是生物测定结果，不代表人的感知质量、安全性、疗效或临床结局。 | +| [MFFI](https://mffi.sjtu.edu.cn/database/search) | 中英文名、感官特征和水中阈值 | 通过 Selenium/Chrome 交互访问，并使用保守的逐行间隔。目前未发现公开且有文档的 API 或速率政策。 | 目前未发现明确的再利用许可证。网页可访问或 robots 允许访问，并不等于获得再发布许可；请保守使用并引用来源。 | +| [ChemicalBook](https://www.chemicalbook.com/) | 旧版气味描述、嗅觉阈值和香型兼容流程 | **默认禁用，并设有许可门槛。**当前 [robots.txt](https://www.chemicalbook.com/robots.txt) 排除了搜索及属性页面路径。连接器保持可见和手动，不会破解或绕过 CAPTCHA。 | 只有在书面许可明确覆盖所需自动访问与再利用时才能运行。`--i-have-permission` 是操作者自己的声明，不代表本项目提供了许可。 | + +## Codex 项目 Skill + +仓库内置了项目级 Skill：`.agents/skills/curate-flavor-data/`。在 Codex 中可直接调用： + +```text +$curate-flavor-data +``` + +该 Skill 会检查工作簿、选择满足需求的最小数据源组合、预览访问及输出影响、执行一个聚焦命令，并核对行数、结构、状态和来源记录。它只是本软件包之上的流程编排指南，不是另一套爬虫，也不会自动赋予数据使用权。 + +也可以直接运行其中只读的工作簿检查工具： + +```bash +python .agents/skills/curate-flavor-data/scripts/inspect_workbook.py compounds.xlsx +``` + +## 旧版兼容入口 + +原有脚本与 Windows 启动器仍然保留，继续支持固定的工作簿布局： + +| 启动器 | 脚本 | 预期工作簿 | 必需列 | 输出 | +| --- | --- | --- | --- | --- | +| `start1.bat` | `nist_excel_tool.py` | `data.xlsx` | `CAS Number`、`Calculated RI` | `data_result.xlsx` | +| `start2.bat` | `name_to_cas.py` | `name.xlsx` | `Name` | `name_with_cas.xlsx` | +| `start3.bat` | `mffi_spider.py` | `max.xlsx` | `CAS Number` | `max_mffi_result.xlsx` | +| `start4.bat` | `cb_spider.py` | `Odor.xlsx` | `CAS Number` | `Odor_cb_result.xlsx` | + +`.bat` 会依次寻找 `.venv`、`myenv`、`venv`，最后才使用系统 `python`。兼容脚本会刻意生成不含来源列的旧版结果，并覆盖固定名称的结果文件；新任务建议使用 CLI，以获得明确路径与覆盖保护。MFFI 和 ChemicalBook 需要本机可用的 Chrome，ChemicalBook 仍会要求输入许可确认短语。 + +## 开发与测试 + +安装开发依赖后，运行离线测试和代码检查： + +```bash +python -m pip install -e ".[dev]" +python -m ruff check . +python -m ruff format --check . +python -m pytest +``` + +CI 会在 Ubuntu 与 Windows 上使用 Python 3.11 和 3.13 执行上述检查。测试通过固定样例或注入客户端运行，不依赖实时网站，也不依赖 Codex 运行环境。 + +## 负责任地使用 + +- 针对你的具体用途核对数据源条款、robots 规则、引用要求和再分发权利。 +- 保持保守请求频率，优先使用缓存。 +- 不得绕过 CAPTCHA、身份验证、付费墙或其他访问控制。 +- 将 `not_found`、`invalid_input`、`http_error`、`network_error`、`parse_error`、`missing_data`、`data_error`、`partial`、`blocked` 和 `skipped` 视为不同结果。 +- 在统计、化学信息学或机器学习工作中使用前，核对来源记录与生物学适用范围。 diff --git a/README.md b/README.md index ed2a139..1cf058c 100644 --- a/README.md +++ b/README.md @@ -1,91 +1,195 @@ -# Flavor-Data-Crawler -Tools for extracting chemical sensory data -# Flavor & Chemical Properties Crawler Tool +# Flavor Data Crawler -这是一个基于 Python 的自动化数据采集工具集，用于从多个化学数据库中批量检索化合物的属性信息。 +[![CI](https://github.com/rastagan-git/Flavor-Data-Crawler/actions/workflows/ci.yml/badge.svg)](https://github.com/rastagan-git/Flavor-Data-Crawler/actions/workflows/ci.yml) +[![Python 3.11+](https://img.shields.io/badge/Python-3.11%2B-3776AB?logo=python&logoColor=white)](https://www.python.org/downloads/) -## 功能模块 (Modules) +**English** · [简体中文](README-CN.md) -1. **NIST WebBook Crawler** - * 根据 CAS 号自动查找保留指数 (Retention Index)。以及根据英文名称查找CAS号 -2. **MFFI Database Crawler** (上海交大风味数据库) - * 抓取化合物的中文名、感官特征 (Sensory Characteristics) 和水中阈值。 -3. **ChemicalBook Crawler** - * 自动化检索气味描述、嗅觉阈值 (Odor Threshold) 和香型 (Odor Type)。 +A provenance-aware toolkit for turning compound workbooks into traceable chemical-sensory datasets. -## 环境依赖 (Requirements) +Flavor Data Crawler connects chemical identity, gas-chromatographic retention indices, odor descriptors, thresholds, and optional olfactory-receptor assay evidence. It preserves the original table, normalizes provider results, and records where each enrichment came from. The resulting tables are practical inputs for downstream statistics, cheminformatics, and carefully scoped machine-learning experiments. -* Python 3.x -* Selenium -* Pandas -* Requests +This project does **not** claim to be an AI system, infer biological effects, or make clinical predictions. M2OR rows are experimental receptor-assay evidence, not human odor perception. -## 使用方法 (Usage) -### 建立虚拟环境 -本项目代码基于虚拟环境的状态下，建议直接使用本项目的虚拟环境代码 +```text +XLSX / CSV / TSV + │ + ▼ +validate identifiers ──► cached provider adapters ──► normalized fields + provenance + │ + ▼ + new, analysis-ready table +``` + +## Why this version + +The original four workbook scripts remain available, but the package now provides one consistent CLI with: + +- exact CAS validation and explicit handling of ambiguous name matches; +- source-level status, URL, retrieval time, cache, version, license, and message fields; +- conservative request pacing, bounded retries, and persistent caching; +- atomic writes, periodic recovery checkpoints, and no accidental overwrite by default; +- optional PubChem, Pyrfume, and M2OR enrichment alongside the original NIST, MFFI, and ChemicalBook workflows. + +## Installation + +Python 3.11 or newer is required. + +### Windows PowerShell + +```powershell +python -m venv .venv +.\.venv\Scripts\Activate.ps1 +python -m pip install --upgrade pip +python -m pip install -e . +``` + +### macOS or Linux +```bash +python3 -m venv .venv +source .venv/bin/activate +python -m pip install --upgrade pip +python -m pip install -e . ``` -# 1. 创建名为 myenv 的虚拟环境 -python -m venv myenv -# 2. 激活虚拟环境 (Windows) -myenv\Scripts\activate +Confirm the installation and review provider modes before a live run: -# 3. 激活虚拟环境 (Mac/Linux) -# source myenv/bin/activate +```bash +flavor-data --version +flavor-data sources ``` -### 安装依赖： - - pip install -r requirements.txt - +## Quick start -### 功能一、根据cas号和计算出的保留指数查找标准保留指数 -你的excel文件名= "data.xlsx" +The CLI accepts `.xlsx`, `.csv`, and `.tsv` tables. Column names are configurable; the examples below use the defaults. -Excel 中存放 CAS 号的列名 (请确保Excel里第一行是列名) -COL_CAS = "CAS Number" +```bash +# Canonical identity, properties, synonyms, CAS identifiers, and sourced odor text +flavor-data pubchem compounds.xlsx --identifier-column "CAS Number" -Excel 中存放 计算保留指数 的列名 -COL_CALC_RI = "Calculated RI" +# Closest NIST RI to an experimentally calculated RI +flavor-data nist-ri data.xlsx \ + --cas-column "CAS Number" \ + --calculated-ri-column "Calculated RI" -结果将写入的新列名 -COL_RESULT = "NIST RI" +# Compound name to CAS through NIST WebBook +flavor-data resolve-cas names.xlsx --name-column "Name" -双击start1.bat +# Selected Pyrfume collections; resolves a CID through PubChem if needed +flavor-data pyrfume compounds.xlsx --archives aromadb,superscent -### 功能二、根据名称查找cas号 -你的 Excel 文件名 = "name.xlsx" +# Optional molecule–olfactory-receptor assay evidence +flavor-data m2or compounds.xlsx --cas-column "CAS Number" -Excel 中存放 物质名称 的列名 (请确保和Excel里一致) -COL_NAME = "Name" +# Interactive browser compatibility source +flavor-data mffi compounds.xlsx --cas-column "CAS Number" -结果将写入的新列名 -COL_RESULT = "Found CAS" +# Permission-gated legacy source; the command asks for explicit confirmation +flavor-data chemicalbook-legacy compounds.xlsx --cas-column "CAS Number" +``` + +On PowerShell, put a multiline command on one line or replace Bash's `\` continuation with PowerShell's backtick. + +## Commands + +| Command | Default input field(s) | Purpose | Default output suffix | +| --- | --- | --- | --- | +| `flavor-data sources` | none | List providers, roles, and access modes. `providers` is an alias. | none | +| `flavor-data nist-ri INPUT` | `CAS Number`, `Calculated RI` | Match the closest value in the original NIST non-polar custom-temperature RI table. | `_nist_result` | +| `flavor-data resolve-cas INPUT` | `Name` | Resolve an unambiguous compound name to a CAS Registry Number through NIST. | `_with_cas` | +| `flavor-data pubchem INPUT` | `CAS Number` | Add CID, names, structure identifiers, selected properties, synonyms, CAS identifiers, and sourced odor annotations. | `_pubchem` | +| `flavor-data pyrfume INPUT` | `PubChem CID`, or `CAS Number` for CID resolution | Match allowlisted pinned archives: `aromadb`, `flavornet`, and/or `superscent`. Defaults to `aromadb,superscent`. | `_pyrfume` | +| `flavor-data m2or INPUT` | `CAS Number` | Aggregate molecule–receptor pairs, responsive pairs, species, human responsive receptors, and study DOIs. | `_m2or` | +| `flavor-data mffi INPUT` | `CAS Number` | Use a visible Chrome session for bilingual names, sensory characteristics, and in-water thresholds. Add `--headless` only when interaction is not needed. | `_mffi_result` | +| `flavor-data chemicalbook-legacy INPUT` | `CAS Number` | Retain the original interactive odor/threshold/type workflow. Disabled until documented permission is confirmed. | `_cb_result` | + +Run `flavor-data COMMAND --help` for column and provider-specific options. Global options must precede the command: -双击start2.bat +```bash +flavor-data --cache-dir .cache/flavor-data --timeout 30 pubchem compounds.xlsx +``` + +### Output, checkpoints, and overwrite safety + +Every table command writes a sibling file by default, keeps the original row order and columns, and adds provider fields. For example, `compounds.xlsx` becomes `compounds_pubchem.xlsx` after a PubChem run. -### 功能三、根据cas号查找中英文名称，气味，水中的嗅觉阈值 -你的excel文件名 = "max.xlsx" +By default, provenance columns include provider status, source URL, retrieval timestamp, cache hit, pinned version, license URL, and a diagnostic message. Use `--no-provenance` only for legacy-shaped output. -Excel 中存放 CAS 号的列名 (请确保Excel里第一行是列名) -COL_CAS = "CAS Number" +```bash +# Choose an output explicitly +flavor-data pubchem compounds.xlsx --output results/compounds_enriched.xlsx -双击start3.bat +# Save a recoverable checkpoint every 10 rows; 0 disables checkpoints +flavor-data pubchem compounds.xlsx --checkpoint-every 10 + +# Replace an existing destination deliberately +flavor-data pubchem compounds.xlsx --output compounds_pubchem.xlsx --force +``` -### 功能四、根据cas号查找嗅觉阈值，气味，香型（自带手动防人机验证） -你的excel文件名 -INPUT_FILE = "Odor.xlsx" +Checkpoints are named like `compounds_pubchem.partial.xlsx`. They are refreshed during the run, preserved if processing is interrupted, and removed after the final output succeeds. Existing destinations cause the command to stop unless `--force` is supplied. Prefer a new output path instead of overwriting the input. -Excel 中存放 CAS 号的列名 (请确保Excel里第一行是列名) -COL_CAS = "CAS Number" +Successful HTTP responses and downloaded snapshots are cached under `~/.cache/flavor-data-crawler` by default. Set `--cache-dir` before the subcommand to use another location. + +## Data sources, access, and rights + +Access policies and dataset terms can change. Re-check the linked provider documentation before a live extraction, publication, or redistribution decision. This repository does not grant rights to third-party data. + +| Source | What this toolkit uses | Access and cache behavior | Rights and scientific limits | +| --- | --- | --- | --- | +| [PubChem PUG REST](https://pubchem.ncbi.nlm.nih.gov/docs/pug-rest) + [PUG-View](https://pubchem.ncbi.nlm.nih.gov/docs/pug-view) | Compound identity, selected properties, synonyms/CAS identifiers, and contributor-sourced odor annotations | Live NCBI APIs with persistent response caching. The client waits 0.25 s between uncached calls—4 requests/s, below PubChem's 5 requests/s ceiling—and retries bounded transient failures. | PubChem aggregates contributor records. Exports retain annotation source names, source URLs, and license URLs; review the [NCBI policies](https://www.ncbi.nlm.nih.gov/home/about/policies/) and each contributor's terms. | +| [NIST Chemistry WebBook, SRD 69](https://webbook.nist.gov/chemistry/) | Retention-index lookup and name-to-CAS resolution | Cached HTML with at least a 5-second delay between uncached requests, following the published [robots.txt](https://webbook.nist.gov/robots.txt). | [NIST Standard Reference Data rights](https://www.nist.gov/srd/public-law) apply. Fetch and cite records on demand; do not treat the service as a freely redistributable bulk dataset. | +| [Pyrfume Public Data Archive](https://github.com/pyrfume/pyrfume-data) | Pinned `aromadb`, `flavornet`, and `superscent` archive files keyed by PubChem CID | Explicit archive allowlist; selected files are downloaded from a pinned commit and cached locally. | Rights are **per manifest and upstream collection**. The repository's code license does not automatically license every dataset. Exports retain manifest source, notes, and license notes. | +| [M2OR](https://github.com/chemosim-lab/M2OR) | Molecule–olfactory-receptor pairs, response labels, species, receptors, and DOIs | Optional pinned CSV snapshot, approximately 43 MB, downloaded on first use and cached; it is not bundled in this repository. | The upstream snapshot is Apache-2.0. These are bioassay observations, not evidence of human perceptual quality, safety, efficacy, or clinical outcome. | +| [MFFI](https://mffi.sjtu.edu.cn/database/search) | Chinese/English names, sensory characteristics, and in-water thresholds | Interactive Selenium/Chrome access with conservative row pacing. No documented public API or rate policy was found. | No documented reuse license was found. A page being accessible—or allowed by robots rules—is not permission to republish its data. Use conservatively and cite the source. | +| [ChemicalBook](https://www.chemicalbook.com/) | Original odor description, odor threshold, and odor-type compatibility workflow | **Disabled by default and permission-gated.** Current [robots.txt](https://www.chemicalbook.com/robots.txt) excludes the search/property routes. The connector stays visible and manual; it never solves or bypasses CAPTCHA. | Run only with documented permission covering the intended automated access and reuse. `--i-have-permission` is an assertion by the operator, not permission supplied by this project. | + +## Project skill for Codex + +The repository includes a project-scoped skill at `.agents/skills/curate-flavor-data/`. In Codex, invoke: + +```text +$curate-flavor-data +``` + +The skill inspects a workbook, chooses the smallest suitable provider set, previews access and output implications, runs one focused command, and verifies row count, schema, statuses, and provenance. It is an orchestration guide around this package—not a separate scraper or an automatic grant of data rights. + +You can run its read-only workbook inspection helper directly: + +```bash +python .agents/skills/curate-flavor-data/scripts/inspect_workbook.py compounds.xlsx +``` + +## Legacy compatibility + +The original scripts and Windows launchers are retained for existing workbook layouts: + +| Launcher | Script | Expected workbook | Required column(s) | Output | +| --- | --- | --- | --- | --- | +| `start1.bat` | `nist_excel_tool.py` | `data.xlsx` | `CAS Number`, `Calculated RI` | `data_result.xlsx` | +| `start2.bat` | `name_to_cas.py` | `name.xlsx` | `Name` | `name_with_cas.xlsx` | +| `start3.bat` | `mffi_spider.py` | `max.xlsx` | `CAS Number` | `max_mffi_result.xlsx` | +| `start4.bat` | `cb_spider.py` | `Odor.xlsx` | `CAS Number` | `Odor_cb_result.xlsx` | + +The `.bat` files prefer `.venv`, then `myenv`, then `venv`, and finally the system `python`. These compatibility scripts intentionally produce the historic, provenance-free column shape and replace their fixed result files. New work should use the CLI for explicit paths and overwrite protection. MFFI and ChemicalBook require a locally available Chrome browser; ChemicalBook still requires the permission phrase. + +## Development and tests + +Install the development dependencies, then run the offline test and lint suite: + +```bash +python -m pip install -e ".[dev]" +python -m ruff check . +python -m ruff format --check . +python -m pytest +``` -双击start4.bat +CI runs those checks on Ubuntu and Windows with Python 3.11 and 3.13. Tests use fixtures or injected clients and do not depend on live provider websites or a Codex runtime. -## 信息来源网站 -https://webbook.nist.gov/chemistry/ -https://mffi.sjtu.edu.cn/database/search?value=Propanal%2C+2-methyl-&keyword=all -https://www.chemicalbook.com/ProductIndex.aspx +## Responsible use -## 注意事项 -本项目仅供学习参考，禁止用于商业用途，合理使用工具，不要给网站服务器带来太多负担 +- Review provider terms, robots policies, citation requirements, and redistribution rights for your exact use case. +- Keep request rates conservative and prefer cached results. +- Never bypass CAPTCHA, authentication, paywalls, or other access controls. +- Treat `not_found`, `invalid_input`, `http_error`, `network_error`, `parse_error`, `missing_data`, `data_error`, `partial`, `blocked`, and `skipped` as different outcomes. +- Validate provenance and biological scope before using exports in statistics, cheminformatics, or machine-learning work. diff --git a/cb_spider.py b/cb_spider.py index c06b295..68cfb02 100644 --- a/cb_spider.py +++ b/cb_spider.py @@ -1,174 +1,36 @@ -import pandas as pd -from selenium import webdriver -from selenium.webdriver.chrome.service import Service -from selenium.webdriver.common.by import By -from selenium.webdriver.support.ui import WebDriverWait -from selenium.webdriver.support import expected_conditions as EC -from webdriver_manager.chrome import ChromeDriverManager -import time -import os - -# ================= 配置区域 ================= -INPUT_FILE = "Odor.xlsx" -COL_CAS = "CAS Number" -# =========================================== - -def init_driver(): - options = webdriver.ChromeOptions() - options.add_argument("--start-maximized") - options.add_argument('--disable-gpu') - options.add_argument('--no-sandbox') - options.add_argument("--log-level=3") - options.add_argument("--disable-notifications") - options.add_argument("--blink-settings=imagesEnabled=false") - - try: - driver_path = ChromeDriverManager(url="https://npmmirror.com/metadata/chromedriver/").install() - except Exception: - driver_path = ChromeDriverManager().install() - - service = Service(driver_path) - driver = webdriver.Chrome(service=service, options=options) - return driver - -def get_chemicalbook_data_with_retry(driver, cas): - """ - 带有手动重试机制的抓取函数 - """ - while True: - try: - # 尝试执行抓取 - return _get_data_logic(driver, cas) - except Exception as e: - # 如果抓取过程中出现严重错误（通常是被验证码拦截导致超时） - print("\n" + "!"*50) - print(f"【暂停警告】在抓取 {cas} 时遇到问题！") - print("可能是出现了：1.人机验证码 2.网络断开 3.页面加载极慢") - print(f"错误信息: {e}") - print("!"*50) - - # 这里暂停，等待用户输入 - user_input = input(">>> 请去浏览器手动解决验证码，完成后在此处按回车重试 (输入 n 跳过此条): ") - - if user_input.lower().strip() == 'n': - # 如果用户输入 n，则返回空数据，跳过这条 - return { - "CB_Odor_Desc": "\\", "CB_Odor_Threshold": "\\", "CB_Odor_Type": "\\" - } - else: - # 否则继续循环，重新尝试 - print(">>> 正在重试...") - continue - -def _get_data_logic(driver, cas): - """ - 核心抓取逻辑（不含重试循环） - """ - data = { - "CB_Odor_Desc": "\\", - "CB_Odor_Threshold": "\\", - "CB_Odor_Type": "\\" - } - - # 1. 访问搜索页 - url = f"https://www.chemicalbook.com/Search.aspx?keyword={cas}" - driver.get(url) - - # 这里我们缩短一点超时时间，方便更快触发“手动验证”提示 - wait = WebDriverWait(driver, 5) - - # 2. 点击链接 - try: - xpath_query = "//a[text()='化学性质' or contains(@href, 'ProductChemicalProperties')]" - target_link = wait.until(EC.element_to_be_clickable((By.XPATH, xpath_query))) - target_link.click() - except Exception: - # 如果找不到链接，这里抛出异常，触发外层的“重试询问” - # 只有确实搜不到的，用户手动确认后可以选择跳过 - raise Exception("未找到'化学性质'链接，可能是被验证码拦截或无数据") - - # 3. 切换窗口 - windows = driver.window_handles - driver.switch_to.window(windows[-1]) - - # 4. 等待详情页 - wait.until(EC.presence_of_element_located((By.TAG_NAME, "th"))) - - # 5. 抓取数据 - try: - ele = driver.find_element(By.XPATH, "//th[contains(text(), '气味')]/following-sibling::td") - data["CB_Odor_Desc"] = ele.text.strip() - except: pass - - try: - ele = driver.find_element(By.XPATH, "//th[contains(text(), '嗅觉阈值')]/following-sibling::td") - data["CB_Odor_Threshold"] = ele.text.strip() - except: pass - - try: - ele = driver.find_element(By.XPATH, "//th[contains(text(), '香型')]/following-sibling::td") - data["CB_Odor_Type"] = ele.text.strip() - except: pass - - # 关闭窗口 - if len(windows) > 1: - driver.close() - driver.switch_to.window(windows[0]) - - return data - -def main(): - print(f"读取文件: {INPUT_FILE} ...") - if not os.path.exists(INPUT_FILE): - print(f"文件不存在: {INPUT_FILE}") - return - - try: - df = pd.read_excel(INPUT_FILE, dtype={COL_CAS: str}) - except Exception as e: - print(f"Excel 读取失败: {e}") - return - - print("正在启动浏览器...") - driver = init_driver() - - try: - total = len(df) - print(f"开始任务，共 {total} 条") - print("-" * 50) - - for index, row in df.iterrows(): - cas = row[COL_CAS] - if pd.isna(cas) or str(cas).strip() == "": continue - cas = str(cas).strip() - - print(f"[{index+1}/{total}] 搜索: {cas} ... ", end="", flush=True) - - # 使用带有重试机制的函数 - result = get_chemicalbook_data_with_retry(driver, cas) - - df.at[index, "CB_Odor_Desc"] = result["CB_Odor_Desc"] - df.at[index, "CB_Odor_Threshold"] = result["CB_Odor_Threshold"] - df.at[index, "CB_Odor_Type"] = result["CB_Odor_Type"] - - found = [] - if result["CB_Odor_Desc"] != "\\": found.append("气味") - if result["CB_Odor_Threshold"] != "\\": found.append("阈值") - if result["CB_Odor_Type"] != "\\": found.append("香型") - - if found: - print(f"成功 ({', '.join(found)})") - else: - print("未找到数据 (或跳过)") - - time.sleep(1.5) - - output_file = INPUT_FILE.replace(".xlsx", "_cb_result.xlsx") - df.to_excel(output_file, index=False) - print(f"完成！保存至: {output_file}") - - finally: - driver.quit() - -if __name__ == "__main__": - main() \ No newline at end of file +"""Permission-gated legacy launcher for the original ChemicalBook workflow.""" + +from flavor_data_crawler.sources.chemicalbook import ( + PERMISSION_PHRASE, + ChemicalBookLegacyClient, +) +from flavor_data_crawler.workflows import run_chemicalbook_legacy + +INPUT_FILE = "Odor.xlsx" +COL_CAS = "CAS Number" + + +def main() -> None: + print( + "ChemicalBook currently disallows these automated routes in robots.txt. " + "Continue only if you have documented permission." + ) + if input(f"Type {PERMISSION_PHRASE!r} to continue: ").strip() != PERMISSION_PHRASE: + print("Cancelled.") + return + output = INPUT_FILE.replace(".xlsx", "_cb_result.xlsx") + with ChemicalBookLegacyClient(permission_confirmed=True) as client: + summary = run_chemicalbook_legacy( + INPUT_FILE, + client, + output_path=output, + cas_column=COL_CAS, + include_provenance=False, + checkpoint_every=5, + force=True, + ) + print(f"Done: {summary.output_path}") + + +if __name__ == "__main__": + main() diff --git a/flavor_data_crawler/__init__.py b/flavor_data_crawler/__init__.py new file mode 100644 index 0000000..4e0b9b5 --- /dev/null +++ b/flavor_data_crawler/__init__.py @@ -0,0 +1,6 @@ +"""Flavor Data Crawler public package.""" + +from flavor_data_crawler.models import LookupResult + +__all__ = ["LookupResult"] +__version__ = "0.2.0" diff --git a/flavor_data_crawler/cli.py b/flavor_data_crawler/cli.py new file mode 100644 index 0000000..d5606d8 --- /dev/null +++ b/flavor_data_crawler/cli.py @@ -0,0 +1,288 @@ +"""Command-line interface for Flavor Data Crawler.""" + +from __future__ import annotations + +import argparse +import sys +from collections.abc import Sequence +from pathlib import Path +from typing import Any + +from flavor_data_crawler import __version__ +from flavor_data_crawler.sources.chemicalbook import ( + PERMISSION_PHRASE, + ChemicalBookLegacyClient, +) +from flavor_data_crawler.sources.mffi import MffiClient +from flavor_data_crawler.workflows import ( + RunSummary, + run_chemicalbook_legacy, + run_m2or, + run_mffi, + run_nist_ri, + run_pubchem, + run_pyrfume, + run_resolve_cas, +) + +SOURCE_TABLE = """\ +Provider Access mode Role Default +PubChem PUG REST/PUG-View Identity, properties, sourced odor text yes +NIST WebBook cached HTML Retention index and name-to-CAS explicit +Pyrfume pinned archive files Curated descriptor collections explicit +M2OR cached 43 MB CSV Odorant-receptor bioassay evidence explicit +MFFI visible browser Bilingual sensory fields and thresholds explicit +ChemicalBook permission-gated Legacy manual compatibility only disabled + +Run `flavor-data --help` for source-specific controls. Data access and +redistribution rights remain source-specific; exported rows retain provenance. +""" + + +def _add_table_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument("input", type=Path, help="Input .xlsx, .csv, or .tsv file") + parser.add_argument( + "-o", "--output", type=Path, help="Output path (defaults to a sibling file)" + ) + parser.add_argument("--force", action="store_true", help="Replace an existing output file") + parser.add_argument( + "--no-provenance", + action="store_true", + help="Omit status and source metadata columns (legacy-style output)", + ) + parser.add_argument( + "--checkpoint-every", + type=int, + default=25, + metavar="N", + help="Write a recoverable partial output every N records; 0 disables it", + ) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="flavor-data", + description=( + "Collect and normalize retention-index, odorant, and olfactory-receptor data " + "with source-level provenance." + ), + ) + parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}") + parser.add_argument("--cache-dir", type=Path, help="Override the provider cache directory") + parser.add_argument( + "--timeout", type=float, default=20, help="Network/browser timeout in seconds" + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + for name in ("sources", "providers"): + source_parser = subparsers.add_parser( + name, help="List provider capabilities and access modes" + ) + source_parser.set_defaults(handler=_handle_sources) + + nist = subparsers.add_parser("nist-ri", help="Match the nearest NIST retention index") + _add_table_arguments(nist) + nist.add_argument("--cas-column", default="CAS Number") + nist.add_argument("--calculated-ri-column", default="Calculated RI") + nist.add_argument("--result-column", default="NIST RI") + nist.set_defaults(handler=_handle_nist_ri) + + resolve = subparsers.add_parser("resolve-cas", help="Resolve compound names through NIST") + _add_table_arguments(resolve) + resolve.add_argument("--name-column", default="Name") + resolve.add_argument("--result-column", default="Found CAS") + resolve.set_defaults(handler=_handle_resolve_cas) + + pubchem = subparsers.add_parser( + "pubchem", help="Add PubChem identifiers, properties, and sourced odor annotations" + ) + _add_table_arguments(pubchem) + pubchem.add_argument("--identifier-column", default="CAS Number") + pubchem.add_argument("--no-odor", action="store_true", help="Skip PUG-View odor annotations") + pubchem.set_defaults(handler=_handle_pubchem) + + pyrfume = subparsers.add_parser( + "pyrfume", help="Match CIDs against selected pinned Pyrfume archive collections" + ) + _add_table_arguments(pyrfume) + pyrfume.add_argument("--cid-column", default="PubChem CID") + pyrfume.add_argument("--identifier-column", default="CAS Number") + pyrfume.add_argument( + "--archives", + default="aromadb,superscent", + help="Comma-separated allowlisted archives (aromadb, flavornet, superscent)", + ) + pyrfume.set_defaults(handler=_handle_pyrfume) + + m2or = subparsers.add_parser( + "m2or", help="Add optional olfactory-receptor evidence from the cached M2OR snapshot" + ) + _add_table_arguments(m2or) + m2or.add_argument("--cas-column", default="CAS Number") + m2or.set_defaults(handler=_handle_m2or) + + mffi = subparsers.add_parser("mffi", help="Run the visible-browser MFFI compatibility source") + _add_table_arguments(mffi) + mffi.add_argument("--cas-column", default="CAS Number") + mffi.add_argument("--headless", action="store_true") + mffi.set_defaults(handler=_handle_mffi) + + chemicalbook = subparsers.add_parser( + "chemicalbook-legacy", + help="Run the permission-gated, manual ChemicalBook compatibility source", + ) + _add_table_arguments(chemicalbook) + chemicalbook.add_argument("--cas-column", default="CAS Number") + chemicalbook.add_argument( + "--i-have-permission", + action="store_true", + help="Assert that your use is authorized despite current robots exclusions", + ) + chemicalbook.set_defaults(handler=_handle_chemicalbook) + return parser + + +def _common_kwargs(args: argparse.Namespace) -> dict[str, Any]: + return { + "output_path": args.output, + "include_provenance": not args.no_provenance, + "checkpoint_every": args.checkpoint_every, + "force": args.force, + } + + +def _http_kwargs(args: argparse.Namespace) -> dict[str, Any]: + values: dict[str, Any] = {"timeout": args.timeout} + if args.cache_dir is not None: + values["cache_dir"] = args.cache_dir + return values + + +def _handle_sources(_: argparse.Namespace) -> int: + print(SOURCE_TABLE) + return 0 + + +def _handle_nist_ri(args: argparse.Namespace) -> RunSummary: + from flavor_data_crawler.sources.nist import NistWebBookClient + + client = NistWebBookClient(**_http_kwargs(args)) + return run_nist_ri( + args.input, + client, + cas_column=args.cas_column, + calculated_ri_column=args.calculated_ri_column, + result_column=args.result_column, + **_common_kwargs(args), + ) + + +def _handle_resolve_cas(args: argparse.Namespace) -> RunSummary: + from flavor_data_crawler.sources.nist import NistWebBookClient + + client = NistWebBookClient(**_http_kwargs(args)) + return run_resolve_cas( + args.input, + client, + name_column=args.name_column, + result_column=args.result_column, + **_common_kwargs(args), + ) + + +def _handle_pubchem(args: argparse.Namespace) -> RunSummary: + from flavor_data_crawler.sources.pubchem import PubChemClient + + client = PubChemClient(**_http_kwargs(args)) + return run_pubchem( + args.input, + client, + identifier_column=args.identifier_column, + include_odor=not args.no_odor, + **_common_kwargs(args), + ) + + +def _handle_pyrfume(args: argparse.Namespace) -> RunSummary: + from flavor_data_crawler.sources.pubchem import PubChemClient + from flavor_data_crawler.sources.pyrfume import PyrfumeArchiveClient + + archive_client = PyrfumeArchiveClient(cache_dir=args.cache_dir, timeout=args.timeout) + pubchem_client = PubChemClient(**_http_kwargs(args)) + archives = [name.strip() for name in args.archives.split(",") if name.strip()] + return run_pyrfume( + args.input, + archive_client, + pubchem_client=pubchem_client, + cid_column=args.cid_column, + identifier_column=args.identifier_column, + archives=archives, + **_common_kwargs(args), + ) + + +def _handle_m2or(args: argparse.Namespace) -> RunSummary: + from flavor_data_crawler.sources.m2or import M2ORClient + + client = M2ORClient(cache_dir=args.cache_dir, timeout=args.timeout) + return run_m2or(args.input, client, cas_column=args.cas_column, **_common_kwargs(args)) + + +def _handle_mffi(args: argparse.Namespace) -> RunSummary: + with MffiClient(timeout=args.timeout, headless=args.headless) as client: + return run_mffi(args.input, client, cas_column=args.cas_column, **_common_kwargs(args)) + + +def _confirm_chemicalbook_permission(args: argparse.Namespace) -> bool: + if args.i_have_permission: + return True + print( + "ChemicalBook currently excludes the automated search/property routes in robots.txt.\n" + "This compatibility connector is not a CAPTCHA bypass and must only be used with " + "documented permission." + ) + return input(f"Type {PERMISSION_PHRASE!r} to continue: ").strip() == PERMISSION_PHRASE + + +def _handle_chemicalbook(args: argparse.Namespace) -> RunSummary | int: + if not _confirm_chemicalbook_permission(args): + print("ChemicalBook compatibility run cancelled.", file=sys.stderr) + return 2 + with ChemicalBookLegacyClient( + permission_confirmed=True, + timeout=args.timeout, + ) as client: + return run_chemicalbook_legacy( + args.input, + client, + cas_column=args.cas_column, + **_common_kwargs(args), + ) + + +def main(argv: Sequence[str] | None = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + try: + if hasattr(args, "input") and not args.input.is_file(): + raise FileNotFoundError(f"Input file does not exist: {args.input}") + result = args.handler(args) + if isinstance(result, RunSummary): + counts = ", ".join(f"{key}={value}" for key, value in result.status_counts.items()) + print(f"Saved {result.rows} rows to {result.output_path}") + print(f"Statuses: {counts or 'none'}") + return 0 + return int(result or 0) + except (FileNotFoundError, FileExistsError, ValueError) as exc: + print(f"Error: {exc}", file=sys.stderr) + return 2 + except KeyboardInterrupt: + print( + "Stopped by user. The latest .partial output is preserved when available.", + file=sys.stderr, + ) + return 130 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/flavor_data_crawler/excel_io.py b/flavor_data_crawler/excel_io.py new file mode 100644 index 0000000..0165e8d --- /dev/null +++ b/flavor_data_crawler/excel_io.py @@ -0,0 +1,96 @@ +"""Safe tabular input and output helpers.""" + +from __future__ import annotations + +import os +import tempfile +from pathlib import Path +from typing import Any + +import pandas as pd + +SUPPORTED_INPUTS = {".csv", ".tsv", ".xlsx"} +FORMULA_PREFIXES = ("=", "+", "-", "@") + + +def read_table(path: str | Path) -> pd.DataFrame: + """Read a supported workbook or delimited text file.""" + + source = Path(path) + if not source.is_file(): + raise FileNotFoundError(f"Input file does not exist: {source}") + suffix = source.suffix.lower() + if suffix not in SUPPORTED_INPUTS: + supported = ", ".join(sorted(SUPPORTED_INPUTS)) + raise ValueError(f"Unsupported input type {suffix!r}; expected one of: {supported}") + if suffix == ".xlsx": + return pd.read_excel(source, dtype=object) + separator = "\t" if suffix == ".tsv" else "," + return pd.read_csv(source, sep=separator, dtype=object, keep_default_na=False) + + +def require_columns(frame: pd.DataFrame, *columns: str) -> None: + """Raise once with every missing column name.""" + + missing = [column for column in columns if column not in frame.columns] + if missing: + available = ", ".join(map(str, frame.columns)) + message = f"Missing required column(s): {', '.join(missing)}. Available: {available}" + raise ValueError(message) + + +def derive_output_path(input_path: str | Path, suffix: str) -> Path: + """Create a sibling output path without touching the source file.""" + + source = Path(input_path) + return source.with_name(f"{source.stem}{suffix}{source.suffix}") + + +def sanitize_excel_cell(value: Any) -> Any: + """Prevent untrusted text from becoming an Excel formula.""" + + if isinstance(value, str) and value.startswith(FORMULA_PREFIXES): + return f"'{value}" + return value + + +def sanitize_frame(frame: pd.DataFrame) -> pd.DataFrame: + """Return a copy whose string cells are safe to open in spreadsheet software.""" + + safe = frame.copy() + for column in safe.columns: + safe[column] = safe[column].map(sanitize_excel_cell) + return safe + + +def write_table(frame: pd.DataFrame, path: str | Path, *, force: bool = False) -> Path: + """Atomically write a table, refusing accidental overwrite by default.""" + + destination = Path(path) + suffix = destination.suffix.lower() + if suffix not in SUPPORTED_INPUTS: + supported = ", ".join(sorted(SUPPORTED_INPUTS)) + raise ValueError(f"Unsupported output type {suffix!r}; expected one of: {supported}") + if destination.exists() and not force: + raise FileExistsError(f"Output already exists: {destination}. Pass --force to replace it.") + destination.parent.mkdir(parents=True, exist_ok=True) + safe = sanitize_frame(frame) + handle = tempfile.NamedTemporaryFile( + prefix=f".{destination.stem}-", + suffix=destination.suffix, + dir=destination.parent, + delete=False, + ) + temporary = Path(handle.name) + handle.close() + try: + if suffix == ".xlsx": + safe.to_excel(temporary, index=False) + else: + separator = "\t" if suffix == ".tsv" else "," + safe.to_csv(temporary, sep=separator, index=False, encoding="utf-8-sig") + os.replace(temporary, destination) + except Exception: + temporary.unlink(missing_ok=True) + raise + return destination diff --git a/flavor_data_crawler/http.py b/flavor_data_crawler/http.py new file mode 100644 index 0000000..9dddd1e --- /dev/null +++ b/flavor_data_crawler/http.py @@ -0,0 +1,362 @@ +"""Polite, retrying HTTP access with a small transparent disk cache.""" + +from __future__ import annotations + +import hashlib +import json +import os +import tempfile +import threading +import time +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from datetime import UTC, datetime +from email.utils import parsedate_to_datetime +from pathlib import Path +from typing import Any + +import requests + +from flavor_data_crawler.models import utc_now_iso + +DEFAULT_CACHE_DIR = Path( + os.environ.get( + "FLAVOR_DATA_CACHE_DIR", + Path.home() / ".cache" / "flavor-data-crawler" / "http", + ) +) +DEFAULT_USER_AGENT = ( + "Flavor-Data-Crawler/0.2 " + "(+https://github.com/rastagan-git/Flavor-Data-Crawler; research client)" +) +RETRYABLE_STATUS_CODES = frozenset({408, 425, 429, 500, 502, 503, 504}) +CACHE_FORMAT_VERSION = 1 + +Params = Mapping[str, object] | Sequence[tuple[str, object]] | None +Timeout = float | tuple[float, float] + + +@dataclass(frozen=True, slots=True) +class RetrievalMetadata: + """Metadata describing how and when an HTTP representation was retrieved.""" + + requested_url: str + url: str + retrieved_at: str + status_code: int + cache_hit: bool + attempts: int + + +@dataclass(frozen=True, slots=True) +class HttpResponse: + """Serializable subset of :class:`requests.Response` used by adapters.""" + + status_code: int + text: str + headers: Mapping[str, str] + metadata: RetrievalMetadata + + @property + def url(self) -> str: + return self.metadata.url + + @property + def ok(self) -> bool: + return 200 <= self.status_code < 400 + + def json(self) -> Any: + """Decode the response body as JSON.""" + + return json.loads(self.text) + + def raise_for_status(self) -> None: + """Raise a requests-compatible error for non-success responses.""" + + if not self.ok: + raise requests.HTTPError( + f"{self.status_code} response for {self.url}", + response=self, + ) + + +class HttpClientError(RuntimeError): + """Raised when every request attempt fails before receiving a response.""" + + def __init__(self, url: str, attempts: int, cause: BaseException) -> None: + super().__init__(f"Request failed after {attempts} attempt(s): {url}: {cause}") + self.url = url + self.attempts = attempts + self.__cause__ = cause + + +class CachedHttpClient: + """A requests-based client with caching, throttling, and bounded retries. + + Only successful GET responses are cached. Cache files contain plain JSON and + are written atomically, so an interrupted run cannot leave a half-written + response behind. ``session``, ``sleep``, and clocks are injectable to keep + provider tests fully offline and deterministic. + """ + + def __init__( + self, + *, + cache_dir: str | Path | None = DEFAULT_CACHE_DIR, + cache_ttl: float | None = 7 * 24 * 60 * 60, + timeout: Timeout = (5.0, 30.0), + min_interval: float = 0.0, + max_retries: int = 3, + backoff_factor: float = 0.5, + session: requests.Session | Any | None = None, + sleep: Callable[[float], None] = time.sleep, + monotonic: Callable[[], float] = time.monotonic, + wall_clock: Callable[[], float] = time.time, + user_agent: str = DEFAULT_USER_AGENT, + ) -> None: + if cache_ttl is not None and cache_ttl < 0: + raise ValueError("cache_ttl must be non-negative or None") + if min_interval < 0: + raise ValueError("min_interval must be non-negative") + if max_retries < 0: + raise ValueError("max_retries must be non-negative") + if backoff_factor < 0: + raise ValueError("backoff_factor must be non-negative") + + self.cache_dir = Path(cache_dir) if cache_dir is not None else None + self.cache_ttl = cache_ttl + self.timeout = timeout + self.min_interval = float(min_interval) + self.max_retries = max_retries + self.backoff_factor = float(backoff_factor) + self.session = session if session is not None else requests.Session() + self.sleep = sleep + self._monotonic = monotonic + self._wall_clock = wall_clock + self._default_headers = {"User-Agent": user_agent} + self._last_request_at: float | None = None + self._throttle_lock = threading.Lock() + + def get( + self, + url: str, + *, + params: Params = None, + headers: Mapping[str, str] | None = None, + timeout: Timeout | None = None, + use_cache: bool = True, + force_refresh: bool = False, + ) -> HttpResponse: + """Retrieve a URL, using a cached successful representation when valid.""" + + return self.request( + "GET", + url, + params=params, + headers=headers, + timeout=timeout, + use_cache=use_cache, + force_refresh=force_refresh, + ) + + def request( + self, + method: str, + url: str, + *, + params: Params = None, + headers: Mapping[str, str] | None = None, + timeout: Timeout | None = None, + use_cache: bool = True, + force_refresh: bool = False, + **kwargs: Any, + ) -> HttpResponse: + """Issue an HTTP request and return its body with retrieval metadata.""" + + normalized_method = method.upper() + requested_url = self._prepare_url(normalized_method, url, params) + cacheable = normalized_method == "GET" and use_cache and self.cache_dir is not None + cache_path = self._cache_path(normalized_method, requested_url) if cacheable else None + + if cache_path is not None and not force_refresh: + cached = self._read_cache(cache_path, requested_url) + if cached is not None: + return cached + + request_headers = {**self._default_headers, **dict(headers or {})} + request_timeout = self.timeout if timeout is None else timeout + last_exception: BaseException | None = None + + for attempt in range(1, self.max_retries + 2): + self._throttle() + try: + raw_response = self.session.request( + normalized_method, + url, + params=params, + headers=request_headers, + timeout=request_timeout, + **kwargs, + ) + except requests.RequestException as exc: + last_exception = exc + if attempt > self.max_retries: + raise HttpClientError(requested_url, attempt, exc) from exc + self.sleep(self._backoff_delay(attempt)) + continue + + status_code = int(raw_response.status_code) + final_url = str(getattr(raw_response, "url", "") or requested_url) + metadata = RetrievalMetadata( + requested_url=requested_url, + url=final_url, + retrieved_at=utc_now_iso(), + status_code=status_code, + cache_hit=False, + attempts=attempt, + ) + response = HttpResponse( + status_code=status_code, + text=str(raw_response.text), + headers={str(key): str(value) for key, value in raw_response.headers.items()}, + metadata=metadata, + ) + + if status_code in RETRYABLE_STATUS_CODES and attempt <= self.max_retries: + retry_after = self._retry_after_seconds(response.headers) + self.sleep(max(self._backoff_delay(attempt), retry_after)) + continue + + if cache_path is not None and response.ok: + self._write_cache(cache_path, response) + return response + + # The loop always returns or raises. This guard makes that invariant explicit. + assert last_exception is not None # pragma: no cover + raise HttpClientError( # pragma: no cover + requested_url, + self.max_retries + 1, + last_exception, + ) + + @staticmethod + def _prepare_url(method: str, url: str, params: Params) -> str: + stable_params: Params + if isinstance(params, Mapping): + stable_params = sorted(params.items(), key=lambda item: item[0]) + else: + stable_params = params + prepared = requests.Request(method=method, url=url, params=stable_params).prepare() + return str(prepared.url or url) + + def _throttle(self) -> None: + if self.min_interval <= 0: + return + with self._throttle_lock: + now = self._monotonic() + if self._last_request_at is None: + self._last_request_at = now + return + earliest = self._last_request_at + self.min_interval + if now < earliest: + self.sleep(earliest - now) + # Deterministic fake sleepers may not advance their paired clock. + now = max(self._monotonic(), earliest) + self._last_request_at = now + + def _backoff_delay(self, attempt: int) -> float: + return self.backoff_factor * (2 ** (attempt - 1)) + + @staticmethod + def _retry_after_seconds(headers: Mapping[str, str]) -> float: + raw = next( + (value for key, value in headers.items() if key.casefold() == "retry-after"), + "", + ).strip() + if not raw: + return 0.0 + try: + return max(0.0, float(raw)) + except ValueError: + try: + when = parsedate_to_datetime(raw) + if when.tzinfo is None: + when = when.replace(tzinfo=UTC) + return max(0.0, (when - datetime.now(UTC)).total_seconds()) + except (TypeError, ValueError, OverflowError): + return 0.0 + + def _cache_path(self, method: str, requested_url: str) -> Path: + assert self.cache_dir is not None + digest = hashlib.sha256(f"{method}\n{requested_url}".encode()).hexdigest() + return self.cache_dir / f"{digest}.json" + + def _read_cache(self, path: Path, requested_url: str) -> HttpResponse | None: + try: + payload = json.loads(path.read_text(encoding="utf-8")) + if payload["format_version"] != CACHE_FORMAT_VERSION: + return None + age = self._wall_clock() - float(payload["stored_at"]) + if self.cache_ttl is not None and age > self.cache_ttl: + return None + status_code = int(payload["status_code"]) + final_url = str(payload["url"]) + return HttpResponse( + status_code=status_code, + text=str(payload["text"]), + headers={str(key): str(value) for key, value in payload["headers"].items()}, + metadata=RetrievalMetadata( + requested_url=requested_url, + url=final_url, + retrieved_at=str(payload["retrieved_at"]), + status_code=status_code, + cache_hit=True, + attempts=0, + ), + ) + except (OSError, json.JSONDecodeError, KeyError, TypeError, ValueError): + return None + + def _write_cache(self, path: Path, response: HttpResponse) -> None: + payload = { + "format_version": CACHE_FORMAT_VERSION, + "stored_at": self._wall_clock(), + "status_code": response.status_code, + "url": response.url, + "retrieved_at": response.metadata.retrieved_at, + "headers": dict(response.headers), + "text": response.text, + } + temporary: Path | None = None + try: + path.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile( + mode="w", + encoding="utf-8", + prefix=f".{path.stem}-", + suffix=".tmp", + dir=path.parent, + delete=False, + ) as handle: + json.dump(payload, handle, ensure_ascii=False, separators=(",", ":")) + temporary = Path(handle.name) + os.replace(temporary, path) + temporary = None + except OSError: + # A read-only or full cache directory must not break data retrieval. + return + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) + + +# A shorter alias is convenient for callers that do not care about implementation details. +PoliteHttpClient = CachedHttpClient + +__all__ = [ + "CachedHttpClient", + "HttpClientError", + "HttpResponse", + "PoliteHttpClient", + "RetrievalMetadata", +] diff --git a/flavor_data_crawler/identifiers.py b/flavor_data_crawler/identifiers.py new file mode 100644 index 0000000..063849c --- /dev/null +++ b/flavor_data_crawler/identifiers.py @@ -0,0 +1,63 @@ +"""Chemical identifier normalization and validation.""" + +from __future__ import annotations + +import re + +CAS_PATTERN = re.compile(r"^(\d{2,7})-(\d{2})-(\d)$") +INVALID_SENTINELS = { + "", + "\\", + "n/a", + "na", + "nan", + "none", + "null", + "not found", + "ambiguous/list found", +} + + +def clean_text(value: object) -> str: + """Normalize spreadsheet text without changing its semantic content.""" + + if value is None: + return "" + text = str(value).replace("\u00a0", " ").strip() + return "" if text.casefold() in INVALID_SENTINELS else text + + +def normalize_cas(value: object) -> str: + """Return a hyphenated CAS number when the input shape is recognizable.""" + + text = clean_text(value).replace(" ", "") + if not text: + return "" + if text.isdigit() and 5 <= len(text) <= 10: + return f"{text[:-3]}-{text[-3:-1]}-{text[-1]}" + match = re.fullmatch(r"(\d{2,7})[-–—](\d{2})[-–—](\d)", text) + return "-".join(match.groups()) if match else text + + +def is_valid_cas(value: object) -> bool: + """Validate CAS syntax and checksum.""" + + cas = normalize_cas(value) + match = CAS_PATTERN.fullmatch(cas) + if not match: + return False + body = "".join(match.groups()[:2]) + check_digit = int(match.group(3)) + checksum = sum(multiplier * int(digit) for multiplier, digit in enumerate(reversed(body), 1)) + return checksum % 10 == check_digit + + +def require_valid_cas(value: object) -> str: + """Return a normalized CAS number or raise a descriptive ValueError.""" + + cas = normalize_cas(value) + if not cas: + raise ValueError("CAS number is empty") + if not is_valid_cas(cas): + raise ValueError(f"Invalid CAS number or checksum: {cas}") + return cas diff --git a/flavor_data_crawler/models.py b/flavor_data_crawler/models.py new file mode 100644 index 0000000..2e1c14f --- /dev/null +++ b/flavor_data_crawler/models.py @@ -0,0 +1,58 @@ +"""Shared result and provenance models.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import UTC, datetime +from typing import Any + + +def utc_now_iso() -> str: + """Return a stable, timezone-aware retrieval timestamp.""" + + return datetime.now(UTC).replace(microsecond=0).isoformat() + + +@dataclass(slots=True) +class LookupResult: + """A provider response plus the minimum provenance needed for an export.""" + + provider: str + values: dict[str, Any] = field(default_factory=dict) + source_url: str = "" + retrieved_at: str = field(default_factory=utc_now_iso) + status: str = "ok" + message: str = "" + cache_hit: bool = False + version: str = "" + license_url: str = "" + + @classmethod + def failure( + cls, + provider: str, + *, + status: str, + message: str, + source_url: str = "", + ) -> LookupResult: + return cls( + provider=provider, + status=status, + message=message, + source_url=source_url, + ) + + def provenance_columns(self, prefix: str | None = None) -> dict[str, Any]: + """Flatten provenance for CSV or workbook output.""" + + label = prefix or self.provider + return { + f"{label} Status": self.status, + f"{label} Source URL": self.source_url, + f"{label} Retrieved At": self.retrieved_at, + f"{label} Cache Hit": self.cache_hit, + f"{label} Version": self.version, + f"{label} License URL": self.license_url, + f"{label} Message": self.message, + } diff --git a/flavor_data_crawler/sources/__init__.py b/flavor_data_crawler/sources/__init__.py new file mode 100644 index 0000000..edf1287 --- /dev/null +++ b/flavor_data_crawler/sources/__init__.py @@ -0,0 +1 @@ +"""Data-source adapters used by Flavor Data Crawler.""" diff --git a/flavor_data_crawler/sources/chemicalbook.py b/flavor_data_crawler/sources/chemicalbook.py new file mode 100644 index 0000000..d793e01 --- /dev/null +++ b/flavor_data_crawler/sources/chemicalbook.py @@ -0,0 +1,127 @@ +"""Permission-gated ChemicalBook browser compatibility adapter. + +ChemicalBook currently disallows the routes used here in robots.txt. This module +therefore never runs unless a user explicitly confirms they have permission. +""" + +from __future__ import annotations + +from urllib.parse import quote_plus + +from selenium import webdriver +from selenium.common.exceptions import TimeoutException, WebDriverException +from selenium.webdriver.common.by import By +from selenium.webdriver.support import expected_conditions as EC +from selenium.webdriver.support.ui import WebDriverWait + +from flavor_data_crawler.identifiers import require_valid_cas +from flavor_data_crawler.models import LookupResult +from flavor_data_crawler.sources.mffi import create_chrome_driver + +CHEMICALBOOK_SEARCH_URL = "https://www.chemicalbook.com/Search.aspx" +CHEMICALBOOK_ROBOTS_URL = "https://www.chemicalbook.com/robots.txt" +PERMISSION_PHRASE = "I HAVE PERMISSION" + + +class ManualVerificationRequired(RuntimeError): + """Signal that the visible browser needs user inspection or CAPTCHA handling.""" + + +class ChemicalBookLegacyClient: + """Retain the original lookup as an explicit, interactive compatibility mode.""" + + def __init__( + self, + driver: webdriver.Chrome | None = None, + *, + permission_confirmed: bool = False, + timeout: float = 8, + ) -> None: + self.permission_confirmed = permission_confirmed + self.driver = driver + self._owns_driver = driver is None + self.timeout = timeout + + def close(self) -> None: + if self._owns_driver and self.driver is not None: + self.driver.quit() + + def __enter__(self) -> ChemicalBookLegacyClient: + return self + + def __exit__(self, *_: object) -> None: + self.close() + + def lookup_cas(self, cas: object) -> LookupResult: + if not self.permission_confirmed: + return LookupResult.failure( + "ChemicalBook", + status="blocked", + message=( + "Automated access is disabled because the current robots.txt disallows the " + "search and product-property routes. Run only with documented permission." + ), + source_url=CHEMICALBOOK_ROBOTS_URL, + ) + try: + normalized = require_valid_cas(cas) + except ValueError as exc: + return LookupResult.failure("ChemicalBook", status="invalid_input", message=str(exc)) + if self.driver is None: + self.driver = create_chrome_driver(headless=False) + + url = f"{CHEMICALBOOK_SEARCH_URL}?keyword={quote_plus(normalized)}" + try: + self.driver.get(url) + wait = WebDriverWait(self.driver, self.timeout) + selector = "//a[text()='化学性质' or contains(@href, 'ProductChemicalProperties')]" + try: + wait.until(EC.element_to_be_clickable((By.XPATH, selector))).click() + except TimeoutException as exc: + raise ManualVerificationRequired( + "The property link was not available. Inspect the visible browser for a " + "CAPTCHA, access block, slow page, or a genuine no-result response." + ) from exc + + windows = self.driver.window_handles + self.driver.switch_to.window(windows[-1]) + wait.until(EC.presence_of_element_located((By.TAG_NAME, "th"))) + detail_url = self.driver.current_url + values = { + "CB_Odor_Desc": self._text_or_sentinel( + "//th[contains(text(), '气味')]/following-sibling::td" + ), + "CB_Odor_Threshold": self._text_or_sentinel( + "//th[contains(text(), '嗅觉阈值')]/following-sibling::td" + ), + "CB_Odor_Type": self._text_or_sentinel( + "//th[contains(text(), '香型')]/following-sibling::td" + ), + } + if len(windows) > 1: + self.driver.close() + self.driver.switch_to.window(windows[0]) + status = "ok" if any(value != "\\" for value in values.values()) else "not_found" + return LookupResult( + provider="ChemicalBook", + values=values, + source_url=detail_url if status == "ok" else url, + status=status, + message="" if status == "ok" else f"No odor fields found for {normalized}", + ) + except ManualVerificationRequired: + raise + except (TimeoutException, WebDriverException) as exc: + return LookupResult.failure( + "ChemicalBook", + status="network_error", + message=str(exc), + source_url=url, + ) + + def _text_or_sentinel(self, xpath: str) -> str: + try: + value = self.driver.find_element(By.XPATH, xpath).text.strip() + except Exception: + value = "" + return value or "\\" diff --git a/flavor_data_crawler/sources/m2or.py b/flavor_data_crawler/sources/m2or.py new file mode 100644 index 0000000..4b47799 --- /dev/null +++ b/flavor_data_crawler/sources/m2or.py @@ -0,0 +1,258 @@ +"""Pinned, cache-backed access to the M2OR odorant/receptor snapshot.""" + +from __future__ import annotations + +import csv +import os +import re +from collections.abc import Callable, Iterable, Mapping +from pathlib import Path +from typing import Any + +import requests + +from flavor_data_crawler.identifiers import normalize_cas, require_valid_cas +from flavor_data_crawler.models import LookupResult + +M2OR_VERSION = "b5cf2f1714b28e591b07539559363d0a6205cf2a" +M2OR_FILENAME = "M2OR_20230428.csv" +M2OR_REPOSITORY_URL = "https://github.com/chemosim-lab/M2OR" +M2OR_DATA_URL = ( + f"https://raw.githubusercontent.com/chemosim-lab/M2OR/{M2OR_VERSION}/{M2OR_FILENAME}" +) +M2OR_LICENSE_URL = "https://www.apache.org/licenses/LICENSE-2.0" + +DownloadFunction = Callable[[str, Path], bytes | str | None] + + +def _default_cache_dir() -> Path: + configured = os.environ.get("FLAVOR_DATA_CRAWLER_CACHE") + if configured: + return Path(configured).expanduser() + return Path.home() / ".cache" / "flavor-data-crawler" + + +def _clean(value: object) -> str: + return "" if value is None else str(value).strip() + + +def _is_responsive(value: object) -> bool: + return _clean(value).casefold() in {"1", "true", "yes", "responsive", "agonist"} + + +def _canonical_doi(value: object) -> str: + doi = _clean(value) + if not doi: + return "" + doi = re.sub(r"^doi\s*:\s*", "", doi, flags=re.IGNORECASE) + doi = re.sub(r"^https?://(?:dx\.)?doi\.org/", "", doi, flags=re.IGNORECASE) + return doi.strip() + + +def _sorted_join(values: Iterable[str]) -> str: + unique = {_clean(value) for value in values if _clean(value)} + return "; ".join(sorted(unique, key=str.casefold)) + + +class M2ORClient: + """Query an exact normalized CAS number in the pinned M2OR CSV. + + The large upstream CSV is not distributed with this package. It is read + from ``data_path`` when supplied, otherwise cached below ``cache_dir`` and + downloaded on first use when ``allow_download`` is true. ``rows`` offers a + lightweight injection point for callers and offline tests. + """ + + def __init__( + self, + cache_dir: str | Path | None = None, + *, + data_path: str | Path | None = None, + allow_download: bool = True, + data_url: str = M2OR_DATA_URL, + timeout: float = 120, + session: requests.Session | None = None, + downloader: DownloadFunction | None = None, + rows: Iterable[Mapping[str, object]] | None = None, + ) -> None: + root = Path(cache_dir).expanduser() if cache_dir else _default_cache_dir() + self.data_path = ( + Path(data_path).expanduser() + if data_path is not None + else root / "m2or" / M2OR_VERSION / M2OR_FILENAME + ) + self.allow_download = allow_download + self.data_url = data_url + self.timeout = timeout + self.session = session or requests.Session() + self.downloader = downloader + self._rows = tuple(dict(row) for row in rows) if rows is not None else None + + def lookup_cas(self, cas: object) -> LookupResult: + """Aggregate receptor-pair evidence for one exact normalized CAS.""" + + try: + normalized_cas = require_valid_cas(cas) + except ValueError as exc: + return self._result(status="invalid_input", message=str(exc)) + + try: + if self._rows is not None: + rows: Iterable[Mapping[str, object]] = self._rows + cache_hit = False + else: + cache_hit = self._ensure_data_file() + rows = self._iter_rows() + values = self._aggregate(rows, normalized_cas) + except FileNotFoundError as exc: + return self._result(status="missing_data", message=str(exc)) + except requests.RequestException as exc: + return self._result(status="network_error", message=str(exc)) + except (OSError, csv.Error, UnicodeError, ValueError) as exc: + return self._result(status="parse_error", message=str(exc)) + + if values["M2OR Pair Count"] == 0: + return self._result( + values=values, + status="not_found", + message=f"No exact M2OR CAS match for {normalized_cas}", + cache_hit=cache_hit, + ) + return self._result(values=values, cache_hit=cache_hit) + + def _aggregate( + self, + rows: Iterable[Mapping[str, object]], + normalized_cas: str, + ) -> dict[str, Any]: + pairs: set[tuple[str, ...]] = set() + responsive_pairs: set[tuple[str, ...]] = set() + species: set[str] = set() + human_receptors: set[str] = set() + dois: set[str] = set() + + for row in rows: + row_cas = normalize_cas(row.get("CAS")) + if row_cas != normalized_cas: + continue + + pair_key = self._pair_key(row) + pairs.add(pair_key) + responsive = _is_responsive(row.get("Responsive")) + if responsive: + responsive_pairs.add(pair_key) + + row_species = _clean(row.get("species") or row.get("Species")) + if row_species: + species.add(row_species) + + if responsive and row_species.casefold() == "homo sapiens": + receptor = _clean( + row.get("Gene ID") + or row.get("Gene Name") + or row.get("Uniprot ID") + or row.get("Sequence") + ) + mutation = _clean(row.get("Mutation")) + if receptor: + human_receptors.add(f"{receptor} ({mutation})" if mutation else receptor) + + doi = _canonical_doi(row.get("DOI")) + if doi: + dois.add(doi) + + return { + "M2OR Pair Count": len(pairs), + "M2OR Responsive Count": len(responsive_pairs), + "M2OR Species": _sorted_join(species), + "M2OR Human Responsive Receptors": _sorted_join(human_receptors), + "M2OR DOIs": _sorted_join(dois), + } + + @staticmethod + def _pair_key(row: Mapping[str, object]) -> tuple[str, ...]: + species = _clean(row.get("species") or row.get("Species")).casefold() + mutation = _clean(row.get("Mutation")).upper() + sequence = re.sub(r"\s+", "", _clean(row.get("Sequence"))).upper() + uniprot = _clean(row.get("Uniprot ID")).upper() + gene = _clean(row.get("Gene ID") or row.get("Gene Name")).upper() + if sequence: + receptor_type, receptor = "sequence", sequence + elif uniprot: + receptor_type, receptor = "uniprot", uniprot + elif gene: + receptor_type, receptor = "gene", gene + else: + # A source row without any receptor identifier cannot be safely + # merged with another unidentified experiment. + receptor_type = "record" + receptor = _clean(row.get("id") or row.get("ID")) + return ( + "pair", + species, + receptor_type, + receptor, + mutation, + ) + + def _iter_rows(self) -> Iterable[Mapping[str, object]]: + with self.data_path.open("r", encoding="utf-8-sig", newline="") as handle: + reader = csv.DictReader(handle, delimiter=";") + if not reader.fieldnames or "CAS" not in reader.fieldnames: + raise csv.Error("M2OR CSV is missing its semicolon-delimited CAS column") + yield from reader + + def _ensure_data_file(self) -> bool: + if self.data_path.is_file() and self.data_path.stat().st_size: + return True + if not self.allow_download: + raise FileNotFoundError( + f"{M2OR_FILENAME} is not cached and downloads are disabled ({self.data_path})" + ) + + self.data_path.parent.mkdir(parents=True, exist_ok=True) + temporary = self.data_path.with_name(f".{self.data_path.name}.{os.getpid()}.part") + temporary.unlink(missing_ok=True) + try: + if self.downloader is not None: + payload = self.downloader(self.data_url, temporary) + if isinstance(payload, str): + temporary.write_text(payload, encoding="utf-8") + elif isinstance(payload, bytes): + temporary.write_bytes(payload) + else: + response = self.session.get(self.data_url, timeout=self.timeout, stream=True) + response.raise_for_status() + with temporary.open("wb") as handle: + for chunk in response.iter_content(chunk_size=1024 * 1024): + if chunk: + handle.write(chunk) + if not temporary.is_file() or not temporary.stat().st_size: + raise OSError(f"Downloader produced no data for {self.data_url}") + temporary.replace(self.data_path) + except Exception: + temporary.unlink(missing_ok=True) + raise + return False + + def _result( + self, + *, + values: dict[str, Any] | None = None, + status: str = "ok", + message: str = "", + cache_hit: bool = False, + ) -> LookupResult: + return LookupResult( + provider="M2OR", + values=values or {}, + source_url=self.data_url, + status=status, + message=message, + cache_hit=cache_hit, + version=M2OR_VERSION, + license_url=M2OR_LICENSE_URL, + ) + + +__all__ = ["M2OR_DATA_URL", "M2OR_VERSION", "M2ORClient"] diff --git a/flavor_data_crawler/sources/mffi.py b/flavor_data_crawler/sources/mffi.py new file mode 100644 index 0000000..c57fa1f --- /dev/null +++ b/flavor_data_crawler/sources/mffi.py @@ -0,0 +1,108 @@ +"""Interactive MFFI browser adapter retained for legacy compatibility.""" + +from __future__ import annotations + +import re +from urllib.parse import quote_plus + +from selenium import webdriver +from selenium.common.exceptions import TimeoutException, WebDriverException +from selenium.webdriver.common.by import By +from selenium.webdriver.support import expected_conditions as EC +from selenium.webdriver.support.ui import WebDriverWait + +from flavor_data_crawler.identifiers import normalize_cas, require_valid_cas +from flavor_data_crawler.models import LookupResult + +MFFI_BASE_URL = "https://mffi.sjtu.edu.cn/database/search" +CAS_IN_TEXT = re.compile(r"\b\d{2,7}-\d{2}-\d\b") + + +def create_chrome_driver(*, headless: bool = False) -> webdriver.Chrome: + """Create Chrome through Selenium Manager without disabling its sandbox.""" + + options = webdriver.ChromeOptions() + options.add_argument("--log-level=3") + options.add_argument("--disable-notifications") + if headless: + options.add_argument("--headless=new") + else: + options.add_argument("--start-maximized") + return webdriver.Chrome(options=options) + + +class MffiClient: + """Look up sensory fields from the SJTU MFFI web interface.""" + + def __init__( + self, + driver: webdriver.Chrome | None = None, + *, + timeout: float = 15, + headless: bool = False, + ) -> None: + self.driver = driver or create_chrome_driver(headless=headless) + self._owns_driver = driver is None + self.timeout = timeout + + def close(self) -> None: + if self._owns_driver: + self.driver.quit() + + def __enter__(self) -> MffiClient: + return self + + def __exit__(self, *_: object) -> None: + self.close() + + def lookup_cas(self, cas: object) -> LookupResult: + try: + normalized = require_valid_cas(cas) + except ValueError as exc: + return LookupResult.failure("MFFI", status="invalid_input", message=str(exc)) + + url = f"{MFFI_BASE_URL}?value={quote_plus(normalized)}&keyword=all" + try: + self.driver.get(url) + rows = WebDriverWait(self.driver, self.timeout).until( + EC.presence_of_all_elements_located((By.CSS_SELECTOR, "tbody tr")) + ) + for row in rows: + cells = row.find_elements(By.TAG_NAME, "td") + if len(cells) < 7: + continue + row_identifiers = { + normalize_cas(match) for match in CAS_IN_TEXT.findall(cells[3].text) + } + if normalized not in row_identifiers: + continue + return LookupResult( + provider="MFFI", + source_url=url, + values={ + "Chinese Name": cells[0].text.strip() or "\\", + "English Name": cells[1].text.strip() or "\\", + "Sensory Characteristics": cells[5].text.strip() or "\\", + "In Water": cells[6].text.strip() or "\\", + }, + ) + return LookupResult.failure( + "MFFI", + status="not_found", + message=f"No exact CAS match for {normalized}", + source_url=url, + ) + except TimeoutException as exc: + return LookupResult.failure( + "MFFI", + status="parse_error", + message=f"Timed out waiting for result rows: {exc}", + source_url=url, + ) + except WebDriverException as exc: + return LookupResult.failure( + "MFFI", + status="network_error", + message=str(exc), + source_url=url, + ) diff --git a/flavor_data_crawler/sources/nist.py b/flavor_data_crawler/sources/nist.py new file mode 100644 index 0000000..75e3710 --- /dev/null +++ b/flavor_data_crawler/sources/nist.py @@ -0,0 +1,314 @@ +"""NIST Chemistry WebBook retention-index and identifier adapter.""" + +from __future__ import annotations + +import math +import re +import time +from collections.abc import Callable +from pathlib import Path +from typing import Any + +from bs4 import BeautifulSoup, Tag + +from flavor_data_crawler.http import ( + DEFAULT_CACHE_DIR, + CachedHttpClient, + HttpClientError, + HttpResponse, +) +from flavor_data_crawler.identifiers import clean_text, is_valid_cas, normalize_cas +from flavor_data_crawler.models import LookupResult + +NIST_CBOOK_URL = "https://webbook.nist.gov/cgi/cbook.cgi" +NIST_LICENSE_URL = "https://www.nist.gov/srd/public-law" +NIST_VERSION = "SRD 69" +TARGET_RI_TABLE_TITLE = "Normal alkane RI, non-polar column, custom temperature program" +CAS_TEXT_PATTERN = re.compile(r"CAS\s+Registry\s+Number\s*:\s*(\d{2,7}-\d{2}-\d)", re.I) +NUMBER_PATTERN = re.compile(r"[+-]?(?:\d{1,3}(?:,\d{3})+|\d+)(?:\.\d+)?") + + +def _normalized_text(value: str) -> str: + return " ".join(value.split()) + + +def _target_ri_table(soup: BeautifulSoup, target_title: str) -> Tag | None: + normalized_title = _normalized_text(target_title).casefold() + for table in soup.find_all("table"): + aria_label = _normalized_text(str(table.get("aria-label", ""))).casefold() + if aria_label == normalized_title: + return table + + for text_node in soup.find_all(string=True): + if normalized_title in _normalized_text(str(text_node)).casefold(): + table = text_node.find_next("table") + if isinstance(table, Tag): + return table + return None + + +def _first_number(text: str) -> float | None: + match = NUMBER_PATTERN.search(text.replace("\u2212", "-")) + if match is None: + return None + try: + return float(match.group(0).replace(",", "")) + except ValueError: + return None + + +def parse_retention_indices( + html: str, + *, + target_title: str = TARGET_RI_TABLE_TITLE, +) -> list[float]: + """Extract the ``I`` column from the original crawler's target RI table.""" + + soup = BeautifulSoup(html, "html.parser") + table = _target_ri_table(soup, target_title) + if table is None: + return [] + + rows = table.find_all("tr") + header_index: int | None = None + ri_column: int | None = None + for row_index, row in enumerate(rows): + labels = [ + _normalized_text(cell.get_text(" ", strip=True)) for cell in row.find_all(["th", "td"]) + ] + for column_index, label in enumerate(labels): + if label == "I": + header_index = row_index + ri_column = column_index + break + if ri_column is not None: + break + + if header_index is None or ri_column is None: + return [] + + values: list[float] = [] + for row in rows[header_index + 1 :]: + cells = row.find_all(["td", "th"]) + if len(cells) <= ri_column: + continue + value = _first_number(cells[ri_column].get_text(" ", strip=True)) + if value is not None: + values.append(value) + return values + + +def parse_nist_cas(html: str) -> str | None: + """Extract a hyphenated CAS Registry Number from a NIST detail page.""" + + soup = BeautifulSoup(html, "html.parser") + for label in soup.find_all("strong"): + if "cas registry number" not in label.get_text(" ", strip=True).casefold(): + continue + parent_text = label.parent.get_text(" ", strip=True) if label.parent else "" + match = CAS_TEXT_PATTERN.search(parent_text) + if match: + return match.group(1) + sibling_text = " ".join(str(sibling) for sibling in label.next_siblings) + match = re.search(r"\d{2,7}-\d{2}-\d", sibling_text) + if match: + return match.group(0) + + match = CAS_TEXT_PATTERN.search(soup.get_text(" ", strip=True)) + return match.group(1) if match else None + + +def is_nist_search_results(html: str) -> bool: + """Return whether NIST responded with an ambiguous search-results page.""" + + soup = BeautifulSoup(html, "html.parser") + title = soup.title.get_text(" ", strip=True) if soup.title else "" + return "search results" in title.casefold() + + +class NistWebBookClient: + """Polite client for the NIST Chemistry WebBook SRD 69.""" + + provider = "NIST WebBook" + + def __init__( + self, + http_client: CachedHttpClient | Any | None = None, + *, + cache_dir: str | Path | None = DEFAULT_CACHE_DIR / "nist", + min_interval: float = 5.0, + timeout: float | tuple[float, float] = (5.0, 30.0), + session: Any | None = None, + sleep: Callable[[float], None] = time.sleep, + ) -> None: + self.http = http_client or CachedHttpClient( + cache_dir=cache_dir, + min_interval=min_interval, + timeout=timeout, + session=session, + sleep=sleep, + ) + + def lookup_ri(self, cas: object, calculated_ri: object) -> LookupResult: + """Find the NIST RI nearest to a caller-supplied calculated RI.""" + + normalized_cas = normalize_cas(cas) + if not normalized_cas or not is_valid_cas(normalized_cas): + return self._failure( + status="invalid_input", + message=f"Invalid CAS Registry Number: {clean_text(cas) or ''}", + ) + try: + target = float(calculated_ri) + if not math.isfinite(target): + raise ValueError + except (TypeError, ValueError): + return self._failure( + status="invalid_input", + message=f"Calculated RI must be a finite number: {calculated_ri!r}", + ) + + try: + response = self.http.get( + NIST_CBOOK_URL, + params={ + "ID": f"C{normalized_cas.replace('-', '')}", + "Units": "SI", + "Mask": "2000", + }, + ) + except HttpClientError as exc: + return self._failure(status="network_error", message=str(exc), source_url=exc.url) + + if response.status_code == 404: + return self._response_failure( + response, + status="not_found", + message=f"NIST has no retention-index page for {normalized_cas}.", + ) + if not response.ok: + return self._response_failure( + response, + status="http_error", + message=f"NIST returned HTTP {response.status_code}.", + ) + + values = parse_retention_indices(response.text) + if not values: + return self._response_failure( + response, + status="not_found", + message=( + f"Target NIST retention-index table was not available for {normalized_cas}." + ), + ) + + closest = min(values, key=lambda value: abs(value - target)) + selected: int | float = int(closest) if closest.is_integer() else closest + return LookupResult( + provider=self.provider, + values={ + "cas": normalized_cas, + "calculated_ri": target, + "retention_index": selected, + "retention_indices": [ + int(value) if value.is_integer() else value for value in values + ], + }, + source_url=response.url, + retrieved_at=response.metadata.retrieved_at, + cache_hit=response.metadata.cache_hit, + version=NIST_VERSION, + license_url=NIST_LICENSE_URL, + ) + + def resolve_name(self, name: object) -> LookupResult: + """Resolve an unambiguous NIST compound-name page to its CAS number.""" + + query = clean_text(name) + if not query: + return self._failure(status="invalid_input", message="Compound name is empty.") + + try: + response = self.http.get( + NIST_CBOOK_URL, + params={"Name": query, "Units": "SI"}, + ) + except HttpClientError as exc: + return self._failure(status="network_error", message=str(exc), source_url=exc.url) + + if response.status_code == 404 or "name not found" in response.text.casefold(): + return self._response_failure( + response, + status="not_found", + message=f"NIST did not find a compound named {query!r}.", + ) + if not response.ok: + return self._response_failure( + response, + status="http_error", + message=f"NIST returned HTTP {response.status_code}.", + ) + if is_nist_search_results(response.text): + return self._response_failure( + response, + status="ambiguous", + message=f"NIST returned multiple search results for {query!r}.", + ) + + cas = parse_nist_cas(response.text) + if cas is None: + return self._response_failure( + response, + status="parse_error", + message="NIST detail page did not contain a CAS Registry Number.", + ) + return LookupResult( + provider=self.provider, + values={"query_name": query, "cas": cas}, + source_url=response.url, + retrieved_at=response.metadata.retrieved_at, + cache_hit=response.metadata.cache_hit, + version=NIST_VERSION, + license_url=NIST_LICENSE_URL, + ) + + def _response_failure( + self, + response: HttpResponse, + *, + status: str, + message: str, + ) -> LookupResult: + return LookupResult( + provider=self.provider, + status=status, + message=message, + source_url=response.url, + retrieved_at=response.metadata.retrieved_at, + cache_hit=response.metadata.cache_hit, + version=NIST_VERSION, + license_url=NIST_LICENSE_URL, + ) + + def _failure(self, *, status: str, message: str, source_url: str = "") -> LookupResult: + return LookupResult( + provider=self.provider, + status=status, + message=message, + source_url=source_url, + version=NIST_VERSION, + license_url=NIST_LICENSE_URL, + ) + + +NISTSource = NistWebBookClient + +__all__ = [ + "NISTSource", + "NistWebBookClient", + "TARGET_RI_TABLE_TITLE", + "is_nist_search_results", + "parse_nist_cas", + "parse_retention_indices", +] diff --git a/flavor_data_crawler/sources/pubchem.py b/flavor_data_crawler/sources/pubchem.py new file mode 100644 index 0000000..9dcf5c7 --- /dev/null +++ b/flavor_data_crawler/sources/pubchem.py @@ -0,0 +1,428 @@ +"""PubChem PUG REST properties and PUG-View odor annotations.""" + +from __future__ import annotations + +import json +import time +from collections.abc import Callable, Iterable, Mapping +from pathlib import Path +from typing import Any +from urllib.parse import quote + +from flavor_data_crawler.http import ( + DEFAULT_CACHE_DIR, + CachedHttpClient, + HttpClientError, + HttpResponse, +) +from flavor_data_crawler.identifiers import clean_text, is_valid_cas, normalize_cas +from flavor_data_crawler.models import LookupResult + +PUBCHEM_PUG_BASE = "https://pubchem.ncbi.nlm.nih.gov/rest/pug" +PUBCHEM_VIEW_BASE = "https://pubchem.ncbi.nlm.nih.gov/rest/pug_view" +PUBCHEM_LICENSE_URL = "https://www.ncbi.nlm.nih.gov/home/about/policies/" +PUBCHEM_VERSION = "PUG REST + PUG-View" +PROPERTY_TAGS = ( + "Title", + "IUPACName", + "MolecularFormula", + "MolecularWeight", + "CanonicalSMILES", + "IsomericSMILES", + "InChI", + "InChIKey", + "XLogP", +) + + +def _first_mapping(value: object) -> Mapping[str, Any] | None: + if isinstance(value, list) and value and isinstance(value[0], Mapping): + return value[0] + return None + + +def parse_pubchem_properties(payload: Mapping[str, Any]) -> dict[str, Any]: + """Flatten the first PUG REST property record into stable local field names.""" + + table = payload.get("PropertyTable") + if not isinstance(table, Mapping): + return {} + record = _first_mapping(table.get("Properties")) + if record is None or "CID" not in record: + return {} + + # PubChem now returns SMILES/ConnectivitySMILES for the legacy request tags. + # Supporting both spellings keeps cached and current API responses compatible. + return { + "cid": record.get("CID"), + "title": record.get("Title", ""), + "iupac_name": record.get("IUPACName", ""), + "molecular_formula": record.get("MolecularFormula", ""), + "molecular_weight": record.get("MolecularWeight", ""), + "canonical_smiles": record.get("ConnectivitySMILES", record.get("CanonicalSMILES", "")), + "isomeric_smiles": record.get("SMILES", record.get("IsomericSMILES", "")), + "inchi": record.get("InChI", ""), + "inchikey": record.get("InChIKey", ""), + "xlogp": record.get("XLogP", ""), + } + + +def parse_pubchem_identifiers(payload: Mapping[str, Any]) -> list[str]: + """Extract unique, valid CAS identifiers from a PUG REST identifiers result.""" + + information_list = payload.get("InformationList") + if not isinstance(information_list, Mapping): + return [] + information = information_list.get("Information") + if not isinstance(information, list): + return [] + + cas_numbers: list[str] = [] + for item in information: + if not isinstance(item, Mapping): + continue + identifiers = item.get("Identifiers") + if not isinstance(identifiers, list): + continue + for identifier in identifiers: + if not isinstance(identifier, Mapping): + continue + if str(identifier.get("Type", "")).casefold() != "cas": + continue + cas = normalize_cas(identifier.get("Identifier")) + if is_valid_cas(cas) and cas not in cas_numbers: + cas_numbers.append(cas) + return cas_numbers + + +def parse_pubchem_synonyms(payload: Mapping[str, Any]) -> list[str]: + """Extract de-duplicated compound synonyms from PUG REST JSON.""" + + information_list = payload.get("InformationList") + if not isinstance(information_list, Mapping): + return [] + information = information_list.get("Information") + if not isinstance(information, list): + return [] + + synonyms: list[str] = [] + for item in information: + if not isinstance(item, Mapping): + continue + raw_synonyms = item.get("Synonym") + if not isinstance(raw_synonyms, list): + continue + for raw in raw_synonyms: + synonym = clean_text(raw) + if synonym and synonym not in synonyms: + synonyms.append(synonym) + return synonyms + + +def _walk_sections(sections: object) -> Iterable[Mapping[str, Any]]: + if not isinstance(sections, list): + return + for section in sections: + if not isinstance(section, Mapping): + continue + yield section + yield from _walk_sections(section.get("Section")) + + +def _value_strings(value: object) -> list[str]: + if not isinstance(value, Mapping): + return [] + strings: list[str] = [] + markup = value.get("StringWithMarkup") + if isinstance(markup, list): + for item in markup: + if isinstance(item, Mapping): + text = clean_text(item.get("String")) + if text: + strings.append(text) + direct = clean_text(value.get("String")) + if direct: + strings.append(direct) + return strings + + +def _reference_numbers(information: Mapping[str, Any]) -> list[str]: + raw = information.get("ReferenceNumber") + if raw is None: + return [] + values = raw if isinstance(raw, list) else [raw] + return [str(value) for value in values] + + +def _append_unique(items: list[str], value: object) -> None: + text = clean_text(value) + if text and text not in items: + items.append(text) + + +def parse_pubchem_odor(payload: Mapping[str, Any]) -> dict[str, Any]: + """Flatten PUG-View Odor text while retaining contributor provenance.""" + + record = payload.get("Record") + if not isinstance(record, Mapping): + return { + "odor": [], + "odor_annotations": [], + "odor_sources": [], + "odor_source_urls": [], + "odor_license_urls": [], + } + + references: dict[str, Mapping[str, Any]] = {} + raw_references = record.get("Reference") + if isinstance(raw_references, list): + for reference in raw_references: + if isinstance(reference, Mapping) and reference.get("ReferenceNumber") is not None: + references[str(reference["ReferenceNumber"])] = reference + + odors: list[str] = [] + annotations: list[dict[str, Any]] = [] + source_names: list[str] = [] + source_urls: list[str] = [] + license_urls: list[str] = [] + + for section in _walk_sections(record.get("Section")): + if str(section.get("TOCHeading", "")).strip().casefold() != "odor": + continue + information_items = section.get("Information") + if not isinstance(information_items, list): + continue + for information in information_items: + if not isinstance(information, Mapping): + continue + texts = _value_strings(information.get("Value")) + reference_numbers = _reference_numbers(information) or [""] + for text in texts: + _append_unique(odors, text) + for number in reference_numbers: + reference = references.get(number, {}) + source_name = clean_text(reference.get("SourceName")) + source_url = clean_text(reference.get("URL")) + license_url = clean_text(reference.get("LicenseURL")) + annotation = { + "text": text, + "reference_number": number, + "source_name": source_name, + "source_url": source_url, + "license_url": license_url, + } + if annotation not in annotations: + annotations.append(annotation) + _append_unique(source_names, source_name) + _append_unique(source_urls, source_url) + _append_unique(license_urls, license_url) + + return { + "odor": odors, + "odor_annotations": annotations, + "odor_sources": source_names, + "odor_source_urls": source_urls, + "odor_license_urls": license_urls, + } + + +class PubChemClient: + """Resolve names/CAS numbers through PUG REST and optional PUG-View odor data.""" + + provider = "PubChem" + + def __init__( + self, + http_client: CachedHttpClient | Any | None = None, + *, + cache_dir: str | Path | None = DEFAULT_CACHE_DIR / "pubchem", + min_interval: float = 0.25, + timeout: float | tuple[float, float] = (5.0, 30.0), + session: Any | None = None, + sleep: Callable[[float], None] = time.sleep, + ) -> None: + self.http = http_client or CachedHttpClient( + cache_dir=cache_dir, + min_interval=min_interval, + timeout=timeout, + session=session, + sleep=sleep, + ) + + def lookup(self, identifier: object, include_odor: bool = True) -> LookupResult: + """Look up a CAS number or compound name and return normalized properties.""" + + query = clean_text(identifier) + if not query: + return self._failure(status="invalid_input", message="Compound identifier is empty.") + + encoded_query = quote(query, safe="") + properties_url = ( + f"{PUBCHEM_PUG_BASE}/compound/name/{encoded_query}/property/" + f"{','.join(PROPERTY_TAGS)}/JSON" + ) + try: + properties_response = self.http.get( + properties_url, + headers={"Accept": "application/json"}, + ) + except HttpClientError as exc: + return self._failure(status="network_error", message=str(exc), source_url=exc.url) + + if properties_response.status_code == 404: + return self._response_failure( + properties_response, + status="not_found", + message=f"PubChem did not resolve {query!r}.", + ) + if not properties_response.ok: + return self._response_failure( + properties_response, + status="http_error", + message=f"PubChem returned HTTP {properties_response.status_code}.", + ) + try: + properties_payload = properties_response.json() + except (json.JSONDecodeError, TypeError) as exc: + return self._response_failure( + properties_response, + status="parse_error", + message=f"PubChem properties response was not valid JSON: {exc}", + ) + if not isinstance(properties_payload, Mapping): + properties = {} + else: + properties = parse_pubchem_properties(properties_payload) + if not properties or properties.get("cid") in (None, ""): + return self._response_failure( + properties_response, + status="parse_error", + message="PubChem properties response did not contain a CID.", + ) + + cid = properties["cid"] + values: dict[str, Any] = { + "query": query, + **properties, + "pubchem_url": f"https://pubchem.ncbi.nlm.nih.gov/compound/{cid}", + "synonyms": [], + "cas_numbers": [], + "odor": [], + "odor_annotations": [], + "odor_sources": [], + "odor_source_urls": [], + "odor_license_urls": [], + } + responses = [properties_response] + warnings: list[str] = [] + + identifiers_url = f"{PUBCHEM_PUG_BASE}/compound/cid/{cid}/identifiers/JSON" + try: + identifiers_response = self.http.get( + identifiers_url, + params={"identifier_type": "CAS"}, + headers={"Accept": "application/json"}, + ) + responses.append(identifiers_response) + if identifiers_response.ok: + payload = identifiers_response.json() + if isinstance(payload, Mapping): + values["cas_numbers"] = parse_pubchem_identifiers(payload) + elif identifiers_response.status_code != 404: + warnings.append(f"CAS identifiers returned HTTP {identifiers_response.status_code}") + except (HttpClientError, json.JSONDecodeError, TypeError) as exc: + warnings.append(f"CAS identifiers unavailable: {exc}") + + synonyms_url = f"{PUBCHEM_PUG_BASE}/compound/cid/{cid}/synonyms/JSON" + try: + synonyms_response = self.http.get( + synonyms_url, + headers={"Accept": "application/json"}, + ) + responses.append(synonyms_response) + if synonyms_response.ok: + payload = synonyms_response.json() + if isinstance(payload, Mapping): + values["synonyms"] = parse_pubchem_synonyms(payload) + elif synonyms_response.status_code != 404: + warnings.append(f"Synonyms returned HTTP {synonyms_response.status_code}") + except (HttpClientError, json.JSONDecodeError, TypeError) as exc: + warnings.append(f"Synonyms unavailable: {exc}") + + normalized_query_cas = normalize_cas(query) + if is_valid_cas(normalized_query_cas) and normalized_query_cas not in values["cas_numbers"]: + values["cas_numbers"].insert(0, normalized_query_cas) + for synonym in values["synonyms"]: + synonym_cas = normalize_cas(synonym) + if is_valid_cas(synonym_cas) and synonym_cas not in values["cas_numbers"]: + values["cas_numbers"].append(synonym_cas) + + if include_odor: + odor_url = f"{PUBCHEM_VIEW_BASE}/data/compound/{cid}/JSON" + try: + odor_response = self.http.get( + odor_url, + params={"heading": "Odor"}, + headers={"Accept": "application/json"}, + ) + responses.append(odor_response) + if odor_response.ok: + payload = odor_response.json() + if isinstance(payload, Mapping): + values.update(parse_pubchem_odor(payload)) + elif odor_response.status_code != 404: + warnings.append(f"Odor annotations returned HTTP {odor_response.status_code}") + except (HttpClientError, json.JSONDecodeError, TypeError) as exc: + warnings.append(f"Odor annotations unavailable: {exc}") + + return LookupResult( + provider=self.provider, + values=values, + source_url=properties_response.url, + retrieved_at=properties_response.metadata.retrieved_at, + status="partial" if warnings else "ok", + message="; ".join(warnings), + cache_hit=all(response.metadata.cache_hit for response in responses), + version=PUBCHEM_VERSION, + license_url=PUBCHEM_LICENSE_URL, + ) + + def _response_failure( + self, + response: HttpResponse, + *, + status: str, + message: str, + ) -> LookupResult: + return LookupResult( + provider=self.provider, + status=status, + message=message, + source_url=response.url, + retrieved_at=response.metadata.retrieved_at, + cache_hit=response.metadata.cache_hit, + version=PUBCHEM_VERSION, + license_url=PUBCHEM_LICENSE_URL, + ) + + def _failure(self, *, status: str, message: str, source_url: str = "") -> LookupResult: + return LookupResult( + provider=self.provider, + status=status, + message=message, + source_url=source_url, + version=PUBCHEM_VERSION, + license_url=PUBCHEM_LICENSE_URL, + ) + + +PubChemSource = PubChemClient + +__all__ = [ + "PubChemClient", + "PubChemSource", + "parse_pubchem_identifiers", + "parse_pubchem_odor", + "parse_pubchem_properties", + "parse_pubchem_synonyms", +] diff --git a/flavor_data_crawler/sources/pyrfume.py b/flavor_data_crawler/sources/pyrfume.py new file mode 100644 index 0000000..690359f --- /dev/null +++ b/flavor_data_crawler/sources/pyrfume.py @@ -0,0 +1,327 @@ +"""Pinned, cache-backed access to a small set of Pyrfume data archives. + +The adapter deliberately downloads data at run time instead of redistributing +Pyrfume archive files. Each archive keeps its own source and rights notes from +``manifest.toml``; the Pyrfume repository must not be treated as granting one +blanket license for every upstream dataset. +""" + +from __future__ import annotations + +import csv +import os +import re +import tomllib +from collections.abc import Callable, Iterable, Mapping +from pathlib import Path +from typing import Any + +import requests + +from flavor_data_crawler.models import LookupResult + +PYRFUME_DATA_VERSION = "8054ea98ed675005ec10e67359902f500e4911b0" +PYRFUME_REPOSITORY_URL = "https://github.com/pyrfume/pyrfume-data" +PYRFUME_SNAPSHOT_URL = f"{PYRFUME_REPOSITORY_URL}/tree/{PYRFUME_DATA_VERSION}" +PYRFUME_RAW_BASE_URL = ( + f"https://raw.githubusercontent.com/pyrfume/pyrfume-data/{PYRFUME_DATA_VERSION}" +) + +# Keep this list intentionally small and review additions archive-by-archive. +# SuperScent has no standardized stimuli.csv or behavior.csv in this snapshot. +ARCHIVE_FILES: dict[str, tuple[str, ...]] = { + "aromadb": ("manifest.toml", "molecules.csv", "stimuli.csv", "behavior.csv"), + "flavornet": ("manifest.toml", "molecules.csv", "stimuli.csv", "behavior.csv"), + "superscent": ("manifest.toml", "molecules.csv"), +} +DEFAULT_ARCHIVES = tuple(ARCHIVE_FILES) + +DownloadFunction = Callable[[str, Path], bytes | str | None] + + +def _default_cache_dir() -> Path: + configured = os.environ.get("FLAVOR_DATA_CRAWLER_CACHE") + if configured: + return Path(configured).expanduser() + return Path.home() / ".cache" / "flavor-data-crawler" + + +def _normalize_cid(value: object) -> str: + """Return a canonical positive PubChem CID, or raise ``ValueError``.""" + + if value is None or isinstance(value, bool): + raise ValueError("PubChem CID is empty") + text = str(value).strip() + if text.endswith(".0") and text[:-2].isdigit(): + text = text[:-2] + if not text or not text.isdigit() or int(text) <= 0: + raise ValueError(f"Invalid PubChem CID: {text or value!s}") + return str(int(text)) + + +def _clean(value: object) -> str: + return "" if value is None else str(value).strip() + + +def _unique_join(values: Iterable[str]) -> str: + seen: set[str] = set() + result: list[str] = [] + for value in values: + cleaned = _clean(value) + key = cleaned.casefold() + if cleaned and key not in seen: + seen.add(key) + result.append(cleaned) + return "; ".join(result) + + +def _descriptor_values(row: Mapping[str, object]) -> list[str]: + for column in ("Filtered Descriptors", "Descriptors", "Raw Descriptors"): + raw = _clean(row.get(column)) + if raw: + return [part.strip() for part in re.split(r"[;,]", raw) if part.strip()] + return [] + + +class PyrfumeArchiveClient: + """Query selected Pyrfume archives by exact PubChem CID. + + Parameters are intentionally injectable for offline use. ``cache_dir`` + controls the persistent cache root, ``base_url`` can target a mirror, and + ``downloader`` may write synthetic test data to the supplied destination. + A downloader may alternatively return ``bytes`` or ``str``. + """ + + def __init__( + self, + cache_dir: str | Path | None = None, + *, + base_url: str = PYRFUME_RAW_BASE_URL, + allow_download: bool = True, + timeout: float = 30, + session: requests.Session | None = None, + downloader: DownloadFunction | None = None, + ) -> None: + self.cache_dir = Path(cache_dir).expanduser() if cache_dir else _default_cache_dir() + self.base_url = base_url.rstrip("/") + self.allow_download = allow_download + self.timeout = timeout + self.session = session or requests.Session() + self.downloader = downloader + + def lookup( + self, + cid: object, + archives: Iterable[str] | str | None = None, + ) -> LookupResult: + """Look up one CID across the requested allowlisted archives.""" + + try: + normalized_cid = _normalize_cid(cid) + selected = self._select_archives(archives) + except ValueError as exc: + return self._result(status="invalid_input", message=str(exc)) + + values: dict[str, Any] = {} + matched: list[str] = [] + errors: list[str] = [] + cache_states: list[bool] = [] + + for archive in selected: + prefix = f"Pyrfume {archive}" + try: + paths: dict[str, Path] = {} + for filename in ARCHIVE_FILES[archive]: + path, cache_hit = self._ensure_file(archive, filename) + paths[filename] = path + cache_states.append(cache_hit) + + manifest = self._read_manifest(paths["manifest.toml"]) + source = manifest.get("source", {}) + if not isinstance(source, Mapping): + source = {} + raw_section = manifest.get("raw", {}) + if not isinstance(raw_section, Mapping): + raw_section = {} + + values.update( + { + f"{prefix} Source Title": _clean(source.get("title")), + f"{prefix} Source Reference": _clean( + source.get("doi") or source.get("url") + ), + f"{prefix} Source Authors": _clean(source.get("authors")), + # Keep copyright and usage caveats verbatim from the manifest. + f"{prefix} Source Notes": _clean(source.get("extra")), + f"{prefix} License Note": _clean(raw_section.get("LICENSE")), + f"{prefix} Manifest URL": self._source_url(archive, "manifest.toml"), + } + ) + + molecule_rows = self._read_csv(paths["molecules.csv"]) + molecules = [ + row for row in molecule_rows if self._row_cid(row.get("CID")) == normalized_cid + ] + present = bool(molecules) + values[f"{prefix} Present"] = present + values[f"{prefix} Name"] = _unique_join( + _clean(row.get("name") or row.get("Name")) for row in molecules + ) + values[f"{prefix} IUPAC Name"] = _unique_join( + _clean(row.get("IUPACName") or row.get("IUPAC Name")) for row in molecules + ) + + descriptors: list[str] = [] + if present and "behavior.csv" in paths: + stimulus_ids = {normalized_cid} + if "stimuli.csv" in paths: + stimulus_ids.update( + _clean(row.get("Stimulus")) + for row in self._read_csv(paths["stimuli.csv"]) + if self._row_cid(row.get("CID")) == normalized_cid + ) + for row in self._read_csv(paths["behavior.csv"]): + if _clean(row.get("Stimulus")) in stimulus_ids: + descriptors.extend(_descriptor_values(row)) + values[f"{prefix} Descriptors"] = _unique_join(descriptors) + + if present: + matched.append(archive) + except FileNotFoundError as exc: + values[f"{prefix} Present"] = False + values[f"{prefix} Descriptors"] = "" + errors.append(f"{archive}: {exc}") + except (OSError, csv.Error, tomllib.TOMLDecodeError, UnicodeError) as exc: + values[f"{prefix} Present"] = False + values[f"{prefix} Descriptors"] = "" + errors.append(f"{archive}: could not parse cached data ({exc})") + except requests.RequestException as exc: + values[f"{prefix} Present"] = False + values[f"{prefix} Descriptors"] = "" + errors.append(f"{archive}: download failed ({exc})") + + values["Pyrfume Archives Matched"] = "; ".join(matched) + if errors: + status = "partial" if len(errors) < len(selected) else "data_error" + message = "; ".join(errors) + elif not matched: + status = "not_found" + message = f"CID {normalized_cid} was not found in {', '.join(selected)}" + else: + status = "ok" + message = "" + return self._result( + values=values, + status=status, + message=message, + cache_hit=bool(cache_states) and all(cache_states), + ) + + @staticmethod + def _select_archives(archives: Iterable[str] | str | None) -> tuple[str, ...]: + if archives is None: + requested: Iterable[str] = DEFAULT_ARCHIVES + elif isinstance(archives, str): + requested = [archives] + else: + requested = archives + selected: list[str] = [] + for archive in requested: + normalized = str(archive).strip().casefold() + if normalized not in ARCHIVE_FILES: + allowed = ", ".join(DEFAULT_ARCHIVES) + raise ValueError(f"Unsupported Pyrfume archive {archive!r}; allowed: {allowed}") + if normalized not in selected: + selected.append(normalized) + if not selected: + raise ValueError("At least one Pyrfume archive must be selected") + return tuple(selected) + + @staticmethod + def _read_manifest(path: Path) -> dict[str, Any]: + with path.open("rb") as handle: + return tomllib.load(handle) + + @staticmethod + def _read_csv(path: Path) -> list[dict[str, str]]: + with path.open("r", encoding="utf-8-sig", newline="") as handle: + reader = csv.DictReader(handle) + if not reader.fieldnames: + raise csv.Error(f"CSV has no header: {path}") + return list(reader) + + @staticmethod + def _row_cid(value: object) -> str: + try: + return _normalize_cid(value) + except ValueError: + return "" + + def _cache_path(self, archive: str, filename: str) -> Path: + return self.cache_dir / "pyrfume-data" / PYRFUME_DATA_VERSION / archive / filename + + def _source_url(self, archive: str, filename: str) -> str: + return f"{self.base_url}/{archive}/{filename}" + + def _ensure_file(self, archive: str, filename: str) -> tuple[Path, bool]: + destination = self._cache_path(archive, filename) + if destination.is_file() and destination.stat().st_size: + return destination, True + if not self.allow_download: + raise FileNotFoundError( + f"{filename} is not cached and downloads are disabled ({destination})" + ) + + destination.parent.mkdir(parents=True, exist_ok=True) + temporary = destination.with_name(f".{destination.name}.{os.getpid()}.part") + temporary.unlink(missing_ok=True) + try: + url = self._source_url(archive, filename) + if self.downloader is not None: + payload = self.downloader(url, temporary) + if isinstance(payload, str): + temporary.write_text(payload, encoding="utf-8") + elif isinstance(payload, bytes): + temporary.write_bytes(payload) + else: + response = self.session.get(url, timeout=self.timeout, stream=True) + response.raise_for_status() + with temporary.open("wb") as handle: + for chunk in response.iter_content(chunk_size=64 * 1024): + if chunk: + handle.write(chunk) + if not temporary.is_file() or not temporary.stat().st_size: + raise OSError(f"Downloader produced no data for {url}") + temporary.replace(destination) + except Exception: + temporary.unlink(missing_ok=True) + raise + return destination, False + + @staticmethod + def _result( + *, + values: dict[str, Any] | None = None, + status: str = "ok", + message: str = "", + cache_hit: bool = False, + ) -> LookupResult: + return LookupResult( + provider="Pyrfume", + values=values or {}, + source_url=PYRFUME_SNAPSHOT_URL, + status=status, + message=message, + cache_hit=cache_hit, + version=PYRFUME_DATA_VERSION, + # Rights differ by archive; manifest fields are returned per archive. + license_url="", + ) + + +__all__ = [ + "ARCHIVE_FILES", + "DEFAULT_ARCHIVES", + "PYRFUME_DATA_VERSION", + "PyrfumeArchiveClient", +] diff --git a/flavor_data_crawler/workflows.py b/flavor_data_crawler/workflows.py new file mode 100644 index 0000000..a71c31a --- /dev/null +++ b/flavor_data_crawler/workflows.py @@ -0,0 +1,431 @@ +"""Workbook-oriented enrichment workflows shared by the CLI and legacy launchers.""" + +from __future__ import annotations + +import json +import time +from collections import Counter +from collections.abc import Callable, Iterable +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import pandas as pd + +from flavor_data_crawler.excel_io import ( + derive_output_path, + read_table, + require_columns, + write_table, +) +from flavor_data_crawler.models import LookupResult +from flavor_data_crawler.sources.chemicalbook import ManualVerificationRequired + +ProgressCallback = Callable[[int, int, str], None] + + +@dataclass(slots=True) +class RunSummary: + output_path: Path + rows: int + status_counts: dict[str, int] + + +def console_progress(current: int, total: int, label: str) -> None: + print(f"[{current}/{total}] {label}", flush=True) + + +def _flatten(value: Any) -> Any: + if isinstance(value, dict) or ( + isinstance(value, (list, tuple)) and any(isinstance(item, dict) for item in value) + ): + return json.dumps(value, ensure_ascii=False, sort_keys=True) + if isinstance(value, (list, tuple, set)): + return "; ".join(str(item) for item in value if str(item).strip()) + return value + + +def _set_cell(frame: pd.DataFrame, index: Any, column: str, value: Any) -> None: + """Create enrichment columns as object dtype so mixed values remain valid.""" + + if column not in frame.columns: + frame[column] = pd.Series([None] * len(frame), index=frame.index, dtype=object) + elif frame[column].dtype != object: + frame[column] = frame[column].astype(object) + frame.at[index, column] = _flatten(value) + + +def _apply_result( + frame: pd.DataFrame, + index: Any, + result: LookupResult, + *, + include_provenance: bool, + prefix: str | None = None, + key_map: dict[str, str] | None = None, +) -> None: + mapping = key_map or {} + for key, value in result.values.items(): + _set_cell(frame, index, mapping.get(key, key), value) + if include_provenance: + _apply_provenance(frame, index, result, prefix=prefix) + + +def _apply_provenance( + frame: pd.DataFrame, + index: Any, + result: LookupResult, + *, + prefix: str | None = None, +) -> None: + for key, value in result.provenance_columns(prefix).items(): + _set_cell(frame, index, key, value) + + +def _prepare_run( + input_path: str | Path, + output_path: str | Path | None, + *, + suffix: str, + force: bool, +) -> tuple[pd.DataFrame, Path]: + frame = read_table(input_path) + destination = Path(output_path) if output_path else derive_output_path(input_path, suffix) + if destination.exists() and not force: + raise FileExistsError(f"Output already exists: {destination}. Pass --force to replace it.") + return frame, destination + + +def _checkpoint( + frame: pd.DataFrame, + destination: Path, + current: int, + checkpoint_every: int, +) -> None: + if checkpoint_every <= 0 or current % checkpoint_every: + return + partial = destination.with_name(f"{destination.stem}.partial{destination.suffix}") + write_table(frame, partial, force=True) + + +def _finish( + frame: pd.DataFrame, + destination: Path, + statuses: Counter[str], + *, + force: bool, +) -> RunSummary: + write_table(frame, destination, force=force) + partial = destination.with_name(f"{destination.stem}.partial{destination.suffix}") + partial.unlink(missing_ok=True) + return RunSummary(destination, len(frame), dict(sorted(statuses.items()))) + + +def _indices(frame: pd.DataFrame) -> Iterable[tuple[int, Any, pd.Series]]: + for position, (index, row) in enumerate(frame.iterrows(), 1): + yield position, index, row + + +def run_nist_ri( + input_path: str | Path, + client: Any, + *, + output_path: str | Path | None = None, + cas_column: str = "CAS Number", + calculated_ri_column: str = "Calculated RI", + result_column: str = "NIST RI", + include_provenance: bool = True, + checkpoint_every: int = 25, + force: bool = False, + progress: ProgressCallback = console_progress, +) -> RunSummary: + frame, destination = _prepare_run(input_path, output_path, suffix="_nist_result", force=force) + require_columns(frame, cas_column, calculated_ri_column) + statuses: Counter[str] = Counter() + total = len(frame) + for current, index, row in _indices(frame): + result = client.lookup_ri(row[cas_column], row[calculated_ri_column]) + _set_cell(frame, index, result_column, result.values.get("retention_index", "\\")) + if include_provenance: + candidates = result.values.get("retention_indices") + if candidates: + _set_cell(frame, index, "NIST RI Candidates", candidates) + _apply_provenance(frame, index, result, prefix="NIST") + statuses[result.status] += 1 + progress(current, total, f"NIST RI: {row[cas_column]} -> {frame.at[index, result_column]}") + _checkpoint(frame, destination, current, checkpoint_every) + return _finish(frame, destination, statuses, force=force) + + +def _legacy_name_status(result: LookupResult) -> str: + if result.status == "ok" and result.values.get("cas"): + return str(result.values["cas"]) + return { + "ambiguous": "Ambiguous/List Found", + "not_found": "Not Found", + "network_error": "Connection Error", + "invalid_input": "\\", + }.get(result.status, "Error") + + +def run_resolve_cas( + input_path: str | Path, + client: Any, + *, + output_path: str | Path | None = None, + name_column: str = "Name", + result_column: str = "Found CAS", + include_provenance: bool = True, + checkpoint_every: int = 25, + force: bool = False, + progress: ProgressCallback = console_progress, +) -> RunSummary: + frame, destination = _prepare_run(input_path, output_path, suffix="_with_cas", force=force) + require_columns(frame, name_column) + statuses: Counter[str] = Counter() + total = len(frame) + for current, index, row in _indices(frame): + result = client.resolve_name(row[name_column]) + _set_cell(frame, index, result_column, _legacy_name_status(result)) + if include_provenance: + _apply_provenance(frame, index, result, prefix="NIST") + statuses[result.status] += 1 + label = f"Resolve name: {row[name_column]} -> {frame.at[index, result_column]}" + progress(current, total, label) + _checkpoint(frame, destination, current, checkpoint_every) + return _finish(frame, destination, statuses, force=force) + + +PUBCHEM_COLUMN_MAP = { + "cid": "PubChem CID", + "title": "PubChem Title", + "iupac_name": "IUPAC Name", + "molecular_formula": "Molecular Formula", + "molecular_weight": "Molecular Weight", + "canonical_smiles": "Canonical SMILES", + "isomeric_smiles": "Isomeric SMILES", + "inchi": "InChI", + "inchikey": "InChIKey", + "xlogp": "XLogP", + "synonyms": "PubChem Synonyms", + "cas_numbers": "PubChem CAS Numbers", + "odor": "PubChem Odor", + "odor_annotations": "PubChem Odor Annotations", + "odor_sources": "PubChem Odor Sources", + "odor_source_urls": "PubChem Odor Source URLs", + "odor_license_urls": "PubChem Odor License URLs", +} + + +def run_pubchem( + input_path: str | Path, + client: Any, + *, + output_path: str | Path | None = None, + identifier_column: str = "CAS Number", + include_odor: bool = True, + include_provenance: bool = True, + checkpoint_every: int = 25, + force: bool = False, + progress: ProgressCallback = console_progress, +) -> RunSummary: + frame, destination = _prepare_run(input_path, output_path, suffix="_pubchem", force=force) + require_columns(frame, identifier_column) + statuses: Counter[str] = Counter() + total = len(frame) + for current, index, row in _indices(frame): + identifier = row[identifier_column] + result = client.lookup(identifier, include_odor=include_odor) + _apply_result( + frame, + index, + result, + include_provenance=include_provenance, + prefix="PubChem", + key_map=PUBCHEM_COLUMN_MAP, + ) + statuses[result.status] += 1 + progress(current, total, f"PubChem: {identifier} -> {result.status}") + _checkpoint(frame, destination, current, checkpoint_every) + return _finish(frame, destination, statuses, force=force) + + +def run_pyrfume( + input_path: str | Path, + archive_client: Any, + *, + pubchem_client: Any | None = None, + output_path: str | Path | None = None, + cid_column: str = "PubChem CID", + identifier_column: str = "CAS Number", + archives: list[str] | None = None, + include_provenance: bool = True, + checkpoint_every: int = 25, + force: bool = False, + progress: ProgressCallback = console_progress, +) -> RunSummary: + frame, destination = _prepare_run(input_path, output_path, suffix="_pyrfume", force=force) + if cid_column not in frame.columns: + require_columns(frame, identifier_column) + if pubchem_client is None: + message = f"{cid_column!r} is absent; a PubChem client is required to resolve CIDs" + raise ValueError(message) + statuses: Counter[str] = Counter() + total = len(frame) + for current, index, row in _indices(frame): + cid = row.get(cid_column) + if (cid is None or str(cid).strip() in {"", "nan"}) and pubchem_client is not None: + identity = pubchem_client.lookup(row[identifier_column], include_odor=False) + cid = identity.values.get("cid") + _apply_result( + frame, + index, + identity, + include_provenance=include_provenance, + prefix="PubChem", + key_map=PUBCHEM_COLUMN_MAP, + ) + if cid is None or str(cid).strip() in {"", "nan"}: + result = LookupResult.failure( + "Pyrfume", status="invalid_input", message="No PubChem CID available" + ) + else: + result = archive_client.lookup(cid, archives=archives) + _apply_result( + frame, + index, + result, + include_provenance=include_provenance, + prefix="Pyrfume", + ) + statuses[result.status] += 1 + progress(current, total, f"Pyrfume CID {cid}: {result.status}") + _checkpoint(frame, destination, current, checkpoint_every) + return _finish(frame, destination, statuses, force=force) + + +def run_m2or( + input_path: str | Path, + client: Any, + *, + output_path: str | Path | None = None, + cas_column: str = "CAS Number", + include_provenance: bool = True, + checkpoint_every: int = 25, + force: bool = False, + progress: ProgressCallback = console_progress, +) -> RunSummary: + frame, destination = _prepare_run(input_path, output_path, suffix="_m2or", force=force) + require_columns(frame, cas_column) + statuses: Counter[str] = Counter() + total = len(frame) + for current, index, row in _indices(frame): + result = client.lookup_cas(row[cas_column]) + _apply_result( + frame, + index, + result, + include_provenance=include_provenance, + prefix="M2OR", + ) + statuses[result.status] += 1 + progress(current, total, f"M2OR: {row[cas_column]} -> {result.status}") + _checkpoint(frame, destination, current, checkpoint_every) + return _finish(frame, destination, statuses, force=force) + + +def run_mffi( + input_path: str | Path, + client: Any, + *, + output_path: str | Path | None = None, + cas_column: str = "CAS Number", + include_provenance: bool = True, + checkpoint_every: int = 10, + delay: float = 2.0, + force: bool = False, + progress: ProgressCallback = console_progress, + sleep: Callable[[float], None] = time.sleep, +) -> RunSummary: + frame, destination = _prepare_run(input_path, output_path, suffix="_mffi_result", force=force) + require_columns(frame, cas_column) + statuses: Counter[str] = Counter() + total = len(frame) + for current, index, row in _indices(frame): + result = client.lookup_cas(row[cas_column]) + if not result.values: + result.values = { + "Chinese Name": "\\", + "English Name": "\\", + "Sensory Characteristics": "\\", + "In Water": "\\", + } + _apply_result( + frame, + index, + result, + include_provenance=include_provenance, + prefix="MFFI", + ) + statuses[result.status] += 1 + progress(current, total, f"MFFI: {row[cas_column]} -> {result.status}") + _checkpoint(frame, destination, current, checkpoint_every) + if delay > 0 and current < total: + sleep(delay) + return _finish(frame, destination, statuses, force=force) + + +def run_chemicalbook_legacy( + input_path: str | Path, + client: Any, + *, + output_path: str | Path | None = None, + cas_column: str = "CAS Number", + include_provenance: bool = True, + checkpoint_every: int = 5, + delay: float = 2.0, + force: bool = False, + progress: ProgressCallback = console_progress, + prompt: Callable[[str], str] = input, + sleep: Callable[[float], None] = time.sleep, +) -> RunSummary: + frame, destination = _prepare_run(input_path, output_path, suffix="_cb_result", force=force) + require_columns(frame, cas_column) + statuses: Counter[str] = Counter() + total = len(frame) + for current, index, row in _indices(frame): + while True: + try: + result = client.lookup_cas(row[cas_column]) + break + except ManualVerificationRequired as exc: + question = f"{exc}\nPress Enter to retry, n to skip, or q to stop: " + answer = prompt(question).strip().lower() + if answer == "q": + raise KeyboardInterrupt("ChemicalBook run stopped by user") from exc + if answer == "n": + result = LookupResult.failure( + "ChemicalBook", + status="skipped", + message="Skipped after manual browser inspection", + ) + break + if not result.values: + result.values = { + "CB_Odor_Desc": "\\", + "CB_Odor_Threshold": "\\", + "CB_Odor_Type": "\\", + } + _apply_result( + frame, + index, + result, + include_provenance=include_provenance, + prefix="ChemicalBook", + ) + statuses[result.status] += 1 + progress(current, total, f"ChemicalBook: {row[cas_column]} -> {result.status}") + _checkpoint(frame, destination, current, checkpoint_every) + if delay > 0 and current < total: + sleep(delay) + return _finish(frame, destination, statuses, force=force) diff --git a/mffi_spider.py b/mffi_spider.py index cdc6c05..571ffef 100644 --- a/mffi_spider.py +++ b/mffi_spider.py @@ -1,135 +1,40 @@ -import pandas as pd -from selenium import webdriver -from selenium.webdriver.chrome.service import Service -from selenium.webdriver.common.by import By -from selenium.webdriver.support.ui import WebDriverWait -from selenium.webdriver.support import expected_conditions as EC -from webdriver_manager.chrome import ChromeDriverManager -import time -import os - -# ================= 配置区域 ================= -INPUT_FILE = "max.xlsx" -COL_CAS = "CAS Number" -# =========================================== - -def init_driver(): - options = webdriver.ChromeOptions() - options.add_argument("--start-maximized") - options.add_argument('--disable-gpu') - options.add_argument('--no-sandbox') - options.add_argument("--log-level=3") - service = Service(ChromeDriverManager().install()) - driver = webdriver.Chrome(service=service, options=options) - return driver - -def get_mffi_data(driver, cas): - # 构造结果字典 - data = { - "Chinese Name": "\\", - "English Name": "\\", - "Sensory Characteristics": "\\", - "In Water": "\\" - } - - try: - # 1. 访问网页 - url = f"https://mffi.sjtu.edu.cn/database/search?value={cas}&keyword=all" - driver.get(url) - - # 2. 等待表格出现 (最多等10秒) - wait = WebDriverWait(driver, 10) - # 只要 tbody 里有行(tr)出现，就开始处理 - rows = wait.until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, "tbody tr"))) - - # 3. 遍历每一行寻找匹配的数据 - for row in rows: - # 获取这一行的所有格子 - cells = row.find_elements(By.TAG_NAME, "td") - - # 如果格子太少，说明是空行或格式不对，跳过 - if len(cells) < 7: - continue - - # 获取这一行里显示的 CAS 号 (第4列，索引是3) - row_cas_text = cells[3].text.strip() - - # 检查 CAS 号是否匹配 (只要包含就可以) - if cas in row_cas_text or row_cas_text in cas: - # 提取我们需要的数据 - data["Chinese Name"] = cells[0].text.strip() - data["English Name"] = cells[1].text.strip() - data["Sensory Characteristics"] = cells[5].text.strip() - data["In Water"] = cells[6].text.strip() - # 找到了就不找了，直接退出循环 - break - - except Exception as e: - # 如果出错（比如超时没找到），什么都不做，直接返回空数据 - # 这样程序就不会崩溃 - pass - - return data - -def main(): - print(f"读取文件: {INPUT_FILE} ...") - - if not os.path.exists(INPUT_FILE): - print(f"[错误] 找不到文件 {INPUT_FILE}") - return - - try: - df = pd.read_excel(INPUT_FILE, dtype={COL_CAS: str}) - except Exception as e: - print(f"[错误] Excel 读取失败: {e}") - return - - print("正在启动浏览器...") - driver = init_driver() - - try: - total = len(df) - print("-" * 50) - print(f"开始任务，共 {total} 条数据") - print("-" * 50) - - for index, row in df.iterrows(): - # 获取 CAS 号并转为字符串 - cas = row[COL_CAS] - - # 如果为空则跳过 - if pd.isna(cas) or str(cas).strip() == "": - continue - - cas = str(cas).strip() - - print(f"[{index+1}/{total}] 搜索: {cas} ... ", end="", flush=True) - - # 执行抓取 - result = get_mffi_data(driver, cas) - - # 填入表格 - df.at[index, "Chinese Name"] = result["Chinese Name"] - df.at[index, "English Name"] = result["English Name"] - df.at[index, "Sensory Characteristics"] = result["Sensory Characteristics"] - df.at[index, "In Water"] = result["In Water"] - - # 打印结果 - if result["Chinese Name"] != "\\": - print(f"成功 -> {result['Chinese Name']}") - else: - print("未找到") - - time.sleep(1) - - # 保存结果 - output_file = INPUT_FILE.replace(".xlsx", "_mffi_result.xlsx") - df.to_excel(output_file, index=False) - print("-" * 50) - print(f"全部完成！结果已保存为: {output_file}") - - finally: - driver.quit() - -if __name__ == "__main__": - main() \ No newline at end of file +"""Legacy launcher for the original MFFI workbook workflow.""" + +from flavor_data_crawler.sources.mffi import MffiClient, create_chrome_driver +from flavor_data_crawler.workflows import run_mffi + +INPUT_FILE = "max.xlsx" +COL_CAS = "CAS Number" + + +def init_driver(): + return create_chrome_driver(headless=False) + + +def get_mffi_data(driver, cas: object) -> dict[str, object]: + result = MffiClient(driver=driver).lookup_cas(cas) + return result.values or { + "Chinese Name": "\\", + "English Name": "\\", + "Sensory Characteristics": "\\", + "In Water": "\\", + } + + +def main() -> None: + output = INPUT_FILE.replace(".xlsx", "_mffi_result.xlsx") + with MffiClient() as client: + summary = run_mffi( + INPUT_FILE, + client, + output_path=output, + cas_column=COL_CAS, + include_provenance=False, + checkpoint_every=10, + force=True, + ) + print(f"Done: {summary.output_path}") + + +if __name__ == "__main__": + main() diff --git a/name_to_cas.py b/name_to_cas.py index 06f6089..5609d7e 100644 --- a/name_to_cas.py +++ b/name_to_cas.py @@ -1,129 +1,40 @@ -import pandas as pd -import requests -from bs4 import BeautifulSoup -import time -import os - -# ================= 配置区域 ================= -# 你的 Excel 文件名 -INPUT_FILE = "name.xlsx" - -# Excel 中存放 物质名称 的列名 (请确保和Excel里一致) -COL_NAME = "Name" - -# 结果将写入的新列名 -COL_RESULT = "Found CAS" -# =========================================== - -def get_cas_by_name(name): - """ - 根据物质名称去 NIST 搜索 CAS 号。 - """ - if pd.isna(name) or str(name).strip() == "": - return "\\" - - clean_name = str(name).strip() - # NIST 的名称搜索 URL 结构 - # 这里的 options=On-off 表示精确匹配逻辑，通常直接搜 Name 即可 - url = "https://webbook.nist.gov/cgi/cbook.cgi" - params = { - "Name": clean_name, - "Units": "SI" - } - - headers = { - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36' - } - - try: - # 发送请求 - response = requests.get(url, params=params, headers=headers, timeout=15) - - # 如果找不到页面，直接返回 - if response.status_code != 200: - return "Connection Error" - - soup = BeautifulSoup(response.text, 'html.parser') - - # 1. 检查是否进入了“搜索结果列表”页面 (即没有直接跳转到物质详情页) - # 如果网页标题包含 "Search Results"，说明名字有歧义或者没完全匹配 - if "Search Results" in soup.title.text: - # 这种情况下，通常我们尝试找第一个链接，或者直接标记为“需人工确认” - # 为了简单，这里我们只抓取完全匹配进入详情页的情况 - return "Ambiguous/List Found" - - # 2. 检查是否是“Name Not Found” - if "Name Not Found" in response.text: - return "Not Found" - - # 3. 在详情页寻找 "CAS Registry Number:" - # NIST 页面结构通常是:
  • CAS Registry Number: 100-52-7
    • - cas_label = soup.find('strong', string="CAS Registry Number:") - - if cas_label: - # 获取 label 后面的文本 - cas_text = cas_label.next_sibling - if cas_text: - return cas_text.strip() - - # 备选方案：如果上面的结构变了，尝试搜索文本 - # 有时候页面结构不同，防止漏抓 - all_text = soup.get_text() - if "CAS Registry Number:" in all_text: - # 简单的文本截取兜底 - import re - match = re.search(r'CAS Registry Number:\s*([\d\-]+)', all_text) - if match: - return match.group(1) - - return "Not Found in Page" - - except Exception as e: - print(f"Error processing {name}: {e}") - return "Error" - -def main(): - print(f"Reading file: {INPUT_FILE} ...") - - if not os.path.exists(INPUT_FILE): - print(f"Error: File not found {INPUT_FILE}") - return - - try: - df = pd.read_excel(INPUT_FILE, dtype=str) # 全部按字符串读取，防止数字名称变型 - except Exception as e: - print(f"Excel read error: {e}") - return - - # 检查列名 - if COL_NAME not in df.columns: - print(f"Error: Column '{COL_NAME}' not found.") - print(f"Current columns: {list(df.columns)}") - return - - print("Searching for CAS numbers... (Please wait)") - - total_rows = len(df) - for index, row in df.iterrows(): - name = row[COL_NAME] - - print(f"[{index+1}/{total_rows}] Searching: {name} ... ", end="", flush=True) - - result = get_cas_by_name(name) - - df.at[index, COL_RESULT] = result - print(f"Found: {result}") - - # 礼貌爬虫，防止封IP - time.sleep(1) - - # 保存结果 - output_filename = INPUT_FILE.replace(".xlsx", "_with_cas.xlsx") - df.to_excel(output_filename, index=False) - - print("-" * 30) - print(f"Done! Saved to: {output_filename}") - print("-" * 30) - -if __name__ == "__main__": - main() \ No newline at end of file +"""Legacy launcher for the original NIST name-to-CAS workbook workflow.""" + +from flavor_data_crawler.sources.nist import NistWebBookClient +from flavor_data_crawler.workflows import run_resolve_cas + +INPUT_FILE = "name.xlsx" +COL_NAME = "Name" +COL_RESULT = "Found CAS" + + +def get_cas_by_name(name: object) -> str: + """Compatibility helper returning the historic text statuses.""" + + result = NistWebBookClient().resolve_name(name) + if result.status == "ok": + return str(result.values.get("cas", "Not Found")) + return { + "ambiguous": "Ambiguous/List Found", + "not_found": "Not Found", + "network_error": "Connection Error", + }.get(result.status, "Error") + + +def main() -> None: + output = INPUT_FILE.replace(".xlsx", "_with_cas.xlsx") + summary = run_resolve_cas( + INPUT_FILE, + NistWebBookClient(), + output_path=output, + name_column=COL_NAME, + result_column=COL_RESULT, + include_provenance=False, + checkpoint_every=25, + force=True, + ) + print(f"Done: {summary.output_path}") + + +if __name__ == "__main__": + main() diff --git a/nist_excel_tool.py b/nist_excel_tool.py index 77dd1a0..fdd8bbc 100644 --- a/nist_excel_tool.py +++ b/nist_excel_tool.py @@ -1,153 +1,36 @@ -import pandas as pd -import requests -from bs4 import BeautifulSoup -import re -import time -import os - -# ================= 配置区域 (请在这里修改) ================= -# 你的 Excel 文件名 (必须带后缀, 如 .xlsx) -INPUT_FILE = "data.xlsx" - -# Excel 中存放 CAS 号的列名 (请确保Excel里第一行是列名) -COL_CAS = "CAS Number" - -# Excel 中存放 计算保留指数 的列名 -COL_CALC_RI = "Calculated RI" - -# 结果将写入的新列名 -COL_RESULT = "NIST RI" -# ======================================================== - -def get_nist_ri(cas, calc_ri): - """ - 根据 CAS 号去 NIST 查找最接近的 RI 值。 - """ - # 如果 CAS 为空或无效，直接返回 \ - if pd.isna(cas) or str(cas).strip() == "": - return "\\" - - clean_cas = str(cas).strip().replace('-', '') - url = f"https://webbook.nist.gov/cgi/cbook.cgi?ID=C{clean_cas}&Units=SI&Mask=2000" - - headers = { - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36' - } - - try: - # 发送请求 - response = requests.get(url, headers=headers, timeout=15) - if response.status_code != 200: - return "\\" - - soup = BeautifulSoup(response.text, 'html.parser') - - # 定位目标表格 - target_title = "Normal alkane RI, non-polar column, custom temperature program" - target_table = soup.find('table', attrs={'aria-label': target_title}) - - # 如果通过 aria-label 找不到，尝试通过文本查找 - if not target_table: - text_matches = soup.find_all(string=re.compile(re.escape(target_title))) - for match in text_matches: - parent = match.parent - next_table = parent.find_next('table') - if next_table: - target_table = next_table - break - - if not target_table: - return "\\" - - # 找到 "I" (保留指数) 所在的列 - header_row = target_table.find('tr') - headers_text = [th.get_text(strip=True) for th in header_row.find_all(['th', 'td'])] - - try: - ri_col_index = headers_text.index('I') - except ValueError: - return "\\" - - ri_values = [] - - # 遍历所有行提取数据 - rows = target_table.find_all('tr')[1:] - for row in rows: - cols = row.find_all(['td', 'th']) - if len(cols) > ri_col_index: - val_text = cols[ri_col_index].get_text(strip=True) - # 清洗非数字字符 - val_clean = re.sub(r'[^\d\.]', '', val_text) - try: - if val_clean: - ri_values.append(float(val_clean)) - except ValueError: - continue - - if not ri_values: - return "\\" - - # 找出与计算值 (calc_ri) 差值最小的那个数 - closest_ri = min(ri_values, key=lambda x: abs(x - float(calc_ri))) - - # 格式化返回（如果是整数显示整数） - if closest_ri.is_integer(): - return int(closest_ri) - else: - return closest_ri - - except Exception as e: - print(f"Error processing CAS {cas}: {e}") - return "\\" - -def main(): - print(f"正在读取文件: {INPUT_FILE} ...") - - if not os.path.exists(INPUT_FILE): - print(f"错误: 找不到文件 {INPUT_FILE}") - return - - # 读取 Excel，强制将 CAS 列作为字符串读取，防止 Excel 自动转成日期 - try: - df = pd.read_excel(INPUT_FILE, dtype={COL_CAS: str}) - except Exception as e: - print(f"读取 Excel 失败: {e}") - return - - # 检查列名是否存在 - if COL_CAS not in df.columns or COL_CALC_RI not in df.columns: - print(f"错误: Excel 中找不到列名 '{COL_CAS}' 或 '{COL_CALC_RI}'。") - print(f"当前 Excel 列名: {list(df.columns)}") - return - - print("开始处理数据，请稍候...") - print("注意: 为防止被 NIST 网站屏蔽，每条数据间会有 1 秒的间隔。") - - # 遍历 DataFrame 处理每一行 - total_rows = len(df) - for index, row in df.iterrows(): - cas = row[COL_CAS] - calc_ri = row[COL_CALC_RI] - - print(f"[{index+1}/{total_rows}] 处理 CAS: {cas} (计算值: {calc_ri}) ... ", end="", flush=True) - - result = get_nist_ri(cas, calc_ri) - - # 写入结果 - df.at[index, COL_RESULT] = result - - print(f"结果: {result}") - - # 休眠 1 秒，礼貌爬虫 - time.sleep(1) - - # 保存结果 - output_filename = INPUT_FILE.replace(".xlsx", "_result.xlsx") - df.to_excel(output_filename, index=False) - - print("-" * 30) - print(f"完成！结果已保存至: {output_filename}") - print("-" * 30) - -if __name__ == "__main__": - main() \ No newline at end of file +"""Legacy launcher for the original NIST retention-index workbook workflow.""" + +from flavor_data_crawler.sources.nist import NistWebBookClient +from flavor_data_crawler.workflows import run_nist_ri + +INPUT_FILE = "data.xlsx" +COL_CAS = "CAS Number" +COL_CALC_RI = "Calculated RI" +COL_RESULT = "NIST RI" + + +def get_nist_ri(cas: object, calc_ri: object) -> object: + """Compatibility helper returning the historic value-or-backslash shape.""" + + result = NistWebBookClient().lookup_ri(cas, calc_ri) + return result.values.get("retention_index", "\\") + + +def main() -> None: + output = INPUT_FILE.replace(".xlsx", "_result.xlsx") + summary = run_nist_ri( + INPUT_FILE, + NistWebBookClient(), + output_path=output, + cas_column=COL_CAS, + calculated_ri_column=COL_CALC_RI, + result_column=COL_RESULT, + include_provenance=False, + checkpoint_every=25, + force=True, + ) + print(f"Done: {summary.output_path}") + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..7c572cc --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,52 @@ +[build-system] +requires = ["setuptools>=69"] +build-backend = "setuptools.build_meta" + +[project] +name = "flavor-data-crawler" +version = "0.2.0" +description = "Provenance-aware enrichment for retention-index, odorant, and olfactory-receptor data" +readme = "README.md" +requires-python = ">=3.11" +dependencies = [ + "beautifulsoup4>=4.12", + "openpyxl>=3.1", + "pandas>=2.1", + "requests>=2.31", + "selenium>=4.17", +] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Science/Research", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Scientific/Engineering :: Bio-Informatics", + "Topic :: Scientific/Engineering :: Chemistry", +] +keywords = ["cheminformatics", "flavor", "odorant", "olfaction", "retention-index"] + +[project.optional-dependencies] +dev = [ + "pytest>=8.0", + "pytest-cov>=5.0", + "ruff>=0.6", +] + +[project.scripts] +flavor-data = "flavor_data_crawler.cli:main" + +[tool.setuptools.packages.find] +include = ["flavor_data_crawler*"] + +[tool.pytest.ini_options] +addopts = "-ra" +testpaths = ["tests"] + +[tool.ruff] +line-length = 100 +target-version = "py311" + +[tool.ruff.lint] +select = ["E", "F", "I", "UP", "B"] diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..aefbcb6 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1 @@ +-e .[dev] diff --git a/requirements.txt b/requirements.txt index dfd6ba9..d6e1198 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1 @@ -pandas -openpyxl -requests -beautifulsoup4 -selenium -webdriver_manager \ No newline at end of file +-e . diff --git a/start1.bat b/start1.bat index 27638ee..7bf6479 100644 --- a/start1.bat +++ b/start1.bat @@ -1,23 +1,19 @@ -@echo off -cd /d "%~dp0" - -:: IMPORTANT: If your folder is named 'venv', change 'myenv' to 'venv' below -if exist myenv\Scripts\activate.bat ( - call myenv\Scripts\activate.bat -) else ( - if exist venv\Scripts\activate.bat ( - call venv\Scripts\activate.bat - ) else ( - echo Error: Virtual environment not found. - echo Please check if the folder is named 'myenv' or 'venv'. - pause - exit /b - ) -) - -echo Environment activated. Running Python script... -python nist_excel_tool.py - -echo. -echo Done. -pause \ No newline at end of file +@echo off +cd /d "%~dp0" +call :find_python +"%PYTHON_EXE%" nist_excel_tool.py +pause +exit /b + +:find_python +set "PYTHON_EXE=python" +if exist ".venv\Scripts\python.exe" ( + set "PYTHON_EXE=.venv\Scripts\python.exe" + exit /b +) +if exist "myenv\Scripts\python.exe" ( + set "PYTHON_EXE=myenv\Scripts\python.exe" + exit /b +) +if exist "venv\Scripts\python.exe" set "PYTHON_EXE=venv\Scripts\python.exe" +exit /b diff --git a/start2.bat b/start2.bat index 9e6137e..9bf1807 100644 --- a/start2.bat +++ b/start2.bat @@ -1,38 +1,19 @@ -@echo off -cd /d "%~dp0" - -echo [STEP 1] Activating Virtual Environment... - -:: Try to activate 'myenv' directly -if exist myenv\Scripts\activate.bat call myenv\Scripts\activate.bat - -:: Try to activate 'venv' directly -if exist venv\Scripts\activate.bat call venv\Scripts\activate.bat - -echo. -echo [STEP 2] Checking Python File... - -:: Check if the file exists before running -if not exist name_to_cas.py goto FileNotFound - -echo. -echo [STEP 3] Running Python Program... -echo ---------------------------------------- -python name_to_cas.py -echo ---------------------------------------- -goto End - -:FileNotFound -echo. -echo [ERROR] CRITICAL ERROR! -echo Python file 'name_to_cas.py' was NOT found in this folder. -echo. -echo PLEASE CHECK: -echo 1. Is the file named 'name_to_cas.py'? -echo 2. Did Windows hide the '.txt' extension? (It might be 'name_to_cas.py.txt') -echo. - -:End -echo. -echo Program Finished. -pause \ No newline at end of file +@echo off +cd /d "%~dp0" +call :find_python +"%PYTHON_EXE%" name_to_cas.py +pause +exit /b + +:find_python +set "PYTHON_EXE=python" +if exist ".venv\Scripts\python.exe" ( + set "PYTHON_EXE=.venv\Scripts\python.exe" + exit /b +) +if exist "myenv\Scripts\python.exe" ( + set "PYTHON_EXE=myenv\Scripts\python.exe" + exit /b +) +if exist "venv\Scripts\python.exe" set "PYTHON_EXE=venv\Scripts\python.exe" +exit /b diff --git a/start3.bat b/start3.bat index 512afd6..608d92e 100644 --- a/start3.bat +++ b/start3.bat @@ -1,30 +1,19 @@ -@echo off -cd /d "%~dp0" - -echo ========================================== -echo MFFI Database Spider (Selenium) -echo ========================================== - -:: 激活虚拟环境 -if exist myenv\Scripts\activate.bat ( - call myenv\Scripts\activate.bat -) else if exist venv\Scripts\activate.bat ( - call venv\Scripts\activate.bat -) else ( - echo [Error] Virtual environment not found. - pause - exit /b -) - -echo. -echo Checking and installing dependencies... -:: 自动安装 selenium 依赖 (第一次运行时需要) -pip install selenium webdriver-manager pandas openpyxl - -echo. -echo Starting Spider... -python mffi_spider.py - -echo. -echo Process Finished. -pause \ No newline at end of file +@echo off +cd /d "%~dp0" +call :find_python +"%PYTHON_EXE%" mffi_spider.py +pause +exit /b + +:find_python +set "PYTHON_EXE=python" +if exist ".venv\Scripts\python.exe" ( + set "PYTHON_EXE=.venv\Scripts\python.exe" + exit /b +) +if exist "myenv\Scripts\python.exe" ( + set "PYTHON_EXE=myenv\Scripts\python.exe" + exit /b +) +if exist "venv\Scripts\python.exe" set "PYTHON_EXE=venv\Scripts\python.exe" +exit /b diff --git a/start4.bat b/start4.bat index ac1a6ff..3392df4 100644 --- a/start4.bat +++ b/start4.bat @@ -1,30 +1,19 @@ -@echo off -cd /d "%~dp0" - -echo ========================================== -echo ChemicalBook Crawler -echo ========================================== - -:: 激活环境 -if exist myenv\Scripts\activate.bat ( - call myenv\Scripts\activate.bat -) else if exist venv\Scripts\activate.bat ( - call venv\Scripts\activate.bat -) else ( - echo [Error] Virtual environment not found. - pause - exit /b -) - -echo. -echo Checking dependencies... -:: 确保依赖都装了 -pip install selenium webdriver-manager pandas openpyxl - -echo. -echo Starting Python Script... -python cb_spider.py - -echo. -echo Process Finished. -pause \ No newline at end of file +@echo off +cd /d "%~dp0" +call :find_python +"%PYTHON_EXE%" cb_spider.py +pause +exit /b + +:find_python +set "PYTHON_EXE=python" +if exist ".venv\Scripts\python.exe" ( + set "PYTHON_EXE=.venv\Scripts\python.exe" + exit /b +) +if exist "myenv\Scripts\python.exe" ( + set "PYTHON_EXE=myenv\Scripts\python.exe" + exit /b +) +if exist "venv\Scripts\python.exe" set "PYTHON_EXE=venv\Scripts\python.exe" +exit /b diff --git a/tests/fixtures/M2OR.synthetic.csv b/tests/fixtures/M2OR.synthetic.csv new file mode 100644 index 0000000..01fbbbf --- /dev/null +++ b/tests/fixtures/M2OR.synthetic.csv @@ -0,0 +1,7 @@ +;id;species;Mutation;Gene ID;Uniprot ID;Sequence;Name;CID;CAS;InChI Key;canonicalSMILES;Parameter;Value;Unit;Value_Screen;Unit_Screen;Responsive;nbr_measurements;Type;Cell_line;Delivery;Assay;Gprotein;Co_transfection;Assay System;Tag;Reference;DOI;Reference Position;Mixture;Norm_Foreign_Key +0;pair-1;homo sapiens;;OR1A1;Q00001;;synthetic molecule;;64-17-5;;;;;;;;1;;test;;;;;;;;Synthetic study A;https://doi.org/10.1000/a;;; +1;experiment-2;homo sapiens;;OR1A1;Q00001;;synthetic molecule;;64-17-5;;;;;;;;0;;test;;;;;;;;Synthetic study A;https://doi.org/10.1000/a;;; +2;pair-2;homo sapiens;;OR2W1;Q00002;;synthetic molecule;;64-17-5;;;;;;;;true;;test;;;;;;;;Synthetic study B;doi:10.1000/b;;; +3;pair-3;Mus musculus;;Olfr1;Q00003;;synthetic molecule;;64-17-5;;;;;;;;agonist;;test;;;;;;;;Synthetic study C;10.1000/c;;; +4;pair-4;homo sapiens;;OR5X1;Q00004;;synthetic molecule;;64-17-5;;;;;;;;0;;test;;;;;;;;Synthetic study D;https://dx.doi.org/10.1000/d;;; +5;pair-5;homo sapiens;;OR9X1;Q00005;;other molecule;;67-56-1;;;;;;;;1;;test;;;;;;;;Other study;10.1000/other;;; diff --git a/tests/test_browser_sources.py b/tests/test_browser_sources.py new file mode 100644 index 0000000..3e25e31 --- /dev/null +++ b/tests/test_browser_sources.py @@ -0,0 +1,27 @@ +from flavor_data_crawler.sources.chemicalbook import ChemicalBookLegacyClient +from flavor_data_crawler.sources.mffi import MffiClient + + +class FakeDriver: + def __init__(self): + self.visited = [] + self.quit_called = False + + def get(self, url): + self.visited.append(url) + + def quit(self): + self.quit_called = True + + +def test_chemicalbook_is_blocked_without_permission(): + result = ChemicalBookLegacyClient().lookup_cas("100-52-7") + assert result.status == "blocked" + assert "robots.txt" in result.message + + +def test_mffi_rejects_invalid_cas_before_browser_use(): + driver = FakeDriver() + result = MffiClient(driver=driver).lookup_cas("100-52-8") + assert result.status == "invalid_input" + assert driver.visited == [] diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..58329c9 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,15 @@ +from flavor_data_crawler.cli import main + + +def test_sources_command_lists_access_modes(capsys): + assert main(["sources"]) == 0 + output = capsys.readouterr().out + assert "PubChem" in output + assert "ChemicalBook" in output + assert "disabled" in output + + +def test_missing_input_returns_user_error(capsys, tmp_path): + code = main(["pubchem", str(tmp_path / "missing.xlsx")]) + assert code == 2 + assert "does not exist" in capsys.readouterr().err diff --git a/tests/test_excel_io.py b/tests/test_excel_io.py new file mode 100644 index 0000000..ace171f --- /dev/null +++ b/tests/test_excel_io.py @@ -0,0 +1,40 @@ +from pathlib import Path + +import openpyxl +import pandas as pd +import pytest + +from flavor_data_crawler.excel_io import ( + read_table, + require_columns, + sanitize_excel_cell, + write_table, +) + + +def test_sanitizes_formula_prefixes_without_touching_numbers(): + assert sanitize_excel_cell("=2+2") == "'=2+2" + assert sanitize_excel_cell("@SUM(A1:A2)") == "'@SUM(A1:A2)" + assert sanitize_excel_cell(42) == 42 + assert sanitize_excel_cell("benzaldehyde") == "benzaldehyde" + + +def test_xlsx_round_trip_is_formula_safe_and_atomic(tmp_path: Path): + output = tmp_path / "safe.xlsx" + frame = pd.DataFrame({"CAS Number": ["100-52-7"], "Remote Text": ["=2+2"]}) + write_table(frame, output) + + workbook = openpyxl.load_workbook(output, data_only=False) + assert workbook.active["B2"].value == "'=2+2" + loaded = read_table(output) + assert loaded.loc[0, "CAS Number"] == "100-52-7" + + +def test_refuses_overwrite_and_reports_missing_columns(tmp_path: Path): + output = tmp_path / "data.csv" + frame = pd.DataFrame({"Name": ["Nonanal"]}) + write_table(frame, output) + with pytest.raises(FileExistsError): + write_table(frame, output) + with pytest.raises(ValueError, match="CAS Number"): + require_columns(frame, "Name", "CAS Number") diff --git a/tests/test_http.py b/tests/test_http.py new file mode 100644 index 0000000..32b899d --- /dev/null +++ b/tests/test_http.py @@ -0,0 +1,184 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +import pytest +import requests + +from flavor_data_crawler.http import CachedHttpClient, HttpClientError + + +@dataclass +class FakeRawResponse: + status_code: int = 200 + text: str = "{}" + headers: dict[str, str] = field(default_factory=dict) + url: str = "https://example.test/data" + + +class QueueSession: + def __init__(self, *outcomes: FakeRawResponse | BaseException) -> None: + self.outcomes = list(outcomes) + self.calls: list[dict[str, Any]] = [] + + def request(self, method: str, url: str, **kwargs: Any) -> FakeRawResponse: + self.calls.append({"method": method, "url": url, **kwargs}) + outcome = self.outcomes.pop(0) + if isinstance(outcome, BaseException): + raise outcome + return outcome + + +class FakeClock: + def __init__(self, now: float = 0.0) -> None: + self.now = now + self.sleeps: list[float] = [] + + def monotonic(self) -> float: + return self.now + + def time(self) -> float: + return 1_000 + self.now + + def sleep(self, seconds: float) -> None: + self.sleeps.append(seconds) + self.now += seconds + + +def test_successful_get_is_reused_from_disk_cache(tmp_path: Any) -> None: + first_session = QueueSession( + FakeRawResponse(text='{"answer": 42}', headers={"Content-Type": "application/json"}) + ) + clock = FakeClock() + first_client = CachedHttpClient( + cache_dir=tmp_path, + session=first_session, + wall_clock=clock.time, + ) + + first = first_client.get( + "https://example.test/data", + params={"z": "last", "a": "first"}, + ) + assert first.json() == {"answer": 42} + assert first.metadata.cache_hit is False + assert first.metadata.attempts == 1 + assert len(first_session.calls) == 1 + + second_session = QueueSession() + second_client = CachedHttpClient( + cache_dir=tmp_path, + session=second_session, + wall_clock=clock.time, + ) + second = second_client.get( + "https://example.test/data", + params={"a": "first", "z": "last"}, + ) + + assert second.json() == {"answer": 42} + assert second.metadata.cache_hit is True + assert second.metadata.attempts == 0 + assert second.metadata.retrieved_at == first.metadata.retrieved_at + assert second_session.calls == [] + + +def test_stale_cache_is_refreshed(tmp_path: Any) -> None: + clock = FakeClock() + first_session = QueueSession(FakeRawResponse(text="old")) + first_client = CachedHttpClient( + cache_dir=tmp_path, + cache_ttl=10, + session=first_session, + wall_clock=clock.time, + ) + first_client.get("https://example.test/data") + clock.now = 11 + + second_session = QueueSession(FakeRawResponse(text="new")) + second_client = CachedHttpClient( + cache_dir=tmp_path, + cache_ttl=10, + session=second_session, + wall_clock=clock.time, + ) + refreshed = second_client.get("https://example.test/data") + + assert refreshed.text == "new" + assert refreshed.metadata.cache_hit is False + assert len(second_session.calls) == 1 + + +def test_rate_limit_and_retry_after_are_applied_without_network(tmp_path: Any) -> None: + clock = FakeClock() + session = QueueSession( + FakeRawResponse(status_code=429, headers={"Retry-After": "1.5"}), + FakeRawResponse(status_code=200, text="ok"), + FakeRawResponse(status_code=200, text="second", url="https://example.test/other"), + ) + client = CachedHttpClient( + cache_dir=tmp_path, + min_interval=0.25, + max_retries=1, + backoff_factor=0.1, + session=session, + sleep=clock.sleep, + monotonic=clock.monotonic, + wall_clock=clock.time, + ) + + response = client.get("https://example.test/data") + client.get("https://example.test/other") + + assert response.text == "ok" + assert response.metadata.attempts == 2 + assert clock.sleeps == pytest.approx([1.5, 0.25]) + assert len(session.calls) == 3 + + +def test_timeout_and_headers_are_forwarded(tmp_path: Any) -> None: + session = QueueSession(FakeRawResponse()) + client = CachedHttpClient(cache_dir=tmp_path, timeout=(2.0, 7.0), session=session) + + client.get("https://example.test/data", headers={"Accept": "application/json"}) + + call = session.calls[0] + assert call["timeout"] == (2.0, 7.0) + assert call["headers"]["Accept"] == "application/json" + assert call["headers"]["User-Agent"].startswith("Flavor-Data-Crawler/") + + +def test_request_exception_retries_then_raises_descriptive_error(tmp_path: Any) -> None: + clock = FakeClock() + session = QueueSession( + requests.ConnectionError("offline"), + requests.Timeout("still offline"), + ) + client = CachedHttpClient( + cache_dir=tmp_path, + max_retries=1, + backoff_factor=0.5, + session=session, + sleep=clock.sleep, + monotonic=clock.monotonic, + ) + + with pytest.raises(HttpClientError, match="2 attempt") as exc_info: + client.get("https://example.test/data") + + assert exc_info.value.attempts == 2 + assert clock.sleeps == [0.5] + assert len(session.calls) == 2 + + +def test_non_retryable_http_error_is_returned_once(tmp_path: Any) -> None: + session = QueueSession(FakeRawResponse(status_code=404, text="missing")) + client = CachedHttpClient(cache_dir=tmp_path, max_retries=3, session=session) + + response = client.get("https://example.test/data") + + assert response.status_code == 404 + assert response.ok is False + assert len(session.calls) == 1 + assert not list(tmp_path.glob("*.json")) diff --git a/tests/test_identifiers.py b/tests/test_identifiers.py new file mode 100644 index 0000000..1c0ee33 --- /dev/null +++ b/tests/test_identifiers.py @@ -0,0 +1,25 @@ +import pytest + +from flavor_data_crawler.identifiers import ( + clean_text, + is_valid_cas, + normalize_cas, + require_valid_cas, +) + + +@pytest.mark.parametrize("cas", ["100-52-7", "64-17-5", "7732-18-5", "5989-54-8"]) +def test_valid_cas_checksums(cas): + assert is_valid_cas(cas) + + +def test_normalizes_digits_and_unicode_spacing(): + assert normalize_cas(" 100527 ") == "100-52-7" + assert normalize_cas("\u00a0100-52-7\u00a0") == "100-52-7" + + +def test_rejects_bad_checksum_and_sentinels(): + assert not is_valid_cas("100-52-8") + assert clean_text("\\") == "" + with pytest.raises(ValueError, match="checksum"): + require_valid_cas("100-52-8") diff --git a/tests/test_m2or.py b/tests/test_m2or.py new file mode 100644 index 0000000..ef10fab --- /dev/null +++ b/tests/test_m2or.py @@ -0,0 +1,88 @@ +from __future__ import annotations + +from pathlib import Path + +from flavor_data_crawler.sources.m2or import ( + M2OR_FILENAME, + M2OR_LICENSE_URL, + M2OR_VERSION, + M2ORClient, +) + +FIXTURE = Path(__file__).parent / "fixtures" / "M2OR.synthetic.csv" + + +def test_lookup_cas_aggregates_distinct_pairs_from_semicolon_csv() -> None: + client = M2ORClient(data_path=FIXTURE, allow_download=False) + + result = client.lookup_cas("64175") + + assert result.status == "ok" + assert result.cache_hit is True + assert result.version == M2OR_VERSION + assert result.license_url == M2OR_LICENSE_URL + assert result.values == { + "M2OR Pair Count": 4, + "M2OR Responsive Count": 3, + "M2OR Species": "homo sapiens; Mus musculus", + "M2OR Human Responsive Receptors": "OR1A1; OR2W1", + "M2OR DOIs": "10.1000/a; 10.1000/b; 10.1000/c; 10.1000/d", + } + + +def test_lookup_cas_downloads_to_versioned_cache_then_runs_offline(tmp_path: Path) -> None: + payload = FIXTURE.read_bytes() + downloads: list[str] = [] + + def downloader(url: str, _destination: Path) -> bytes: + downloads.append(url) + return payload + + first = M2ORClient(cache_dir=tmp_path, downloader=downloader).lookup_cas("64-17-5") + assert first.status == "ok" + assert first.cache_hit is False + assert len(downloads) == 1 + cached_file = tmp_path / "m2or" / M2OR_VERSION / M2OR_FILENAME + assert cached_file.read_bytes() == payload + + second = M2ORClient(cache_dir=tmp_path, allow_download=False).lookup_cas("64-17-5") + assert second.status == "ok" + assert second.cache_hit is True + assert len(downloads) == 1 + + +def test_lookup_cas_supports_injected_rows_and_exact_matching() -> None: + rows = [ + { + "id": "a", + "CAS": "64-17-5", + "species": "homo sapiens", + "Gene ID": "OR1A1", + "Responsive": "1", + "DOI": "https://doi.org/10.1000/example", + }, + { + "id": "b", + "CAS": "164-17-5", + "species": "homo sapiens", + "Gene ID": "SHOULD_NOT_MATCH", + "Responsive": "1", + }, + ] + + result = M2ORClient(rows=rows, allow_download=False).lookup_cas("64-17-5") + + assert result.status == "ok" + assert result.values["M2OR Pair Count"] == 1 + assert result.values["M2OR Human Responsive Receptors"] == "OR1A1" + + +def test_lookup_cas_reports_invalid_missing_and_not_found(tmp_path: Path) -> None: + missing = M2ORClient(cache_dir=tmp_path, allow_download=False) + assert missing.lookup_cas("64-17-5").status == "missing_data" + + client = M2ORClient(data_path=FIXTURE, allow_download=False) + assert client.lookup_cas("not-cas").status == "invalid_input" + not_found = client.lookup_cas("50-00-0") + assert not_found.status == "not_found" + assert not_found.values["M2OR Pair Count"] == 0 diff --git a/tests/test_nist.py b/tests/test_nist.py new file mode 100644 index 0000000..0f302d3 --- /dev/null +++ b/tests/test_nist.py @@ -0,0 +1,140 @@ +from __future__ import annotations + +from typing import Any + +from flavor_data_crawler.http import HttpResponse, RetrievalMetadata +from flavor_data_crawler.sources.nist import ( + NistWebBookClient, + is_nist_search_results, + parse_nist_cas, + parse_retention_indices, +) + +RI_HTML = """ +Benzaldehyde + + +
    I
    9999
    + + + + + +
    ColumnIReference
    DB-11,100A
    DB-51200.5 ± 2B
    DB-5not reportedC
    + +""" + +NAME_HTML = """ +Benzaldehyde + + +""" + + +def response( + text: str, + *, + status: int = 200, + cache_hit: bool = False, + url: str = "https://webbook.nist.gov/cgi/cbook.cgi?example=1", +) -> HttpResponse: + return HttpResponse( + status_code=status, + text=text, + headers={}, + metadata=RetrievalMetadata( + requested_url=url, + url=url, + retrieved_at="2026-07-19T00:00:00+00:00", + status_code=status, + cache_hit=cache_hit, + attempts=0 if cache_hit else 1, + ), + ) + + +class FakeHttp: + def __init__(self, *responses: HttpResponse) -> None: + self.responses = list(responses) + self.calls: list[dict[str, Any]] = [] + + def get(self, url: str, **kwargs: Any) -> HttpResponse: + self.calls.append({"url": url, **kwargs}) + return self.responses.pop(0) + + +def test_parse_retention_indices_only_uses_original_target_table() -> None: + assert parse_retention_indices(RI_HTML) == [1100.0, 1200.5] + + +def test_parse_retention_indices_falls_back_from_heading_to_next_table() -> None: + html = """ +

    Normal alkane RI, non-polar column, custom temperature program

    +
    TemperatureI
    ramp876
    + """ + assert parse_retention_indices(html) == [876.0] + + +def test_name_page_helpers_distinguish_detail_and_search_results() -> None: + assert parse_nist_cas(NAME_HTML) == "100-52-7" + assert is_nist_search_results(NAME_HTML) is False + assert is_nist_search_results("Search Results") is True + + +def test_lookup_ri_selects_nearest_value_and_returns_provenance() -> None: + fake_http = FakeHttp(response(RI_HTML, cache_hit=True)) + client = NistWebBookClient(http_client=fake_http) + + result = client.lookup_ri("100527", 1180) + + assert result.status == "ok" + assert result.values["cas"] == "100-52-7" + assert result.values["retention_index"] == 1200.5 + assert result.values["retention_indices"] == [1100, 1200.5] + assert result.cache_hit is True + assert result.version == "SRD 69" + assert result.source_url.startswith("https://webbook.nist.gov/") + params = fake_http.calls[0]["params"] + assert params == {"ID": "C100527", "Units": "SI", "Mask": "2000"} + + +def test_lookup_ri_rejects_bad_inputs_without_http() -> None: + fake_http = FakeHttp() + client = NistWebBookClient(http_client=fake_http) + + bad_cas = client.lookup_ri("123-45-6", 1000) + bad_ri = client.lookup_ri("100-52-7", "not a number") + + assert bad_cas.status == "invalid_input" + assert bad_ri.status == "invalid_input" + assert fake_http.calls == [] + + +def test_resolve_name_returns_cas_and_query() -> None: + fake_http = FakeHttp(response(NAME_HTML)) + client = NistWebBookClient(http_client=fake_http) + + result = client.resolve_name(" Benzaldehyde ") + + assert result.status == "ok" + assert result.values == {"query_name": "Benzaldehyde", "cas": "100-52-7"} + assert fake_http.calls[0]["params"] == {"Name": "Benzaldehyde", "Units": "SI"} + + +def test_resolve_name_reports_ambiguous_and_missing_pages() -> None: + fake_http = FakeHttp( + response("Search Resultsmatches"), + response("Name Not FoundName Not Found"), + ) + client = NistWebBookClient(http_client=fake_http) + + ambiguous = client.resolve_name("oil") + missing = client.resolve_name("definitely-not-a-compound") + + assert ambiguous.status == "ambiguous" + assert missing.status == "not_found" + + +def test_nist_default_request_interval_is_at_least_five_seconds(tmp_path: Any) -> None: + client = NistWebBookClient(cache_dir=tmp_path) + assert client.http.min_interval >= 5.0 diff --git a/tests/test_pubchem.py b/tests/test_pubchem.py new file mode 100644 index 0000000..c9f10e1 --- /dev/null +++ b/tests/test_pubchem.py @@ -0,0 +1,230 @@ +from __future__ import annotations + +import json +from typing import Any + +from flavor_data_crawler.http import HttpResponse, RetrievalMetadata +from flavor_data_crawler.sources.pubchem import ( + PubChemClient, + parse_pubchem_identifiers, + parse_pubchem_odor, + parse_pubchem_properties, + parse_pubchem_synonyms, +) + +PROPERTIES = { + "PropertyTable": { + "Properties": [ + { + "CID": 240, + "Title": "Benzaldehyde", + "IUPACName": "benzaldehyde", + "MolecularFormula": "C7H6O", + "MolecularWeight": "106.12", + "ConnectivitySMILES": "C1=CC=C(C=C1)C=O", + "SMILES": "C1=CC=C(C=C1)C=O", + "InChI": "InChI=1S/C7H6O/c8-6-7-4-2-1-3-5-7/h1-6H", + "InChIKey": "HUMNYLRZRPPJDN-UHFFFAOYSA-N", + "XLogP": 1.5, + } + ] + } +} + +IDENTIFIERS = { + "InformationList": { + "Information": [ + { + "CID": 240, + "Identifiers": [ + {"Type": "CAS", "Identifier": "100-52-7"}, + {"Type": "CAS", "Identifier": "not-a-cas"}, + ], + } + ] + } +} + +SYNONYMS = { + "InformationList": { + "Information": [{"CID": 240, "Synonym": ["Benzaldehyde", "100-52-7", "Benzaldehyde"]}] + } +} + +ODOR = { + "Record": { + "RecordNumber": 240, + "Section": [ + { + "TOCHeading": "Chemical and Physical Properties", + "Section": [ + { + "TOCHeading": "Experimental Properties", + "Section": [ + { + "TOCHeading": "Odor", + "Information": [ + { + "ReferenceNumber": 126, + "Value": { + "StringWithMarkup": [ + {"String": "Characteristic almond odor"} + ] + }, + }, + { + "ReferenceNumber": 126, + "Value": { + "StringWithMarkup": [ + {"String": "Odor of bitter almond"} + ] + }, + }, + ], + } + ], + } + ], + } + ], + "Reference": [ + { + "ReferenceNumber": 126, + "SourceName": "Hazardous Substances Data Bank (HSDB)", + "URL": "https://pubchem.ncbi.nlm.nih.gov/source/hsdb/388", + "LicenseURL": "https://www.nlm.nih.gov/web_policies.html", + } + ], + } +} + + +def response( + payload: object, + *, + status: int = 200, + cache_hit: bool = True, + url: str = "https://pubchem.ncbi.nlm.nih.gov/rest/pug/example", +) -> HttpResponse: + return HttpResponse( + status_code=status, + text=json.dumps(payload), + headers={"Content-Type": "application/json"}, + metadata=RetrievalMetadata( + requested_url=url, + url=url, + retrieved_at="2026-07-19T00:00:00+00:00", + status_code=status, + cache_hit=cache_hit, + attempts=0 if cache_hit else 1, + ), + ) + + +class FakeHttp: + def __init__(self, *responses: HttpResponse) -> None: + self.responses = list(responses) + self.calls: list[dict[str, Any]] = [] + + def get(self, url: str, **kwargs: Any) -> HttpResponse: + self.calls.append({"url": url, **kwargs}) + return self.responses.pop(0) + + +def test_pure_pubchem_parsers_handle_current_api_shapes() -> None: + properties = parse_pubchem_properties(PROPERTIES) + assert properties["cid"] == 240 + assert properties["canonical_smiles"] == "C1=CC=C(C=C1)C=O" + assert properties["isomeric_smiles"] == "C1=CC=C(C=C1)C=O" + assert parse_pubchem_identifiers(IDENTIFIERS) == ["100-52-7"] + assert parse_pubchem_synonyms(SYNONYMS) == ["Benzaldehyde", "100-52-7"] + + +def test_odor_parser_preserves_contributor_urls_and_licenses() -> None: + odor = parse_pubchem_odor(ODOR) + + assert odor["odor"] == ["Characteristic almond odor", "Odor of bitter almond"] + assert odor["odor_sources"] == ["Hazardous Substances Data Bank (HSDB)"] + assert odor["odor_source_urls"] == ["https://pubchem.ncbi.nlm.nih.gov/source/hsdb/388"] + assert odor["odor_license_urls"] == ["https://www.nlm.nih.gov/web_policies.html"] + assert odor["odor_annotations"][0] == { + "text": "Characteristic almond odor", + "reference_number": "126", + "source_name": "Hazardous Substances Data Bank (HSDB)", + "source_url": "https://pubchem.ncbi.nlm.nih.gov/source/hsdb/388", + "license_url": "https://www.nlm.nih.gov/web_policies.html", + } + + +def test_lookup_combines_properties_identifiers_synonyms_and_odor() -> None: + fake_http = FakeHttp( + response(PROPERTIES, url="https://pubchem.ncbi.nlm.nih.gov/rest/pug/properties"), + response(IDENTIFIERS), + response(SYNONYMS), + response(ODOR, url="https://pubchem.ncbi.nlm.nih.gov/rest/pug_view/odor"), + ) + client = PubChemClient(http_client=fake_http) + + result = client.lookup("benzaldehyde") + + assert result.status == "ok" + assert result.values["query"] == "benzaldehyde" + assert result.values["cid"] == 240 + assert result.values["cas_numbers"] == ["100-52-7"] + assert result.values["synonyms"] == ["Benzaldehyde", "100-52-7"] + assert result.values["odor"] == ["Characteristic almond odor", "Odor of bitter almond"] + assert result.values["odor_sources"] == ["Hazardous Substances Data Bank (HSDB)"] + assert result.cache_hit is True + assert result.source_url.endswith("/properties") + assert len(fake_http.calls) == 4 + assert fake_http.calls[1]["params"] == {"identifier_type": "CAS"} + assert fake_http.calls[3]["params"] == {"heading": "Odor"} + + +def test_lookup_accepts_cas_and_can_skip_pug_view() -> None: + fake_http = FakeHttp(response(PROPERTIES), response(IDENTIFIERS), response(SYNONYMS)) + client = PubChemClient(http_client=fake_http) + + result = client.lookup("100527", include_odor=False) + + assert result.status == "ok" + assert result.values["cas_numbers"] == ["100-52-7"] + assert result.values["odor"] == [] + assert len(fake_http.calls) == 3 + assert "/compound/name/100527/property/" in fake_http.calls[0]["url"] + + +def test_missing_odor_is_not_a_primary_lookup_failure() -> None: + fake_http = FakeHttp( + response(PROPERTIES), + response(IDENTIFIERS), + response(SYNONYMS), + response({"Fault": {"Message": "No data"}}, status=404), + ) + client = PubChemClient(http_client=fake_http) + + result = client.lookup("benzaldehyde") + + assert result.status == "ok" + assert result.values["odor"] == [] + assert result.message == "" + + +def test_primary_not_found_and_malformed_payloads_are_explicit() -> None: + fake_http = FakeHttp( + response({"Fault": {}}, status=404), + response({"PropertyTable": {"Properties": []}}), + ) + client = PubChemClient(http_client=fake_http) + + missing = client.lookup("not-real") + malformed = client.lookup("also-not-real") + + assert missing.status == "not_found" + assert malformed.status == "parse_error" + assert len(fake_http.calls) == 2 + + +def test_pubchem_default_rate_is_below_five_requests_per_second(tmp_path: Any) -> None: + client = PubChemClient(cache_dir=tmp_path) + assert client.http.min_interval > 0.2 diff --git a/tests/test_pyrfume.py b/tests/test_pyrfume.py new file mode 100644 index 0000000..e014d38 --- /dev/null +++ b/tests/test_pyrfume.py @@ -0,0 +1,121 @@ +from __future__ import annotations + +from pathlib import Path + +from flavor_data_crawler.sources.pyrfume import ( + ARCHIVE_FILES, + PYRFUME_DATA_VERSION, + PyrfumeArchiveClient, +) + +SYNTHETIC_ARCHIVE_FILES = { + "aromadb/manifest.toml": """ +[source] +doi = "10.0000/example-aromadb" +title = "Synthetic Aroma Archive" +authors = "Example Authors" +extra = "Copyright remains with the Example Aroma Consortium" + +[raw] +LICENSE = "Consult the source owner before reuse" +""".lstrip(), + "aromadb/molecules.csv": ( + "CID,MolecularWeight,IsomericSMILES,IUPACName,name\n" + "176,60.05,CC(=O)O,acetic acid,synthetic vinegar molecule\n" + ), + "aromadb/stimuli.csv": "Stimulus,CID\nstimulus-a,176\n", + "aromadb/behavior.csv": ( + "Stimulus,Raw Descriptors,Filtered Descriptors,Modifiers\n" + 'stimulus-a,"sharp, sour","sour, pungent",\n' + ), + "flavornet/manifest.toml": """ +[source] +url = "https://example.test/flavornet" +title = "Synthetic Flavor Archive" +authors = "Test Curators" +extra = "Example archive copyright notice" + +[raw] +""".lstrip(), + "flavornet/molecules.csv": ( + "CID,MolecularWeight,IsomericSMILES,IUPACName,name\n" + "176,60.05,CC(=O)O,acetic acid,synthetic acetic acid\n" + ), + "flavornet/stimuli.csv": "Stimulus,CID\n176,176\n", + "flavornet/behavior.csv": "Stimulus,Descriptors\n176,sour;acidic\n", + "superscent/manifest.toml": """ +[source] +url = "https://example.test/superscent" +title = "Synthetic SuperScent List" +authors = "" +extra = "" + +[raw] +LICENSE = "" +""".lstrip(), + "superscent/molecules.csv": ( + "CID,MolecularWeight,IsomericSMILES,IUPACName,name\n" + "176,60.05,CC(=O)O,acetic acid,synthetic super molecule\n" + ), +} + + +def _synthetic_downloader(url: str, _destination: Path) -> bytes: + archive_file = "/".join(url.split("/")[-2:]) + return SYNTHETIC_ARCHIVE_FILES[archive_file].encode("utf-8") + + +def test_lookup_downloads_allowlisted_files_and_reuses_cache(tmp_path: Path) -> None: + client = PyrfumeArchiveClient(cache_dir=tmp_path, downloader=_synthetic_downloader) + + result = client.lookup("176.0") + + assert result.status == "ok" + assert result.cache_hit is False + assert result.version == PYRFUME_DATA_VERSION + assert result.values["Pyrfume Archives Matched"] == "aromadb; flavornet; superscent" + assert result.values["Pyrfume aromadb Present"] is True + assert result.values["Pyrfume aromadb Descriptors"] == "sour; pungent" + assert result.values["Pyrfume flavornet Descriptors"] == "sour; acidic" + assert result.values["Pyrfume superscent Present"] is True + assert result.values["Pyrfume superscent Descriptors"] == "" + assert result.values["Pyrfume aromadb Source Notes"] == ( + "Copyright remains with the Example Aroma Consortium" + ) + assert result.values["Pyrfume aromadb License Note"] == ( + "Consult the source owner before reuse" + ) + + snapshot_cache = tmp_path / "pyrfume-data" / PYRFUME_DATA_VERSION + expected = { + snapshot_cache / archive / filename + for archive, filenames in ARCHIVE_FILES.items() + for filename in filenames + } + assert all(path.is_file() for path in expected) + + offline = PyrfumeArchiveClient(cache_dir=tmp_path, allow_download=False) + cached_result = offline.lookup(176) + assert cached_result.status == "ok" + assert cached_result.cache_hit is True + assert cached_result.values == result.values + + +def test_lookup_is_exact_and_can_select_one_archive(tmp_path: Path) -> None: + client = PyrfumeArchiveClient(cache_dir=tmp_path, downloader=_synthetic_downloader) + + result = client.lookup(17, archives="aromadb") + + assert result.status == "not_found" + assert result.values["Pyrfume aromadb Present"] is False + assert result.values["Pyrfume Archives Matched"] == "" + assert not (tmp_path / "pyrfume-data" / PYRFUME_DATA_VERSION / "flavornet").exists() + + +def test_lookup_rejects_invalid_cid_and_non_allowlisted_archive(tmp_path: Path) -> None: + client = PyrfumeArchiveClient(cache_dir=tmp_path, allow_download=False) + + assert client.lookup("not-a-cid").status == "invalid_input" + unsupported = client.lookup(176, archives=["not-reviewed"]) + assert unsupported.status == "invalid_input" + assert "allowed: aromadb, flavornet, superscent" in unsupported.message diff --git a/tests/test_skill.py b/tests/test_skill.py new file mode 100644 index 0000000..cff35cf --- /dev/null +++ b/tests/test_skill.py @@ -0,0 +1,36 @@ +import json +import subprocess +import sys +from pathlib import Path + +import pandas as pd + +REPO_ROOT = Path(__file__).resolve().parents[1] +SKILL_ROOT = REPO_ROOT / ".agents" / "skills" / "curate-flavor-data" + + +def test_skill_metadata_and_interface_are_complete(): + skill = (SKILL_ROOT / "SKILL.md").read_text(encoding="utf-8") + interface = (SKILL_ROOT / "agents" / "openai.yaml").read_text(encoding="utf-8") + assert skill.startswith("---\nname: curate-flavor-data\n") + assert "TODO" not in skill + assert "description:" in skill.split("---", 2)[1] + assert 'display_name: "Curate Flavor Data"' in interface + assert "$curate-flavor-data" in interface + + +def test_skill_inspector_reports_identifier_quality(tmp_path: Path): + workbook = tmp_path / "input.xlsx" + pd.DataFrame({"CAS Number": ["100-52-7", "", "100-52-8"]}).to_excel(workbook, index=False) + script = SKILL_ROOT / "scripts" / "inspect_workbook.py" + completed = subprocess.run( + [sys.executable, str(script), str(workbook)], + cwd=REPO_ROOT, + check=True, + capture_output=True, + text=True, + ) + report = json.loads(completed.stdout) + assert report["rows"] == 3 + assert report["cas"]["valid"] == 1 + assert report["cas"]["invalid"] == 1 diff --git a/tests/test_workflows.py b/tests/test_workflows.py new file mode 100644 index 0000000..ca582a4 --- /dev/null +++ b/tests/test_workflows.py @@ -0,0 +1,94 @@ +from pathlib import Path + +import pandas as pd +import pytest + +from flavor_data_crawler.excel_io import read_table +from flavor_data_crawler.models import LookupResult +from flavor_data_crawler.workflows import run_nist_ri, run_pubchem + + +class FakeNist: + def __init__(self): + self.calls = 0 + + def lookup_ri(self, cas, calculated_ri): + self.calls += 1 + if cas == "100-52-7": + return LookupResult( + provider="NIST", + values={"retention_index": 955}, + source_url="https://example.test/nist/100-52-7", + ) + return LookupResult.failure("NIST", status="not_found", message="missing") + + +class FakePubChem: + def lookup(self, identifier, include_odor=True): + return LookupResult( + provider="PubChem", + values={ + "cid": 240, + "title": "Benzaldehyde", + "inchikey": "HUMNYLRZRPPJDN-UHFFFAOYSA-N", + "odor": ["=formula-like", "almond"], + "odor_sources": ["Example contributor"], + }, + source_url="https://pubchem.ncbi.nlm.nih.gov/compound/240", + license_url="https://example.test/license", + ) + + +def _silent(*_): + return None + + +def test_legacy_nist_contract_preserves_rows_and_columns(tmp_path: Path): + source = tmp_path / "input.xlsx" + destination = tmp_path / "output.xlsx" + pd.DataFrame( + {"CAS Number": ["100-52-7", "62016-37-9"], "Calculated RI": [955.2, 962.3]} + ).to_excel(source, index=False) + + summary = run_nist_ri( + source, + FakeNist(), + output_path=destination, + include_provenance=False, + checkpoint_every=0, + progress=_silent, + ) + output = read_table(destination) + assert summary.rows == 2 + assert list(output.columns) == ["CAS Number", "Calculated RI", "NIST RI"] + assert output.loc[0, "NIST RI"] == 955 + assert output.loc[1, "NIST RI"] == "\\" + + +def test_pubchem_maps_fields_and_sanitizes_remote_text(tmp_path: Path): + source = tmp_path / "input.xlsx" + destination = tmp_path / "pubchem.xlsx" + pd.DataFrame({"CAS Number": ["100-52-7"]}).to_excel(source, index=False) + summary = run_pubchem( + source, + FakePubChem(), + output_path=destination, + checkpoint_every=0, + progress=_silent, + ) + output = read_table(destination) + assert summary.status_counts == {"ok": 1} + assert output.loc[0, "PubChem CID"] == 240 + assert output.loc[0, "PubChem Odor"].startswith("'") + assert output.loc[0, "PubChem Source URL"].endswith("/240") + + +def test_existing_output_is_rejected_before_provider_calls(tmp_path: Path): + source = tmp_path / "input.xlsx" + destination = tmp_path / "output.xlsx" + pd.DataFrame({"CAS Number": ["100-52-7"], "Calculated RI": [955]}).to_excel(source, index=False) + destination.write_bytes(b"occupied") + client = FakeNist() + with pytest.raises(FileExistsError): + run_nist_ri(source, client, output_path=destination, progress=_silent) + assert client.calls == 0