|
| 1 | +"""Literature retrieval skill — a REAL offline retrieval over a curated domain |
| 2 | +knowledge base. |
| 3 | +
|
| 4 | +This replaces the earlier honest mock. It stays true to the project's identity: |
| 5 | +zero third-party dependencies, fully offline, and **deterministic** (the same |
| 6 | +query always returns the same ranked results — which is what lets the |
| 7 | +`result_is_reproducible` contract hold). Retrieval is stdlib TF-IDF cosine over a |
| 8 | +small, curated knowledge base of domain findings. |
| 9 | +
|
| 10 | +Deliberately NOT a live web/API search (arXiv/Crossref/Semantic Scholar): a |
| 11 | +network call would break offline execution and determinism, and the whole point |
| 12 | +of this layer is reproducibility. Entries are concept-level domain knowledge, not |
| 13 | +fabricated paper citations — every match is honestly labelled with its retrieval |
| 14 | +score and knowledge-base id. |
| 15 | +""" |
| 16 | + |
| 17 | +from __future__ import annotations |
| 18 | + |
| 19 | +import math |
| 20 | +import re |
| 21 | +from collections import Counter |
| 22 | + |
| 23 | +from ..schemas import EvidenceItem, EvidenceKind |
| 24 | + |
| 25 | +# --- curated domain knowledge base (topical findings, not fake citations) ------ |
| 26 | +# Each entry: id, domain, finding (the claim), keywords/text used for matching. |
| 27 | +KNOWLEDGE_BASE: list[dict] = [ |
| 28 | + {"id": "rfb-catholyte-metrics", "domain": "electrochemistry", |
| 29 | + "finding": "Nonaqueous redox-flow catholyte viability is governed jointly by the redox-potential window, active-species solubility, and long-term cycling stability — no single metric is sufficient.", |
| 30 | + "text": "nonaqueous redox flow battery catholyte viability redox potential window solubility cycling stability energy density capacity fade active species"}, |
| 31 | + {"id": "rfb-capacity-fade", "domain": "electrochemistry", |
| 32 | + "finding": "Capacity fade in organic redox-flow catholytes is dominated by chemical decomposition of the charged state, so cycling-stability data is required before viability claims.", |
| 33 | + "text": "capacity fade organic redoxmer decomposition charged state cycling stability degradation flow battery dialkoxyarene"}, |
| 34 | + {"id": "rfb-solubility-tradeoff", "domain": "electrochemistry", |
| 35 | + "finding": "Solubility of the redox-active species sets the achievable energy density; predicted solubility should be confirmed by measurement before design decisions.", |
| 36 | + "text": "solubility energy density redox active species prediction measurement nonaqueous electrolyte concentration"}, |
| 37 | + {"id": "cv-redox-potential", "domain": "electrochemistry", |
| 38 | + "finding": "Cyclic voltammetry is the standard method to characterize redox potential and reversibility of a candidate redoxmer.", |
| 39 | + "text": "cyclic voltammetry redox potential reversibility characterization electrochemistry scan rate peak separation"}, |
| 40 | + {"id": "stem-phase-id", "domain": "microscopy", |
| 41 | + "finding": "Crystalline-phase identification from STEM requires lattice-resolved imaging (d-spacings and symmetry) or a companion diffraction measurement; texture descriptors alone are insufficient.", |
| 42 | + "text": "STEM scanning transmission electron microscopy crystalline phase identification lattice spacing d-spacing symmetry diffraction imaging"}, |
| 43 | + {"id": "eels-eds-composition", "domain": "microscopy", |
| 44 | + "finding": "Local composition in electron microscopy is established by EELS or EDS, not by contrast/texture alone.", |
| 45 | + "text": "EELS EDS composition electron microscopy elemental analysis spectroscopy contrast texture region"}, |
| 46 | + {"id": "microscopy-sampling-bias", "domain": "microscopy", |
| 47 | + "finding": "A single imaged region may not be representative; phase claims should account for sampling bias across the specimen.", |
| 48 | + "text": "sampling bias representative region specimen heterogeneity microscopy field of view phase distribution"}, |
| 49 | + {"id": "xrd-phase-id", "domain": "materials", |
| 50 | + "finding": "X-ray diffraction identifies crystalline phases by matching reflection positions and intensities; Rietveld refinement quantifies phase fractions.", |
| 51 | + "text": "x-ray diffraction XRD phase identification reflection peak position intensity Rietveld refinement crystalline"}, |
| 52 | + {"id": "tio2-anatase-rutile", "domain": "materials", |
| 53 | + "finding": "Anatase and rutile TiO2 are distinguished by characteristic XRD reflections (e.g. anatase (101) vs rutile (110)); single-phase claims require the absence of the competing phase's peaks.", |
| 54 | + "text": "TiO2 titanium dioxide anatase rutile XRD reflection 101 110 single phase secondary phase peak"}, |
| 55 | + {"id": "single-phase-confirmation", "domain": "materials", |
| 56 | + "finding": "A single-phase assignment is only supported when no secondary-phase reflections are detectable above the noise floor.", |
| 57 | + "text": "single phase confirmation secondary phase reflections detection limit noise floor purity XRD"}, |
| 58 | + {"id": "measurement-vs-prediction", "domain": "general", |
| 59 | + "finding": "Treating a model prediction as if it were a direct measurement is a common source of over-confidence; the two carry different evidential weight.", |
| 60 | + "text": "measurement prediction model evidence provenance confidence over-confidence evidence kind mismatch"}, |
| 61 | + {"id": "uncertainty-before-acceptance", "domain": "general", |
| 62 | + "finding": "Explicit uncertainty quantification should precede any acceptance decision; unquantified confidence is not the same as validated confidence.", |
| 63 | + "text": "uncertainty quantification acceptance decision confidence validation evidence quality reasoning"}, |
| 64 | + {"id": "missing-evidence-gating", "domain": "general", |
| 65 | + "finding": "When a required measurement is absent, the responsible action is to acquire it rather than conclude — missing evidence should gate acceptance.", |
| 66 | + "text": "missing evidence required measurement gating acquire before conclude decision human review responsible"}, |
| 67 | + {"id": "provenance-reproducibility", "domain": "general", |
| 68 | + "finding": "Every scientific claim should carry provenance and be reproducible from a recorded trace; irreproducible reasoning cannot be audited.", |
| 69 | + "text": "provenance reproducibility trace replay audit scientific claim record deterministic"}, |
| 70 | + {"id": "human-in-the-loop", "domain": "general", |
| 71 | + "finding": "Autonomous scientific agents should route consequential actions through a human-in-the-loop review gate before any experiment or write action.", |
| 72 | + "text": "human in the loop review gate autonomous agent experiment write action safety approval"}, |
| 73 | +] |
| 74 | + |
| 75 | +_STOP = { |
| 76 | + "the", "a", "an", "and", "or", "of", "to", "in", "is", "are", "for", "on", |
| 77 | + "this", "that", "with", "as", "by", "be", "it", "at", "from", "what", "should", |
| 78 | + "i", "next", "does", "do", "can", "we", "how", "which", "was", "were", "not", |
| 79 | +} |
| 80 | + |
| 81 | + |
| 82 | +def _tokens(text: str) -> list[str]: |
| 83 | + return [w for w in re.split(r"[^a-z0-9]+", (text or "").lower()) if w and w not in _STOP] |
| 84 | + |
| 85 | + |
| 86 | +# Precompute IDF + per-document TF-IDF vectors from the fixed KB (deterministic). |
| 87 | +def _build_index(): |
| 88 | + docs = [_tokens(f"{e['finding']} {e['text']} {e['domain']}") for e in KNOWLEDGE_BASE] |
| 89 | + n = len(docs) |
| 90 | + df: Counter = Counter() |
| 91 | + for d in docs: |
| 92 | + for w in set(d): |
| 93 | + df[w] += 1 |
| 94 | + idf = {w: math.log((n + 1) / (c + 1)) + 1.0 for w, c in df.items()} |
| 95 | + vecs = [] |
| 96 | + for d in docs: |
| 97 | + tf = Counter(d) |
| 98 | + vec = {w: (tf[w] / len(d)) * idf.get(w, 0.0) for w in tf} if d else {} |
| 99 | + vecs.append(vec) |
| 100 | + return idf, vecs |
| 101 | + |
| 102 | + |
| 103 | +_IDF, _DOC_VECS = _build_index() |
| 104 | + |
| 105 | + |
| 106 | +def _cosine(qv: dict, dv: dict) -> float: |
| 107 | + if not qv or not dv: |
| 108 | + return 0.0 |
| 109 | + dot = sum(qv[w] * dv.get(w, 0.0) for w in qv) |
| 110 | + nq = math.sqrt(sum(v * v for v in qv.values())) |
| 111 | + nd = math.sqrt(sum(v * v for v in dv.values())) |
| 112 | + return dot / (nq * nd) if nq and nd else 0.0 |
| 113 | + |
| 114 | + |
| 115 | +def search(query: str, domain: str | None = None, k: int = 2) -> list[EvidenceItem]: |
| 116 | + """Return the top-k knowledge-base matches for the query as LITERATURE evidence. |
| 117 | + Deterministic: pure function of (query, domain, fixed KB).""" |
| 118 | + qtok = _tokens(f"{query} {domain or ''}") |
| 119 | + if not qtok: |
| 120 | + return [] |
| 121 | + qtf = Counter(qtok) |
| 122 | + qv = {w: (qtf[w] / len(qtok)) * _IDF.get(w, 0.0) for w in qtf} |
| 123 | + |
| 124 | + scored = [] |
| 125 | + for i, e in enumerate(KNOWLEDGE_BASE): |
| 126 | + s = _cosine(qv, _DOC_VECS[i]) |
| 127 | + if domain and e["domain"] == domain: |
| 128 | + s *= 1.15 # gentle in-domain boost |
| 129 | + scored.append((s, i)) |
| 130 | + # deterministic ordering: score desc, then KB order |
| 131 | + scored.sort(key=lambda t: (-t[0], t[1])) |
| 132 | + |
| 133 | + out: list[EvidenceItem] = [] |
| 134 | + for s, i in scored[: max(0, k)]: |
| 135 | + if s <= 0.01: |
| 136 | + continue |
| 137 | + e = KNOWLEDGE_BASE[i] |
| 138 | + out.append( |
| 139 | + EvidenceItem( |
| 140 | + claim=e["finding"], |
| 141 | + kind=EvidenceKind.LITERATURE, |
| 142 | + value=None, |
| 143 | + confidence=round(min(0.6, 0.2 + s), 3), |
| 144 | + source=f"knowledge_base:{e['id']}", |
| 145 | + caveats=( |
| 146 | + f"Retrieved from a curated offline domain knowledge base " |
| 147 | + f"(TF-IDF match {round(s, 3)}); not a live literature API." |
| 148 | + ), |
| 149 | + ) |
| 150 | + ) |
| 151 | + return out |
0 commit comments