Merge pull request #15 from closedloop-ai/campaign-prd-739-20260915-c1

feat(code-intel): no completeness claim without data (PLN-2027 PR 1)

Author: wongk

src/lemoncrow/gateway/adapters/mcp_server.py

@@ -162,11 +162,18 @@
     tool_web_fetch,
 )
 from lemoncrow.gateway.adapters.mcp_branding import icon_metadata
-from lemoncrow.infra.code_intel.completeness import CODE_OP_MATCH_KINDS, CODE_OP_OBJECTIVES, OBJECTIVE_RANKED
+from lemoncrow.infra.code_intel.completeness import (
+    CODE_OP_MATCH_KINDS,
+    CODE_OP_OBJECTIVES,
+    DATA_UNAVAILABLE,
+    OBJECTIVE_PARTIAL,
+    OBJECTIVE_RANKED,
+)
 from lemoncrow.infra.code_intel.freshness import (  # noqa: F401  (IndexRebuilding re-exported for handlers/tests)
     FRESHNESS_REBUILT,
     IndexRebuilding,
     VersionedEngineCache,
+    reset_readiness_probes,
 )
 from lemoncrow.infra.runtime.run_ledger import (
     RunLedger,
@@ -1131,6 +1138,7 @@ def _reset_runtime_cache_for_testing() -> None:
     _COMPACT_ADVISE_CACHE.clear()
     _last_blocked_plan_hash_by_session.clear()
     _code_engine_cache.clear()
+    reset_readiness_probes()
     _scoped_context_cache.clear()
 
 
@@ -8994,7 +9002,12 @@ def _maybe_attach_code_rendered(op: str, payload: dict[str, Any], *, render_comp
     # top-N read as a complete set is how a confident wrong finding gets filed.
     objective = CODE_OP_OBJECTIVES.get(op)
     if objective is not None:
-        result.setdefault("objective", objective)
+        # Edge data that was never there to look up is not "no edges found":
+        # `empty` keeps the op's claim, `unavailable` cannot make it.
+        if result.get("data_status") == DATA_UNAVAILABLE:
+            result["objective"] = OBJECTIVE_PARTIAL
+        else:
+            result.setdefault("objective", objective)
 
     # ...and say how the edges were matched. Exhaustive is necessary, not
     # sufficient: both edge stores are name-keyed, so a complete enumeration of

src/lemoncrow/infra/code_intel/change_impact.py

@@ -32,7 +32,14 @@
 from pathlib import Path
 from typing import Any
 
-from lemoncrow.infra.code_intel.completeness import MATCH_NAME, MATCH_RESOLVED, OBJECTIVE_EXHAUSTIVE
+from lemoncrow.infra.code_intel.completeness import (
+    DATA_AVAILABLE,
+    DATA_UNAVAILABLE,
+    MATCH_NAME,
+    MATCH_RESOLVED,
+    objective_for_data,
+)
+from lemoncrow.infra.code_intel.freshness import require_ready
 from lemoncrow.infra.code_intel.store import CodeIntelStore, SymbolRow
 
 __all__ = [
@@ -190,13 +197,20 @@ class ChangeImpactReport:
     impacted_total: int
     truncated: bool
     unindexed_paths: tuple[str, ...]
+    #: ``unavailable`` when the call graph the caller lookups read was never
+    #: built: every ``callers`` count is then a zero nobody measured, and
+    #: ``reason`` says what was missing.
+    data_status: str = DATA_AVAILABLE
+    reason: str | None = None
 
     def to_dict(self) -> dict[str, Any]:
-        return {
+        payload: dict[str, Any] = {
             # Name-keyed matching over-reports and never misses, so the caller
             # list is a superset of the truth -- exhaustive in the sense that
-            # matters for impact analysis.
-            "objective": OBJECTIVE_EXHAUSTIVE,
+            # matters for impact analysis. Only over a call graph that exists,
+            # though: with no edges to reverse, every changed symbol reads as
+            # having no callers, and that is missing data rather than an answer.
+            "objective": objective_for_data(self.data_status == DATA_AVAILABLE),
             "repo_root": self.repo_root,
             "base_ref": self.base_ref,
             "diff_ref": self.diff_ref,
@@ -209,7 +223,11 @@ def to_dict(self) -> dict[str, Any]:
             "impacted_total": self.impacted_total,
             "truncated": self.truncated,
             "unindexed_paths": list(self.unindexed_paths),
+            "data_status": self.data_status,
         }
+        if self.reason is not None:
+            payload["reason"] = self.reason
+        return payload
 
 
 # --------------------------------------------------------------------------- #
@@ -465,14 +483,27 @@ def analyze_changes(
     A site that reaches two changed symbols is reported once per symbol. That is
     not double counting: "who calls what" has two answers there, and collapsing
     them would silently drop one.
+
+    Raises :class:`~lemoncrow.infra.code_intel.freshness.IndexRebuilding` while
+    the index is mid-write and
+    :class:`~lemoncrow.infra.code_intel.store.CodeIntelUnavailable` when it is
+    absent. With no call graph to reverse, the report still maps the diff onto
+    changed symbols but states ``data_status: "unavailable"`` and a ``partial``
+    objective, because the zero callers it would otherwise report were never
+    measured.
     """
     root = Path(repo_root).resolve()
     depth = max(1, int(depth))
     limit = max(1, int(limit))
     diff_ref, changes = collect_changes(root, base_ref=base_ref, paths=paths)
+    # After the diff, which reports a directory that is not a git worktree as the
+    # caller's error to fix first; before the first store read, so a torn or
+    # empty index raises rather than mapping the diff onto nothing.
+    require_ready(root)
 
     with CodeIntelStore(root) as store:
         index_version = store.engine_state("index_version")
+        call_graph_gap = store.call_graph_gap()
         indexed_paths = {row.file_path for row in store.files()}
 
         pending: list[tuple[SymbolRow, str]] = []
@@ -576,4 +607,6 @@ def analyze_changes(
         impacted_total=len(impacted),
         truncated=len(impacted) > limit,
         unindexed_paths=tuple(unindexed),
+        data_status=DATA_AVAILABLE if call_graph_gap is None else DATA_UNAVAILABLE,
+        reason=call_graph_gap,
     )

src/lemoncrow/infra/code_intel/completeness.py

@@ -72,12 +72,15 @@
 __all__ = [
     "CODE_OP_MATCH_KINDS",
     "CODE_OP_OBJECTIVES",
+    "DATA_AVAILABLE",
+    "DATA_UNAVAILABLE",
     "MATCH_NAME",
     "MATCH_RESOLVED",
     "OBJECTIVE_EXHAUSTIVE",
     "OBJECTIVE_PARTIAL",
     "OBJECTIVE_RANKED",
     "objective_for_coverage",
+    "objective_for_data",
     "with_match_kind",
     "with_objective",
 ]
@@ -90,7 +93,7 @@
 
 #: Exhaustive over what was examined -- but what was examined is not everything.
 #:
-#: Only derived surfaces can be in this state. They answer from stored results,
+#: Derived surfaces reach this state through coverage. They answer from stored results,
 #: so the question "was the whole subject looked at" is separate from "was the
 #: whole answer returned", and ``truncated`` only ever spoke to the second.
 #:
@@ -103,6 +106,13 @@
 #: consumer honouring the documented predicate got a false negative. So the
 #: field the contract *does* make authoritative carries it: below full
 #: coverage the objective is no longer exhaustive.
+#:
+#: The second way to examine less than everything is for the data the answer
+#: depends on to be unavailable. A reverse lookup over a call graph that was
+#: never built returns ``[]`` -- the value a symbol with no callers returns --
+#: and ``code_changes`` reported that as zero callers for every changed symbol,
+#: exhaustive and untruncated. :func:`objective_for_data` makes that partial
+#: before coverage is consulted.
 OBJECTIVE_PARTIAL = "partial"
 
 #: Engine-backed ``code`` ops we have evidence for. ``pattern`` and ``node`` are
@@ -139,6 +149,16 @@
 }
 
 
+#: The data an enumeration depends on was there to be read. Mirrors the engine's
+#: call-graph ``data_status``, whose third value, ``"empty"``, means "looked up
+#: and none found" -- a real answer, which stays exhaustive.
+DATA_AVAILABLE = "available"
+
+#: The data an enumeration depends on was never built, so an empty result is not
+#: evidence of absence. See :func:`objective_for_data`.
+DATA_UNAVAILABLE = "unavailable"
+
+
 def objective_for_coverage(coverage: float | None, superseded: int | None = None) -> str:
     """The objective a stored-result surface may claim about this answer.
 
@@ -161,6 +181,24 @@ def objective_for_coverage(coverage: float | None, superseded: int | None = None
     return OBJECTIVE_EXHAUSTIVE
 
 
+def objective_for_data(available: bool, coverage: float | None = None, superseded: int | None = None) -> str:
+    """The objective an enumerative answer may claim, given whether its data existed.
+
+    *available* false means the lookup had nothing to read -- the call graph or
+    import table the answer depends on was never built -- so an empty result is
+    not evidence of absence, and the answer is :data:`OBJECTIVE_PARTIAL`
+    whatever else is true. Otherwise this defers to
+    :func:`objective_for_coverage`.
+
+    Unavailable is not empty. Edges that were looked up with none found are a
+    real answer and stay exhaustive; only data that was never there to look up
+    downgrades the claim.
+    """
+    if not available:
+        return OBJECTIVE_PARTIAL
+    return objective_for_coverage(coverage, superseded)
+
+
 def with_objective(payload: dict[str, Any], objective: str) -> dict[str, Any]:
     """Stamp *payload* in place and return it."""
     payload["objective"] = objective

src/lemoncrow/infra/code_intel/coverage.py

@@ -30,6 +30,7 @@
 from typing import Any
 
 from lemoncrow.infra.code_intel.completeness import OBJECTIVE_EXHAUSTIVE
+from lemoncrow.infra.code_intel.freshness import require_ready
 from lemoncrow.infra.code_intel.languages import language_for_path
 from lemoncrow.infra.code_intel.store import CodeIntelStore, FileRow
 
@@ -179,8 +180,15 @@ def check_coverage(paths: list[str] | None = None, repo_root: Path | str = ".")
     With no *paths*, the candidate set is every git-tracked file plus everything
     already in the index -- not a filesystem walk, which would drag in build
     output and virtualenvs the indexer never looked at.
+
+    Raises :class:`~lemoncrow.infra.code_intel.freshness.IndexRebuilding` while
+    the index is mid-write, and
+    :class:`~lemoncrow.infra.code_intel.store.CodeIntelUnavailable` when it is
+    absent: verdicts judged against a torn index report real files as missing,
+    and an absent one has nothing to judge against.
     """
     root = Path(repo_root).expanduser().resolve()
+    require_ready(root)
 
     with CodeIntelStore(root) as store:
         snapshot = store.snapshot()

src/lemoncrow/infra/code_intel/file_graph.py

@@ -30,7 +30,8 @@
 from types import TracebackType
 from typing import Any
 
-from lemoncrow.infra.code_intel.completeness import OBJECTIVE_EXHAUSTIVE
+from lemoncrow.infra.code_intel.completeness import DATA_AVAILABLE, DATA_UNAVAILABLE, objective_for_data
+from lemoncrow.infra.code_intel.freshness import require_ready
 from lemoncrow.infra.code_intel.store import CodeIntelStore, IndexSnapshot
 
 __all__ = ["FileGraph", "open_file_graph"]
@@ -141,6 +142,7 @@ def __init__(self, store: CodeIntelStore, repo_root: Path) -> None:
         self.repo_root: Path = repo_root
         self._store: CodeIntelStore = store
         self._snapshot: IndexSnapshot = store.snapshot()
+        self._import_gap: str | None = store.import_gap()
         self._files: dict[str, str] = {row.file_path: row.language for row in store.files()}
         self._edges: _Edges = self._build_edges()
         self._import_languages: frozenset[str] = frozenset(
@@ -241,15 +243,20 @@ def _envelope(self) -> dict[str, Any]:
 
         Every operation here enumerates rather than ranks -- the import graph is
         walked exhaustively and only the *returned list* is ever cut -- so the
-        objective is stamped once, for all five.
+        objective is stamped once, for all five: exhaustive over an import table
+        that has rows, partial over one that has none.
         """
-        return {
-            "objective": OBJECTIVE_EXHAUSTIVE,
+        envelope: dict[str, Any] = {
+            "objective": objective_for_data(self._import_gap is None),
             "analyzed_files": len(self._files),
             "resolved_edges": self._edges.resolved,
             "unresolved_edges": self._edges.unresolved,
             "engine_index_version": self._snapshot.index_version,
+            "data_status": DATA_AVAILABLE if self._import_gap is None else DATA_UNAVAILABLE,
         }
+        if self._import_gap is not None:
+            envelope["reason"] = self._import_gap
+        return envelope
 
     def _scope(self, paths: list[str] | None) -> frozenset[str] | None:
         """Normalise a caller's path filter to repo-relative prefixes."""
@@ -476,6 +483,13 @@ def _normalise(self, path: str) -> str:
 
 
 def open_file_graph(repo_root: Path | str = ".") -> FileGraph:
-    """Open a :class:`FileGraph` over *repo_root*'s code index."""
+    """Open a :class:`FileGraph` over *repo_root*'s code index.
+
+    Raises :class:`~lemoncrow.infra.code_intel.freshness.IndexRebuilding` while
+    the index is mid-write and
+    :class:`~lemoncrow.infra.code_intel.store.CodeIntelUnavailable` when it is
+    absent or empty, rather than analysing nothing.
+    """
     root = Path(repo_root).expanduser().resolve()
+    require_ready(root)
     return FileGraph(CodeIntelStore(root), root)

src/lemoncrow/infra/code_intel/freshness.py

@@ -16,6 +16,9 @@
   error.
 * :class:`VersionedEngineCache` -- a process cache whose entries are stamped
   with the generation they were built against, and rebuilt on mismatch.
+* :func:`require_ready` -- the same probe, throttled the same way, as a gate
+  for the tools that read the databases directly instead of through a cached
+  engine.
 
 The rule both serve is **fail loud, never empty**. An empty result set from an
 index that is mid-rebuild is indistinguishable from a true negative, which
@@ -40,7 +43,7 @@
 from pathlib import Path
 from typing import Any
 
-from lemoncrow.infra.code_intel.store import CODE_CONTEXT_DB, workspace_dir
+from lemoncrow.infra.code_intel.store import CODE_CONTEXT_DB, CodeIntelUnavailable, workspace_dir
 
 __all__ = [
     "DEFAULT_RECHECK_SECONDS",
@@ -57,6 +60,8 @@
     "IndexState",
     "VersionedEngineCache",
     "index_state",
+    "require_ready",
+    "reset_readiness_probes",
 ]
 
 logger = logging.getLogger(__name__)
@@ -354,3 +359,46 @@ def __len__(self) -> int:
 
     def __contains__(self, key: object) -> bool:
         return key in self._entries
+
+
+#: Readiness probes for the store-backed tools, throttled exactly like the engine
+#: cache's. One entry per distinct repo root this process has queried -- the bound
+#: ``_code_engine_cache`` already has -- cleared by :func:`reset_readiness_probes`.
+_readiness = VersionedEngineCache("store_readiness")
+
+
+def require_ready(repo_root: Path | str = ".") -> IndexState:
+    """Raise unless the index under *repo_root* can answer; return its state.
+
+    ``code_changes``, ``code_query``, ``code_coverage_check`` and the file-graph
+    analytics open the engine's databases directly, so the engine cache's
+    rebuild check never ran for them: mid-reindex they read a torn index and
+    returned what was left, an empty answer delivered as a complete one. This
+    is that check, applied where they start.
+
+    ``rebuilding`` raises :class:`IndexRebuilding`. ``absent`` raises
+    :class:`~lemoncrow.infra.code_intel.store.CodeIntelUnavailable`: the probe
+    cannot tell a workspace that was never indexed from one whose index was
+    emptied for an engine migration and has not repopulated yet, and neither
+    has anything to enumerate. Both fail loud; neither returns empty.
+
+    The probe is re-read at most once per :data:`DEFAULT_RECHECK_SECONDS`, the
+    staleness bound the engine cache already accepts, so a hot query path does
+    not open SQLite an extra time per call.
+    """
+    root = Path(repo_root).expanduser().resolve()
+    state = _readiness.state_for(root)
+    if state.status == STATUS_REBUILDING:
+        raise IndexRebuilding(root, state.detail)
+    if state.status == STATUS_ABSENT:
+        raise CodeIntelUnavailable(
+            f"code index for {root} is unavailable ({state.detail}): the workspace has not been "
+            "indexed, or its index is being migrated and has not repopulated yet -- run "
+            "`lc code index`, or retry shortly"
+        )
+    return state
+
+
+def reset_readiness_probes() -> None:
+    """Forget every throttled readiness probe, so the next gate re-reads disk."""
+    _readiness.clear()