Add component-level incremental updater (C3) behind --update

Replace file-level cache invalidation with a component-level updater:
diff the saved dependency graph against the new one (body hash,
signature, edges, rename pairing), repair the module tree (renames,
deletions, routing rules 1-3, routing agent for orphans, growth check),
build a per-leaf change report (Own / Up / RefCh / Tree), check the
fallback ratios, then run one editing agent per active leaf, in
dependency order, restricted to a write set enforced by
str_replace_editor. Missing pages are regenerated by the normal
pipeline, a stale-name scan fixes leftovers, and every decision,
verdict and call cost lands in update_record.json.

`codewiki generate --update` now runs the new updater (rung 3);
`--update-rung 0` keeps the previous file-level behaviour as the
baseline, rungs 1/2/3b are ablations, and --tau-*/--k-hop/
--max-diff-tokens override the hyperparameters.

Also: LLMBackend.run_update_agent + last_usage on both backends,
allowed_write_paths on CodeWikiDeps, absolute/.. path rejection in the
pydantic-ai editor path, and a guard on the module-level stdout rewrap
in str_replace_editor that broke pytest capture.

Author: anhnh2002

.gitignore

@@ -80,3 +80,10 @@ Thumbs.db
 *.log
 *.bak
 .codewiki/
+!tests/updater_toy.py
+!tests/test_updater_graph_diff.py
+!tests/test_updater_tree_repair.py
+!tests/test_updater_reference_index.py
+!tests/test_updater_change_report.py
+!tests/test_updater_write_guard.py
+!tests/test_updater_orchestrator.py

codewiki/cli/adapters/doc_generator.py

@@ -186,6 +186,17 @@ async def _run_backend_generation(self, backend_config: BackendConfig):
         # Create documentation generator
         doc_generator = DocumentationGenerator(backend_config, commit_id=self.commit_id)
 
+        # Incremental update (component-level): keep the previous graph before
+        # the builder overwrites it, so the updater can diff old against new.
+        update_opts = self._update_options()
+        prev_graph_path = None
+        if update_opts is not None:
+            from codewiki.src.be.updater.graph_store import snapshot_old_graph
+
+            prev_graph_path = snapshot_old_graph(
+                backend_config.dependency_graph_dir, str(self.repo_path)
+            )
+
         if self.verbose:
             self.progress_tracker.update_stage(0.5, "Parsing source files...")
 
@@ -211,6 +222,16 @@ async def _run_backend_generation(self, backend_config: BackendConfig):
 
         self.progress_tracker.complete_stage()
 
+        if update_opts is not None:
+            outcome = await self._run_incremental_update(
+                backend_config, doc_generator, update_opts, prev_graph_path, components, leaf_nodes
+            )
+            if outcome in ("incremental", "no_change"):
+                return
+            # full_fallback / detector_failure: preserve the old docs, rebuild from scratch
+            self._move_docs_aside()
+            components, leaf_nodes = doc_generator.graph_builder.build_dependency_graph()
+
         # Stage 2: Module Clustering
         self.progress_tracker.start_stage(2, "Module Clustering")
         if self.verbose:
@@ -334,6 +355,7 @@ async def _run_backend_generation(self, backend_config: BackendConfig):
 
             # Create metadata
             doc_generator.create_documentation_metadata(working_dir, components, len(leaf_nodes))
+            self._merge_update_summary(working_dir)
 
             # Collect generated files
             for file_path in os.listdir(working_dir):
@@ -356,6 +378,111 @@ async def _run_backend_generation(self, backend_config: BackendConfig):
 
         self.progress_tracker.complete_stage()
 
+    # ------------------------------------------------------------------
+    # Incremental update helpers
+    # ------------------------------------------------------------------
+
+    def _update_options(self):
+        """Return ``UpdateOptions`` when this run is a component-level update, else None."""
+        if not self.config.get("update"):
+            return None
+        raw = dict(self.config.get("update_options") or {})
+        rung = str(raw.pop("rung", "3") or "3")
+        if rung == "0":
+            return None  # legacy path handled in generate.py
+        if not (self.output_dir / "module_tree.json").exists():
+            return None  # nothing to update against: normal generation
+        from codewiki.src.be.updater.options import UpdateOptions
+
+        return UpdateOptions.from_rung(rung, **raw)
+
+    async def _run_incremental_update(
+        self, backend_config, doc_generator, update_opts, prev_graph_path, components, leaf_nodes
+    ) -> str:
+        import json
+
+        from codewiki.src.be.updater.orchestrator import IncrementalUpdater
+
+        self.progress_tracker.start_stage(2, "Incremental Update")
+        prev_commit = None
+        try:
+            with open(self.output_dir / "metadata.json", encoding="utf-8") as f:
+                prev_commit = (json.load(f).get("generation_info") or {}).get("commit_id")
+        except (OSError, json.JSONDecodeError, AttributeError):
+            pass
+        revision = {
+            "old_commit": prev_commit,
+            "new_commit": self.commit_id,
+            "repo_path": str(self.repo_path),
+        }
+        updater = IncrementalUpdater(
+            backend_config, doc_generator.backend, doc_generator, update_opts
+        )
+        record = await updater.run(prev_graph_path, components, leaf_nodes, revision)
+        self._last_update_record = record
+        summary = record.summary()
+        if self.verbose:
+            self.progress_tracker.update_stage(
+                0.9,
+                f"Update outcome: {record.outcome} (diff {summary['diff_counts']}, "
+                f"{summary['n_active']} active leaves, {summary['n_calls']} LLM calls)",
+            )
+        working_dir = str(self.output_dir.absolute())
+        if record.outcome in ("incremental", "no_change"):
+            doc_generator.create_documentation_metadata(working_dir, components, len(leaf_nodes))
+            self._merge_update_summary(working_dir)
+            for file_path in os.listdir(working_dir):
+                if file_path.endswith((".md", ".json")):
+                    self.job.files_generated.append(file_path)
+            tree_path = os.path.join(working_dir, "module_tree.json")
+            if os.path.exists(tree_path):
+                with open(tree_path, encoding="utf-8") as f:
+                    self.job.module_count = len(json.load(f))
+            missing_docs = doc_generator.validate_generated_docs(working_dir)
+            if missing_docs:
+                raise IncompleteGenerationError(
+                    "Incremental update finished but these module docs are missing: "
+                    + ", ".join(f"{name}.md" for name in missing_docs),
+                    missing_modules=missing_docs,
+                )
+        self.progress_tracker.complete_stage()
+        return record.outcome
+
+    def _move_docs_aside(self) -> None:
+        """Keep the previous docs (and the failed attempt's record) next to the output dir."""
+        import shutil
+
+        suffix = (self.commit_id or "unknown")[:8]
+        target = self.output_dir.parent / f"{self.output_dir.name}.prev-{suffix}"
+        n = 2
+        while target.exists():
+            target = self.output_dir.parent / f"{self.output_dir.name}.prev-{suffix}-{n}"
+            n += 1
+        shutil.move(str(self.output_dir), str(target))
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+        self._preserved_docs_dir = str(target)
+        record = getattr(self, "_last_update_record", None)
+        if record is not None:
+            record.detector_notes.append(f"previous docs moved to {target}")
+        if self.verbose:
+            self.progress_tracker.update_stage(0.1, f"Previous docs preserved at {target}")
+
+    def _merge_update_summary(self, working_dir: str) -> None:
+        record = getattr(self, "_last_update_record", None)
+        if record is None:
+            return
+        from codewiki.src.be.updater.record import merge_into_metadata
+
+        summary = record.summary()
+        preserved = getattr(self, "_preserved_docs_dir", None)
+        if preserved:
+            summary["previous_docs"] = preserved
+        merge_into_metadata(working_dir, summary)
+        try:
+            record.save(working_dir)
+        except OSError:
+            pass
+
     def _run_html_generation(self):
         """Run HTML generation stage."""
         self.progress_tracker.start_stage(4, "HTML Generation")

codewiki/cli/commands/generate.py

@@ -330,6 +330,49 @@ def _find_affected(tree, parent_names=None):
     is_flag=True,
     help="Incremental update: only regenerate modules affected by changes since last generation",
 )
+@click.option(
+    "--update-rung",
+    type=click.Choice(["0", "1", "2", "3", "3b"]),
+    default="3",
+    show_default=True,
+    help=(
+        "Incremental updater variant used with --update. 0 = legacy file-level "
+        "invalidation; 1-3 = component-level updater ablation rungs; 3b = rung 3 with 2 hops."
+    ),
+)
+@click.option(
+    "--tau-ren", type=float, default=None, help="Rename similarity threshold (default 0.95)."
+)
+@click.option(
+    "--tau-nb", type=float, default=None, help="Neighbour-majority routing share (default 0.5)."
+)
+@click.option(
+    "--tau-grow",
+    type=float,
+    default=None,
+    help="Leaf growth share that re-clusters (default 0.33).",
+)
+@click.option(
+    "--tau-full",
+    type=float,
+    default=None,
+    help="Active-leaf share that forces a full build (default 0.5).",
+)
+@click.option(
+    "--tau-tree",
+    type=float,
+    default=None,
+    help="Structural-change share that forces a full build (default 0.3).",
+)
+@click.option(
+    "--k-hop", type=int, default=None, help="Dependency hops followed for Up (default 1)."
+)
+@click.option(
+    "--max-diff-tokens",
+    type=int,
+    default=None,
+    help="Cap on one component diff in a report (default 8000).",
+)
 @click.option(
     "--compare-to",
     type=str,
@@ -361,6 +404,14 @@ def generate_command(
     artifact_exclude: str | None = None,
     update: bool = False,
     compare_to: str | None = None,
+    update_rung: str = "3",
+    tau_ren: float | None = None,
+    tau_nb: float | None = None,
+    tau_grow: float | None = None,
+    tau_full: float | None = None,
+    tau_tree: float | None = None,
+    k_hop: int | None = None,
+    max_diff_tokens: int | None = None,
 ):
     """
     Generate comprehensive documentation for a code repository.
@@ -485,12 +536,23 @@ def generate_command(
                     "No changes detected since last generation. Documentation is up to date."
                 )
                 sys.exit(EXIT_SUCCESS)
-            if changed_files is not None:
+            if changed_files is not None and update_rung == "0":
                 logger.info(
                     f"  Detected {len(changed_files)} changed files — regenerating affected modules."
                 )
                 # Remove cached module docs for affected files so they get regenerated
                 _invalidate_affected_modules(output_dir, changed_files, logger, verbose)
+            elif update_rung != "0":
+                if changed_files is not None:
+                    logger.info(
+                        f"  Detected {len(changed_files)} changed files — running the "
+                        f"component-level updater (rung {update_rung})."
+                    )
+                else:
+                    logger.info(
+                        f"  Git diff unavailable — the component-level updater (rung {update_rung}) "
+                        f"will compare the saved dependency graph instead."
+                    )
 
         # Check for existing documentation
         if (
@@ -665,6 +727,18 @@ def generate_command(
                 "artifacts_enabled": artifacts,
                 "artifact_token_budget": artifact_token_budget,
                 "with_prose": with_prose,
+                # Incremental updater (runtime-only)
+                "update": update,
+                "update_options": {
+                    "rung": update_rung,
+                    "tau_ren": tau_ren,
+                    "tau_nb": tau_nb,
+                    "tau_grow": tau_grow,
+                    "tau_full": tau_full,
+                    "tau_tree": tau_tree,
+                    "k_hop": k_hop,
+                    "max_diff_tokens": max_diff_tokens,
+                },
             },
             verbose=verbose,
             generate_html=github_pages,

codewiki/src/be/agent_tools/deps.py

@@ -2,6 +2,7 @@
 from codewiki.src.be.dependency_analyzer.models.core import Node
 from codewiki.src.config import Config
 
+
 @dataclass
 class CodeWikiDeps:
     absolute_docs_path: str
@@ -14,4 +15,8 @@ class CodeWikiDeps:
     max_depth: int
     current_depth: int
     config: Config  # LLM configuration
-    custom_instructions: str = None
+    custom_instructions: str = None
+    # Incremental updates: when set, ``str_replace_editor`` refuses any write
+    # (create / str_replace / insert / undo_edit) to a docs file whose
+    # absolute, resolved path is not in this set. ``None`` = unrestricted.
+    allowed_write_paths: set[str] | None = None

codewiki/src/be/agent_tools/str_replace_editor.py

@@ -5,6 +5,7 @@
 import io
 import json
 import logging
+import os
 import re
 import shlex
 import subprocess
@@ -50,7 +51,13 @@ def _coerce_json_string(value):
 # There are some super strange "ascii can't decode x" errors,
 # that can be solved with setting the default encoding for stdout
 # (note that python3.6 doesn't have the reconfigure method)
-sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8")
+# Only rewrap when stdout is not already UTF-8. Replacing sys.stdout
+# unconditionally drops the previous wrapper, which closes the underlying
+# buffer when garbage-collected; under pytest that kills output capture.
+if (getattr(sys.stdout, "encoding", "") or "").lower().replace("-", "") != "utf8" and hasattr(
+    sys.stdout, "buffer"
+):
+    sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8")
 
 TRUNCATED_MESSAGE: str = "<response clipped><NOTE>To save on context only part of this file has been shown to you. You should retry this tool after you have searched inside the file with `grep -n` in order to find the line numbers of what you are looking for.</NOTE>"
 MAX_RESPONSE_LEN: int = 16000
@@ -803,6 +810,35 @@ def _make_output(
         )
 
 
+def check_write_allowed(absolute_path: str, allowed: "set[str] | None") -> str | None:
+    """Return an error message when ``absolute_path`` is outside the write set.
+
+    ``allowed`` holds absolute paths; both sides are resolved so symlinks and
+    ``..`` segments cannot slip a write past the guard. ``None`` means the
+    agent may write anywhere under its working dir (normal generation).
+    """
+    if allowed is None:
+        return None
+    try:
+        resolved = str(Path(absolute_path).resolve())
+    except OSError:
+        resolved = os.path.abspath(absolute_path)
+    allowed_resolved = set()
+    for a in allowed:
+        try:
+            allowed_resolved.add(str(Path(a).resolve()))
+        except OSError:
+            allowed_resolved.add(os.path.abspath(a))
+    if resolved in allowed_resolved:
+        return None
+    names = sorted(os.path.basename(a) for a in allowed_resolved)
+    return (
+        f"Error: {os.path.basename(absolute_path)!r} is not in this agent's write set. "
+        f"You may only edit these pages: {names}. Everything else must stay untouched; "
+        f"if it needs a change, say so in your final verdict instead."
+    )
+
+
 async def str_replace_editor(
     ctx: RunContext[CodeWikiDeps],
     working_dir: Literal["repo", "docs"],
@@ -842,14 +878,30 @@ async def str_replace_editor(
         path = file
 
     tool = EditTool(ctx.deps.registry, ctx.deps.absolute_docs_path)
-    if working_dir == "docs":
-        absolute_path = str(Path(ctx.deps.absolute_docs_path) / path)
-    else:
-        absolute_path = str(Path(ctx.deps.absolute_repo_path) / path)
+    # Absolute paths would resolve *outside* the chosen working dir
+    # (``Path(base) / "/abs"`` == ``/abs``); force relative paths like the caw path does.
+    if os.path.isabs(path):
+        return (
+            f"Error: `path` must be relative to `working_dir` ({working_dir!r}), "
+            f"got absolute path {path!r}."
+        )
+    base_dir = ctx.deps.absolute_docs_path if working_dir == "docs" else ctx.deps.absolute_repo_path
+    absolute_path = str(Path(base_dir) / path)
+    try:
+        Path(absolute_path).resolve().relative_to(Path(base_dir).resolve())
+    except ValueError:
+        return (
+            f"Error: resolved path {absolute_path!r} escapes working_dir={working_dir!r} "
+            f"root {base_dir!r}."
+        )
 
     # validate command
     if command != "view" and working_dir == "repo":
         return "The `view` command is the only allowed command when `working_dir` is `repo`."
+    if command != "view":
+        denied = check_write_allowed(absolute_path, getattr(ctx.deps, "allowed_write_paths", None))
+        if denied:
+            return denied
 
     tool(
         command=command,