Merge pull request #111 from FSoft-AI4Code/feat/component-level-updater

Feat/component level updater

Author: anhnh2002

.gitignore

@@ -80,3 +80,10 @@ Thumbs.db
 *.log
 *.bak
 .codewiki/
+!tests/updater_toy.py
+!tests/test_updater_graph_diff.py
+!tests/test_updater_tree_repair.py
+!tests/test_updater_reference_index.py
+!tests/test_updater_change_report.py
+!tests/test_updater_write_guard.py
+!tests/test_updater_orchestrator.py

codewiki/cli/adapters/doc_generator.py

@@ -186,6 +186,17 @@ async def _run_backend_generation(self, backend_config: BackendConfig):
         # Create documentation generator
         doc_generator = DocumentationGenerator(backend_config, commit_id=self.commit_id)
 
+        # Incremental update (component-level): keep the previous graph before
+        # the builder overwrites it, so the updater can diff old against new.
+        update_opts = self._update_options()
+        prev_graph_path = None
+        if update_opts is not None:
+            from codewiki.src.be.updater.graph_store import snapshot_old_graph
+
+            prev_graph_path = snapshot_old_graph(
+                backend_config.dependency_graph_dir, str(self.repo_path)
+            )
+
         if self.verbose:
             self.progress_tracker.update_stage(0.5, "Parsing source files...")
 
@@ -211,6 +222,27 @@ async def _run_backend_generation(self, backend_config: BackendConfig):
 
         self.progress_tracker.complete_stage()
 
+        if update_opts is not None:
+            outcome = await self._run_incremental_update(
+                backend_config, doc_generator, update_opts, prev_graph_path, components, leaf_nodes
+            )
+            if outcome in ("incremental", "no_change"):
+                # The builder wrote the new graph under this checkout's name. Drop graphs
+                # left by earlier checkouts so the next update has exactly one to pick.
+                from codewiki.src.be.updater.graph_store import (
+                    graph_file_path,
+                    prune_superseded_graphs,
+                )
+
+                prune_superseded_graphs(
+                    backend_config.dependency_graph_dir,
+                    keep=graph_file_path(backend_config.dependency_graph_dir, str(self.repo_path)),
+                )
+                return
+            # full_fallback / detector_failure: preserve the old docs, rebuild from scratch
+            self._move_docs_aside()
+            components, leaf_nodes = doc_generator.graph_builder.build_dependency_graph()
+
         # Stage 2: Module Clustering
         self.progress_tracker.start_stage(2, "Module Clustering")
         if self.verbose:
@@ -334,6 +366,7 @@ async def _run_backend_generation(self, backend_config: BackendConfig):
 
             # Create metadata
             doc_generator.create_documentation_metadata(working_dir, components, len(leaf_nodes))
+            self._merge_update_summary(working_dir)
 
             # Collect generated files
             for file_path in os.listdir(working_dir):
@@ -356,6 +389,111 @@ async def _run_backend_generation(self, backend_config: BackendConfig):
 
         self.progress_tracker.complete_stage()
 
+    # ------------------------------------------------------------------
+    # Incremental update helpers
+    # ------------------------------------------------------------------
+
+    def _update_options(self):
+        """Return ``UpdateOptions`` when this run is a component-level update, else None."""
+        if not self.config.get("update"):
+            return None
+        raw = dict(self.config.get("update_options") or {})
+        rung = str(raw.pop("rung", "3") or "3")
+        if rung == "0":
+            return None  # legacy path handled in generate.py
+        if not (self.output_dir / "module_tree.json").exists():
+            return None  # nothing to update against: normal generation
+        from codewiki.src.be.updater.options import UpdateOptions
+
+        return UpdateOptions.from_rung(rung, **raw)
+
+    async def _run_incremental_update(
+        self, backend_config, doc_generator, update_opts, prev_graph_path, components, leaf_nodes
+    ) -> str:
+        import json
+
+        from codewiki.src.be.updater.orchestrator import IncrementalUpdater
+
+        self.progress_tracker.start_stage(2, "Incremental Update")
+        prev_commit = None
+        try:
+            with open(self.output_dir / "metadata.json", encoding="utf-8") as f:
+                prev_commit = (json.load(f).get("generation_info") or {}).get("commit_id")
+        except (OSError, json.JSONDecodeError, AttributeError):
+            pass
+        revision = {
+            "old_commit": prev_commit,
+            "new_commit": self.commit_id,
+            "repo_path": str(self.repo_path),
+        }
+        updater = IncrementalUpdater(
+            backend_config, doc_generator.backend, doc_generator, update_opts
+        )
+        record = await updater.run(prev_graph_path, components, leaf_nodes, revision)
+        self._last_update_record = record
+        summary = record.summary()
+        if self.verbose:
+            self.progress_tracker.update_stage(
+                0.9,
+                f"Update outcome: {record.outcome} (diff {summary['diff_counts']}, "
+                f"{summary['n_active']} active leaves, {summary['n_calls']} LLM calls)",
+            )
+        working_dir = str(self.output_dir.absolute())
+        if record.outcome in ("incremental", "no_change"):
+            doc_generator.create_documentation_metadata(working_dir, components, len(leaf_nodes))
+            self._merge_update_summary(working_dir)
+            for file_path in os.listdir(working_dir):
+                if file_path.endswith((".md", ".json")):
+                    self.job.files_generated.append(file_path)
+            tree_path = os.path.join(working_dir, "module_tree.json")
+            if os.path.exists(tree_path):
+                with open(tree_path, encoding="utf-8") as f:
+                    self.job.module_count = len(json.load(f))
+            missing_docs = doc_generator.validate_generated_docs(working_dir)
+            if missing_docs:
+                raise IncompleteGenerationError(
+                    "Incremental update finished but these module docs are missing: "
+                    + ", ".join(f"{name}.md" for name in missing_docs),
+                    missing_modules=missing_docs,
+                )
+        self.progress_tracker.complete_stage()
+        return record.outcome
+
+    def _move_docs_aside(self) -> None:
+        """Keep the previous docs (and the failed attempt's record) next to the output dir."""
+        import shutil
+
+        suffix = (self.commit_id or "unknown")[:8]
+        target = self.output_dir.parent / f"{self.output_dir.name}.prev-{suffix}"
+        n = 2
+        while target.exists():
+            target = self.output_dir.parent / f"{self.output_dir.name}.prev-{suffix}-{n}"
+            n += 1
+        shutil.move(str(self.output_dir), str(target))
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+        self._preserved_docs_dir = str(target)
+        record = getattr(self, "_last_update_record", None)
+        if record is not None:
+            record.detector_notes.append(f"previous docs moved to {target}")
+        if self.verbose:
+            self.progress_tracker.update_stage(0.1, f"Previous docs preserved at {target}")
+
+    def _merge_update_summary(self, working_dir: str) -> None:
+        record = getattr(self, "_last_update_record", None)
+        if record is None:
+            return
+        from codewiki.src.be.updater.record import merge_into_metadata
+
+        summary = record.summary()
+        preserved = getattr(self, "_preserved_docs_dir", None)
+        if preserved:
+            summary["previous_docs"] = preserved
+        merge_into_metadata(working_dir, summary)
+        try:
+            record.save(working_dir)
+        except OSError:
+            pass
+
     def _run_html_generation(self):
         """Run HTML generation stage."""
         self.progress_tracker.start_stage(4, "HTML Generation")

codewiki/cli/commands/generate.py

@@ -330,6 +330,49 @@ def _find_affected(tree, parent_names=None):
     is_flag=True,
     help="Incremental update: only regenerate modules affected by changes since last generation",
 )
+@click.option(
+    "--update-rung",
+    type=click.Choice(["0", "1", "2", "3", "3b"]),
+    default="3",
+    show_default=True,
+    help=(
+        "Incremental updater variant used with --update. 0 = legacy file-level "
+        "invalidation; 1-3 = component-level updater ablation rungs; 3b = rung 3 with 2 hops."
+    ),
+)
+@click.option(
+    "--tau-ren", type=float, default=None, help="Rename similarity threshold (default 0.95)."
+)
+@click.option(
+    "--tau-nb", type=float, default=None, help="Neighbour-majority routing share (default 0.5)."
+)
+@click.option(
+    "--tau-grow",
+    type=float,
+    default=None,
+    help="Leaf growth share that re-clusters (default 0.33).",
+)
+@click.option(
+    "--tau-full",
+    type=float,
+    default=None,
+    help="Active-leaf share that forces a full build (default 0.5).",
+)
+@click.option(
+    "--tau-tree",
+    type=float,
+    default=None,
+    help="Structural-change share that forces a full build (default 0.3).",
+)
+@click.option(
+    "--k-hop", type=int, default=None, help="Dependency hops followed for Up (default 1)."
+)
+@click.option(
+    "--max-diff-tokens",
+    type=int,
+    default=None,
+    help="Cap on one component diff in a report (default 8000).",
+)
 @click.option(
     "--compare-to",
     type=str,
@@ -361,6 +404,14 @@ def generate_command(
     artifact_exclude: str | None = None,
     update: bool = False,
     compare_to: str | None = None,
+    update_rung: str = "3",
+    tau_ren: float | None = None,
+    tau_nb: float | None = None,
+    tau_grow: float | None = None,
+    tau_full: float | None = None,
+    tau_tree: float | None = None,
+    k_hop: int | None = None,
+    max_diff_tokens: int | None = None,
 ):
     """
     Generate comprehensive documentation for a code repository.
@@ -485,12 +536,23 @@ def generate_command(
                     "No changes detected since last generation. Documentation is up to date."
                 )
                 sys.exit(EXIT_SUCCESS)
-            if changed_files is not None:
+            if changed_files is not None and update_rung == "0":
                 logger.info(
                     f"  Detected {len(changed_files)} changed files — regenerating affected modules."
                 )
                 # Remove cached module docs for affected files so they get regenerated
                 _invalidate_affected_modules(output_dir, changed_files, logger, verbose)
+            elif update_rung != "0":
+                if changed_files is not None:
+                    logger.info(
+                        f"  Detected {len(changed_files)} changed files — running the "
+                        f"component-level updater (rung {update_rung})."
+                    )
+                else:
+                    logger.info(
+                        f"  Git diff unavailable — the component-level updater (rung {update_rung}) "
+                        f"will compare the saved dependency graph instead."
+                    )
 
         # Check for existing documentation
         if (
@@ -665,6 +727,18 @@ def generate_command(
                 "artifacts_enabled": artifacts,
                 "artifact_token_budget": artifact_token_budget,
                 "with_prose": with_prose,
+                # Incremental updater (runtime-only)
+                "update": update,
+                "update_options": {
+                    "rung": update_rung,
+                    "tau_ren": tau_ren,
+                    "tau_nb": tau_nb,
+                    "tau_grow": tau_grow,
+                    "tau_full": tau_full,
+                    "tau_tree": tau_tree,
+                    "k_hop": k_hop,
+                    "max_diff_tokens": max_diff_tokens,
+                },
             },
             verbose=verbose,
             generate_html=github_pages,

codewiki/src/be/agent_tools/deps.py

@@ -2,6 +2,7 @@
 from codewiki.src.be.dependency_analyzer.models.core import Node
 from codewiki.src.config import Config
 
+
 @dataclass
 class CodeWikiDeps:
     absolute_docs_path: str
@@ -14,4 +15,8 @@ class CodeWikiDeps:
     max_depth: int
     current_depth: int
     config: Config  # LLM configuration
-    custom_instructions: str = None
+    custom_instructions: str = None
+    # Incremental updates: when set, ``str_replace_editor`` refuses any write
+    # (create / str_replace / insert / undo_edit) to a docs file whose
+    # absolute, resolved path is not in this set. ``None`` = unrestricted.
+    allowed_write_paths: set[str] | None = None