Reference children docs by path in overview prompts instead of inlining

Parent/repo overview prompts embedded the entire module tree — every
module's components list — as indent-4 JSON plus the full markdown of every
child doc. On large repos (wazuh: ~2.2M chars) that exceeds codex's hard
per-turn input cap of 1,048,576 chars, so every overview call failed with
input_too_large, surfaced misleadingly as "LLM returned empty content".

- build_overview_structure now strips components lists recursively and sets
  an absolute docs_path per child instead of inlining docs (None when the
  doc is missing). Wazuh-scale prompts drop from ~2.2M to ~83K chars.
- REPO_OVERVIEW_PROMPT / MODULE_OVERVIEW_PROMPT instruct the agent to read
  the referenced files itself (backend.complete runs with ToolGroup.READER,
  so file reads are available). Placeholders are unchanged, keeping the MCP
  prompt_server rendering path working.
- Pin coding-agent-wrapper to the fork branch carrying the matching codex
  provider fixes (stdin prompts, resume sandbox flag, and surfacing CLI
  failures that were silently returned as empty turns).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: anhnh2002

codewiki/src/be/documentation_generator.py

@@ -117,9 +117,17 @@ def is_leaf_module(self, module_info: Dict[str, Any]) -> bool:
 
     def build_overview_structure(self, module_tree: Dict[str, Any], module_path: List[str],
                                  working_dir: str) -> Dict[str, Any]:
-        """Build structure for overview generation with 1-depth children docs and target indicator."""
-        
+        """Build structure for overview generation with 1-depth children doc paths and target indicator.
+
+        Children docs are referenced by absolute file path (``docs_path``)
+        rather than inlined, and ``components`` lists are stripped: inlining
+        the full tree plus docs blew past provider input caps (codex rejects
+        turns over 1,048,576 chars) on large repos. The overview agent reads
+        the referenced files itself.
+        """
+
         processed_module_tree = deepcopy(module_tree)
+        self._strip_components(processed_module_tree)
         module_info = processed_module_tree
         for path_part in module_path:
             module_info = module_info[path_part]
@@ -134,13 +142,25 @@ def build_overview_structure(self, module_tree: Dict[str, Any], module_path: Lis
         for child_name, child_info in module_info.items():
             child_docs_path = self._resolve_child_docs_path(working_dir, child_name)
             if child_docs_path is not None:
-                child_info["docs"] = file_manager.load_text(child_docs_path)
+                child_info["docs_path"] = child_docs_path
             else:
                 logger.warning(f"Module docs not found at {os.path.join(working_dir, f'{child_name}.md')}")
-                child_info["docs"] = ""
+                child_info["docs_path"] = None
 
         return processed_module_tree
 
+    @classmethod
+    def _strip_components(cls, tree: Dict[str, Any]) -> None:
+        """Recursively drop ``components`` lists — they dominate the tree's
+        serialized size and add nothing to an overview prompt."""
+        for module_info in tree.values():
+            if not isinstance(module_info, dict):
+                continue
+            module_info.pop("components", None)
+            children = module_info.get("children")
+            if isinstance(children, dict):
+                cls._strip_components(children)
+
     @staticmethod
     def _resolve_child_docs_path(working_dir: str, child_name: str) -> str | None:
         """Resolve the on-disk path for a child module's .md doc.
@@ -274,17 +294,18 @@ async def generate_parent_module_docs(self, module_path: List[str],
             logger.info(f"✓ Parent docs already exists at {parent_docs_path}")
             return module_tree
 
-        # Create repo structure with 1-depth children docs and target indicator
+        # Create repo structure with 1-depth children doc paths and target indicator
         repo_structure = self.build_overview_structure(module_tree, module_path, working_dir)
 
         prompt = MODULE_OVERVIEW_PROMPT.format(
             module_name=module_name,
-            repo_structure=json.dumps(repo_structure, indent=4)
+            repo_structure=json.dumps(repo_structure, indent=2)
         ) if len(module_path) >= 1 else REPO_OVERVIEW_PROMPT.format(
             repo_name=module_name,
-            repo_structure=json.dumps(repo_structure, indent=4)
+            repo_structure=json.dumps(repo_structure, indent=2)
         )
-        
+        logger.debug(f"Overview prompt for {module_name}: {len(prompt)} chars")
+
         try:
             parent_docs = self.backend.complete(prompt)
             if not parent_docs:

codewiki/src/be/prompt_template.py

@@ -98,11 +98,13 @@
 - The end-to-end architecture of the repository visualized by mermaid diagrams
 - The references to the core modules documentation
 
-Provide `{repo_name}` repo structure and its core modules documentation:
+Provide `{repo_name}` repo structure:
 <REPO_STRUCTURE>
 {repo_structure}
 </REPO_STRUCTURE>
 
+The core modules' documentation is NOT inlined above. Each top-level module carries a `docs_path` field with the absolute path to its documentation file — read those files with your file-reading tools before writing the overview (skip entries whose `docs_path` is null).
+
 Please generate the overview of the `{repo_name}` repository in markdown format with the following structure:
 <OVERVIEW>
 overview_content
@@ -117,11 +119,13 @@
 - The architecture of the module visualized by mermaid diagrams
 - The references to the core components documentation
 
-Provide repo structure and core components documentation of the `{module_name}` module:
+Provide repo structure of the `{module_name}` module (marked with `is_target_for_overview_generation`):
 <REPO_STRUCTURE>
 {repo_structure}
 </REPO_STRUCTURE>
 
+The child modules' documentation is NOT inlined above. Each child of the target module carries a `docs_path` field with the absolute path to its documentation file — read those files with your file-reading tools before writing the overview (skip entries whose `docs_path` is null).
+
 Please generate the overview of the `{module_name}` module in markdown format with the following structure:
 <OVERVIEW>
 overview_content

pyproject.toml

@@ -58,7 +58,7 @@ dependencies = [
     "python-multipart>=0.0.20",
     "colorama>=0.4.6",
     "logfire>=4.1.0",
-    "coding-agent-wrapper>=0.1.2",
+    "coding-agent-wrapper @ git+https://github.com/anhnh2002/caw@fix/codex-exec-robustness",
     "mcp>=1.0.0"
 ]
 

tests/test_overview_structure.py

@@ -0,0 +1,78 @@
+"""Tests for build_overview_structure's prompt-size discipline.
+
+Overview prompts used to inline the whole module tree (every module's
+``components`` list) plus the full markdown of every child doc. On large
+repos that blew past provider input caps — codex rejects any turn over
+1,048,576 characters — and every parent/repo overview failed. The structure
+must now strip ``components`` and reference children docs by absolute
+``docs_path`` instead of inlining them.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+
+from codewiki.src.be.documentation_generator import DocumentationGenerator
+
+CODEX_INPUT_CAP = 1_048_576
+
+
+def _generator() -> DocumentationGenerator:
+    # build_overview_structure touches neither config nor backend; skip the
+    # heavyweight __init__ (graph builder, backend resolution).
+    return DocumentationGenerator.__new__(DocumentationGenerator)
+
+
+def _make_tree(n_modules: int, n_components: int) -> dict:
+    return {
+        f"module_{i}": {
+            "components": [f"src/file_{i}_{j}.c::func_{j}" for j in range(n_components)],
+            "children": {
+                f"module_{i}_child": {
+                    "components": [f"src/file_{i}_{j}.c::helper_{j}" for j in range(n_components)],
+                    "children": {},
+                }
+            },
+        }
+        for i in range(n_modules)
+    }
+
+
+def test_components_are_stripped_at_every_depth(tmp_path):
+    tree = _make_tree(n_modules=3, n_components=5)
+    result = _generator().build_overview_structure(tree, [], str(tmp_path))
+    assert "components" not in json.dumps(result)
+    # The original tree is untouched (deepcopy semantics).
+    assert tree["module_0"]["components"]
+
+
+def test_children_docs_referenced_by_absolute_path_not_inlined(tmp_path):
+    tree = _make_tree(n_modules=2, n_components=1)
+    doc = tmp_path / "module_0.md"
+    doc.write_text("# module_0\n\nA very long body that must not appear in the prompt.")
+
+    result = _generator().build_overview_structure(tree, [], str(tmp_path))
+
+    assert result["module_0"]["docs_path"] == str(doc)
+    assert os.path.isabs(result["module_0"]["docs_path"])
+    assert result["module_1"]["docs_path"] is None  # missing doc -> null, no crash
+    assert "must not appear" not in json.dumps(result)
+
+
+def test_target_module_is_marked(tmp_path):
+    tree = _make_tree(n_modules=2, n_components=1)
+    result = _generator().build_overview_structure(tree, ["module_1"], str(tmp_path))
+    assert result["module_1"]["is_target_for_overview_generation"] is True
+    # Children of the target get docs_path entries.
+    assert "docs_path" in result["module_1"]["children"]["module_1_child"]
+
+
+def test_wazuh_scale_tree_stays_under_codex_input_cap(tmp_path):
+    # Modeled on the real failure: ~30 top-level modules, hundreds of
+    # components each, which serialized to ~2.1M chars before the fix.
+    tree = _make_tree(n_modules=30, n_components=600)
+    assert len(json.dumps(tree, indent=2)) > CODEX_INPUT_CAP  # the old shape overflowed
+
+    result = _generator().build_overview_structure(tree, [], str(tmp_path))
+    assert len(json.dumps(result, indent=2)) < CODEX_INPUT_CAP / 10