Skip to content

Commit 840e1f4

Browse files
authored
Merge pull request #77 from FSoft-AI4Code/fix/leaf-nodes-collapse_name-collisions
add function-leaf nodes for non OOP projects and prefix for identical…
2 parents 763d08f + db46447 commit 840e1f4

13 files changed

Lines changed: 430 additions & 103 deletions

‎codewiki/cli/adapters/doc_generator.py‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616

1717
from codewiki.cli.utils.progress import ProgressTracker
1818
from codewiki.cli.models.job import DocumentationJob, LLMConfig
19-
from codewiki.cli.utils.errors import APIError
19+
from codewiki.cli.utils.errors import APIError, IncompleteGenerationError
2020

2121
# Import backend modules
2222
from codewiki.src.be.documentation_generator import DocumentationGenerator
@@ -257,6 +257,10 @@ async def _run_backend_generation(self, backend_config: BackendConfig):
257257
backend_config,
258258
completer=lambda p: doc_generator.backend.complete(p, model=cluster_model),
259259
)
260+
# Only freshly clustered trees are deduped: renaming a cached
261+
# key whose .md already exists would orphan the doc.
262+
from codewiki.src.be.module_naming import dedupe_module_tree_names
263+
module_tree = dedupe_module_tree_names(module_tree)
260264
file_manager.save_json(module_tree, first_module_tree_path)
261265

262266
file_manager.save_json(module_tree, module_tree_path)
@@ -303,10 +307,21 @@ async def _run_backend_generation(self, backend_config: BackendConfig):
303307
for file_path in os.listdir(working_dir):
304308
if file_path.endswith('.md') or file_path.endswith('.json'):
305309
self.job.files_generated.append(file_path)
306-
310+
307311
except Exception as e:
308312
raise APIError(f"Documentation generation failed: {e}")
309-
313+
314+
# Reconcile the final module tree against the docs on disk so name
315+
# collisions or failed sub-agents can't pass as success (issue #76).
316+
# Outside the try/except above so it is not reported as an API error.
317+
missing_docs = doc_generator.validate_generated_docs(working_dir)
318+
if missing_docs:
319+
raise IncompleteGenerationError(
320+
"Documentation generation finished but these module docs are missing: "
321+
+ ", ".join(f"{name}.md" for name in missing_docs),
322+
missing_modules=missing_docs,
323+
)
324+
310325
self.progress_tracker.complete_stage()
311326

312327
def _run_html_generation(self):

‎codewiki/cli/commands/generate.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
ConfigurationError,
1616
RepositoryError,
1717
APIError,
18+
IncompleteGenerationError,
1819
handle_error,
1920
EXIT_SUCCESS,
2021
)
@@ -614,6 +615,15 @@ def generate_command(
614615
logger.error(e.message)
615616
logger.error(f"Traceback: {traceback.format_exc()}")
616617
sys.exit(e.exit_code)
618+
except IncompleteGenerationError as e:
619+
click.secho(f"\n✗ {e.message}", fg="red", err=True)
620+
for module_name in e.missing_modules:
621+
click.secho(f" - {module_name}.md", fg="red", err=True)
622+
click.echo(
623+
"Re-run the same command to resume generation for the missing modules.",
624+
err=True,
625+
)
626+
sys.exit(e.exit_code)
617627
except KeyboardInterrupt:
618628
click.echo("\n\nInterrupted by user")
619629
sys.exit(130)

‎codewiki/cli/utils/errors.py‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,11 +56,19 @@ def __init__(self, message: str):
5656

5757
class FileSystemError(CodeWikiError):
5858
"""File system-related errors."""
59-
59+
6060
def __init__(self, message: str):
6161
super().__init__(message, EXIT_FILESYSTEM_ERROR)
6262

6363

64+
class IncompleteGenerationError(CodeWikiError):
65+
"""Generation finished but some module docs are missing on disk."""
66+
67+
def __init__(self, message: str, missing_modules: list = None):
68+
self.missing_modules = missing_modules or []
69+
super().__init__(message, EXIT_GENERAL_ERROR)
70+
71+
6472
def handle_error(error: Exception, verbose: bool = False) -> int:
6573
"""
6674
Handle errors and return appropriate exit code.

‎codewiki/src/be/agent_tools/generate_sub_module_documentations.py‎

Lines changed: 41 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
1+
import os
2+
13
from pydantic_ai import RunContext, Tool, Agent
24

35
from codewiki.src.be.agent_tools.deps import CodeWikiDeps
6+
from codewiki.src.be.module_naming import normalize_sub_module_specs
47
from codewiki.src.be.agent_tools.read_code_components import read_code_components_tool
58
from codewiki.src.be.agent_tools.str_replace_editor import str_replace_editor_tool
69
from codewiki.src.be.llm_services import create_fallback_models
@@ -20,25 +23,41 @@ async def generate_sub_module_documentation(
2023
"""Delegate documentation generation of sub-modules to sub-agents. Each sub-module will be documented separately.
2124
2225
Args:
23-
sub_module_specs: A dictionary mapping sub-module names to their core component IDs.
26+
sub_module_specs: A dictionary mapping sub-module names to their core component IDs.
2427
Example: {"authentication": ["auth_handler.py::AuthHandler", "auth_middleware.py::verify_token"], "database": ["db_client.py::DBClient", "models.py::UserModel"]}
2528
Each key is a descriptive sub-module name, and the value is a list of component IDs from the current module's core components that belong to that sub-module.
29+
Sub-module names must be unique across the whole wiki; a name that is already used by another
30+
module is automatically prefixed with the current module name, and the tool result reports the
31+
final file names actually saved.
2632
"""
2733

2834
deps = ctx.deps
2935
previous_module_name = deps.current_module_name
30-
36+
3137
# Create fallback models from config
3238
fallback_models = create_fallback_models(deps.config)
3339

40+
# Resolve name collisions against the module tree and files already on disk
41+
# before touching the tree (issue #76): docs live in one flat directory.
42+
name_map = normalize_sub_module_specs(
43+
sub_module_specs,
44+
previous_module_name,
45+
deps.module_tree,
46+
deps.absolute_docs_path,
47+
)
48+
final_specs = {
49+
name_map[requested_name]: core_component_ids
50+
for requested_name, core_component_ids in sub_module_specs.items()
51+
}
52+
3453
# add the sub-module to the module tree
3554
value = deps.module_tree
3655
for key in deps.path_to_current_module:
3756
value = value[key]["children"]
38-
for sub_module_name, core_component_ids in sub_module_specs.items():
57+
for sub_module_name, core_component_ids in final_specs.items():
3958
value[sub_module_name] = {"components": core_component_ids, "children": {}}
40-
41-
for sub_module_name, core_component_ids in sub_module_specs.items():
59+
60+
for sub_module_name, core_component_ids in final_specs.items():
4261

4362
# Create visual indentation for nested modules
4463
indent = " " * deps.current_depth
@@ -88,7 +107,23 @@ async def generate_sub_module_documentation(
88107
# restore the previous module name
89108
deps.current_module_name = previous_module_name
90109

91-
return f"Generate successfully. Documentations: {', '.join([key + '.md' for key in sub_module_specs.keys()])} are saved in the working directory."
110+
# Report what actually landed on disk so the parent agent links real filenames.
111+
saved = []
112+
missing = []
113+
for requested_name, final_name in name_map.items():
114+
entry = f"{final_name}.md"
115+
if final_name != requested_name:
116+
entry += f" (requested '{requested_name}', renamed to avoid a collision)"
117+
if os.path.exists(os.path.join(deps.absolute_docs_path, f"{final_name}.md")):
118+
saved.append(entry)
119+
else:
120+
missing.append(entry)
121+
122+
report = f"Saved documentations: {', '.join(saved) if saved else 'none'}."
123+
if missing:
124+
report += f" MISSING (generation did not produce these files): {', '.join(missing)}."
125+
logger.warning("Sub-module documentation missing after generation: %s", ", ".join(missing))
126+
return report
92127

93128

94129
generate_sub_module_documentation_tool = Tool(

‎codewiki/src/be/caw_toolkit.py‎

Lines changed: 37 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
from mcp.server.fastmcp import Context
2929

3030
from codewiki.src.be.agent_tools.deps import CodeWikiDeps
31+
from codewiki.src.be.module_naming import normalize_sub_module_specs
3132

3233
if TYPE_CHECKING:
3334
from codewiki.src.be.caw_backend import CawBackend
@@ -232,7 +233,10 @@ async def str_replace_editor(
232233
"is documented separately.\n"
233234
"sub_module_specs: a dictionary mapping sub-module names to their core component IDs. "
234235
"Example: {'authentication': ['auth_handler.py::AuthHandler'], "
235-
"'database': ['db_client.py::DBClient']}"
236+
"'database': ['db_client.py::DBClient']}\n"
237+
"Sub-module names must be unique across the whole wiki; a name already used by another "
238+
"module is automatically prefixed with the current module name, and the tool result "
239+
"reports the final file names actually saved."
236240
)
237241
)
238242
async def generate_sub_module_documentation(
@@ -272,15 +276,28 @@ def _run_sub_modules(self, sub_module_specs: dict[str, list[str]]) -> str:
272276
deps = self._deps
273277
previous_module_name = deps.current_module_name
274278

279+
# Resolve name collisions against the module tree and files already on
280+
# disk before touching the tree (issue #76): docs live in one flat directory.
281+
name_map = normalize_sub_module_specs(
282+
sub_module_specs,
283+
previous_module_name,
284+
deps.module_tree,
285+
deps.absolute_docs_path,
286+
)
287+
final_specs = {
288+
name_map[requested_name]: core_ids
289+
for requested_name, core_ids in sub_module_specs.items()
290+
}
291+
275292
# Add sub-modules to the in-memory module tree.
276293
value = deps.module_tree
277294
for key in deps.path_to_current_module:
278295
value = value[key]["children"]
279-
for sub_name, core_ids in sub_module_specs.items():
296+
for sub_name, core_ids in final_specs.items():
280297
value[sub_name] = {"components": core_ids, "children": {}}
281298

282299
try:
283-
for sub_name, core_ids in sub_module_specs.items():
300+
for sub_name, core_ids in final_specs.items():
284301
indent = " " * deps.current_depth
285302
arrow = "└─" if deps.current_depth > 0 else "→"
286303
logger.info("%s%s Generating documentation for sub-module: %s", indent, arrow, sub_name)
@@ -309,8 +326,20 @@ def _run_sub_modules(self, sub_module_specs: dict[str, list[str]]) -> str:
309326
finally:
310327
deps.current_module_name = previous_module_name
311328

312-
return (
313-
"Generate successfully. Documentations: "
314-
+ ", ".join(key + ".md" for key in sub_module_specs.keys())
315-
+ " are saved in the working directory."
316-
)
329+
# Report what actually landed on disk so the parent agent links real filenames.
330+
saved = []
331+
missing = []
332+
for requested_name, final_name in name_map.items():
333+
entry = f"{final_name}.md"
334+
if final_name != requested_name:
335+
entry += f" (requested '{requested_name}', renamed to avoid a collision)"
336+
if os.path.exists(os.path.join(deps.absolute_docs_path, f"{final_name}.md")):
337+
saved.append(entry)
338+
else:
339+
missing.append(entry)
340+
341+
report = f"Saved documentations: {', '.join(saved) if saved else 'none'}."
342+
if missing:
343+
report += f" MISSING (generation did not produce these files): {', '.join(missing)}."
344+
logger.warning("Sub-module documentation missing after generation: %s", ", ".join(missing))
345+
return report

‎codewiki/src/be/cluster_modules.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,10 @@
1212

1313
Completer = Callable[[str], str]
1414

15+
# When whole-repo mode is chosen but leaf entry points touch fewer than this
16+
# fraction of parsed files, warn that coverage depends on agent exploration.
17+
LOW_COVERAGE_RATIO = 0.5
18+
1519

1620
def format_potential_core_components(leaf_nodes: List[str], components: Dict[str, Node]) -> tuple[str, str]:
1721
"""
@@ -95,6 +99,22 @@ def cluster_modules(
9599
input_tokens,
96100
threshold,
97101
)
102+
if current_module_name is None:
103+
leaf_files = {
104+
components[leaf_node].relative_path
105+
for leaf_node in leaf_nodes
106+
if leaf_node in components
107+
}
108+
all_files = {c.relative_path for c in components.values()}
109+
if all_files and len(leaf_files) / len(all_files) < LOW_COVERAGE_RATIO:
110+
logger.warning(
111+
"Leaf-node entry points cover only %d of %d parsed files (%.0f%%). "
112+
"Whole-repository documentation will start from these entry points and "
113+
"rely on agent exploration to reach the rest of the codebase.",
114+
len(leaf_files),
115+
len(all_files),
116+
100 * len(leaf_files) / len(all_files),
117+
)
98118
return {}
99119

100120
prompt = format_cluster_prompt(potential_core_components, current_module_tree, current_module_name)

‎codewiki/src/be/dependency_analyzer/dependency_graphs_builder.py‎

Lines changed: 6 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
from codewiki.src.config import Config
44
from codewiki.src.be.dependency_analyzer.ast_parser import DependencyParser
55
from codewiki.src.be.dependency_analyzer.topo_sort import build_graph_from_components, get_leaf_nodes
6+
from codewiki.src.be.dependency_analyzer.leaf_selection import compute_valid_leaf_types, filter_leaf_nodes
67
from codewiki.src.utils import file_manager
78

89
import logging
@@ -70,33 +71,9 @@ def build_dependency_graph(self) -> tuple[Dict[str, Any], List[str]]:
7071
leaf_nodes = get_leaf_nodes(graph, components)
7172

7273
# check if leaf_nodes are in components, only keep the ones that are in components
73-
# and type is one of the following: class, interface, struct (or function for C-based projects)
74-
75-
# Determine if we should include functions based on available component types
76-
available_types = set()
77-
for comp in components.values():
78-
available_types.add(comp.component_type)
79-
80-
# Valid types for leaf nodes - include functions for C-based codebases
81-
valid_types = {"class", "interface", "struct"}
82-
# If no classes/interfaces/structs are found, include functions
83-
if not available_types.intersection(valid_types):
84-
valid_types.add("function")
85-
86-
keep_leaf_nodes = []
87-
for leaf_node in leaf_nodes:
88-
# Skip any leaf nodes that are clearly error strings or invalid identifiers
89-
if not isinstance(leaf_node, str) or leaf_node.strip() == "" or any(err_keyword in leaf_node.lower() for err_keyword in ['error', 'exception', 'failed', 'invalid']):
90-
logger.warning(f"Skipping invalid leaf node identifier: '{leaf_node}'")
91-
continue
92-
93-
if leaf_node in components:
94-
if components[leaf_node].component_type in valid_types:
95-
keep_leaf_nodes.append(leaf_node)
96-
else:
97-
# logger.debug(f"Leaf node {leaf_node} is a {components[leaf_node].component_type}, removing it")
98-
pass
99-
else:
100-
logger.warning(f"Leaf node {leaf_node} not found in components, removing it")
101-
74+
# and type is one of the following: class, interface, struct (or function when
75+
# functions carry the architecture — see leaf_selection)
76+
valid_types = compute_valid_leaf_types(components)
77+
keep_leaf_nodes = filter_leaf_nodes(leaf_nodes, components, valid_types)
78+
10279
return components, keep_leaf_nodes

0 commit comments

Comments
 (0)