Skip to content

Commit 972dba9

Browse files
committed
Add module clustering diagnostics
1 parent 3ab1b98 commit 972dba9

3 files changed

Lines changed: 124 additions & 15 deletions

File tree

‎codewiki/cli/adapters/doc_generator.py‎

Lines changed: 35 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -207,7 +207,10 @@ async def _run_backend_generation(self, backend_config: BackendConfig):
207207
self.progress_tracker.update_stage(0.5, "Clustering modules with LLM...")
208208

209209
# Import clustering function
210-
from codewiki.src.be.cluster_modules import cluster_modules
210+
from codewiki.src.be.cluster_modules import (
211+
cluster_modules,
212+
get_clustering_input_token_count,
213+
)
211214
from codewiki.src.utils import file_manager
212215
from codewiki.src.config import FIRST_MODULE_TREE_FILENAME, MODULE_TREE_FILENAME
213216

@@ -223,7 +226,27 @@ async def _run_backend_generation(self, backend_config: BackendConfig):
223226
self.progress_tracker.update_stage(0.5, "Loaded cached module tree")
224227
else:
225228
if self.verbose:
226-
self.progress_tracker.update_stage(0.3, f"Clustering {len(leaf_nodes)} leaf nodes with LLM...")
229+
clustering_tokens = get_clustering_input_token_count(
230+
leaf_nodes, components
231+
)
232+
self.progress_tracker.update_stage(
233+
0.3,
234+
(
235+
f"Preparing {len(leaf_nodes)} leaf nodes for clustering "
236+
f"({clustering_tokens} tokens, threshold "
237+
f"{backend_config.max_token_per_module})"
238+
),
239+
)
240+
if clustering_tokens <= backend_config.max_token_per_module:
241+
self.progress_tracker.update_stage(
242+
0.4,
243+
"Skipping LLM clustering; selected leaf nodes fit within the module token threshold",
244+
)
245+
else:
246+
self.progress_tracker.update_stage(
247+
0.4,
248+
"Clustering modules with LLM...",
249+
)
227250
cluster_model = backend_config.cluster_model or None
228251
module_tree = cluster_modules(
229252
leaf_nodes,
@@ -237,7 +260,16 @@ async def _run_backend_generation(self, backend_config: BackendConfig):
237260
self.job.module_count = len(module_tree)
238261

239262
if self.verbose:
240-
self.progress_tracker.update_stage(1.0, f"Created {len(module_tree)} modules")
263+
if len(module_tree) == 0:
264+
self.progress_tracker.update_stage(
265+
1.0,
266+
"Created 0 modules; continuing in whole-repository documentation mode",
267+
)
268+
else:
269+
self.progress_tracker.update_stage(
270+
1.0,
271+
f"Created {len(module_tree)} modules",
272+
)
241273
for mod_name in sorted(module_tree.keys()):
242274
file_count = len(module_tree[mod_name]) if isinstance(module_tree[mod_name], list) else "?"
243275
self.progress_tracker.update_stage(1.0, f" Module: {mod_name} ({file_count} files)")
@@ -313,4 +345,3 @@ def _finalize_job(self):
313345
# Create our own if backend didn't
314346
with open(metadata_path, 'w') as f:
315347
f.write(self.job.to_json())
316-

‎codewiki/src/be/cluster_modules.py‎

Lines changed: 65 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,16 @@ def format_potential_core_components(leaf_nodes: List[str], components: Dict[str
4343
return potential_core_components, potential_core_components_with_code
4444

4545

46+
def get_clustering_input_token_count(
47+
leaf_nodes: List[str], components: Dict[str, Node]
48+
) -> int:
49+
"""Count the tokens used to decide whether a module needs clustering."""
50+
_, potential_core_components_with_code = format_potential_core_components(
51+
leaf_nodes, components
52+
)
53+
return count_tokens(potential_core_components_with_code)
54+
55+
4656
def cluster_modules(
4757
leaf_nodes: List[str],
4858
components: Dict[str, Node],
@@ -62,13 +72,38 @@ def cluster_modules(
6272
subscription-mode (caw) routing. If ``None``, falls back to
6373
``call_llm`` for backward compatibility with direct callers.
6474
"""
65-
potential_core_components, potential_core_components_with_code = format_potential_core_components(leaf_nodes, components)
66-
67-
if count_tokens(potential_core_components_with_code) <= config.max_token_per_module:
68-
logger.debug(f"Skipping clustering for {current_module_name} because the potential core components are too few: {count_tokens(potential_core_components_with_code)} tokens")
75+
potential_core_components, potential_core_components_with_code = (
76+
format_potential_core_components(leaf_nodes, components)
77+
)
78+
input_tokens = count_tokens(potential_core_components_with_code)
79+
threshold = config.max_token_per_module
80+
module_label = current_module_name or "repository"
81+
82+
logger.info(
83+
"Module clustering input for %s: %d leaf nodes, %d tokens, threshold %d",
84+
module_label,
85+
len(leaf_nodes),
86+
input_tokens,
87+
threshold,
88+
)
89+
90+
if input_tokens <= threshold:
91+
logger.info(
92+
"Skipping LLM module clustering for %s because %d tokens fit within the "
93+
"%d-token threshold; using whole-module documentation mode.",
94+
module_label,
95+
input_tokens,
96+
threshold,
97+
)
6998
return {}
7099

71100
prompt = format_cluster_prompt(potential_core_components, current_module_tree, current_module_name)
101+
logger.info(
102+
"Requesting LLM module clustering for %s because %d tokens exceed the %d-token threshold.",
103+
module_label,
104+
input_tokens,
105+
threshold,
106+
)
72107
if completer is not None:
73108
response = completer(prompt)
74109
else:
@@ -77,7 +112,12 @@ def cluster_modules(
77112
#parse the response
78113
try:
79114
if "<GROUPED_COMPONENTS>" not in response or "</GROUPED_COMPONENTS>" not in response:
80-
logger.error(f"Invalid LLM response format - missing component tags: {response[:200]}...")
115+
logger.warning(
116+
"Invalid LLM clustering response for %s: missing <GROUPED_COMPONENTS> "
117+
"tags; falling back to whole-module documentation. Response preview: %s...",
118+
module_label,
119+
response[:200],
120+
)
81121
return {}
82122

83123
response_content = response.split("<GROUPED_COMPONENTS>")[1].split("</GROUPED_COMPONENTS>")[0]
@@ -88,15 +128,32 @@ def cluster_modules(
88128
return {}
89129

90130
except Exception as e:
91-
logger.error(f"Failed to parse LLM response: {e}. Response: {response[:200]}...")
131+
logger.warning(
132+
"Failed to parse LLM clustering response for %s; falling back to "
133+
"whole-module documentation. Error: %s. Response preview: %s...",
134+
module_label,
135+
e,
136+
response[:200],
137+
)
92138
logger.error(f"Traceback: {traceback.format_exc()}")
93139
return {}
94140

95141
# check if the module tree is valid
96142
if len(module_tree) <= 1:
97-
logger.debug(f"Skipping clustering for {current_module_name} because the module tree is too small: {len(module_tree)} modules")
143+
logger.info(
144+
"Skipping LLM clustering result for %s because it produced only "
145+
"%d module(s); using whole-module documentation mode.",
146+
module_label,
147+
len(module_tree),
148+
)
98149
return {}
99150

151+
logger.info(
152+
"LLM module clustering for %s produced %d top-level modules.",
153+
module_label,
154+
len(module_tree),
155+
)
156+
100157
if current_module_tree == {}:
101158
current_module_tree = module_tree
102159
else:
@@ -131,4 +188,4 @@ def cluster_modules(
131188
)
132189
current_module_path.pop()
133190

134-
return module_tree
191+
return module_tree

‎codewiki/src/be/documentation_generator.py‎

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,10 @@
1515
REPO_OVERVIEW_PROMPT,
1616
MODULE_OVERVIEW_PROMPT,
1717
)
18-
from codewiki.src.be.cluster_modules import cluster_modules
18+
from codewiki.src.be.cluster_modules import (
19+
cluster_modules,
20+
get_clustering_input_token_count,
21+
)
1922
from codewiki.src.config import (
2023
Config,
2124
FIRST_MODULE_TREE_FILENAME,
@@ -318,6 +321,15 @@ async def run(self) -> None:
318321
module_tree = file_manager.load_json(first_module_tree_path)
319322
else:
320323
logger.debug(f"Module tree not found at {module_tree_path}, clustering modules")
324+
clustering_tokens = get_clustering_input_token_count(
325+
leaf_nodes, components
326+
)
327+
logger.info(
328+
"Preparing %d leaf nodes for module clustering (%d tokens, threshold %d)",
329+
len(leaf_nodes),
330+
clustering_tokens,
331+
self.config.max_token_per_module,
332+
)
321333
# Bind cluster_model into the completer so the backend uses the
322334
# configured clustering model (separate from main_model) when
323335
# one is set. Caw mode's cluster_model is typically empty —
@@ -333,7 +345,16 @@ async def run(self) -> None:
333345

334346
file_manager.save_json(module_tree, module_tree_path)
335347

336-
logger.debug(f"Grouped components into {len(module_tree)} modules")
348+
if len(module_tree) == 0:
349+
logger.info(
350+
"Module clustering produced no top-level modules; continuing in "
351+
"whole-repository documentation mode"
352+
)
353+
else:
354+
logger.info(
355+
"Grouped components into %d top-level modules",
356+
len(module_tree),
357+
)
337358

338359
# Generate module documentation using dynamic programming approach
339360
# This processes leaf modules first, then parent modules
@@ -349,4 +370,4 @@ async def run(self) -> None:
349370
except Exception as e:
350371
logger.error(f"Documentation generation failed: {str(e)}")
351372
logger.error(f"Traceback: {traceback.format_exc()}")
352-
raise
373+
raise

0 commit comments

Comments
 (0)