Skip to content

Commit d94cacf

Browse files
authored
Merge pull request #85 from FSoft-AI4Code/imp/modular-docs
Imp/modular docs
2 parents 21a5607 + e46db45 commit d94cacf

15 files changed

Lines changed: 4204 additions & 925 deletions

‎codewiki/cli/adapters/doc_generator.py‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -215,6 +215,7 @@ async def _run_backend_generation(self, backend_config: BackendConfig):
215215
from codewiki.src.be.cluster_modules import (
216216
cluster_modules,
217217
get_clustering_input_token_count,
218+
super_group_modules,
218219
)
219220
from codewiki.src.utils import file_manager
220221
from codewiki.src.config import FIRST_MODULE_TREE_FILENAME, MODULE_TREE_FILENAME
@@ -227,6 +228,12 @@ async def _run_backend_generation(self, backend_config: BackendConfig):
227228
try:
228229
if os.path.exists(first_module_tree_path):
229230
module_tree = file_manager.load_json(first_module_tree_path)
231+
# Never clobber an existing module_tree.json with the cached
232+
# first tree: it carries sub-modules inserted by agents in
233+
# earlier (resumed) runs, and those modules short-circuit on
234+
# their existing .md without re-inserting their children.
235+
if not os.path.exists(module_tree_path):
236+
file_manager.save_json(module_tree, module_tree_path)
230237
if self.verbose:
231238
self.progress_tracker.update_stage(0.5, "Loaded cached module tree")
232239
else:
@@ -259,13 +266,19 @@ async def _run_backend_generation(self, backend_config: BackendConfig):
259266
backend_config,
260267
completer=lambda p: doc_generator.backend.complete(p, model=cluster_model),
261268
)
269+
if module_tree:
270+
module_tree = super_group_modules(
271+
module_tree,
272+
backend_config,
273+
completer=lambda p: doc_generator.backend.complete(p, model=cluster_model),
274+
)
262275
# Only freshly clustered trees are deduped: renaming a cached
263276
# key whose .md already exists would orphan the doc.
264277
from codewiki.src.be.module_naming import dedupe_module_tree_names
265278
module_tree = dedupe_module_tree_names(module_tree)
266279
file_manager.save_json(module_tree, first_module_tree_path)
280+
file_manager.save_json(module_tree, module_tree_path)
267281

268-
file_manager.save_json(module_tree, module_tree_path)
269282
self.job.module_count = len(module_tree)
270283

271284
if self.verbose:

‎codewiki/src/be/agent_tools/generate_sub_module_documentations.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ async def generate_sub_module_documentation(
6767

6868
num_tokens = count_tokens(format_potential_core_components(core_component_ids, ctx.deps.components)[-1])
6969

70-
if is_complex_module(ctx.deps.components, core_component_ids) and ctx.deps.current_depth < ctx.deps.max_depth and num_tokens >= ctx.deps.config.max_token_per_leaf_module:
70+
if is_complex_module(ctx.deps.components, core_component_ids) and ctx.deps.current_depth < ctx.deps.max_depth and num_tokens >= ctx.deps.config.max_token_per_leaf_module * ctx.deps.current_depth:
7171
sub_agent = Agent(
7272
model=fallback_models,
7373
name=sub_module_name,

‎codewiki/src/be/cluster_modules.py‎

Lines changed: 171 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,15 @@
11
from typing import List, Dict, Any, Callable, Optional
22
from collections import defaultdict
3+
import ast
34
import logging
45
import traceback
56
logger = logging.getLogger(__name__)
67

78
from codewiki.src.be.dependency_analyzer.models.core import Node
89
from codewiki.src.be.llm_services import call_llm
910
from codewiki.src.be.utils import count_tokens
10-
from codewiki.src.config import Config
11-
from codewiki.src.be.prompt_template import format_cluster_prompt
11+
from codewiki.src.config import Config, DEFAULT_MIN_MODULES_FOR_SUPER_GROUPING
12+
from codewiki.src.be.prompt_template import format_cluster_prompt, format_super_group_prompt
1213

1314
Completer = Callable[[str], str]
1415

@@ -209,3 +210,171 @@ def cluster_modules(
209210
current_module_path.pop()
210211

211212
return module_tree
213+
214+
215+
def _common_path_prefix(paths: List[str]) -> str:
216+
"""Longest common directory prefix of the given relative paths."""
217+
split_paths = [p.strip("/").split("/") for p in paths if p]
218+
if not split_paths:
219+
return ""
220+
common = []
221+
for parts in zip(*split_paths):
222+
if all(part == parts[0] for part in parts):
223+
common.append(parts[0])
224+
else:
225+
break
226+
return "/".join(common)
227+
228+
229+
def _parse_super_group_response(response: str) -> Optional[Dict[str, Any]]:
230+
if "<GROUPED_MODULES>" not in response or "</GROUPED_MODULES>" not in response:
231+
logger.warning(
232+
"Invalid super-grouping response: missing <GROUPED_MODULES> tags; "
233+
"keeping the flat module tree. Response preview: %s...",
234+
response[:200],
235+
)
236+
return None
237+
try:
238+
grouping = ast.literal_eval(
239+
response.split("<GROUPED_MODULES>")[1].split("</GROUPED_MODULES>")[0]
240+
)
241+
except Exception as e:
242+
logger.warning(
243+
"Failed to parse super-grouping response; keeping the flat module "
244+
"tree. Error: %s. Response preview: %s...",
245+
e,
246+
response[:200],
247+
)
248+
return None
249+
if not isinstance(grouping, dict):
250+
logger.warning(
251+
"Invalid super-grouping format - expected dict, got %s; keeping the "
252+
"flat module tree.",
253+
type(grouping),
254+
)
255+
return None
256+
return grouping
257+
258+
259+
def super_group_modules(
260+
module_tree: Dict[str, Any],
261+
config: Config,
262+
completer: Optional[Completer] = None,
263+
) -> Dict[str, Any]:
264+
"""
265+
Group a flat top level of modules into higher-level architectural subsystems.
266+
267+
Runs one extra LLM pass over the top-level module names (with their paths
268+
and components) and nests the flat modules as children of the subsystems it
269+
proposes. Any invalid or non-consolidating response leaves the tree
270+
unchanged, so this pass can never make the structure worse.
271+
"""
272+
min_modules = getattr(
273+
config, "min_modules_for_super_grouping", DEFAULT_MIN_MODULES_FOR_SUPER_GROUPING
274+
)
275+
if min_modules <= 0:
276+
logger.info(
277+
"Super-grouping disabled (min_modules_for_super_grouping=%d).", min_modules
278+
)
279+
return module_tree
280+
if len(module_tree) <= min_modules:
281+
logger.info(
282+
"Skipping super-grouping: %d top-level modules fit within the "
283+
"%d-module threshold.",
284+
len(module_tree),
285+
min_modules,
286+
)
287+
return module_tree
288+
289+
prompt = format_super_group_prompt(module_tree)
290+
logger.info(
291+
"Requesting super-grouping of %d top-level modules into architectural "
292+
"subsystems.",
293+
len(module_tree),
294+
)
295+
if completer is not None:
296+
response = completer(prompt)
297+
else:
298+
response = call_llm(prompt, config, model=config.cluster_model)
299+
300+
grouping = _parse_super_group_response(response)
301+
if grouping is None:
302+
return module_tree
303+
304+
# Validate assignments: unknown modules are dropped, duplicates keep their
305+
# first assignment, unassigned modules stay at the top level.
306+
assigned = set()
307+
subsystems: Dict[str, List[str]] = {}
308+
for subsystem_name, info in grouping.items():
309+
members = info.get("modules") if isinstance(info, dict) else None
310+
if not isinstance(members, list):
311+
logger.warning(
312+
"Skipping subsystem '%s' in super-grouping response: no valid "
313+
"'modules' list.",
314+
subsystem_name,
315+
)
316+
continue
317+
valid_members = []
318+
for member in members:
319+
if member not in module_tree:
320+
logger.warning(
321+
"Skipping unknown module '%s' in subsystem '%s'.",
322+
member,
323+
subsystem_name,
324+
)
325+
continue
326+
if member in assigned:
327+
logger.warning(
328+
"Module '%s' assigned to multiple subsystems; keeping its "
329+
"first assignment.",
330+
member,
331+
)
332+
continue
333+
assigned.add(member)
334+
valid_members.append(member)
335+
if valid_members:
336+
subsystems[subsystem_name] = valid_members
337+
338+
unassigned = [name for name in module_tree if name not in assigned]
339+
340+
# A single-module subsystem is just a rename; keep the module itself.
341+
top_level_count = len(subsystems) + len(unassigned)
342+
if len(subsystems) <= 1 or top_level_count >= len(module_tree):
343+
logger.info(
344+
"Skipping super-grouping result: %d subsystem(s) over %d modules "
345+
"provide no real consolidation.",
346+
len(subsystems),
347+
len(module_tree),
348+
)
349+
return module_tree
350+
351+
result: Dict[str, Any] = {}
352+
for subsystem_name, members in subsystems.items():
353+
if len(members) == 1:
354+
result[members[0]] = module_tree[members[0]]
355+
continue
356+
components = []
357+
seen = set()
358+
for member in members:
359+
for component in module_tree[member].get("components", []):
360+
if component not in seen:
361+
seen.add(component)
362+
components.append(component)
363+
result[subsystem_name] = {
364+
"path": _common_path_prefix(
365+
[module_tree[member].get("path", "") for member in members]
366+
),
367+
"components": components,
368+
"children": {member: module_tree[member] for member in members},
369+
}
370+
for name in unassigned:
371+
result[name] = module_tree[name]
372+
373+
logger.info(
374+
"Super-grouping consolidated %d top-level modules into %d entries "
375+
"(%d subsystems).",
376+
len(module_tree),
377+
len(result),
378+
len(subsystems),
379+
)
380+
return result

‎codewiki/src/be/documentation_generator.py‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
from codewiki.src.be.cluster_modules import (
1919
cluster_modules,
2020
get_clustering_input_token_count,
21+
super_group_modules,
2122
)
2223
from codewiki.src.config import (
2324
Config,
@@ -329,6 +330,12 @@ async def run(self) -> None:
329330
if os.path.exists(first_module_tree_path):
330331
logger.debug(f"Module tree found at {first_module_tree_path}")
331332
module_tree = file_manager.load_json(first_module_tree_path)
333+
# Never clobber an existing module_tree.json with the cached
334+
# first tree: it carries sub-modules inserted by agents in
335+
# earlier (resumed) runs, and those modules short-circuit on
336+
# their existing .md without re-inserting their children.
337+
if not os.path.exists(module_tree_path):
338+
file_manager.save_json(module_tree, module_tree_path)
332339
else:
333340
logger.debug(f"Module tree not found at {module_tree_path}, clustering modules")
334341
clustering_tokens = get_clustering_input_token_count(
@@ -351,12 +358,17 @@ async def run(self) -> None:
351358
self.config,
352359
completer=lambda p: self.backend.complete(p, model=cluster_model),
353360
)
361+
if module_tree:
362+
module_tree = super_group_modules(
363+
module_tree,
364+
self.config,
365+
completer=lambda p: self.backend.complete(p, model=cluster_model),
366+
)
354367
# Only freshly clustered trees are deduped: renaming a cached
355368
# key whose .md already exists would orphan the doc.
356369
module_tree = dedupe_module_tree_names(module_tree)
357370
file_manager.save_json(module_tree, first_module_tree_path)
358-
359-
file_manager.save_json(module_tree, module_tree_path)
371+
file_manager.save_json(module_tree, module_tree_path)
360372

361373
if len(module_tree) == 0:
362374
logger.info(

‎codewiki/src/be/prompt_template.py‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -202,6 +202,44 @@
202202
</GROUPED_COMPONENTS>
203203
""".strip()
204204

205+
SUPER_GROUP_PROMPT = """
206+
Here is the flat list of top-level modules of a repository. Each module has a path and its core components:
207+
208+
<MODULES>
209+
{formatted_modules}
210+
</MODULES>
211+
212+
The module list is flat: conceptually related modules sit next to unrelated ones. Please group these modules into higher-level architectural subsystems so the repository reads as an architecture rather than a directory listing.
213+
214+
Guidelines:
215+
- Each subsystem is a set of modules that together fulfil one architectural responsibility (e.g. a processing pipeline, a user-facing interface layer, a shared platform/foundation).
216+
- Name each subsystem by its architectural role, not by a directory name.
217+
- There must be significantly fewer subsystems than modules; do not simply mirror the module list back.
218+
- Subsystems should contain multiple modules. A single-module subsystem is just a rename — keep a module alone ONLY when it truly shares no architectural responsibility with any other module.
219+
- Every module must be assigned to exactly one subsystem.
220+
- Return the module names EXACTLY as given — do NOT rename or shorten them.
221+
222+
Firstly reason about the modules' responsibilities and relationships and identify the architectural layers they form, then return a grouping that MATCHES your reasoning in the following format:
223+
<GROUPED_MODULES>
224+
{{
225+
"subsystem_name_1": {{
226+
"modules": [
227+
<module_name_1>,
228+
<module_name_2>,
229+
...
230+
]
231+
}},
232+
"subsystem_name_2": {{
233+
"modules": [
234+
<module_name_3>,
235+
...
236+
]
237+
}},
238+
...
239+
}}
240+
</GROUPED_MODULES>
241+
""".strip()
242+
205243
FILTER_FOLDERS_PROMPT = """
206244
Here is the list of relative paths of files, folders in 2-depth of project {project_name}:
207245
```
@@ -374,6 +412,19 @@ def _format_module_tree(module_tree: dict[str, any], indent: int = 0):
374412
return CLUSTER_MODULE_PROMPT.format(potential_core_components=potential_core_components, module_tree=formatted_module_tree, module_name=module_name)
375413

376414

415+
def format_super_group_prompt(module_tree: Dict[str, Any]) -> str:
416+
"""
417+
Format the super-grouping prompt with the flat top-level modules of a tree.
418+
"""
419+
lines = []
420+
for name, info in module_tree.items():
421+
path = info.get("path", "")
422+
lines.append(f"{name} (path: {path})" if path else name)
423+
for component in info.get("components", []):
424+
lines.append(f"\t{component}")
425+
return SUPER_GROUP_PROMPT.format(formatted_modules="\n".join(lines))
426+
427+
377428
def format_system_prompt(module_name: str, custom_instructions: str = None) -> str:
378429
"""
379430
Format the system prompt with module name and optional custom instructions.

‎codewiki/src/config.py‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,10 @@
1717
# Default max token settings
1818
DEFAULT_MAX_TOKENS = 32_768
1919
DEFAULT_MAX_TOKEN_PER_MODULE = 36_369
20-
DEFAULT_MAX_TOKEN_PER_LEAF_MODULE = 16_000
20+
DEFAULT_MAX_TOKEN_PER_LEAF_MODULE = 4_000
21+
# Super-group the flat top level into architectural subsystems only when it
22+
# has more than this many modules; 0 or negative disables the pass.
23+
DEFAULT_MIN_MODULES_FOR_SUPER_GROUPING = 3
2124
# Legacy constants (for backward compatibility)
2225
MAX_TOKEN_PER_MODULE = DEFAULT_MAX_TOKEN_PER_MODULE
2326
MAX_TOKEN_PER_LEAF_MODULE = DEFAULT_MAX_TOKEN_PER_LEAF_MODULE
@@ -70,6 +73,7 @@ class Config:
7073
max_tokens: int = DEFAULT_MAX_TOKENS
7174
max_token_per_module: int = DEFAULT_MAX_TOKEN_PER_MODULE
7275
max_token_per_leaf_module: int = DEFAULT_MAX_TOKEN_PER_LEAF_MODULE
76+
min_modules_for_super_grouping: int = DEFAULT_MIN_MODULES_FOR_SUPER_GROUPING
7377
# Prompt caching for agentic/multi-turn calls (auto-disables per model if
7478
# the provider rejects cache_control markers)
7579
prompt_caching: bool = True
@@ -177,6 +181,7 @@ def from_cli(
177181
max_tokens: int = DEFAULT_MAX_TOKENS,
178182
max_token_per_module: int = DEFAULT_MAX_TOKEN_PER_MODULE,
179183
max_token_per_leaf_module: int = DEFAULT_MAX_TOKEN_PER_LEAF_MODULE,
184+
min_modules_for_super_grouping: int = DEFAULT_MIN_MODULES_FOR_SUPER_GROUPING,
180185
max_depth: int = MAX_DEPTH,
181186
agent_instructions: Optional[Dict[str, Any]] = None,
182187
use_gitignore: bool = True,
@@ -200,6 +205,9 @@ def from_cli(
200205
max_tokens: Maximum tokens for LLM response
201206
max_token_per_module: Maximum tokens per module for clustering
202207
max_token_per_leaf_module: Maximum tokens per leaf module
208+
min_modules_for_super_grouping: Super-group the top level into
209+
subsystems only when it has more than this many modules
210+
(0 or negative disables the pass)
203211
max_depth: Maximum depth for hierarchical decomposition
204212
agent_instructions: Custom agent instructions dict
205213
use_gitignore: Whether to apply Git ignore rules
@@ -229,6 +237,7 @@ def from_cli(
229237
max_tokens=max_tokens,
230238
max_token_per_module=max_token_per_module,
231239
max_token_per_leaf_module=max_token_per_leaf_module,
240+
min_modules_for_super_grouping=min_modules_for_super_grouping,
232241
agent_instructions=agent_instructions,
233242
use_gitignore=use_gitignore,
234243
prompt_caching=prompt_caching,

0 commit comments

Comments
 (0)