Skip to content

Commit 7dd6623

Browse files
committed
add super group stage for more modular generated docs
1 parent a7f7223 commit 7dd6623

6 files changed

Lines changed: 261 additions & 12 deletions

File tree

‎codewiki/src/be/agent_tools/generate_sub_module_documentations.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ async def generate_sub_module_documentation(
6767

6868
num_tokens = count_tokens(format_potential_core_components(core_component_ids, ctx.deps.components)[-1])
6969

70-
if is_complex_module(ctx.deps.components, core_component_ids) and ctx.deps.current_depth < ctx.deps.max_depth and num_tokens >= ctx.deps.config.max_token_per_leaf_module:
70+
if is_complex_module(ctx.deps.components, core_component_ids) and ctx.deps.current_depth < ctx.deps.max_depth and num_tokens >= ctx.deps.config.max_token_per_leaf_module * ctx.deps.current_depth:
7171
sub_agent = Agent(
7272
model=fallback_models,
7373
name=sub_module_name,

‎codewiki/src/be/cluster_modules.py‎

Lines changed: 171 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,15 @@
11
from typing import List, Dict, Any, Callable, Optional
22
from collections import defaultdict
3+
import ast
34
import logging
45
import traceback
56
logger = logging.getLogger(__name__)
67

78
from codewiki.src.be.dependency_analyzer.models.core import Node
89
from codewiki.src.be.llm_services import call_llm
910
from codewiki.src.be.utils import count_tokens
10-
from codewiki.src.config import Config
11-
from codewiki.src.be.prompt_template import format_cluster_prompt
11+
from codewiki.src.config import Config, DEFAULT_MIN_MODULES_FOR_SUPER_GROUPING
12+
from codewiki.src.be.prompt_template import format_cluster_prompt, format_super_group_prompt
1213

1314
Completer = Callable[[str], str]
1415

@@ -209,3 +210,171 @@ def cluster_modules(
209210
current_module_path.pop()
210211

211212
return module_tree
213+
214+
215+
def _common_path_prefix(paths: List[str]) -> str:
216+
"""Longest common directory prefix of the given relative paths."""
217+
split_paths = [p.strip("/").split("/") for p in paths if p]
218+
if not split_paths:
219+
return ""
220+
common = []
221+
for parts in zip(*split_paths):
222+
if all(part == parts[0] for part in parts):
223+
common.append(parts[0])
224+
else:
225+
break
226+
return "/".join(common)
227+
228+
229+
def _parse_super_group_response(response: str) -> Optional[Dict[str, Any]]:
230+
if "<GROUPED_MODULES>" not in response or "</GROUPED_MODULES>" not in response:
231+
logger.warning(
232+
"Invalid super-grouping response: missing <GROUPED_MODULES> tags; "
233+
"keeping the flat module tree. Response preview: %s...",
234+
response[:200],
235+
)
236+
return None
237+
try:
238+
grouping = ast.literal_eval(
239+
response.split("<GROUPED_MODULES>")[1].split("</GROUPED_MODULES>")[0]
240+
)
241+
except Exception as e:
242+
logger.warning(
243+
"Failed to parse super-grouping response; keeping the flat module "
244+
"tree. Error: %s. Response preview: %s...",
245+
e,
246+
response[:200],
247+
)
248+
return None
249+
if not isinstance(grouping, dict):
250+
logger.warning(
251+
"Invalid super-grouping format - expected dict, got %s; keeping the "
252+
"flat module tree.",
253+
type(grouping),
254+
)
255+
return None
256+
return grouping
257+
258+
259+
def super_group_modules(
260+
module_tree: Dict[str, Any],
261+
config: Config,
262+
completer: Optional[Completer] = None,
263+
) -> Dict[str, Any]:
264+
"""
265+
Group a flat top level of modules into higher-level architectural subsystems.
266+
267+
Runs one extra LLM pass over the top-level module names (with their paths
268+
and components) and nests the flat modules as children of the subsystems it
269+
proposes. Any invalid or non-consolidating response leaves the tree
270+
unchanged, so this pass can never make the structure worse.
271+
"""
272+
min_modules = getattr(
273+
config, "min_modules_for_super_grouping", DEFAULT_MIN_MODULES_FOR_SUPER_GROUPING
274+
)
275+
if min_modules <= 0:
276+
logger.info(
277+
"Super-grouping disabled (min_modules_for_super_grouping=%d).", min_modules
278+
)
279+
return module_tree
280+
if len(module_tree) <= min_modules:
281+
logger.info(
282+
"Skipping super-grouping: %d top-level modules fit within the "
283+
"%d-module threshold.",
284+
len(module_tree),
285+
min_modules,
286+
)
287+
return module_tree
288+
289+
prompt = format_super_group_prompt(module_tree)
290+
logger.info(
291+
"Requesting super-grouping of %d top-level modules into architectural "
292+
"subsystems.",
293+
len(module_tree),
294+
)
295+
if completer is not None:
296+
response = completer(prompt)
297+
else:
298+
response = call_llm(prompt, config, model=config.cluster_model)
299+
300+
grouping = _parse_super_group_response(response)
301+
if grouping is None:
302+
return module_tree
303+
304+
# Validate assignments: unknown modules are dropped, duplicates keep their
305+
# first assignment, unassigned modules stay at the top level.
306+
assigned = set()
307+
subsystems: Dict[str, List[str]] = {}
308+
for subsystem_name, info in grouping.items():
309+
members = info.get("modules") if isinstance(info, dict) else None
310+
if not isinstance(members, list):
311+
logger.warning(
312+
"Skipping subsystem '%s' in super-grouping response: no valid "
313+
"'modules' list.",
314+
subsystem_name,
315+
)
316+
continue
317+
valid_members = []
318+
for member in members:
319+
if member not in module_tree:
320+
logger.warning(
321+
"Skipping unknown module '%s' in subsystem '%s'.",
322+
member,
323+
subsystem_name,
324+
)
325+
continue
326+
if member in assigned:
327+
logger.warning(
328+
"Module '%s' assigned to multiple subsystems; keeping its "
329+
"first assignment.",
330+
member,
331+
)
332+
continue
333+
assigned.add(member)
334+
valid_members.append(member)
335+
if valid_members:
336+
subsystems[subsystem_name] = valid_members
337+
338+
unassigned = [name for name in module_tree if name not in assigned]
339+
340+
# A single-module subsystem is just a rename; keep the module itself.
341+
top_level_count = len(subsystems) + len(unassigned)
342+
if len(subsystems) <= 1 or top_level_count >= len(module_tree):
343+
logger.info(
344+
"Skipping super-grouping result: %d subsystem(s) over %d modules "
345+
"provide no real consolidation.",
346+
len(subsystems),
347+
len(module_tree),
348+
)
349+
return module_tree
350+
351+
result: Dict[str, Any] = {}
352+
for subsystem_name, members in subsystems.items():
353+
if len(members) == 1:
354+
result[members[0]] = module_tree[members[0]]
355+
continue
356+
components = []
357+
seen = set()
358+
for member in members:
359+
for component in module_tree[member].get("components", []):
360+
if component not in seen:
361+
seen.add(component)
362+
components.append(component)
363+
result[subsystem_name] = {
364+
"path": _common_path_prefix(
365+
[module_tree[member].get("path", "") for member in members]
366+
),
367+
"components": components,
368+
"children": {member: module_tree[member] for member in members},
369+
}
370+
for name in unassigned:
371+
result[name] = module_tree[name]
372+
373+
logger.info(
374+
"Super-grouping consolidated %d top-level modules into %d entries "
375+
"(%d subsystems).",
376+
len(module_tree),
377+
len(result),
378+
len(subsystems),
379+
)
380+
return result

‎codewiki/src/be/documentation_generator.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
from codewiki.src.be.cluster_modules import (
1919
cluster_modules,
2020
get_clustering_input_token_count,
21+
super_group_modules,
2122
)
2223
from codewiki.src.config import (
2324
Config,
@@ -351,6 +352,12 @@ async def run(self) -> None:
351352
self.config,
352353
completer=lambda p: self.backend.complete(p, model=cluster_model),
353354
)
355+
if module_tree:
356+
module_tree = super_group_modules(
357+
module_tree,
358+
self.config,
359+
completer=lambda p: self.backend.complete(p, model=cluster_model),
360+
)
354361
# Only freshly clustered trees are deduped: renaming a cached
355362
# key whose .md already exists would orphan the doc.
356363
module_tree = dedupe_module_tree_names(module_tree)

‎codewiki/src/be/prompt_template.py‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -202,6 +202,44 @@
202202
</GROUPED_COMPONENTS>
203203
""".strip()
204204

205+
SUPER_GROUP_PROMPT = """
206+
Here is the flat list of top-level modules of a repository. Each module has a path and its core components:
207+
208+
<MODULES>
209+
{formatted_modules}
210+
</MODULES>
211+
212+
The module list is flat: conceptually related modules sit next to unrelated ones. Please group these modules into higher-level architectural subsystems so the repository reads as an architecture rather than a directory listing.
213+
214+
Guidelines:
215+
- Each subsystem is a set of modules that together fulfil one architectural responsibility (e.g. a processing pipeline, a user-facing interface layer, a shared platform/foundation).
216+
- Name each subsystem by its architectural role, not by a directory name.
217+
- There must be significantly fewer subsystems than modules; do not simply mirror the module list back.
218+
- Subsystems should contain multiple modules. A single-module subsystem is just a rename — keep a module alone ONLY when it truly shares no architectural responsibility with any other module.
219+
- Every module must be assigned to exactly one subsystem.
220+
- Return the module names EXACTLY as given — do NOT rename or shorten them.
221+
222+
Firstly reason about the modules' responsibilities and relationships and identify the architectural layers they form, then return a grouping that MATCHES your reasoning in the following format:
223+
<GROUPED_MODULES>
224+
{{
225+
"subsystem_name_1": {{
226+
"modules": [
227+
<module_name_1>,
228+
<module_name_2>,
229+
...
230+
]
231+
}},
232+
"subsystem_name_2": {{
233+
"modules": [
234+
<module_name_3>,
235+
...
236+
]
237+
}},
238+
...
239+
}}
240+
</GROUPED_MODULES>
241+
""".strip()
242+
205243
FILTER_FOLDERS_PROMPT = """
206244
Here is the list of relative paths of files, folders in 2-depth of project {project_name}:
207245
```
@@ -374,6 +412,19 @@ def _format_module_tree(module_tree: dict[str, any], indent: int = 0):
374412
return CLUSTER_MODULE_PROMPT.format(potential_core_components=potential_core_components, module_tree=formatted_module_tree, module_name=module_name)
375413

376414

415+
def format_super_group_prompt(module_tree: Dict[str, Any]) -> str:
416+
"""
417+
Format the super-grouping prompt with the flat top-level modules of a tree.
418+
"""
419+
lines = []
420+
for name, info in module_tree.items():
421+
path = info.get("path", "")
422+
lines.append(f"{name} (path: {path})" if path else name)
423+
for component in info.get("components", []):
424+
lines.append(f"\t{component}")
425+
return SUPER_GROUP_PROMPT.format(formatted_modules="\n".join(lines))
426+
427+
377428
def format_system_prompt(module_name: str, custom_instructions: str = None) -> str:
378429
"""
379430
Format the system prompt with module name and optional custom instructions.

‎codewiki/src/config.py‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,10 @@
1717
# Default max token settings
1818
DEFAULT_MAX_TOKENS = 32_768
1919
DEFAULT_MAX_TOKEN_PER_MODULE = 36_369
20-
DEFAULT_MAX_TOKEN_PER_LEAF_MODULE = 16_000
20+
DEFAULT_MAX_TOKEN_PER_LEAF_MODULE = 4_000
21+
# Super-group the flat top level into architectural subsystems only when it
22+
# has more than this many modules; 0 or negative disables the pass.
23+
DEFAULT_MIN_MODULES_FOR_SUPER_GROUPING = 3
2124
# Legacy constants (for backward compatibility)
2225
MAX_TOKEN_PER_MODULE = DEFAULT_MAX_TOKEN_PER_MODULE
2326
MAX_TOKEN_PER_LEAF_MODULE = DEFAULT_MAX_TOKEN_PER_LEAF_MODULE
@@ -70,6 +73,7 @@ class Config:
7073
max_tokens: int = DEFAULT_MAX_TOKENS
7174
max_token_per_module: int = DEFAULT_MAX_TOKEN_PER_MODULE
7275
max_token_per_leaf_module: int = DEFAULT_MAX_TOKEN_PER_LEAF_MODULE
76+
min_modules_for_super_grouping: int = DEFAULT_MIN_MODULES_FOR_SUPER_GROUPING
7377
# Prompt caching for agentic/multi-turn calls (auto-disables per model if
7478
# the provider rejects cache_control markers)
7579
prompt_caching: bool = True
@@ -177,6 +181,7 @@ def from_cli(
177181
max_tokens: int = DEFAULT_MAX_TOKENS,
178182
max_token_per_module: int = DEFAULT_MAX_TOKEN_PER_MODULE,
179183
max_token_per_leaf_module: int = DEFAULT_MAX_TOKEN_PER_LEAF_MODULE,
184+
min_modules_for_super_grouping: int = DEFAULT_MIN_MODULES_FOR_SUPER_GROUPING,
180185
max_depth: int = MAX_DEPTH,
181186
agent_instructions: Optional[Dict[str, Any]] = None,
182187
use_gitignore: bool = True,
@@ -200,6 +205,9 @@ def from_cli(
200205
max_tokens: Maximum tokens for LLM response
201206
max_token_per_module: Maximum tokens per module for clustering
202207
max_token_per_leaf_module: Maximum tokens per leaf module
208+
min_modules_for_super_grouping: Super-group the top level into
209+
subsystems only when it has more than this many modules
210+
(0 or negative disables the pass)
203211
max_depth: Maximum depth for hierarchical decomposition
204212
agent_instructions: Custom agent instructions dict
205213
use_gitignore: Whether to apply Git ignore rules
@@ -229,6 +237,7 @@ def from_cli(
229237
max_tokens=max_tokens,
230238
max_token_per_module=max_token_per_module,
231239
max_token_per_leaf_module=max_token_per_leaf_module,
240+
min_modules_for_super_grouping=min_modules_for_super_grouping,
232241
agent_instructions=agent_instructions,
233242
use_gitignore=use_gitignore,
234243
prompt_caching=prompt_caching,

‎codewiki/templates/github_pages/viewer_template.html‎

Lines changed: 21 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,18 @@
147147
color: white;
148148
}
149149

150+
.nav-item.nav-parent {
151+
font-weight: 600;
152+
font-size: 13px;
153+
color: #64748b;
154+
text-transform: uppercase;
155+
letter-spacing: 0.05em;
156+
}
157+
158+
.nav-item.nav-parent.active {
159+
color: white;
160+
}
161+
150162
.nav-subsection {
151163
margin-left: 15px;
152164
margin-top: 5px;
@@ -466,17 +478,18 @@ <h4>Generation Info</h4>
466478
function buildNavItem(key, data, depth) {
467479
const indentClass = depth > 0 ? 'nav-subsection' : 'nav-section';
468480
const fileName = `${key}.md`;
469-
481+
const hasChildren = data.children && Object.keys(data.children).length > 0;
482+
470483
let html = `<div class="${indentClass}">`;
471-
472-
if (data.components && data.components.length > 0) {
473-
html += `<div class="nav-item" data-file="${fileName}">
474-
${formatNavTitle(key)}
475-
</div>`;
476-
} else if (depth === 0) {
484+
485+
// Every module in the tree has a generated doc (parents get a
486+
// synthesized overview), so render a link whenever the node has
487+
// components or children. Only the synthetic fallback tree's bare
488+
// top-level node stays a plain header.
489+
if (depth === 0 && !hasChildren && !(data.components && data.components.length > 0)) {
477490
html += `<h3>${formatNavTitle(key)}</h3>`;
478491
} else {
479-
html += `<div class="nav-item" data-file="${fileName}">
492+
html += `<div class="nav-item${depth === 0 && hasChildren ? ' nav-parent' : ''}" data-file="${fileName}">
480493
${formatNavTitle(key)}
481494
</div>`;
482495
}

0 commit comments

Comments
 (0)