Skip to content

Commit 0815b54

Browse files
committed
docs: mirror the module tree in the docs folder by default
Pages now live in folders that follow the module tree: a module's page sits next to the folder holding its sub-modules (auth.md, auth/login.md), and overview.md stays at the root. Module names stay unique across the wiki, so the module-tree key is still the page's filename stem. - doc_layout: one place that maps modules to page paths, finds pages in either layout, and after each run moves misplaced pages and rewrites links between pages to correct relative paths - prompts give each agent its page path and every module's page path; the sub-module tool reports links relative to the parent page - --flat keeps the old single-folder layout for small models - the layout is recorded in metadata.json and --update keeps it; docs without a recorded layout are treated as flat - the updater, GitHub Pages viewer, web app (now with a path traversal guard), MCP tools (doc_path in processing_order.json) and the wiki-generator skill follow the layout - '&' is no longer allowed in module names: agents shell-escape it and write pages into a stray folder Closes #125 (hierarchical output; OKF export not included)
1 parent c8a2bf1 commit 0815b54

36 files changed

Lines changed: 1337 additions & 243 deletions

‎CHANGELOG.md‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,18 @@ uses [Semantic Versioning](https://semver.org/).
1313
struct-literal, and call edges. `#[cfg(test)]` modules are skipped, and
1414
`build.rs` is classified as a build artifact.
1515

16+
### Changed
17+
18+
- **Docs mirror the module tree** ([#125](https://github.com/FSoft-AI4Code/CodeWiki/issues/125)).
19+
A module's page now sits next to the folder that holds its sub-modules
20+
(`auth.md`, `auth/login.md`) instead of every page sharing one flat folder.
21+
Links between pages are relative. After each run, pages saved in the wrong
22+
folder are moved and wrong links are repaired. `--flat` keeps the old
23+
layout for small models. The layout is recorded in `metadata.json`, and
24+
`--update` keeps it, so existing flat docs stay flat. The GitHub Pages
25+
viewer, the web app and the MCP tools (`doc_path` in
26+
`processing_order.json`) follow the layout.
27+
1628
## [2.0.0] - 2026-09-18
1729

1830
The first release since 1.0.1. Everything that landed on `main` in between is

‎README.md‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -164,7 +164,8 @@ configuration in CodeWiki.
164164
```
165165
./docs/
166166
├── overview.md # start here
167-
├── <module>.md ... # one page per module, leaves and parents
167+
├── <module>.md ... # one page per top-level module
168+
├── <module>/<sub-module>.md ... # sub-module pages, in folders mirroring the module tree
168169
├── module_tree.json # the module hierarchy
169170
├── first_module_tree.json # clustering result before super-grouping
170171
├── metadata.json # model, version, commit, statistics
@@ -174,6 +175,11 @@ configuration in CodeWiki.
174175
└── index.html # viewer (with --github-pages)
175176
```
176177

178+
Pages mirror the module tree: a module's page sits next to the folder holding
179+
its sub-modules (`auth.md`, `auth/login.md`). `--flat` puts every page in
180+
`./docs` instead, which helps small models that get relative links wrong.
181+
`--update` keeps the layout the docs were generated with.
182+
177183
This repository's own output is checked in under [`./docs/`](./docs/).
178184

179185
## Benchmark results

‎codewiki/cli/adapters/doc_generator.py‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,19 @@
1919

2020
# Import backend modules
2121
from codewiki.src.be.documentation_generator import DocumentationGenerator
22+
from codewiki.src.be.doc_layout import list_doc_files
23+
from codewiki.src.config import DEFAULT_LAYOUT
2224
from codewiki.src.config import Config as BackendConfig
2325
from codewiki.src.config import set_cli_context
2426

2527

28+
def _generated_files(working_dir: str) -> list[str]:
29+
"""Docs-relative paths of the pages and the JSON files in the docs root."""
30+
pages = sorted(list_doc_files(working_dir).values())
31+
jsons = sorted(f for f in os.listdir(working_dir) if f.endswith(".json"))
32+
return pages + jsons
33+
34+
2635
class CLIDocumentationGenerator:
2736
"""
2837
CLI adapter for documentation generation with progress reporting.
@@ -154,6 +163,7 @@ def generate(self) -> DocumentationJob:
154163
artifacts_enabled=self.config.get("artifacts_enabled", True),
155164
artifact_token_budget=self.config.get("artifact_token_budget", 200_000),
156165
with_prose=self.config.get("with_prose", False),
166+
layout=self.config.get("layout", DEFAULT_LAYOUT),
157167
)
158168

159169
# Run backend documentation generation
@@ -372,9 +382,7 @@ async def _run_backend_generation(self, backend_config: BackendConfig):
372382
self._merge_update_summary(working_dir)
373383

374384
# Collect generated files
375-
for file_path in os.listdir(working_dir):
376-
if file_path.endswith((".md", ".json")):
377-
self.job.files_generated.append(file_path)
385+
self.job.files_generated.extend(_generated_files(working_dir))
378386

379387
except Exception as e: # noqa: BLE001 — surfaced to the user as an APIError
380388
raise APIError(f"Documentation generation failed: {e}")
@@ -448,9 +456,7 @@ async def _run_incremental_update(
448456
prior_history = self._read_update_history(working_dir)
449457
doc_generator.create_documentation_metadata(working_dir, components, len(leaf_nodes))
450458
self._merge_update_summary(working_dir, prior_history)
451-
for file_path in os.listdir(working_dir):
452-
if file_path.endswith((".md", ".json")):
453-
self.job.files_generated.append(file_path)
459+
self.job.files_generated.extend(_generated_files(working_dir))
454460
tree_path = os.path.join(working_dir, "module_tree.json")
455461
if os.path.exists(tree_path):
456462
with open(tree_path, encoding="utf-8") as f:

‎codewiki/cli/commands/generate.py‎

Lines changed: 29 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
"""
44

55
import logging
6+
import os
67
import sys
78
import time
89
import traceback
@@ -24,6 +25,8 @@
2425
)
2526
from codewiki.cli.utils.instructions import display_post_generation_instructions
2627
from codewiki.cli.utils.logging import create_logger
28+
from codewiki.src.be.doc_layout import find_doc, list_doc_files, read_layout, remove_doc
29+
from codewiki.src.config import DEFAULT_LAYOUT, LAYOUT_FLAT
2730
from codewiki.src.language import resolve_update_language
2831
from codewiki.cli.utils.repo_validator import (
2932
check_writable_output,
@@ -204,13 +207,13 @@ def _find_affected(tree, parent_names=None):
204207
if modules_to_invalidate:
205208
modules_to_invalidate.add("overview")
206209

207-
# Delete affected module docs
210+
# Delete affected module docs (wherever the docs layout put them)
208211
for mod_name in modules_to_invalidate:
209-
doc_path = output_dir / f"{mod_name}.md"
210-
if doc_path.exists():
211-
doc_path.unlink()
212+
doc_path = find_doc(str(output_dir), mod_name, module_tree)
213+
if doc_path is not None:
214+
remove_doc(str(output_dir), doc_path)
212215
if verbose:
213-
logger.debug(f"Invalidated: {doc_path.name}")
216+
logger.debug(f"Invalidated: {os.path.relpath(doc_path, output_dir)}")
214217

215218
if verbose:
216219
logger.debug(f"Invalidated {len(modules_to_invalidate)} modules for regeneration.")
@@ -353,6 +356,13 @@ def _find_affected(tree, parent_names=None):
353356
is_flag=True,
354357
help="Also read the root README and docs/ as a `prose` artifact class (off by default)",
355358
)
359+
@click.option(
360+
"--flat",
361+
is_flag=True,
362+
help="Save every page in the output directory root instead of folders mirroring the "
363+
"module tree (for small models that struggle with relative links). --update keeps the "
364+
"layout the docs were generated with",
365+
)
356366
@click.option(
357367
"--artifact-exclude",
358368
type=str,
@@ -447,6 +457,7 @@ def generate_command(
447457
artifacts: bool = True,
448458
artifact_token_budget: int = 200_000,
449459
with_prose: bool = False,
460+
flat: bool = False,
450461
artifact_exclude: str | None = None,
451462
update: bool = False,
452463
compare_to: str | None = None,
@@ -585,6 +596,16 @@ def generate_command(
585596
)
586597
except ValueError as e:
587598
raise ConfigurationError(str(e)) from None
599+
layout = LAYOUT_FLAT if flat else DEFAULT_LAYOUT
600+
if updating_existing_docs:
601+
# Keep updated pages where the existing docs put them
602+
stored_layout = read_layout(str(output_dir))
603+
if flat and stored_layout != LAYOUT_FLAT:
604+
logger.warning(
605+
f"These docs use the {stored_layout} layout; --flat is ignored by --update. "
606+
"Rerun without --update to regenerate them flat."
607+
)
608+
layout = stored_layout
588609
if update and output_dir.exists():
589610
changed_files = _detect_changed_files(
590611
repo_path, output_dir, logger, verbose, compare_to=compare_to
@@ -616,7 +637,7 @@ def generate_command(
616637
if (
617638
not update
618639
and output_dir.exists()
619-
and list(output_dir.glob("*.md"))
640+
and list_doc_files(str(output_dir))
620641
and not click.confirm(
621642
f"\n{output_dir} already contains documentation. Overwrite?", default=True
622643
)
@@ -777,6 +798,8 @@ def generate_command(
777798
"artifacts_enabled": artifacts,
778799
"artifact_token_budget": artifact_token_budget,
779800
"with_prose": with_prose,
801+
# Docs layout (runtime-only; --update keeps the stored one)
802+
"layout": layout,
780803
# Incremental updater (runtime-only)
781804
"update": update,
782805
"update_options": {

‎codewiki/cli/html_generator.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -156,12 +156,21 @@ def generate(
156156
page_titles = extract_page_titles(str(docs_dir), module_tree)
157157
page_titles_json = json.dumps(page_titles, ensure_ascii=False).replace("</", "<\\/")
158158

159+
# Where each page lives (nested folders or the docs root)
160+
doc_paths = {}
161+
if docs_dir:
162+
from codewiki.src.be.doc_layout import doc_path_map
163+
164+
doc_paths = doc_path_map(str(docs_dir), module_tree)
165+
doc_paths_json = json.dumps(doc_paths, ensure_ascii=False).replace("</", "<\\/")
166+
159167
# Replace placeholders
160168
html_content = template_content
161169
replacements = {
162170
"{{TITLE}}": self._escape_html(title),
163171
"{{HTML_LANG}}": language_tag(language),
164172
"{{PAGE_TITLES_JSON}}": page_titles_json,
173+
"{{DOC_PATHS_JSON}}": doc_paths_json,
165174
"{{REPO_LINK}}": repo_link,
166175
"{{SHOW_INFO}}": show_info,
167176
"{{INFO_CONTENT}}": info_content,

‎codewiki/mcp/server.py‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -150,7 +150,11 @@ def _fine_grained_tools() -> list[Tool]:
150150
},
151151
"filename": {
152152
"type": "string",
153-
"description": "Filename for the doc (e.g., 'auth_module.md')",
153+
"description": (
154+
"Path of the doc relative to the output dir: the module's "
155+
"doc_path from processing_order.json (e.g., 'auth.md' or "
156+
"'auth/login.md'; folders are created as needed)"
157+
),
154158
},
155159
"content": {
156160
"type": "string",
@@ -531,6 +535,8 @@ async def _legacy_generate_docs(arguments: dict[str, Any]) -> list[TextContent]:
531535

532536
set_cli_context(True)
533537

538+
from codewiki.src.be.doc_layout import docs_layout, list_doc_files
539+
534540
backend_config = BackendConfig.from_cli(
535541
repo_path=str(repo_path),
536542
output_dir=str(output_dir),
@@ -544,6 +550,7 @@ async def _legacy_generate_docs(arguments: dict[str, Any]) -> list[TextContent]:
544550
max_tokens=config.max_tokens,
545551
agent_instructions=agent_instructions or None,
546552
use_gitignore=arguments.get("use_gitignore", True),
553+
layout=docs_layout(str(output_dir)),
547554
)
548555

549556
from codewiki.cli.utils.repo_validator import get_git_commit_hash
@@ -554,9 +561,9 @@ async def _legacy_generate_docs(arguments: dict[str, Any]) -> list[TextContent]:
554561
)
555562
await doc_gen.run()
556563

557-
generated_files = []
564+
generated_files = sorted(list_doc_files(str(output_dir)).values())
558565
for f in output_dir.iterdir():
559-
if f.suffix in (".md", ".json", ".html"):
566+
if f.suffix in (".json", ".html"):
560567
generated_files.append(f.name)
561568

562569
result = {
@@ -654,10 +661,18 @@ def _write_generation_metadata(session: SessionState) -> None:
654661
except (json.JSONDecodeError, OSError):
655662
pass
656663

664+
from codewiki.src.be.doc_layout import docs_layout, organize_docs
665+
666+
# Keep the layout of existing docs; new docs get the default. Move
667+
# misplaced pages and fix links between pages before stamping it.
668+
layout = docs_layout(str(output_dir))
669+
organize_docs(str(output_dir), layout)
670+
657671
existing["generation_info"] = {
658672
**existing.get("generation_info", {}),
659673
"commit_id": commit_id,
660674
"timestamp": datetime.now().isoformat(),
675+
"layout": layout,
661676
}
662677
metadata_path.write_text(
663678
json.dumps(existing, indent=2, ensure_ascii=False),

‎codewiki/mcp/tools/module_tree.py‎

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,8 @@
1313
from typing import Any, Dict, List, Tuple
1414

1515
from codewiki.mcp.session import SessionStore
16-
from codewiki.src.config import FIRST_MODULE_TREE_FILENAME, MODULE_TREE_FILENAME
16+
from codewiki.src.be.doc_layout import docs_layout, module_doc_relpath
17+
from codewiki.src.config import DEFAULT_LAYOUT, FIRST_MODULE_TREE_FILENAME, MODULE_TREE_FILENAME
1718

1819
logger = logging.getLogger(__name__)
1920

@@ -30,12 +31,15 @@ def _cap(ids: List[str]) -> Tuple[List[str], bool]:
3031

3132

3233
def _get_processing_order(
33-
module_tree: Dict[str, Any], parent_path: List[str] | None = None
34+
module_tree: Dict[str, Any],
35+
parent_path: List[str] | None = None,
36+
layout: str = DEFAULT_LAYOUT,
3437
) -> List[Dict[str, Any]]:
3538
"""Compute leaf-first processing order from a module tree.
3639
37-
Returns a list of dicts with module path, name, leaf status, and
38-
component/children info.
40+
Returns a list of dicts with module path, name, leaf status,
41+
component/children info and ``doc_path`` (the page to write, relative to
42+
the output dir, for the docs ``layout``).
3943
"""
4044
if parent_path is None:
4145
parent_path = []
@@ -54,6 +58,7 @@ def _collect(tree: Dict[str, Any], path: List[str]) -> None:
5458
"module": module_name,
5559
"path": current_path,
5660
"is_leaf": False,
61+
"doc_path": module_doc_relpath(current_path, layout),
5762
"children": list(children.keys()),
5863
"components": module_info.get("components", []),
5964
}
@@ -64,6 +69,7 @@ def _collect(tree: Dict[str, Any], path: List[str]) -> None:
6469
"module": module_name,
6570
"path": current_path,
6671
"is_leaf": True,
72+
"doc_path": module_doc_relpath(current_path, layout),
6773
"components": module_info.get("components", []),
6874
}
6975
)
@@ -189,7 +195,7 @@ def handle_save_module_tree(
189195
logger.info("save_module_tree for session %s: %s", session_id, note)
190196

191197
# Compute processing order and write to workspace file
192-
order = _get_processing_order(module_tree)
198+
order = _get_processing_order(module_tree, layout=docs_layout(session.output_dir))
193199
order_file = None
194200
if session.workspace is not None:
195201
order_path = session.workspace.write_json("processing_order.json", order)
@@ -206,7 +212,9 @@ def handle_save_module_tree(
206212
"Read the processing_order.json file for the leaf-first generation order. "
207213
"Process leaf modules first (is_leaf=true), then parent modules. "
208214
"For each leaf module: get_prompt('system_leaf') + read_code_components + write_doc_file. "
209-
"For each parent module: get_prompt('overview_module') + write_doc_file."
215+
"For each parent module: get_prompt('overview_module') + write_doc_file. "
216+
"Save each module's page at its doc_path (sub-folders are created automatically) "
217+
"and link pages with paths relative to the linking page."
210218
),
211219
}
212220
if warning:
@@ -237,7 +245,7 @@ def handle_get_processing_order(
237245
else:
238246
return json.dumps({"error": "Module tree not found. Call save_module_tree first."})
239247

240-
order = _get_processing_order(module_tree)
248+
order = _get_processing_order(module_tree, layout=docs_layout(session.output_dir))
241249

242250
# Write to workspace file
243251
order_file = None

0 commit comments

Comments
 (0)