Skip to content

Commit a089266

Browse files
authored
Merge pull request #110 from kiboook/fix/mermaid-py-docs-comment
docs: correct mermaid-py behavior in comments (remote API, not local Node.js)
2 parents 7fe4745 + 8be46ef commit a089266

2 files changed

Lines changed: 58 additions & 36 deletions

File tree

‎codewiki/src/be/utils.py‎

Lines changed: 49 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77
from typing import List, Tuple
88
import logging
99
import tiktoken
10-
import traceback
1110

1211

1312
logger = logging.getLogger(__name__)
@@ -29,10 +28,12 @@ def set_main_loop(loop: asyncio.AbstractEventLoop) -> None:
2928
_main_loop = loop
3029
_main_loop_thread_ident = threading.get_ident()
3130

31+
3232
# ------------------------------------------------------------
3333
# ---------------------- Complexity Check --------------------
3434
# ------------------------------------------------------------
3535

36+
3637
def is_complex_module(components: dict[str, any], core_component_ids: list[str]) -> bool:
3738
files = set()
3839
for component_id in core_component_ids:
@@ -50,6 +51,7 @@ def is_complex_module(components: dict[str, any], core_component_ids: list[str])
5051

5152
enc = tiktoken.encoding_for_model("gpt-4")
5253

54+
5355
def count_tokens(text: str) -> int:
5456
"""
5557
Count the number of tokens in a text.
@@ -63,10 +65,11 @@ def count_tokens(text: str) -> int:
6365
# ---------------------- Mermaid Validation -----------------
6466
# ------------------------------------------------------------
6567

68+
6669
async def validate_mermaid_diagrams(md_file_path: str, relative_path: str) -> str:
6770
"""
6871
Validate all Mermaid diagrams in a markdown file.
69-
72+
7073
Args:
7174
md_file_path: Path to the markdown file to check
7275
relative_path: Relative path to the markdown file
@@ -80,68 +83,70 @@ async def validate_mermaid_diagrams(md_file_path: str, relative_path: str) -> st
8083
file_path = Path(md_file_path)
8184
if not file_path.exists():
8285
return f"Error: File '{md_file_path}' does not exist"
83-
84-
content = file_path.read_text(encoding='utf-8')
85-
86+
87+
content = file_path.read_text(encoding="utf-8")
88+
8689
# Extract all mermaid code blocks
8790
mermaid_blocks = extract_mermaid_blocks(content)
88-
91+
8992
if not mermaid_blocks:
9093
return "No mermaid diagrams found in the file"
91-
94+
9295
# Validate each mermaid diagram sequentially to avoid segfaults
9396
errors = []
9497
for i, (line_start, diagram_content) in enumerate(mermaid_blocks, 1):
9598
error_msg = await validate_single_diagram(diagram_content, i, line_start)
9699
if error_msg:
97100
errors.append("\n")
98101
errors.append(error_msg)
99-
102+
100103
# if errors:
101104
# logger.debug(f"Mermaid syntax errors found in file: {md_file_path}: {errors}")
102-
105+
103106
if errors:
104-
return "Mermaid syntax errors found in file: " + relative_path + "\n" + "\n".join(errors)
107+
return (
108+
"Mermaid syntax errors found in file: " + relative_path + "\n" + "\n".join(errors)
109+
)
105110
else:
106111
return "All mermaid diagrams in file: " + relative_path + " are syntax correct"
107-
112+
108113
except Exception as e:
109114
return f"Error processing file: {str(e)}"
110115

111116

112117
def extract_mermaid_blocks(content: str) -> List[Tuple[int, str]]:
113118
"""
114119
Extract all mermaid code blocks from markdown content.
115-
120+
116121
Returns:
117122
List of tuples containing (line_number, diagram_content)
118123
"""
119124
mermaid_blocks = []
120-
lines = content.split('\n')
125+
lines = content.split("\n")
121126
i = 0
122-
127+
123128
while i < len(lines):
124129
line = lines[i].strip()
125-
130+
126131
# Look for mermaid code block start
127-
if line == '```mermaid' or line.startswith('```mermaid'):
132+
if line == "```mermaid" or line.startswith("```mermaid"):
128133
start_line = i + 1
129134
diagram_lines = []
130135
i += 1
131-
136+
132137
# Collect lines until we find the closing ```
133138
while i < len(lines):
134-
if lines[i].strip() == '```':
139+
if lines[i].strip() == "```":
135140
break
136141
diagram_lines.append(lines[i])
137142
i += 1
138-
143+
139144
if diagram_lines: # Only add non-empty diagrams
140-
diagram_content = '\n'.join(diagram_lines)
145+
diagram_content = "\n".join(diagram_lines)
141146
mermaid_blocks.append((start_line, diagram_content))
142-
147+
143148
i += 1
144-
149+
145150
return mermaid_blocks
146151

147152

@@ -151,9 +156,13 @@ def extract_mermaid_blocks(content: str) -> List[Tuple[int, str]]:
151156
# Skip it proactively so SpiderMonkey is never loaded into the process.
152157
_PYTHONMONKEY_BROKEN = sys.version_info >= (3, 12)
153158

154-
# mermaid-py spawns a Node.js subprocess that can hang indefinitely (e.g. when
155-
# Node.js is missing or the mermaid CLI is misconfigured). Enabled by default;
156-
# set MERMAID_VALIDATE=0 to disable.
159+
# mermaid-py validates diagrams by sending them to a remote rendering service
160+
# (https://mermaid.ink by default, overridable via the MERMAID_INK_SERVER env
161+
# var understood by mermaid-py itself) rather than spawning a local Node.js
162+
# subprocess. Note that this sends diagram content to that third-party service.
163+
# It can still hang or fail when the service is unreachable (no network egress,
164+
# DNS/firewall issues, or an outage). Enabled by default; set MERMAID_VALIDATE=0
165+
# to disable.
157166
_MERMAID_PY_BROKEN = os.environ.get("MERMAID_VALIDATE", "1") == "0"
158167
_MERMAID_PY_PROBED = True # Skip probing — rely on env var
159168

@@ -178,16 +187,14 @@ async def _try_pythonmonkey_parse(diagram_content: str) -> str | None:
178187
return None
179188

180189
old_stderr = sys.stderr
181-
sys.stderr = open(os.devnull, 'w')
190+
sys.stderr = open(os.devnull, "w")
182191
try:
183192
if (
184193
_main_loop is not None
185194
and _main_loop.is_running()
186195
and threading.get_ident() != _main_loop_thread_ident
187196
):
188-
fut = asyncio.run_coroutine_threadsafe(
189-
parse_mermaid_py(diagram_content), _main_loop
190-
)
197+
fut = asyncio.run_coroutine_threadsafe(parse_mermaid_py(diagram_content), _main_loop)
191198
await asyncio.wrap_future(fut)
192199
else:
193200
await parse_mermaid_py(diagram_content)
@@ -218,6 +225,7 @@ def _parse_via_mermaid_py(diagram_content: str) -> str:
218225
text, otherwise a successful SVG gets reported as a parse error.
219226
"""
220227
import mermaid as md
228+
221229
try:
222230
md.Mermaid(diagram_content)
223231
return ""
@@ -244,7 +252,9 @@ async def validate_single_diagram(diagram_content: str, diagram_num: int, line_s
244252
# Validation disabled/unavailable is not a syntax error — a
245253
# non-empty return here would make callers report valid diagrams
246254
# as broken and send agents into fix loops.
247-
logger.debug("Diagram %d: validation skipped (mermaid-py disabled or unavailable)", diagram_num)
255+
logger.debug(
256+
"Diagram %d: validation skipped (mermaid-py disabled or unavailable)", diagram_num
257+
)
248258
return ""
249259
try:
250260
core_error = await asyncio.wait_for(
@@ -254,28 +264,32 @@ async def validate_single_diagram(diagram_content: str, diagram_num: int, line_s
254264
except asyncio.TimeoutError:
255265
# Inconclusive, not a parse failure. Latch so the remaining
256266
# diagrams (and later calls) don't each block 15s on the same
257-
# broken Node.js setup.
267+
# unreachable rendering service.
258268
_MERMAID_PY_BROKEN = True
259-
logger.warning("Diagram %d: mermaid validation timed out (15s); skipping further validation", diagram_num)
269+
logger.warning(
270+
"Diagram %d: mermaid validation timed out (15s); skipping further validation",
271+
diagram_num,
272+
)
260273
return ""
261274
except Exception as e:
262275
return f" Diagram {diagram_num}: Exception during validation - {str(e)}"
263276

264277
if not core_error:
265278
return ""
266279

267-
line_match = re.search(r'line (\d+)', core_error)
280+
line_match = re.search(r"line (\d+)", core_error)
268281
if line_match:
269282
error_line_in_diagram = int(line_match.group(1))
270283
actual_line_in_file = line_start + error_line_in_diagram
271-
newline = '\n'
284+
newline = "\n"
272285
return f"Diagram {diagram_num}: Parse error on line {actual_line_in_file}:{newline}{newline.join(core_error.split(newline)[1:])}"
273286
return f"Diagram {diagram_num}: {core_error}"
274287

275288

276289
if __name__ == "__main__":
277290
# Test with the provided file
278291
import asyncio
292+
279293
test_file = "output/docs/SWE_agent-docs/agent_hooks.md"
280294
result = asyncio.run(validate_mermaid_diagrams(test_file, "agent_hooks.md"))
281-
print(result)
295+
print(result)

‎pyproject.toml‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,10 @@ dependencies = [
6565
]
6666

6767
[external]
68-
# Node.js is required for mermaid-py which validates mermaid diagrams in generated documentation
68+
# Node.js/npm are required at *install* time by PythonMonkey (pulled in via
69+
# mermaid-parser-py), whose pminit helper shells out to npm to populate its JS
70+
# dependencies. mermaid-py itself does not need Node.js: it validates diagrams
71+
# with an HTTP call to a remote rendering service (mermaid.ink by default).
6972
build-requires = [
7073
{ name = "nodejs", version = ">=14.0.0" }
7174
]
@@ -128,6 +131,11 @@ disallow_incomplete_defs = false
128131
line-length = 100
129132
target-version = "py312"
130133

134+
[tool.ruff.lint]
135+
# ruff 0.16 widened its default rule set considerably; pin the previous
136+
# defaults so an unpinned `pip install ruff` in CI stays stable.
137+
select = ["E4", "E7", "E9", "F"]
138+
131139
[tool.pytest.ini_options]
132140
testpaths = ["tests"]
133141
norecursedirs = [".venv", "atlascloud", "titan-sight"]

0 commit comments

Comments
 (0)