|
1 | 1 | from typing import List, Dict, Any, Callable, Optional |
2 | 2 | from collections import defaultdict |
| 3 | +import ast |
3 | 4 | import logging |
4 | 5 | import traceback |
5 | 6 | logger = logging.getLogger(__name__) |
6 | 7 |
|
7 | 8 | from codewiki.src.be.dependency_analyzer.models.core import Node |
8 | 9 | from codewiki.src.be.llm_services import call_llm |
9 | 10 | from codewiki.src.be.utils import count_tokens |
10 | | -from codewiki.src.config import Config |
11 | | -from codewiki.src.be.prompt_template import format_cluster_prompt |
| 11 | +from codewiki.src.config import Config, DEFAULT_MIN_MODULES_FOR_SUPER_GROUPING |
| 12 | +from codewiki.src.be.prompt_template import format_cluster_prompt, format_super_group_prompt |
12 | 13 |
|
13 | 14 | Completer = Callable[[str], str] |
14 | 15 |
|
@@ -209,3 +210,171 @@ def cluster_modules( |
209 | 210 | current_module_path.pop() |
210 | 211 |
|
211 | 212 | return module_tree |
| 213 | + |
| 214 | + |
| 215 | +def _common_path_prefix(paths: List[str]) -> str: |
| 216 | + """Longest common directory prefix of the given relative paths.""" |
| 217 | + split_paths = [p.strip("/").split("/") for p in paths if p] |
| 218 | + if not split_paths: |
| 219 | + return "" |
| 220 | + common = [] |
| 221 | + for parts in zip(*split_paths): |
| 222 | + if all(part == parts[0] for part in parts): |
| 223 | + common.append(parts[0]) |
| 224 | + else: |
| 225 | + break |
| 226 | + return "/".join(common) |
| 227 | + |
| 228 | + |
| 229 | +def _parse_super_group_response(response: str) -> Optional[Dict[str, Any]]: |
| 230 | + if "<GROUPED_MODULES>" not in response or "</GROUPED_MODULES>" not in response: |
| 231 | + logger.warning( |
| 232 | + "Invalid super-grouping response: missing <GROUPED_MODULES> tags; " |
| 233 | + "keeping the flat module tree. Response preview: %s...", |
| 234 | + response[:200], |
| 235 | + ) |
| 236 | + return None |
| 237 | + try: |
| 238 | + grouping = ast.literal_eval( |
| 239 | + response.split("<GROUPED_MODULES>")[1].split("</GROUPED_MODULES>")[0] |
| 240 | + ) |
| 241 | + except Exception as e: |
| 242 | + logger.warning( |
| 243 | + "Failed to parse super-grouping response; keeping the flat module " |
| 244 | + "tree. Error: %s. Response preview: %s...", |
| 245 | + e, |
| 246 | + response[:200], |
| 247 | + ) |
| 248 | + return None |
| 249 | + if not isinstance(grouping, dict): |
| 250 | + logger.warning( |
| 251 | + "Invalid super-grouping format - expected dict, got %s; keeping the " |
| 252 | + "flat module tree.", |
| 253 | + type(grouping), |
| 254 | + ) |
| 255 | + return None |
| 256 | + return grouping |
| 257 | + |
| 258 | + |
| 259 | +def super_group_modules( |
| 260 | + module_tree: Dict[str, Any], |
| 261 | + config: Config, |
| 262 | + completer: Optional[Completer] = None, |
| 263 | +) -> Dict[str, Any]: |
| 264 | + """ |
| 265 | + Group a flat top level of modules into higher-level architectural subsystems. |
| 266 | +
|
| 267 | + Runs one extra LLM pass over the top-level module names (with their paths |
| 268 | + and components) and nests the flat modules as children of the subsystems it |
| 269 | + proposes. Any invalid or non-consolidating response leaves the tree |
| 270 | + unchanged, so this pass can never make the structure worse. |
| 271 | + """ |
| 272 | + min_modules = getattr( |
| 273 | + config, "min_modules_for_super_grouping", DEFAULT_MIN_MODULES_FOR_SUPER_GROUPING |
| 274 | + ) |
| 275 | + if min_modules <= 0: |
| 276 | + logger.info( |
| 277 | + "Super-grouping disabled (min_modules_for_super_grouping=%d).", min_modules |
| 278 | + ) |
| 279 | + return module_tree |
| 280 | + if len(module_tree) <= min_modules: |
| 281 | + logger.info( |
| 282 | + "Skipping super-grouping: %d top-level modules fit within the " |
| 283 | + "%d-module threshold.", |
| 284 | + len(module_tree), |
| 285 | + min_modules, |
| 286 | + ) |
| 287 | + return module_tree |
| 288 | + |
| 289 | + prompt = format_super_group_prompt(module_tree) |
| 290 | + logger.info( |
| 291 | + "Requesting super-grouping of %d top-level modules into architectural " |
| 292 | + "subsystems.", |
| 293 | + len(module_tree), |
| 294 | + ) |
| 295 | + if completer is not None: |
| 296 | + response = completer(prompt) |
| 297 | + else: |
| 298 | + response = call_llm(prompt, config, model=config.cluster_model) |
| 299 | + |
| 300 | + grouping = _parse_super_group_response(response) |
| 301 | + if grouping is None: |
| 302 | + return module_tree |
| 303 | + |
| 304 | + # Validate assignments: unknown modules are dropped, duplicates keep their |
| 305 | + # first assignment, unassigned modules stay at the top level. |
| 306 | + assigned = set() |
| 307 | + subsystems: Dict[str, List[str]] = {} |
| 308 | + for subsystem_name, info in grouping.items(): |
| 309 | + members = info.get("modules") if isinstance(info, dict) else None |
| 310 | + if not isinstance(members, list): |
| 311 | + logger.warning( |
| 312 | + "Skipping subsystem '%s' in super-grouping response: no valid " |
| 313 | + "'modules' list.", |
| 314 | + subsystem_name, |
| 315 | + ) |
| 316 | + continue |
| 317 | + valid_members = [] |
| 318 | + for member in members: |
| 319 | + if member not in module_tree: |
| 320 | + logger.warning( |
| 321 | + "Skipping unknown module '%s' in subsystem '%s'.", |
| 322 | + member, |
| 323 | + subsystem_name, |
| 324 | + ) |
| 325 | + continue |
| 326 | + if member in assigned: |
| 327 | + logger.warning( |
| 328 | + "Module '%s' assigned to multiple subsystems; keeping its " |
| 329 | + "first assignment.", |
| 330 | + member, |
| 331 | + ) |
| 332 | + continue |
| 333 | + assigned.add(member) |
| 334 | + valid_members.append(member) |
| 335 | + if valid_members: |
| 336 | + subsystems[subsystem_name] = valid_members |
| 337 | + |
| 338 | + unassigned = [name for name in module_tree if name not in assigned] |
| 339 | + |
| 340 | + # A single-module subsystem is just a rename; keep the module itself. |
| 341 | + top_level_count = len(subsystems) + len(unassigned) |
| 342 | + if len(subsystems) <= 1 or top_level_count >= len(module_tree): |
| 343 | + logger.info( |
| 344 | + "Skipping super-grouping result: %d subsystem(s) over %d modules " |
| 345 | + "provide no real consolidation.", |
| 346 | + len(subsystems), |
| 347 | + len(module_tree), |
| 348 | + ) |
| 349 | + return module_tree |
| 350 | + |
| 351 | + result: Dict[str, Any] = {} |
| 352 | + for subsystem_name, members in subsystems.items(): |
| 353 | + if len(members) == 1: |
| 354 | + result[members[0]] = module_tree[members[0]] |
| 355 | + continue |
| 356 | + components = [] |
| 357 | + seen = set() |
| 358 | + for member in members: |
| 359 | + for component in module_tree[member].get("components", []): |
| 360 | + if component not in seen: |
| 361 | + seen.add(component) |
| 362 | + components.append(component) |
| 363 | + result[subsystem_name] = { |
| 364 | + "path": _common_path_prefix( |
| 365 | + [module_tree[member].get("path", "") for member in members] |
| 366 | + ), |
| 367 | + "components": components, |
| 368 | + "children": {member: module_tree[member] for member in members}, |
| 369 | + } |
| 370 | + for name in unassigned: |
| 371 | + result[name] = module_tree[name] |
| 372 | + |
| 373 | + logger.info( |
| 374 | + "Super-grouping consolidated %d top-level modules into %d entries " |
| 375 | + "(%d subsystems).", |
| 376 | + len(module_tree), |
| 377 | + len(result), |
| 378 | + len(subsystems), |
| 379 | + ) |
| 380 | + return result |
0 commit comments