@@ -43,6 +43,16 @@ def format_potential_core_components(leaf_nodes: List[str], components: Dict[str
4343 return potential_core_components , potential_core_components_with_code
4444
4545
46+ def get_clustering_input_token_count (
47+ leaf_nodes : List [str ], components : Dict [str , Node ]
48+ ) -> int :
49+ """Count the tokens used to decide whether a module needs clustering."""
50+ _ , potential_core_components_with_code = format_potential_core_components (
51+ leaf_nodes , components
52+ )
53+ return count_tokens (potential_core_components_with_code )
54+
55+
4656def cluster_modules (
4757 leaf_nodes : List [str ],
4858 components : Dict [str , Node ],
@@ -62,13 +72,38 @@ def cluster_modules(
6272 subscription-mode (caw) routing. If ``None``, falls back to
6373 ``call_llm`` for backward compatibility with direct callers.
6474 """
65- potential_core_components , potential_core_components_with_code = format_potential_core_components (leaf_nodes , components )
66-
67- if count_tokens (potential_core_components_with_code ) <= config .max_token_per_module :
68- logger .debug (f"Skipping clustering for { current_module_name } because the potential core components are too few: { count_tokens (potential_core_components_with_code )} tokens" )
75+ potential_core_components , potential_core_components_with_code = (
76+ format_potential_core_components (leaf_nodes , components )
77+ )
78+ input_tokens = count_tokens (potential_core_components_with_code )
79+ threshold = config .max_token_per_module
80+ module_label = current_module_name or "repository"
81+
82+ logger .info (
83+ "Module clustering input for %s: %d leaf nodes, %d tokens, threshold %d" ,
84+ module_label ,
85+ len (leaf_nodes ),
86+ input_tokens ,
87+ threshold ,
88+ )
89+
90+ if input_tokens <= threshold :
91+ logger .info (
92+ "Skipping LLM module clustering for %s because %d tokens fit within the "
93+ "%d-token threshold; using whole-module documentation mode." ,
94+ module_label ,
95+ input_tokens ,
96+ threshold ,
97+ )
6998 return {}
7099
71100 prompt = format_cluster_prompt (potential_core_components , current_module_tree , current_module_name )
101+ logger .info (
102+ "Requesting LLM module clustering for %s because %d tokens exceed the %d-token threshold." ,
103+ module_label ,
104+ input_tokens ,
105+ threshold ,
106+ )
72107 if completer is not None :
73108 response = completer (prompt )
74109 else :
@@ -77,7 +112,12 @@ def cluster_modules(
77112 #parse the response
78113 try :
79114 if "<GROUPED_COMPONENTS>" not in response or "</GROUPED_COMPONENTS>" not in response :
80- logger .error (f"Invalid LLM response format - missing component tags: { response [:200 ]} ..." )
115+ logger .warning (
116+ "Invalid LLM clustering response for %s: missing <GROUPED_COMPONENTS> "
117+ "tags; falling back to whole-module documentation. Response preview: %s..." ,
118+ module_label ,
119+ response [:200 ],
120+ )
81121 return {}
82122
83123 response_content = response .split ("<GROUPED_COMPONENTS>" )[1 ].split ("</GROUPED_COMPONENTS>" )[0 ]
@@ -88,15 +128,32 @@ def cluster_modules(
88128 return {}
89129
90130 except Exception as e :
91- logger .error (f"Failed to parse LLM response: { e } . Response: { response [:200 ]} ..." )
131+ logger .warning (
132+ "Failed to parse LLM clustering response for %s; falling back to "
133+ "whole-module documentation. Error: %s. Response preview: %s..." ,
134+ module_label ,
135+ e ,
136+ response [:200 ],
137+ )
92138 logger .error (f"Traceback: { traceback .format_exc ()} " )
93139 return {}
94140
95141 # check if the module tree is valid
96142 if len (module_tree ) <= 1 :
97- logger .debug (f"Skipping clustering for { current_module_name } because the module tree is too small: { len (module_tree )} modules" )
143+ logger .info (
144+ "Skipping LLM clustering result for %s because it produced only "
145+ "%d module(s); using whole-module documentation mode." ,
146+ module_label ,
147+ len (module_tree ),
148+ )
98149 return {}
99150
151+ logger .info (
152+ "LLM module clustering for %s produced %d top-level modules." ,
153+ module_label ,
154+ len (module_tree ),
155+ )
156+
100157 if current_module_tree == {}:
101158 current_module_tree = module_tree
102159 else :
@@ -131,4 +188,4 @@ def cluster_modules(
131188 )
132189 current_module_path .pop ()
133190
134- return module_tree
191+ return module_tree
0 commit comments