Skip to content

Commit 06e6d01

Browse files
authored
Merge pull request #64 from FSoft-AI4Code/improv/c-cpp-java-parser
improve dependency analyzer symbol resolution
2 parents 972dba9 + ef59c23 commit 06e6d01

7 files changed

Lines changed: 1199 additions & 143 deletions

File tree

‎codewiki/src/be/dependency_analyzer/analysis/call_graph_analyzer.py‎

Lines changed: 204 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -6,16 +6,23 @@
66
across different programming languages in a repository.
77
"""
88

9-
from typing import Dict, List
9+
from typing import Dict, List, Optional
1010
import logging
1111
import traceback
1212
import time
1313
import signal
14+
import re
15+
from collections import defaultdict
1416
from pathlib import Path
1517
from contextlib import contextmanager
1618
from codewiki.src.be.dependency_analyzer.models.core import Node, CallRelationship
1719
from codewiki.src.be.dependency_analyzer.utils.patterns import CODE_EXTENSIONS
1820
from codewiki.src.be.dependency_analyzer.utils.security import safe_open_text
21+
from codewiki.src.be.dependency_analyzer.utils.external_symbols import (
22+
CPP_STANDARD_HEADERS,
23+
is_external_symbol,
24+
is_macro_name,
25+
)
1926

2027
logger = logging.getLogger(__name__)
2128

@@ -69,6 +76,7 @@ def analyze_code_files(self, code_files: List[Dict], base_dir: str) -> Dict:
6976

7077
self.functions = {}
7178
self.call_relationships = []
79+
code_files = self._route_contextual_headers(code_files, base_dir)
7280

7381
files_analyzed = 0
7482
files_failed = 0
@@ -149,6 +157,54 @@ def traverse(tree):
149157
traverse(file_tree)
150158
return code_files
151159

160+
def _route_contextual_headers(self, code_files: List[Dict], base_dir: str) -> List[Dict]:
161+
"""Route ambiguous .h headers per file.
162+
163+
A header is parsed as C++ when its own content shows C++ signals, or
164+
when the repository is C++-only (so even a signal-free header cannot be
165+
C). In a mixed C/C++ repository, a plain C header stays routed as C.
166+
"""
167+
cpp_extensions = {".cpp", ".cc", ".cxx", ".c++", ".hpp", ".hxx", ".h++"}
168+
has_cpp_files = any(
169+
file_info.get("extension", "").lower() in cpp_extensions
170+
or file_info.get("language") == "cpp"
171+
for file_info in code_files
172+
)
173+
has_c_files = any(
174+
file_info.get("extension", "").lower() == ".c" for file_info in code_files
175+
)
176+
177+
routed_files = []
178+
for file_info in code_files:
179+
routed = dict(file_info)
180+
if routed.get("extension", "").lower() == ".h":
181+
if self._header_has_cpp_signal(base_dir, routed["path"]):
182+
routed["language"] = "cpp"
183+
elif has_cpp_files and not has_c_files:
184+
routed["language"] = "cpp"
185+
routed_files.append(routed)
186+
return routed_files
187+
188+
def _header_has_cpp_signal(self, base_dir: str, relative_path: str) -> bool:
189+
base = Path(base_dir)
190+
try:
191+
content = safe_open_text(base, base / relative_path)
192+
except Exception:
193+
return False
194+
195+
if re.search(
196+
r"\b(?:namespace\s+[A-Za-z_{:]|class\s+[A-Za-z_]|template\s*<"
197+
r"|typename\b|(?:public|private|protected)\s*:)",
198+
content,
199+
):
200+
return True
201+
if "::" in content:
202+
return True
203+
for header in CPP_STANDARD_HEADERS:
204+
if f"#include <{header}>" in content:
205+
return True
206+
return False
207+
152208
def _analyze_code_file(self, repo_dir: str, file_info: Dict):
153209
"""
154210
Analyze a single code file based on its language.
@@ -406,42 +462,156 @@ def _resolve_call_relationships(self):
406462
Attempts to match function calls to actual function definitions,
407463
handling cross-language calls where possible.
408464
"""
409-
func_lookup = {}
465+
indexes = self._build_resolution_indexes()
410466
for func_id, func_info in self.functions.items():
411-
func_lookup[func_id] = func_id
412-
func_lookup[func_info.name] = func_id
413-
if func_info.component_id:
414-
func_lookup[func_info.component_id] = func_id
415-
# Extract short name: handle both new (path::Name) and legacy (path.Name) formats
416-
if "::" in func_info.component_id:
417-
method_name = func_info.component_id.split("::")[-1]
418-
else:
419-
method_name = func_info.component_id.split(".")[-1]
420-
if method_name not in func_lookup:
421-
func_lookup[method_name] = func_id
467+
if not func_info.language:
468+
file_ext = Path(func_info.file_path).suffix.lower()
469+
func_info.language = CODE_EXTENSIONS.get(file_ext)
422470

423471
resolved_count = 0
424472
for relationship in self.call_relationships:
425-
callee_name = relationship.callee
473+
if relationship.is_resolved and relationship.callee in self.functions:
474+
continue
426475

427-
if callee_name in func_lookup:
428-
relationship.callee = func_lookup[callee_name]
476+
resolved_id = self._resolve_callee(relationship, indexes)
477+
if resolved_id:
478+
relationship.callee = resolved_id
429479
relationship.is_resolved = True
430480
resolved_count += 1
431-
elif "::" in callee_name or "." in callee_name:
432-
if callee_name in func_lookup:
433-
relationship.callee = func_lookup[callee_name]
434-
relationship.is_resolved = True
435-
resolved_count += 1
436-
else:
437-
if "::" in callee_name:
438-
method_name = callee_name.split("::")[-1]
439-
else:
440-
method_name = callee_name.split(".")[-1]
441-
if method_name in func_lookup:
442-
relationship.callee = func_lookup[method_name]
443-
relationship.is_resolved = True
444-
resolved_count += 1
481+
482+
java_packages = self._java_project_packages()
483+
self.call_relationships = [
484+
relationship
485+
for relationship in self.call_relationships
486+
if relationship.is_resolved
487+
or not self._is_external_callee(
488+
self._caller_language(relationship.caller),
489+
relationship.callee,
490+
java_packages,
491+
)
492+
]
493+
494+
def _java_project_packages(self) -> set:
495+
packages = set()
496+
for func_info in self.functions.values():
497+
if func_info.language == "java":
498+
package = self._java_package_for_node(func_info)
499+
if package:
500+
packages.add(package)
501+
return packages
502+
503+
def _is_external_callee(self, language: Optional[str], callee: str, java_packages: set) -> bool:
504+
"""Classify a still-unresolved callee as external, after project
505+
resolution has had its chance.
506+
507+
Rules are generic, not name lists: prefix/standard-library knowledge in
508+
is_external_symbol, the C/C++ ALL_CAPS macro convention (macros are
509+
never components, so such calls can never resolve), and Java package
510+
origin — a dotted name qualified to a package with no prefix relation
511+
to any project package came from a third-party import.
512+
"""
513+
if is_external_symbol(language, callee):
514+
return True
515+
if language in ("c", "cpp") and is_macro_name(callee):
516+
return True
517+
if language == "java" and "." in callee and java_packages:
518+
package = callee.rsplit(".", 1)[0]
519+
if not any(
520+
package == project
521+
or package.startswith(project + ".")
522+
or project.startswith(package + ".")
523+
for project in java_packages
524+
):
525+
return True
526+
return False
527+
528+
def _build_resolution_indexes(self) -> Dict[str, Dict[str, List[str]]]:
529+
exact: Dict[str, List[str]] = defaultdict(list)
530+
simple: Dict[str, List[str]] = defaultdict(list)
531+
532+
def add(index: Dict[str, List[str]], key: Optional[str], func_id: str) -> None:
533+
if key and func_id not in index[key]:
534+
index[key].append(func_id)
535+
536+
for func_id, func_info in self.functions.items():
537+
add(exact, func_id, func_id)
538+
add(exact, func_info.component_id, func_id)
539+
add(exact, func_info.qualified_name, func_id)
540+
add(exact, func_info.name, func_id)
541+
542+
names = {func_info.name}
543+
if func_info.component_id:
544+
names.add(func_info.component_id.split("::")[-1])
545+
if func_info.qualified_name:
546+
names.add(func_info.qualified_name.split(".")[-1])
547+
parts = func_info.qualified_name.split(".")
548+
if len(parts) >= 2:
549+
names.add(".".join(parts[-2:]))
550+
551+
for name in names:
552+
add(simple, name, func_id)
553+
if name and "." in name:
554+
add(simple, name.split(".")[-1], func_id)
555+
556+
return {"exact": exact, "simple": simple}
557+
558+
def _resolve_callee(self, relationship: CallRelationship, indexes: Dict[str, Dict[str, List[str]]]) -> Optional[str]:
559+
callee_name = relationship.callee
560+
561+
exact_match = self._unique_match(indexes["exact"], callee_name)
562+
if exact_match:
563+
return exact_match
564+
565+
if "::" in callee_name:
566+
suffix = callee_name.split("::")[-1]
567+
exact_match = self._unique_match(indexes["exact"], suffix)
568+
if exact_match:
569+
return exact_match
570+
simple_match = self._unique_match(indexes["simple"], suffix)
571+
if simple_match:
572+
return simple_match
573+
574+
if "." in callee_name:
575+
exact_match = self._unique_match(indexes["exact"], callee_name)
576+
if exact_match:
577+
return exact_match
578+
simple_match = self._unique_match(indexes["simple"], callee_name)
579+
if simple_match:
580+
return simple_match
581+
tail_match = self._unique_match(indexes["simple"], callee_name.split(".")[-1])
582+
if tail_match:
583+
return tail_match
584+
585+
caller = self.functions.get(relationship.caller)
586+
if caller and caller.language == "java" and "." not in callee_name:
587+
package = self._java_package_for_node(caller)
588+
if package:
589+
same_package_match = self._unique_match(indexes["exact"], f"{package}.{callee_name}")
590+
if same_package_match:
591+
return same_package_match
592+
593+
return self._unique_match(indexes["simple"], callee_name)
594+
595+
def _unique_match(self, index: Dict[str, List[str]], key: str) -> Optional[str]:
596+
matches = index.get(key, [])
597+
return matches[0] if len(matches) == 1 else None
598+
599+
def _java_package_for_node(self, node: Node) -> str:
600+
qualified_name = node.qualified_name or ""
601+
parts = qualified_name.split(".")
602+
if len(parts) < 2:
603+
return ""
604+
if node.component_type == "method" and len(parts) >= 3:
605+
return ".".join(parts[:-2])
606+
return ".".join(parts[:-1])
607+
608+
def _caller_language(self, caller_id: str) -> Optional[str]:
609+
caller = self.functions.get(caller_id)
610+
if caller and caller.language:
611+
return caller.language
612+
if caller:
613+
return CODE_EXTENSIONS.get(Path(caller.file_path).suffix.lower())
614+
return None
445615

446616
def _deduplicate_relationships(self):
447617
"""
@@ -480,15 +650,16 @@ def _generate_visualization_data(self) -> Dict:
480650
node_classes.append("node-function")
481651

482652
file_ext = Path(func_info.file_path).suffix.lower()
653+
language = func_info.language or CODE_EXTENSIONS.get(file_ext, "unknown")
483654
if file_ext == ".py":
484655
node_classes.append("lang-python")
485656
elif file_ext == ".js":
486657
node_classes.append("lang-javascript")
487658
elif file_ext == ".ts":
488659
node_classes.append("lang-typescript")
489-
elif file_ext in [".c", ".h"]:
660+
elif language == "c":
490661
node_classes.append("lang-c")
491-
elif file_ext in [".cpp", ".cc", ".cxx", ".hpp", ".hxx"]:
662+
elif language == "cpp" or file_ext in [".cpp", ".cc", ".cxx", ".c++", ".hpp", ".hxx", ".h++"]:
492663
node_classes.append("lang-cpp")
493664
elif file_ext in [".kt", ".kts"]:
494665
node_classes.append("lang-kotlin")
@@ -502,7 +673,7 @@ def _generate_visualization_data(self) -> Dict:
502673
"label": func_info.name,
503674
"file": func_info.file_path,
504675
"type": func_info.node_type or "function",
505-
"language": CODE_EXTENSIONS.get(file_ext, "unknown"),
676+
"language": language,
506677
},
507678
"classes": " ".join(node_classes),
508679
}
@@ -616,4 +787,3 @@ def _select_most_connected_nodes(self, target_count: int):
616787
for rel in self.call_relationships
617788
if rel.caller in selected_func_ids and rel.callee in selected_func_ids
618789
]
619-

‎codewiki/src/be/dependency_analyzer/analyzers/c.py‎

Lines changed: 12 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -132,7 +132,9 @@ def _extract_nodes(self, node, top_level_nodes, lines):
132132
base_classes=None,
133133
class_name=None,
134134
display_name=f"{node_type} {node_name}",
135-
component_id=component_id
135+
component_id=component_id,
136+
language="c",
137+
qualified_name=node_name
136138
)
137139

138140
if node_type in ["function", "struct"]:
@@ -159,17 +161,18 @@ def _extract_relationships(self, node, top_level_nodes):
159161
if containing_function:
160162
containing_function_id = self._get_component_id(containing_function)
161163

162-
# Get called function name
164+
# Get called function name. External/libc filtering happens in
165+
# CallGraphAnalyzer after cross-file resolution, so a project
166+
# function that shadows a libc name still gets its edges.
163167
function_node = next((c for c in node.children if c.type == "identifier"), None)
164168
if function_node:
165169
called_function = function_node.text.decode()
166-
if not self._is_system_function(called_function):
167-
self.call_relationships.append(CallRelationship(
168-
caller=containing_function_id,
169-
callee=called_function, # Use simple name for cross-file resolution
170-
call_line=node.start_point[0]+1,
171-
is_resolved=False # Let CallGraphAnalyzer resolve
172-
))
170+
self.call_relationships.append(CallRelationship(
171+
caller=containing_function_id,
172+
callee=called_function, # Use simple name for cross-file resolution
173+
call_line=node.start_point[0]+1,
174+
is_resolved=False # Let CallGraphAnalyzer resolve
175+
))
173176

174177
# 2. function uses global variables
175178
if node.type == "identifier":
@@ -206,16 +209,6 @@ def _find_containing_function(self, node, top_level_nodes):
206209
return func_name
207210
current = current.parent
208211
return None
209-
210-
def _is_system_function(self, func_name: str) -> bool:
211-
"""Check if function is a system/library function."""
212-
# Common C library functions
213-
system_functions = {
214-
"printf", "scanf", "malloc", "free", "strlen", "strcpy", "strcmp",
215-
"memcpy", "memset", "exit", "abort", "fopen", "fclose", "fread", "fwrite",
216-
"SDL_Init", "SDL_CreateWindow", "SDL_Log", "SDL_GetError", "SDL_Quit"
217-
}
218-
return func_name in system_functions
219212

220213
def analyze_c_file(file_path: str, content: str, repo_path: str = None) -> Tuple[List[Node], List[CallRelationship]]:
221214
analyzer = TreeSitterCAnalyzer(file_path, content, repo_path)

0 commit comments

Comments
 (0)