Skip to content

Commit ef59c23

Browse files
committed
Improve dependency analyzer symbol resolution
1 parent 9284774 commit ef59c23

5 files changed

Lines changed: 477 additions & 97 deletions

File tree

‎codewiki/src/be/dependency_analyzer/analysis/call_graph_analyzer.py‎

Lines changed: 56 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
from codewiki.src.be.dependency_analyzer.utils.external_symbols import (
2222
CPP_STANDARD_HEADERS,
2323
is_external_symbol,
24+
is_macro_name,
2425
)
2526

2627
logger = logging.getLogger(__name__)
@@ -157,19 +158,29 @@ def traverse(tree):
157158
return code_files
158159

159160
def _route_contextual_headers(self, code_files: List[Dict], base_dir: str) -> List[Dict]:
160-
"""Route ambiguous .h headers as C++ when repo or file content indicates C++."""
161+
"""Route ambiguous .h headers per file.
162+
163+
A header is parsed as C++ when its own content shows C++ signals, or
164+
when the repository is C++-only (so even a signal-free header cannot be
165+
C). In a mixed C/C++ repository, a plain C header stays routed as C.
166+
"""
161167
cpp_extensions = {".cpp", ".cc", ".cxx", ".c++", ".hpp", ".hxx", ".h++"}
162168
has_cpp_files = any(
163169
file_info.get("extension", "").lower() in cpp_extensions
164170
or file_info.get("language") == "cpp"
165171
for file_info in code_files
166172
)
173+
has_c_files = any(
174+
file_info.get("extension", "").lower() == ".c" for file_info in code_files
175+
)
167176

168177
routed_files = []
169178
for file_info in code_files:
170179
routed = dict(file_info)
171180
if routed.get("extension", "").lower() == ".h":
172-
if has_cpp_files or self._header_has_cpp_signal(base_dir, routed["path"]):
181+
if self._header_has_cpp_signal(base_dir, routed["path"]):
182+
routed["language"] = "cpp"
183+
elif has_cpp_files and not has_c_files:
173184
routed["language"] = "cpp"
174185
routed_files.append(routed)
175186
return routed_files
@@ -181,9 +192,13 @@ def _header_has_cpp_signal(self, base_dir: str, relative_path: str) -> bool:
181192
except Exception:
182193
return False
183194

184-
if re.search(r"\b(namespace|class|template|typename|public|private|protected)\b", content):
195+
if re.search(
196+
r"\b(?:namespace\s+[A-Za-z_{:]|class\s+[A-Za-z_]|template\s*<"
197+
r"|typename\b|(?:public|private|protected)\s*:)",
198+
content,
199+
):
185200
return True
186-
if "::" in content or "std::" in content:
201+
if "::" in content:
187202
return True
188203
for header in CPP_STANDARD_HEADERS:
189204
if f"#include <{header}>" in content:
@@ -464,16 +479,52 @@ def _resolve_call_relationships(self):
464479
relationship.is_resolved = True
465480
resolved_count += 1
466481

482+
java_packages = self._java_project_packages()
467483
self.call_relationships = [
468484
relationship
469485
for relationship in self.call_relationships
470486
if relationship.is_resolved
471-
or not is_external_symbol(
487+
or not self._is_external_callee(
472488
self._caller_language(relationship.caller),
473489
relationship.callee,
490+
java_packages,
474491
)
475492
]
476493

494+
def _java_project_packages(self) -> set:
495+
packages = set()
496+
for func_info in self.functions.values():
497+
if func_info.language == "java":
498+
package = self._java_package_for_node(func_info)
499+
if package:
500+
packages.add(package)
501+
return packages
502+
503+
def _is_external_callee(self, language: Optional[str], callee: str, java_packages: set) -> bool:
504+
"""Classify a still-unresolved callee as external, after project
505+
resolution has had its chance.
506+
507+
Rules are generic, not name lists: prefix/standard-library knowledge in
508+
is_external_symbol, the C/C++ ALL_CAPS macro convention (macros are
509+
never components, so such calls can never resolve), and Java package
510+
origin — a dotted name qualified to a package with no prefix relation
511+
to any project package came from a third-party import.
512+
"""
513+
if is_external_symbol(language, callee):
514+
return True
515+
if language in ("c", "cpp") and is_macro_name(callee):
516+
return True
517+
if language == "java" and "." in callee and java_packages:
518+
package = callee.rsplit(".", 1)[0]
519+
if not any(
520+
package == project
521+
or package.startswith(project + ".")
522+
or project.startswith(package + ".")
523+
for project in java_packages
524+
):
525+
return True
526+
return False
527+
477528
def _build_resolution_indexes(self) -> Dict[str, Dict[str, List[str]]]:
478529
exact: Dict[str, List[str]] = defaultdict(list)
479530
simple: Dict[str, List[str]] = defaultdict(list)

‎codewiki/src/be/dependency_analyzer/analyzers/c.py‎

Lines changed: 9 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77
from tree_sitter import Parser, Language
88
import tree_sitter_c
99
from codewiki.src.be.dependency_analyzer.models.core import Node, CallRelationship
10-
from codewiki.src.be.dependency_analyzer.utils.external_symbols import is_external_symbol
1110

1211
logger = logging.getLogger(__name__)
1312

@@ -162,17 +161,18 @@ def _extract_relationships(self, node, top_level_nodes):
162161
if containing_function:
163162
containing_function_id = self._get_component_id(containing_function)
164163

165-
# Get called function name
164+
# Get called function name. External/libc filtering happens in
165+
# CallGraphAnalyzer after cross-file resolution, so a project
166+
# function that shadows a libc name still gets its edges.
166167
function_node = next((c for c in node.children if c.type == "identifier"), None)
167168
if function_node:
168169
called_function = function_node.text.decode()
169-
if not self._is_system_function(called_function):
170-
self.call_relationships.append(CallRelationship(
171-
caller=containing_function_id,
172-
callee=called_function, # Use simple name for cross-file resolution
173-
call_line=node.start_point[0]+1,
174-
is_resolved=False # Let CallGraphAnalyzer resolve
175-
))
170+
self.call_relationships.append(CallRelationship(
171+
caller=containing_function_id,
172+
callee=called_function, # Use simple name for cross-file resolution
173+
call_line=node.start_point[0]+1,
174+
is_resolved=False # Let CallGraphAnalyzer resolve
175+
))
176176

177177
# 2. function uses global variables
178178
if node.type == "identifier":
@@ -209,10 +209,6 @@ def _find_containing_function(self, node, top_level_nodes):
209209
return func_name
210210
current = current.parent
211211
return None
212-
213-
def _is_system_function(self, func_name: str) -> bool:
214-
"""Check if function is a system/library function."""
215-
return is_external_symbol("c", func_name)
216212

217213
def analyze_c_file(file_path: str, content: str, repo_path: str = None) -> Tuple[List[Node], List[CallRelationship]]:
218214
analyzer = TreeSitterCAnalyzer(file_path, content, repo_path)

‎codewiki/src/be/dependency_analyzer/analyzers/cpp.py‎

Lines changed: 105 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -8,21 +8,13 @@
88
from tree_sitter import Parser, Language
99
import tree_sitter_cpp
1010
from codewiki.src.be.dependency_analyzer.models.core import Node, CallRelationship
11-
from codewiki.src.be.dependency_analyzer.utils.external_symbols import is_external_symbol
11+
from codewiki.src.be.dependency_analyzer.utils.external_symbols import (
12+
is_external_symbol,
13+
is_macro_name,
14+
)
1215

1316
logger = logging.getLogger(__name__)
1417

15-
# ALL_CAPS tokens that are common standard constants/keywords, not declaration
16-
# macros, so they must never be stripped from specifier position.
17-
_NON_MACRO_UPPER = {
18-
"FALSE",
19-
"TRUE",
20-
"NULL",
21-
"EOF",
22-
"EXIT_SUCCESS",
23-
"EXIT_FAILURE",
24-
}
25-
2618
# An ALL_CAPS attribute/specifier macro sitting in front of a declaration, e.g.
2719
# `EXPORT_API void foo()` or `CONSTEXPR auto bar()`. Matched at line start or
2820
# after a structural delimiter so identifiers in expression position are left
@@ -31,23 +23,16 @@
3123
_SPECIFIER_MACRO_RE = re.compile(r"(^\s*|[{};>,]\s*)([A-Z][A-Z0-9_]*[A-Z0-9])(\s+)(?=[A-Za-z_~])")
3224
# Same, but for function-like specifier macros such as `VISIBILITY("default") void f()`.
3325
_SPECIFIER_MACRO_CALL_RE = re.compile(r"(^\s*|[{};>,]\s*)([A-Z][A-Z0-9_]*[A-Z0-9])\s*\([^()]*\)(\s+)(?=[A-Za-z_~])")
26+
# An export/visibility macro between a class-like keyword and the type name,
27+
# e.g. `class LIB_API logger {`. Without this the macro is taken as the type
28+
# name and the real name is lost.
29+
_KEYWORD_MACRO_RE = re.compile(r"\b(class|struct|union|enum)(\s+)([A-Z][A-Z0-9_]*[A-Z0-9])\s+(?=[A-Za-z_~])")
3430
# A line that is nothing but a bare ALL_CAPS macro (optionally a macro call),
3531
# e.g. namespace-bracket macros like `LIB_BEGIN_NAMESPACE`. Left in place these
3632
# break parsing of the declaration that follows, so the line is blanked. Begin/
3733
# end pairs are both removed, keeping any braces they expand to balanced.
3834
_STANDALONE_MACRO_RE = re.compile(r"^\s*([A-Z][A-Z0-9_]*[A-Z0-9])(\s*\([^()]*\))?\s*$")
3935

40-
41-
_ALL_CAPS_RE = re.compile(r"^[A-Z][A-Z0-9_]*$")
42-
43-
44-
def _is_macro_name(token: str) -> bool:
45-
"""Heuristic: an ALL_CAPS identifier (with an underscore or 4+ chars) reads
46-
as a macro by C/C++ naming convention, not a function or type."""
47-
if not token or not _ALL_CAPS_RE.match(token):
48-
return False
49-
return (len(token) >= 4 or "_" in token) and token not in _NON_MACRO_UPPER
50-
5136
class TreeSitterCppAnalyzer:
5237
def __init__(self, file_path: str, content: str, repo_path: str = None):
5338
self.file_path = Path(file_path)
@@ -91,9 +76,7 @@ def _analyze(self):
9176
language_capsule = tree_sitter_cpp.language()
9277
cpp_language = Language(language_capsule)
9378
parser = Parser(cpp_language)
94-
parse_content = self._normalize_for_parser(self.content)
95-
tree = parser.parse(bytes(parse_content, "utf8"))
96-
root = tree.root_node
79+
root = self._parse_with_macro_recovery(parser)
9780
lines = self.content.splitlines()
9881

9982
top_level_nodes = {}
@@ -104,6 +87,39 @@ def _analyze(self):
10487
# extract relationships between top-level nodes
10588
self._extract_relationships(root, top_level_nodes)
10689

90+
def _parse_with_macro_recovery(self, parser):
91+
"""Parse the original source; if it has syntax errors, retry with macro
92+
normalization and keep whichever parse has fewer errors.
93+
94+
Normalization strips ALL_CAPS tokens by naming convention, which is
95+
wrong for code whose *types* are ALL_CAPS (e.g. Win32 `HANDLE`/`DWORD`).
96+
Comparing error counts makes the heuristic self-correcting: clean files
97+
are never touched, and normalization is only kept when it demonstrably
98+
recovers structure.
99+
"""
100+
tree = parser.parse(bytes(self.content, "utf8"))
101+
if not tree.root_node.has_error:
102+
return tree.root_node
103+
104+
normalized = self._normalize_for_parser(self.content)
105+
if normalized == self.content:
106+
return tree.root_node
107+
108+
normalized_tree = parser.parse(bytes(normalized, "utf8"))
109+
if self._count_parse_errors(normalized_tree.root_node) < self._count_parse_errors(tree.root_node):
110+
return normalized_tree.root_node
111+
return tree.root_node
112+
113+
def _count_parse_errors(self, root) -> int:
114+
errors = 0
115+
stack = [root]
116+
while stack:
117+
node = stack.pop()
118+
if node.is_error or node.is_missing:
119+
errors += 1
120+
stack.extend(node.children)
121+
return errors
122+
107123
def _normalize_for_parser(self, content: str) -> str:
108124
"""Strip ALL_CAPS attribute/specifier macros that sit in front of a
109125
declaration so tree-sitter can recover the underlying signature. This is
@@ -116,17 +132,21 @@ def _normalize_for_parser(self, content: str) -> str:
116132
for line in content.splitlines():
117133
updated = line
118134
standalone = _STANDALONE_MACRO_RE.match(updated)
119-
if standalone and _is_macro_name(standalone.group(1)):
135+
if standalone and is_macro_name(standalone.group(1)):
120136
normalized_lines.append("")
121137
continue
122138
for pattern in (_SPECIFIER_MACRO_CALL_RE, _SPECIFIER_MACRO_RE):
123139
previous = None
124140
while previous != updated:
125141
previous = updated
126142
updated = pattern.sub(
127-
lambda m: (m.group(1) + m.group(3)) if _is_macro_name(m.group(2)) else m.group(0),
143+
lambda m: (m.group(1) + m.group(3)) if is_macro_name(m.group(2)) else m.group(0),
128144
updated,
129145
)
146+
updated = _KEYWORD_MACRO_RE.sub(
147+
lambda m: (m.group(1) + m.group(2)) if is_macro_name(m.group(3)) else m.group(0),
148+
updated,
149+
)
130150
normalized_lines.append(updated)
131151
return "\n".join(normalized_lines)
132152

@@ -194,6 +214,21 @@ def _extract_nodes(self, node, top_level_nodes, lines):
194214
elif child.type == "identifier":
195215
node_name = child.text.decode()
196216
break
217+
elif node.type == "alias_declaration":
218+
# using name = type; — aliases are real API surface (e.g. a
219+
# library's public alias for an internal template), so they are
220+
# extracted as components and can resolve call/type references.
221+
node_type = "type_alias"
222+
for child in node.children:
223+
if child.type == "type_identifier":
224+
node_name = child.text.decode()
225+
break
226+
elif node.type == "type_definition":
227+
# typedef ... name; — the alias name is the trailing type_identifier
228+
node_type = "type_alias"
229+
identifiers = [c for c in node.children if c.type == "type_identifier"]
230+
if identifiers:
231+
node_name = identifiers[-1].text.decode()
197232
elif node.type == "namespace_definition":
198233
node_type = "namespace"
199234
found_namespace_keyword = False
@@ -241,7 +276,7 @@ def _extract_nodes(self, node, top_level_nodes, lines):
241276
top_level_nodes[f"{containing_class}.{node_name}"] = node_obj
242277
top_level_nodes.setdefault(node_name, node_obj)
243278

244-
if node_type in ["class", "struct", "function", "method"]:
279+
if node_type in ["class", "struct", "function", "method", "type_alias"]:
245280
self.nodes.append(node_obj)
246281

247282
# Recursively process children
@@ -356,7 +391,24 @@ def _extract_relationships(self, node, top_level_nodes):
356391
call_line=node.start_point[0]+1,
357392
is_resolved=True
358393
))
359-
elif not self._is_system_function(called_function):
394+
elif receiver_name is not None:
395+
# A member call whose receiver type could not be
396+
# resolved: a name matching an STL member here is
397+
# overwhelmingly likely external, so suppress it.
398+
if not self._is_system_function(called_function):
399+
self.call_relationships.append(CallRelationship(
400+
caller=containing_function_id,
401+
callee=called_function,
402+
call_line=node.start_point[0]+1,
403+
is_resolved=False
404+
))
405+
elif (
406+
not is_macro_name(called_function)
407+
and called_function not in self._find_template_parameters(node)
408+
):
409+
# Plain calls are emitted for cross-file resolution;
410+
# external filtering happens centrally after the
411+
# project resolver has had its chance.
360412
self.call_relationships.append(CallRelationship(
361413
caller=containing_function_id,
362414
callee=called_function,
@@ -368,10 +420,13 @@ def _extract_relationships(self, node, top_level_nodes):
368420
# Find the containing class
369421
containing_class = self._find_containing_class(node)
370422
if containing_class:
423+
template_params = self._find_template_parameters(node)
371424
# Extract base class names
372425
for child in node.children:
373426
if child.type == "type_identifier":
374427
base_class = child.text.decode()
428+
if base_class in template_params or is_macro_name(base_class):
429+
continue
375430
containing_class_id = self._get_component_id(containing_class)
376431
self.call_relationships.append(CallRelationship(
377432
caller=containing_class_id,
@@ -559,6 +614,25 @@ def _find_containing_class(self, node):
559614
current = current.parent
560615
return None
561616

617+
def _find_template_parameters(self, node) -> set:
618+
"""Collect template type-parameter names in scope at this node, so a
619+
reference to `T`/`Char`/... is not reported as an unresolved project
620+
symbol."""
621+
params = set()
622+
current = node.parent
623+
while current:
624+
if current.type == "template_declaration":
625+
param_list = next(
626+
(c for c in current.children if c.type == "template_parameter_list"), None
627+
)
628+
if param_list:
629+
for param in param_list.children:
630+
for child in getattr(param, "children", []):
631+
if child.type == "type_identifier":
632+
params.add(child.text.decode())
633+
current = current.parent
634+
return params
635+
562636
def _is_system_function(self, func_name: str) -> bool:
563637
"""Check if a call target is external rather than a project function.
564638
@@ -571,7 +645,7 @@ def _is_system_function(self, func_name: str) -> bool:
571645
"""
572646
if is_external_symbol("cpp", func_name):
573647
return True
574-
return _is_macro_name(func_name)
648+
return is_macro_name(func_name)
575649

576650
def _find_method_component(self, method_name, top_level_nodes, class_name: str = None):
577651
if class_name:

0 commit comments

Comments
 (0)