Skip to content

Commit 75f5a76

Browse files
authored
Add stacktraces column to calculate_numeric_gap API
Differential Revision: D94713076 Pull Request resolved: pytorch#17764
1 parent 6412f55 commit 75f5a76

4 files changed

Lines changed: 406 additions & 69 deletions

File tree

‎devtools/inspector/_inspector.py‎

Lines changed: 143 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@
3333
import numpy as np
3434
import pandas as pd
3535

36+
import torch
37+
3638
from executorch.devtools.debug_format.et_schema import OperatorGraph, OperatorNode
3739
from executorch.devtools.etdump.schema_flatcc import (
3840
DebugEvent,
@@ -1164,33 +1166,35 @@ def _consume_etrecord(self) -> None:
11641166
index
11651167
]
11661168

1167-
def _get_aot_intermediate_outputs_and_op_names(
1169+
def _resolve_reference_graph(
11681170
self,
11691171
reference_graph: Optional[str] = None,
1170-
disable_debug_handle_valdiation: bool = False,
1171-
) -> Tuple[Dict[DebugHandle, Any], Dict[DebugHandle, List[str]]]:
1172+
disable_debug_handle_validation: bool = False,
1173+
) -> Tuple[torch.fx.GraphModule, str]:
11721174
"""
1173-
Capture intermediate outputs only if _representative_inputs are provided
1174-
when using bundled program to create the etrecord.
1175+
Resolve the reference graph module to use for AOT operations.
1176+
1177+
This method centralizes the logic for determining which graph module to use,
1178+
ensuring consistency across all methods that need the reference graph.
11751179
11761180
Args:
1177-
reference_graph_name: Name of the graph to use as the reference for intermediate
1178-
output capture. Must be one of:
1179-
- "exported_program": Uses the ATen dialect exported program. Requires
1180-
successful debug handle backpropagation, otherwise raises an error.
1181-
- "edge_dialect_exported_program": Uses the Edge dialect program directly.
1182-
- Any other string: Fetches from graph_map (e.g., "edge_after_transform/forward"
1183-
for post-custom-transform graph when transform_passes are applied in
1184-
to_edge_transform_and_lower with generate_etrecord=True).
1185-
disable_debug_handle_valdiation: If True, skip debug handle validation.
1181+
reference_graph: The name of the reference graph. Options:
1182+
- None: Auto-select (try exported_program first, fall back to edge_dialect)
1183+
- "exported_program": Use ATen dialect with debug handle backpropagation
1184+
- "edge_dialect_exported_program": Use Edge dialect directly
1185+
- Any other string: Look up in graph_map
1186+
disable_debug_handle_validation: If True, skip debug handle validation for
1187+
exported_program.
11861188
11871189
Returns:
1188-
Tuple of (intermediate_outputs, debug_handle_to_op_names) dictionaries.
1190+
Tuple of (graph_module, resolved_graph_name) where resolved_graph_name is
1191+
the actual graph used.
11891192
11901193
Raises:
1191-
ValueError: If the specified reference_graph_name is not available or if
1194+
ValueError: If the specified reference_graph is not available or if
11921195
debug handle backpropagation fails for "exported_program".
11931196
"""
1197+
resolved_graph_name = reference_graph
11941198

11951199
# Determine the reference graph to use
11961200
if reference_graph is None or reference_graph == "exported_program":
@@ -1199,39 +1203,36 @@ def _get_aot_intermediate_outputs_and_op_names(
11991203
self._etrecord.exported_program,
12001204
self._etrecord.export_graph_id,
12011205
self._etrecord.edge_dialect_program,
1202-
disable_debug_handle_valdiation,
1206+
disable_debug_handle_validation,
12031207
):
1204-
reference_graph = "exported_program"
1208+
resolved_graph_name = "exported_program"
12051209
elif reference_graph is None:
12061210
log.warning(
12071211
"Either ATen dialect exported program is not in ETRecord, or debug handle "
12081212
"backpropagation failed. Falling back to 'edge_dialect_exported_program'."
12091213
)
1210-
reference_graph = "edge_dialect_exported_program"
1214+
resolved_graph_name = "edge_dialect_exported_program"
12111215
else:
12121216
raise ValueError(
12131217
"Cannot use 'exported_program': Debug handle backpropagation failed or exported program is unavailable. "
12141218
"Please check if the exported program is available in ETRecord, or try to disable debug handle validation."
12151219
)
1216-
if reference_graph == "edge_dialect_exported_program":
1217-
# Explicitly requested edge_dialect_exported_program
1220+
1221+
if resolved_graph_name == "edge_dialect_exported_program":
12181222
export_program = self._etrecord.edge_dialect_program
12191223
log.info(
1220-
"Using 'edge_dialect_exported_program' (Edge dialect) as reference graph for intermediate output capture"
1224+
"Using 'edge_dialect_exported_program' (Edge dialect) as reference graph"
12211225
)
1222-
elif reference_graph == "exported_program":
1226+
elif resolved_graph_name == "exported_program":
12231227
export_program = self._etrecord.exported_program
1224-
log.info(
1225-
"Using 'exported_program' (ATen dialect) as reference graph for intermediate output capture"
1226-
)
1228+
log.info("Using 'exported_program' (ATen dialect) as reference graph")
12271229
else:
12281230
# Try to fetch from graph_map
1229-
# If no method name is provided (no "/" in the name), try adding "/forward" as default
1230-
lookup_name = reference_graph
1231-
if "/" not in reference_graph:
1232-
lookup_name = f"{reference_graph}/forward"
1231+
lookup_name = resolved_graph_name
1232+
if "/" not in resolved_graph_name:
1233+
lookup_name = f"{resolved_graph_name}/forward"
12331234
log.info(
1234-
f"No method name specified in '{reference_graph}', "
1235+
f"No method name specified in '{resolved_graph_name}', "
12351236
f"using '{lookup_name}' as default"
12361237
)
12371238

@@ -1240,9 +1241,7 @@ def _get_aot_intermediate_outputs_and_op_names(
12401241
and lookup_name in self._etrecord.graph_map
12411242
):
12421243
export_program = self._etrecord.graph_map[lookup_name]
1243-
log.info(
1244-
f"Using '{lookup_name}' from graph_map as reference graph for intermediate output capture"
1245-
)
1244+
log.info(f"Using '{lookup_name}' from graph_map as reference graph")
12461245
else:
12471246
available_graphs = (
12481247
list(self._etrecord.graph_map.keys())
@@ -1254,16 +1253,82 @@ def _get_aot_intermediate_outputs_and_op_names(
12541253
f"Available options: 'exported_program', 'edge_dialect_exported_program', "
12551254
f"or one of the graphs in graph_map: {available_graphs}"
12561255
)
1257-
graph_module = export_program.module()
1256+
1257+
return export_program.module(), resolved_graph_name
1258+
1259+
def _get_aot_intermediate_outputs_and_op_names(
1260+
self,
1261+
reference_graph_module: torch.fx.GraphModule,
1262+
) -> Tuple[Dict[DebugHandle, Any], Dict[DebugHandle, List[str]]]:
1263+
"""
1264+
Capture intermediate outputs and operator name mappings from the given graph module.
1265+
1266+
Args:
1267+
reference_graph_module: The resolved reference graph module to use.
1268+
1269+
Returns:
1270+
Tuple of (intermediate_outputs, debug_handle_to_op_names) dictionaries.
1271+
"""
12581272
aot_debug_handle_to_op_name = get_aot_debug_handle_to_op_name_mapping(
1259-
graph_module
1273+
reference_graph_module
12601274
)
1261-
capturer = IntermediateOutputCapturer(graph_module)
1275+
capturer = IntermediateOutputCapturer(reference_graph_module)
12621276
aot_intermediate_outputs = capturer.run_and_capture(
12631277
self._etrecord._representative_inputs
12641278
)
12651279
return aot_intermediate_outputs, aot_debug_handle_to_op_name
12661280

1281+
def _get_aot_debug_handle_to_stack_traces(
1282+
self,
1283+
reference_graph_module: torch.fx.GraphModule,
1284+
resolved_graph_name: str,
1285+
) -> Dict[DebugHandle, Dict[str, Optional[str]]]:
1286+
"""
1287+
Get a mapping from debug handle to stack traces from the given graph module.
1288+
1289+
Args:
1290+
reference_graph_module: The resolved reference graph module to use.
1291+
resolved_graph_name: The name of the graph (for warning messages).
1292+
1293+
Returns:
1294+
Dict[DebugHandle, Dict[str, Optional[str]]]: A dictionary mapping debug handles
1295+
to dictionaries of {op_name: stack_trace}.
1296+
"""
1297+
from executorch.devtools.inspector._inspector_utils import NodeFilter
1298+
1299+
node_filters = [
1300+
NodeFilter("debug_handle", "call_function", exclude_ops=["getitem"])
1301+
]
1302+
1303+
result: Dict[DebugHandle, Dict[str, Optional[str]]] = {}
1304+
has_any_stack_trace = False
1305+
1306+
for node in reference_graph_module.graph.nodes:
1307+
if all(filter.matches(node) for filter in node_filters):
1308+
debug_handle = node.meta["debug_handle"]
1309+
key = (
1310+
(debug_handle,)
1311+
if isinstance(debug_handle, int)
1312+
else tuple(debug_handle)
1313+
)
1314+
stack_trace = node.meta.get("stack_trace")
1315+
if stack_trace is not None:
1316+
has_any_stack_trace = True
1317+
1318+
if key in result:
1319+
result[key][node.name] = stack_trace
1320+
else:
1321+
result[key] = {node.name: stack_trace}
1322+
1323+
if not has_any_stack_trace and result:
1324+
log.warning(
1325+
f"No stack traces found in reference_graph '{resolved_graph_name}'. "
1326+
"The 'stacktraces' column will contain None values. "
1327+
"Ensure the model was exported with stack trace information preserved."
1328+
)
1329+
1330+
return result
1331+
12671332
# TODO: Make it more extensible to further merge overlapping debug handles
12681333
def _get_runtime_intermediate_outputs_and_op_names(
12691334
self,
@@ -1506,17 +1571,29 @@ def calculate_numeric_gap(
15061571
15071572
Returns:
15081573
pd.DataFrame: A DataFrame listing corresponding operator intermediate outputs from both stages and their computed numerical gaps.
1574+
The DataFrame includes a "stacktraces" column where each entry is a dict mapping operator names to their stack traces.
15091575
"""
1576+
# First, resolve the reference graph to use
1577+
reference_graph_module, resolved_graph_name = self._resolve_reference_graph(
1578+
reference_graph,
1579+
disable_debug_handle_valdiation,
1580+
)
1581+
1582+
# Get intermediate outputs and op names from the resolved graph
15101583
aot_intermediate_outputs, aot_debug_handle_to_op_names = (
1511-
self._get_aot_intermediate_outputs_and_op_names(
1512-
reference_graph,
1513-
disable_debug_handle_valdiation,
1514-
)
1584+
self._get_aot_intermediate_outputs_and_op_names(reference_graph_module)
15151585
)
15161586
if len(aot_intermediate_outputs) == 0 or len(aot_debug_handle_to_op_names) == 0:
15171587
raise ValueError(
15181588
"Missing etrecord or missing representative inputs within etrecord, both of which are required for calculating numerical gap"
15191589
)
1590+
1591+
# Get the stack trace mapping from the resolved graph
1592+
aot_debug_handle_to_stack_traces = self._get_aot_debug_handle_to_stack_traces(
1593+
reference_graph_module,
1594+
resolved_graph_name,
1595+
)
1596+
15201597
# The runtime_op_names will be used later to map runtime debug_handle to op_name
15211598
runtime_intermediate_outputs, runtime_debug_handle_to_op_names = (
15221599
self._get_runtime_intermediate_outputs_and_op_names()
@@ -1543,8 +1620,32 @@ def calculate_numeric_gap(
15431620
raise ValueError(f"Unsupported distance metric {distance!r}")
15441621

15451622
# Delegate to comparator's compare method (includes preprocessing)
1546-
return comparator.compare(
1623+
df = comparator.compare(
15471624
mapping,
15481625
aot_debug_handle_to_op_names,
15491626
runtime_debug_handle_to_op_names,
15501627
)
1628+
1629+
# Add stacktraces column by looking up each row's debug handle
1630+
# We need to map from aot_ops back to debug handles to get stack traces
1631+
def get_stacktraces_for_row(aot_ops: List[str]) -> Dict[str, Optional[str]]:
1632+
"""Find stack traces for the given aot_ops by looking up in all debug handles."""
1633+
result: Dict[str, Optional[str]] = {}
1634+
for op_name in aot_ops:
1635+
# Search through debug handle mappings to find the stack trace for this op
1636+
for (
1637+
_,
1638+
stack_traces_dict,
1639+
) in aot_debug_handle_to_stack_traces.items():
1640+
if op_name in stack_traces_dict:
1641+
result[op_name] = stack_traces_dict[op_name]
1642+
break
1643+
else:
1644+
# Op not found in any debug handle's stack traces
1645+
result[op_name] = None
1646+
return result
1647+
1648+
if len(df) > 0:
1649+
df["stacktraces"] = df["aot_ops"].apply(get_stacktraces_for_row)
1650+
1651+
return df

0 commit comments

Comments
 (0)