3333import numpy as np
3434import pandas as pd
3535
36+ import torch
37+
3638from executorch .devtools .debug_format .et_schema import OperatorGraph , OperatorNode
3739from executorch .devtools .etdump .schema_flatcc import (
3840 DebugEvent ,
@@ -1164,33 +1166,35 @@ def _consume_etrecord(self) -> None:
11641166 index
11651167 ]
11661168
1167- def _get_aot_intermediate_outputs_and_op_names (
1169+ def _resolve_reference_graph (
11681170 self ,
11691171 reference_graph : Optional [str ] = None ,
1170- disable_debug_handle_valdiation : bool = False ,
1171- ) -> Tuple [Dict [ DebugHandle , Any ], Dict [ DebugHandle , List [ str ]] ]:
1172+ disable_debug_handle_validation : bool = False ,
1173+ ) -> Tuple [torch . fx . GraphModule , str ]:
11721174 """
1173- Capture intermediate outputs only if _representative_inputs are provided
1174- when using bundled program to create the etrecord.
1175+ Resolve the reference graph module to use for AOT operations.
1176+
1177+ This method centralizes the logic for determining which graph module to use,
1178+ ensuring consistency across all methods that need the reference graph.
11751179
11761180 Args:
1177- reference_graph_name: Name of the graph to use as the reference for intermediate
1178- output capture. Must be one of:
1179- - "exported_program": Uses the ATen dialect exported program. Requires
1180- successful debug handle backpropagation, otherwise raises an error.
1181- - "edge_dialect_exported_program": Uses the Edge dialect program directly.
1182- - Any other string: Fetches from graph_map (e.g., "edge_after_transform/forward"
1183- for post-custom-transform graph when transform_passes are applied in
1184- to_edge_transform_and_lower with generate_etrecord=True).
1185- disable_debug_handle_valdiation: If True, skip debug handle validation.
1181+ reference_graph: The name of the reference graph. Options:
1182+ - None: Auto-select (try exported_program first, fall back to edge_dialect)
1183+ - "exported_program": Use ATen dialect with debug handle backpropagation
1184+ - "edge_dialect_exported_program": Use Edge dialect directly
1185+ - Any other string: Look up in graph_map
1186+ disable_debug_handle_validation: If True, skip debug handle validation for
1187+ exported_program.
11861188
11871189 Returns:
1188- Tuple of (intermediate_outputs, debug_handle_to_op_names) dictionaries.
1190+ Tuple of (graph_module, resolved_graph_name) where resolved_graph_name is
1191+ the actual graph used.
11891192
11901193 Raises:
1191- ValueError: If the specified reference_graph_name is not available or if
1194+ ValueError: If the specified reference_graph is not available or if
11921195 debug handle backpropagation fails for "exported_program".
11931196 """
1197+ resolved_graph_name = reference_graph
11941198
11951199 # Determine the reference graph to use
11961200 if reference_graph is None or reference_graph == "exported_program" :
@@ -1199,39 +1203,36 @@ def _get_aot_intermediate_outputs_and_op_names(
11991203 self ._etrecord .exported_program ,
12001204 self ._etrecord .export_graph_id ,
12011205 self ._etrecord .edge_dialect_program ,
1202- disable_debug_handle_valdiation ,
1206+ disable_debug_handle_validation ,
12031207 ):
1204- reference_graph = "exported_program"
1208+ resolved_graph_name = "exported_program"
12051209 elif reference_graph is None :
12061210 log .warning (
12071211 "Either ATen dialect exported program is not in ETRecord, or debug handle "
12081212 "backpropagation failed. Falling back to 'edge_dialect_exported_program'."
12091213 )
1210- reference_graph = "edge_dialect_exported_program"
1214+ resolved_graph_name = "edge_dialect_exported_program"
12111215 else :
12121216 raise ValueError (
12131217 "Cannot use 'exported_program': Debug handle backpropagation failed or exported program is unavailable. "
12141218 "Please check if the exported program is available in ETRecord, or try to disable debug handle validation."
12151219 )
1216- if reference_graph == "edge_dialect_exported_program" :
1217- # Explicitly requested edge_dialect_exported_program
1220+
1221+ if resolved_graph_name == " edge_dialect_exported_program" :
12181222 export_program = self ._etrecord .edge_dialect_program
12191223 log .info (
1220- "Using 'edge_dialect_exported_program' (Edge dialect) as reference graph for intermediate output capture "
1224+ "Using 'edge_dialect_exported_program' (Edge dialect) as reference graph"
12211225 )
1222- elif reference_graph == "exported_program" :
1226+ elif resolved_graph_name == "exported_program" :
12231227 export_program = self ._etrecord .exported_program
1224- log .info (
1225- "Using 'exported_program' (ATen dialect) as reference graph for intermediate output capture"
1226- )
1228+ log .info ("Using 'exported_program' (ATen dialect) as reference graph" )
12271229 else :
12281230 # Try to fetch from graph_map
1229- # If no method name is provided (no "/" in the name), try adding "/forward" as default
1230- lookup_name = reference_graph
1231- if "/" not in reference_graph :
1232- lookup_name = f"{ reference_graph } /forward"
1231+ lookup_name = resolved_graph_name
1232+ if "/" not in resolved_graph_name :
1233+ lookup_name = f"{ resolved_graph_name } /forward"
12331234 log .info (
1234- f"No method name specified in '{ reference_graph } ', "
1235+ f"No method name specified in '{ resolved_graph_name } ', "
12351236 f"using '{ lookup_name } ' as default"
12361237 )
12371238
@@ -1240,9 +1241,7 @@ def _get_aot_intermediate_outputs_and_op_names(
12401241 and lookup_name in self ._etrecord .graph_map
12411242 ):
12421243 export_program = self ._etrecord .graph_map [lookup_name ]
1243- log .info (
1244- f"Using '{ lookup_name } ' from graph_map as reference graph for intermediate output capture"
1245- )
1244+ log .info (f"Using '{ lookup_name } ' from graph_map as reference graph" )
12461245 else :
12471246 available_graphs = (
12481247 list (self ._etrecord .graph_map .keys ())
@@ -1254,16 +1253,82 @@ def _get_aot_intermediate_outputs_and_op_names(
12541253 f"Available options: 'exported_program', 'edge_dialect_exported_program', "
12551254 f"or one of the graphs in graph_map: { available_graphs } "
12561255 )
1257- graph_module = export_program .module ()
1256+
1257+ return export_program .module (), resolved_graph_name
1258+
1259+ def _get_aot_intermediate_outputs_and_op_names (
1260+ self ,
1261+ reference_graph_module : torch .fx .GraphModule ,
1262+ ) -> Tuple [Dict [DebugHandle , Any ], Dict [DebugHandle , List [str ]]]:
1263+ """
1264+ Capture intermediate outputs and operator name mappings from the given graph module.
1265+
1266+ Args:
1267+ reference_graph_module: The resolved reference graph module to use.
1268+
1269+ Returns:
1270+ Tuple of (intermediate_outputs, debug_handle_to_op_names) dictionaries.
1271+ """
12581272 aot_debug_handle_to_op_name = get_aot_debug_handle_to_op_name_mapping (
1259- graph_module
1273+ reference_graph_module
12601274 )
1261- capturer = IntermediateOutputCapturer (graph_module )
1275+ capturer = IntermediateOutputCapturer (reference_graph_module )
12621276 aot_intermediate_outputs = capturer .run_and_capture (
12631277 self ._etrecord ._representative_inputs
12641278 )
12651279 return aot_intermediate_outputs , aot_debug_handle_to_op_name
12661280
1281+ def _get_aot_debug_handle_to_stack_traces (
1282+ self ,
1283+ reference_graph_module : torch .fx .GraphModule ,
1284+ resolved_graph_name : str ,
1285+ ) -> Dict [DebugHandle , Dict [str , Optional [str ]]]:
1286+ """
1287+ Get a mapping from debug handle to stack traces from the given graph module.
1288+
1289+ Args:
1290+ reference_graph_module: The resolved reference graph module to use.
1291+ resolved_graph_name: The name of the graph (for warning messages).
1292+
1293+ Returns:
1294+ Dict[DebugHandle, Dict[str, Optional[str]]]: A dictionary mapping debug handles
1295+ to dictionaries of {op_name: stack_trace}.
1296+ """
1297+ from executorch .devtools .inspector ._inspector_utils import NodeFilter
1298+
1299+ node_filters = [
1300+ NodeFilter ("debug_handle" , "call_function" , exclude_ops = ["getitem" ])
1301+ ]
1302+
1303+ result : Dict [DebugHandle , Dict [str , Optional [str ]]] = {}
1304+ has_any_stack_trace = False
1305+
1306+ for node in reference_graph_module .graph .nodes :
1307+ if all (filter .matches (node ) for filter in node_filters ):
1308+ debug_handle = node .meta ["debug_handle" ]
1309+ key = (
1310+ (debug_handle ,)
1311+ if isinstance (debug_handle , int )
1312+ else tuple (debug_handle )
1313+ )
1314+ stack_trace = node .meta .get ("stack_trace" )
1315+ if stack_trace is not None :
1316+ has_any_stack_trace = True
1317+
1318+ if key in result :
1319+ result [key ][node .name ] = stack_trace
1320+ else :
1321+ result [key ] = {node .name : stack_trace }
1322+
1323+ if not has_any_stack_trace and result :
1324+ log .warning (
1325+ f"No stack traces found in reference_graph '{ resolved_graph_name } '. "
1326+ "The 'stacktraces' column will contain None values. "
1327+ "Ensure the model was exported with stack trace information preserved."
1328+ )
1329+
1330+ return result
1331+
12671332 # TODO: Make it more extensible to further merge overlapping debug handles
12681333 def _get_runtime_intermediate_outputs_and_op_names (
12691334 self ,
@@ -1506,17 +1571,29 @@ def calculate_numeric_gap(
15061571
15071572 Returns:
15081573 pd.DataFrame: A DataFrame listing corresponding operator intermediate outputs from both stages and their computed numerical gaps.
1574+ The DataFrame includes a "stacktraces" column where each entry is a dict mapping operator names to their stack traces.
15091575 """
1576+ # First, resolve the reference graph to use
1577+ reference_graph_module , resolved_graph_name = self ._resolve_reference_graph (
1578+ reference_graph ,
1579+ disable_debug_handle_valdiation ,
1580+ )
1581+
1582+ # Get intermediate outputs and op names from the resolved graph
15101583 aot_intermediate_outputs , aot_debug_handle_to_op_names = (
1511- self ._get_aot_intermediate_outputs_and_op_names (
1512- reference_graph ,
1513- disable_debug_handle_valdiation ,
1514- )
1584+ self ._get_aot_intermediate_outputs_and_op_names (reference_graph_module )
15151585 )
15161586 if len (aot_intermediate_outputs ) == 0 or len (aot_debug_handle_to_op_names ) == 0 :
15171587 raise ValueError (
15181588 "Missing etrecord or missing representative inputs within etrecord, both of which are required for calculating numerical gap"
15191589 )
1590+
1591+ # Get the stack trace mapping from the resolved graph
1592+ aot_debug_handle_to_stack_traces = self ._get_aot_debug_handle_to_stack_traces (
1593+ reference_graph_module ,
1594+ resolved_graph_name ,
1595+ )
1596+
15201597 # The runtime_op_names will be used later to map runtime debug_handle to op_name
15211598 runtime_intermediate_outputs , runtime_debug_handle_to_op_names = (
15221599 self ._get_runtime_intermediate_outputs_and_op_names ()
@@ -1543,8 +1620,32 @@ def calculate_numeric_gap(
15431620 raise ValueError (f"Unsupported distance metric { distance !r} " )
15441621
15451622 # Delegate to comparator's compare method (includes preprocessing)
1546- return comparator .compare (
1623+ df = comparator .compare (
15471624 mapping ,
15481625 aot_debug_handle_to_op_names ,
15491626 runtime_debug_handle_to_op_names ,
15501627 )
1628+
1629+ # Add stacktraces column by looking up each row's debug handle
1630+ # We need to map from aot_ops back to debug handles to get stack traces
1631+ def get_stacktraces_for_row (aot_ops : List [str ]) -> Dict [str , Optional [str ]]:
1632+ """Find stack traces for the given aot_ops by looking up in all debug handles."""
1633+ result : Dict [str , Optional [str ]] = {}
1634+ for op_name in aot_ops :
1635+ # Search through debug handle mappings to find the stack trace for this op
1636+ for (
1637+ _ ,
1638+ stack_traces_dict ,
1639+ ) in aot_debug_handle_to_stack_traces .items ():
1640+ if op_name in stack_traces_dict :
1641+ result [op_name ] = stack_traces_dict [op_name ]
1642+ break
1643+ else :
1644+ # Op not found in any debug handle's stack traces
1645+ result [op_name ] = None
1646+ return result
1647+
1648+ if len (df ) > 0 :
1649+ df ["stacktraces" ] = df ["aot_ops" ].apply (get_stacktraces_for_row )
1650+
1651+ return df
0 commit comments