Skip to content

Commit c7c4073

Browse files
authored
solve false negtive gap for multiple output operators (pytorch#16729)
Differential Revision: D91201882 Pull Request resolved: pytorch#16763
1 parent 092f42f commit c7c4073

3 files changed

Lines changed: 358 additions & 76 deletions

File tree

‎devtools/inspector/_inspector_utils.py‎

Lines changed: 189 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -751,6 +751,168 @@ def dfs(node_id, component):
751751
return connected_components
752752

753753

754+
def _map_sequence_aot_output(
755+
aot_intermediate_output: Sequence,
756+
runtime_intermediate_output: Any,
757+
negative_index: int,
758+
) -> Tuple[Tuple, Tuple]:
759+
"""
760+
Handle the case when aot_intermediate_output is a Sequence.
761+
762+
Returns:
763+
Tuple of (aot_intermediate_output as tuple, mapped runtime output as tuple)
764+
"""
765+
if not isinstance(runtime_intermediate_output, Sequence):
766+
raise TypeError(
767+
"runtime intermediate output should be a sequence when aot intermediate output is a sequence"
768+
)
769+
last_element = runtime_intermediate_output[negative_index]
770+
# TODO: this (last_element = list) is never really the case because runtime never returns output as a list
771+
# for delegate case.
772+
if isinstance(last_element, list) and all(
773+
isinstance(t, torch.Tensor) for t in last_element
774+
):
775+
# If the last element is a list of tensors (delegate case)
776+
aot_mapped_runtime_intermediate_output = last_element
777+
elif isinstance(last_element, torch.Tensor):
778+
# If the last element is a tensor, as is always the case for runtime.
779+
# However, now we have a strange condition where aot_intermediate_output is a list of tensors
780+
# while runtime_intermediate_output is a single tensor. So we should never really come here.
781+
# TODO: fix this
782+
aot_mapped_runtime_intermediate_output = runtime_intermediate_output
783+
else:
784+
raise ValueError(
785+
"The last element of runtime argument list must be a tensor or a list of tensors when aot intermediate output is a sequence"
786+
)
787+
# List can't be used as a key, so convert to tuple
788+
return tuple(aot_intermediate_output), tuple(aot_mapped_runtime_intermediate_output)
789+
790+
791+
def _find_matching_runtime_output_by_shape_and_dtype(
792+
aot_intermediate_output: torch.Tensor,
793+
runtime_intermediate_output: Sequence,
794+
) -> Any:
795+
"""
796+
Find the runtime output that matches the AOT output shape and dtype.
797+
Used for multi-output operations (like native_layer_norm.out, native_dropout.out).
798+
799+
Returns:
800+
The matching runtime output, or runtime_intermediate_output[-1] as fallback.
801+
"""
802+
# Find all runtime outputs that match the AOT shape
803+
matching_indices = []
804+
for idx, runtime_out in enumerate(runtime_intermediate_output):
805+
if isinstance(runtime_out, torch.Tensor):
806+
if runtime_out.shape == aot_intermediate_output.shape:
807+
matching_indices.append(idx)
808+
809+
if len(matching_indices) == 1:
810+
# Exactly one shape match - use it (native multi-output case like layer_norm)
811+
return runtime_intermediate_output[matching_indices[0]]
812+
813+
if len(matching_indices) > 1:
814+
# Multiple shape matches - try to distinguish by dtype
815+
# For native_dropout, output is float and mask is bool; prefer matching dtype
816+
dtype_matching_indices = []
817+
for idx in matching_indices:
818+
runtime_out = runtime_intermediate_output[idx]
819+
if isinstance(runtime_out, torch.Tensor):
820+
if runtime_out.dtype == aot_intermediate_output.dtype:
821+
dtype_matching_indices.append(idx)
822+
823+
if len(dtype_matching_indices) == 1:
824+
# Exactly one dtype match - use it (e.g., dropout case where mask is bool)
825+
return runtime_intermediate_output[dtype_matching_indices[0]]
826+
827+
# No unique match found, return the last element as fallback
828+
return runtime_intermediate_output[-1]
829+
830+
831+
def _map_non_sequence_aot_output(
832+
aot_intermediate_output: Any,
833+
runtime_intermediate_output: Any,
834+
num_outputs: int,
835+
negative_index: int,
836+
) -> Any:
837+
"""
838+
Handle the case when aot_intermediate_output is NOT a Sequence.
839+
840+
Returns:
841+
The mapped runtime intermediate output.
842+
"""
843+
if not isinstance(runtime_intermediate_output, Sequence):
844+
return runtime_intermediate_output
845+
846+
# Use the last element of the runtime output as fallback if no match is found
847+
aot_mapped_runtime_intermediate_output = runtime_intermediate_output[negative_index]
848+
849+
# delegate runtime call and AOT intermediate is not a sequence.
850+
# For multi-output operations (like native_layer_norm.out, native_dropout.out),
851+
# the runtime captures all outputs but AOT only captures the primary output.
852+
# We need to find the runtime output that matches the AOT output shape and dtype.
853+
if (
854+
num_outputs == 1
855+
and len(runtime_intermediate_output) > 1
856+
and isinstance(aot_intermediate_output, torch.Tensor)
857+
):
858+
aot_mapped_runtime_intermediate_output = (
859+
_find_matching_runtime_output_by_shape_and_dtype(
860+
aot_intermediate_output, runtime_intermediate_output
861+
)
862+
)
863+
864+
return aot_mapped_runtime_intermediate_output
865+
866+
867+
def _process_single_runtime_output(
868+
aot_list: List[Tuple[DebugHandle, Any]],
869+
runtime_debug_handle: DebugHandle,
870+
runtime_intermediate_output: Any,
871+
num_outputs: int,
872+
output_index: int,
873+
) -> Optional[Tuple[Tuple[DebugHandle, Any], Tuple[DebugHandle, Any]]]:
874+
"""
875+
Process a single runtime output and map it to the corresponding AOT output.
876+
877+
Returns:
878+
A tuple of ((aot_debug_handle, aot_output), (runtime_debug_handle, runtime_output))
879+
or None if the mapping should be skipped.
880+
"""
881+
negative_index = -1 * (output_index + 1)
882+
883+
# Combine aot debug handles into a single key
884+
aot_combined_debug_handle, aot_intermediate_output = (
885+
_combine_aot_overlapped_intermediate_outputs(
886+
aot_list,
887+
(runtime_debug_handle, runtime_intermediate_output, num_outputs),
888+
negative_index,
889+
)
890+
)
891+
892+
if aot_combined_debug_handle == (-1,):
893+
# Skip this mapping if the aot combined debug handle and runtime debug handle do not exact match.
894+
return None
895+
896+
if isinstance(aot_intermediate_output, Sequence):
897+
aot_intermediate_output, aot_mapped_runtime_intermediate_output = (
898+
_map_sequence_aot_output(
899+
aot_intermediate_output, runtime_intermediate_output, negative_index
900+
)
901+
)
902+
else:
903+
aot_mapped_runtime_intermediate_output = _map_non_sequence_aot_output(
904+
aot_intermediate_output,
905+
runtime_intermediate_output,
906+
num_outputs,
907+
negative_index,
908+
)
909+
910+
return (
911+
(aot_combined_debug_handle, aot_intermediate_output),
912+
(runtime_debug_handle, aot_mapped_runtime_intermediate_output),
913+
)
914+
915+
754916
def map_runtime_aot_intermediate_outputs(
755917
aot_intermediate_outputs: Dict[DebugHandle, Any],
756918
runtime_intermediate_outputs: Dict[DebugHandle, Tuple[Any, int]],
@@ -789,84 +951,36 @@ def map_runtime_aot_intermediate_outputs(
789951
if nodes[node_id].source == NodeSource.RUNTIME
790952
]
791953

792-
# Map only if both AOT and runtime data are present.
793-
if len(aot_list) != 0 and len(runtime_list) != 0:
794-
# The size of runtime_list should be 1 because all AOT debug_handles are tuples with one element.
795-
# Additionally, runtime debug handles have already undergone pre-processing to merge overlapping debug_hanldes.
796-
# As a result, there shouldn't be any 1-to-n or n-to-n (AOT to runtime) mappings.
797-
if len(runtime_list) != 1:
798-
raise ValueError(
799-
f"Expected only one runtime debug handle, but found {len(runtime_list)}: {runtime_list}"
800-
)
954+
if len(aot_list) == 0 or len(runtime_list) == 0:
955+
# Skip this mapping if there are no AOT or runtime data.
956+
continue
801957

802-
runtime_debug_handle, runtime_intermediate_output, num_outputs = (
803-
runtime_list[0]
958+
# The size of runtime_list should be 1 because all AOT debug_handles are tuples with one element.
959+
# Additionally, runtime debug handles have already undergone pre-processing to merge overlapping debug_hanldes.
960+
# As a result, there shouldn't be any 1-to-n or n-to-n (AOT to runtime) mappings.
961+
if len(runtime_list) != 1:
962+
raise ValueError(
963+
f"Expected only one runtime debug handle, but found {len(runtime_list)}: {runtime_list}"
804964
)
805-
# iterate through each of the output from runtime,
806-
# get the corresponding debug handle
807-
# and map it to the aot debug handle
808-
# and create a dictionary that maps aot debug handle + aot output to
809-
# runtime debug handle + runtime output
810-
# Note this works only for delegate case for now.
811-
for i in range(num_outputs):
812-
813-
negative_index = -1 * (i + 1)
814-
aot_mapped_runtime_intermediate_output = runtime_intermediate_output
815-
# Combine aot debug handles into a single key
816-
aot_combined_debug_handle, aot_intermediate_output = (
817-
_combine_aot_overlapped_intermediate_outputs(
818-
aot_list, runtime_list[0], negative_index
819-
)
820-
)
821-
822-
if aot_combined_debug_handle == (-1,):
823-
# Skip this mapping if the aot combined debug handle and runtime debug handle do not exact match.
824-
continue
825-
826-
if isinstance(aot_intermediate_output, Sequence):
827-
if not isinstance(runtime_intermediate_output, Sequence):
828-
raise TypeError(
829-
"runtime intermediate output should be a sequence when aot intermediate output is a sequence"
830-
)
831-
last_element = runtime_intermediate_output[negative_index]
832-
# TODO: this (last_element = list) is never really the case because runtime never returns output as a list
833-
# for delegate case.
834-
if isinstance(last_element, list) and all(
835-
isinstance(t, torch.Tensor) for t in last_element
836-
):
837-
# If the last element is a list of tensors (delegate case)
838-
aot_mapped_runtime_intermediate_output = last_element
839-
elif isinstance(last_element, torch.Tensor):
840-
# If the last element is a tensor, as is always the case for runtime.
841-
# However, now we have a strange condition where aot_intermediate_output is a list of tensors
842-
# while runtime_intermediate_output is a single tensor. So we should never really come here.
843-
# TODO: fix this
844-
aot_mapped_runtime_intermediate_output = (
845-
runtime_intermediate_output
846-
)
847-
else:
848-
raise ValueError(
849-
"The last element of runtime argument list must be a tensor or a list of tensors when aot intermediate output is a sequence"
850-
)
851-
# List can't be used as a key, so convert to tuple
852-
aot_intermediate_output = tuple(aot_intermediate_output)
853-
aot_mapped_runtime_intermediate_output = tuple(
854-
aot_mapped_runtime_intermediate_output
855-
)
856965

857-
elif isinstance(runtime_intermediate_output, Sequence):
858-
# delegate runtime call and AOT intermediate is not a sequence, just take the last element from runtime list
859-
aot_mapped_runtime_intermediate_output = (
860-
runtime_intermediate_output[negative_index]
861-
)
862-
863-
# Create a mapping between runtime and aot
864-
aot_runtime_mapping[
865-
(aot_combined_debug_handle, aot_intermediate_output)
866-
] = (
867-
runtime_debug_handle,
868-
aot_mapped_runtime_intermediate_output,
869-
)
966+
runtime_debug_handle, runtime_intermediate_output, num_outputs = runtime_list[0]
967+
# iterate through each of the output from runtime,
968+
# get the corresponding debug handle
969+
# and map it to the aot debug handle
970+
# and create a dictionary that maps aot debug handle + aot output to
971+
# runtime debug handle + runtime output
972+
# Note this works only for delegate case for now.
973+
for i in range(num_outputs):
974+
result = _process_single_runtime_output(
975+
aot_list,
976+
runtime_debug_handle,
977+
runtime_intermediate_output,
978+
num_outputs,
979+
i,
980+
)
981+
if result is not None:
982+
aot_key, runtime_value = result
983+
aot_runtime_mapping[aot_key] = runtime_value
870984

871985
return aot_runtime_mapping
872986

‎devtools/inspector/tests/TARGETS‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,14 +11,17 @@ python_unittest(
1111
ci.buckconfig("executorch.event_tracer_enabled", "true"),
1212
),
1313
deps = [
14+
"//executorch/backends/xnnpack/partition:xnnpack_partitioner",
15+
"//executorch/backends/xnnpack/utils:xnnpack_utils",
1416
"//executorch/devtools:lib",
1517
"//executorch/devtools/debug_format:et_schema",
1618
"//executorch/devtools/etdump:schema_flatcc",
1719
"//executorch/devtools/etrecord/tests:etrecord_test_library",
1820
"//executorch/devtools/inspector:inspector",
1921
"//executorch/devtools/inspector:lib",
20-
"//executorch/exir:lib",
2122
"//executorch/devtools/inspector/tests:inspector_test_utils",
23+
"//executorch/exir:lib",
24+
"//executorch/runtime:runtime",
2225
],
2326
)
2427

0 commit comments

Comments
 (0)