Skip to content

Commit 0e543a3

Browse files
authored
[ExecuTorch][Core AI][1/x] Graph pass: int64/float64 dtype narrowing (pytorch#21509)
Adds `NarrowToCoreAIDtypesPass`, which casts int64/float64 graph inputs to 32-bit at the boundary and widens 64-bit outputs back. This lets index style inputs such as token ids be delegated, since Core AI supports only up to 32-bit dtypes. The model's external I/O dtypes are preserved. Includes unit tests.
1 parent e94faaa commit 0e543a3

4 files changed

Lines changed: 216 additions & 0 deletions

File tree

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
from executorch.backends.apple.coreai.passes.narrow_dtypes import (
8+
NarrowToCoreAIDtypesPass,
9+
)
10+
11+
__all__ = ["NarrowToCoreAIDtypesPass"]
Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
"""Boundary dtype-narrowing pass for the Core AI backend."""
8+
9+
import torch
10+
from executorch.exir.dialects.edge._ops import EdgeOpOverload
11+
from executorch.exir.pass_base import PassResult
12+
from torch.fx import GraphModule
13+
from torch.fx.passes.fake_tensor_prop import FakeTensorProp
14+
15+
# Core AI represents only up-to-32-bit dtypes (coreai-torch narrows int64->si32
16+
# and float64->f32 internally), so it can't carry a 64-bit tensor across the
17+
# delegate boundary.
18+
_NARROW = {torch.int64: torch.int32, torch.float64: torch.float32}
19+
_SIXTY_FOUR_BIT = (torch.int64, torch.float64)
20+
21+
22+
class NarrowToCoreAIDtypesPass:
23+
"""Narrow int64/float64 to 32-bit inside the graph while preserving I/O.
24+
25+
Casts int64/float64 graph inputs to 32-bit right after the placeholder,
26+
re-propagates dtypes, then casts 32-bit values feeding int64/float64 outputs
27+
back to 64-bit. The model's external input/output dtypes stay unchanged,
28+
while the interior (what Core AI sees) becomes 32-bit.
29+
30+
The inserted ``_to_copy`` casts keep a 64-bit operand/result, so
31+
:class:`CoreAIPartitioner` (which rejects nodes with int64/float64 tensors)
32+
leaves them outside the delegate; the delegate boundary then sees only 32-bit
33+
tensors that match what coreai declares.
34+
35+
Runs pre-partition (e.g. via ``get_default_passes``) at either the ATen or
36+
edge level (it picks the matching ``_to_copy`` overload).
37+
"""
38+
39+
def __call__(self, graph_module: GraphModule) -> PassResult:
40+
graph = graph_module.graph
41+
placeholders = [n for n in graph.nodes if n.op == "placeholder"]
42+
if not any(_is_64bit(p.meta.get("val")) for p in placeholders):
43+
return PassResult(graph_module, False)
44+
45+
uses_edge = any(
46+
isinstance(node.target, EdgeOpOverload)
47+
for node in graph.nodes
48+
if node.op == "call_function"
49+
)
50+
if uses_edge:
51+
from executorch.exir.dialects._ops import ops as exir_ops
52+
53+
to_copy = exir_ops.edge.aten._to_copy.default
54+
else:
55+
to_copy = torch.ops.aten._to_copy.default
56+
57+
output_node = next(n for n in graph.nodes if n.op == "output")
58+
# Remember which outputs were 64-bit so we can restore them afterwards.
59+
orig_output_dtypes = [
60+
val.dtype if _is_64bit(val := _node_val(a)) else None
61+
for a in output_node.args[0]
62+
]
63+
64+
# 1. Narrow 64-bit inputs right after the placeholder.
65+
for placeholder in placeholders:
66+
val = placeholder.meta.get("val")
67+
if not _is_64bit(val):
68+
continue
69+
narrow_dtype = _NARROW[val.dtype]
70+
with graph.inserting_after(placeholder):
71+
cast = graph.call_function(
72+
to_copy, (placeholder,), {"dtype": narrow_dtype}
73+
)
74+
placeholder.replace_all_uses_with(cast)
75+
cast.args = (placeholder,) # restore the cast's own (detached) input
76+
77+
# 2. Re-propagate dtypes through the (now-32-bit) interior.
78+
FakeTensorProp(graph_module).propagate(*[p.meta["val"] for p in placeholders])
79+
80+
# 3. Widen 32-bit values feeding originally-64-bit outputs back.
81+
out_args = list(output_node.args[0])
82+
for i, arg in enumerate(out_args):
83+
orig_dtype = orig_output_dtypes[i]
84+
if orig_dtype is None or not isinstance(arg, torch.fx.Node):
85+
continue
86+
cur = _node_val(arg)
87+
if isinstance(cur, torch.Tensor) and cur.dtype != orig_dtype:
88+
with graph.inserting_before(output_node):
89+
widen = graph.call_function(to_copy, (arg,), {"dtype": orig_dtype})
90+
# Set by hand: this runs after the propagate above, so nothing
91+
# else fills it in.
92+
widen.meta["val"] = cur.to(orig_dtype)
93+
out_args[i] = widen
94+
output_node.args = (tuple(out_args),)
95+
96+
graph_module.recompile()
97+
return PassResult(graph_module, True)
98+
99+
100+
def _node_val(arg):
101+
return arg.meta.get("val") if isinstance(arg, torch.fx.Node) else None
102+
103+
104+
def _is_64bit(val) -> bool:
105+
return isinstance(val, torch.Tensor) and val.dtype in _SIXTY_FOUR_BIT
Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
"""Unit tests for :class:`NarrowToCoreAIDtypesPass`."""
8+
9+
import unittest
10+
11+
import torch
12+
import torch.nn as nn
13+
14+
from executorch.backends.apple.coreai.passes.narrow_dtypes import (
15+
NarrowToCoreAIDtypesPass,
16+
)
17+
from executorch.exir import to_edge
18+
19+
20+
def _edge_gm(module, example_inputs):
21+
ep = torch.export.export(module.eval(), example_inputs)
22+
return to_edge(ep).exported_program().graph_module
23+
24+
25+
def _placeholders(gm):
26+
return [n for n in gm.graph.nodes if n.op == "placeholder"]
27+
28+
29+
def _copy_nodes(gm):
30+
return [
31+
n
32+
for n in gm.graph.nodes
33+
if n.op == "call_function" and "_to_copy" in str(n.target)
34+
]
35+
36+
37+
def _dtype(node):
38+
val = node.meta.get("val")
39+
return getattr(val, "dtype", None)
40+
41+
42+
class _Add(nn.Module):
43+
def forward(self, x):
44+
return x + x
45+
46+
47+
class NarrowToCoreAIDtypesPassTest(unittest.TestCase):
48+
def test_int64_boundary_narrowed_and_widened(self):
49+
# x + x with an int64 input: output dtype follows the input, so the pass
50+
# must narrow the input to i32 and widen the output back to i64.
51+
gm = _edge_gm(_Add(), (torch.randint(-8, 8, (2, 8), dtype=torch.int64),))
52+
before = len(_copy_nodes(gm))
53+
result = NarrowToCoreAIDtypesPass()(gm)
54+
self.assertTrue(result.modified)
55+
56+
# External input placeholder dtype is unchanged.
57+
self.assertEqual(_dtype(_placeholders(gm)[0]), torch.int64)
58+
# Boundary casts were inserted (input narrow + output widen).
59+
cast_dtypes = {_dtype(c) for c in _copy_nodes(gm)}
60+
self.assertGreater(len(_copy_nodes(gm)), before)
61+
self.assertIn(torch.int32, cast_dtypes) # input narrowed to i32
62+
self.assertIn(torch.int64, cast_dtypes) # output widened back to i64
63+
# External output dtype preserved as int64.
64+
out = gm.graph.output_node().args[0][0]
65+
self.assertEqual(_dtype(out), torch.int64)
66+
67+
def test_interior_is_narrowed_to_32bit(self):
68+
gm = _edge_gm(_Add(), (torch.randint(-8, 8, (2, 8), dtype=torch.int64),))
69+
NarrowToCoreAIDtypesPass()(gm)
70+
add = next(
71+
n
72+
for n in gm.graph.nodes
73+
if n.op == "call_function" and "add" in str(n.target)
74+
)
75+
self.assertEqual(_dtype(add), torch.int32)
76+
77+
def test_float64_narrowed_io_preserved(self):
78+
gm = _edge_gm(_Add(), (torch.randn(2, 8, dtype=torch.float64),))
79+
result = NarrowToCoreAIDtypesPass()(gm)
80+
self.assertTrue(result.modified)
81+
self.assertEqual(_dtype(_placeholders(gm)[0]), torch.float64) # I/O kept
82+
cast_dtypes = {_dtype(c) for c in _copy_nodes(gm)}
83+
self.assertIn(torch.float32, cast_dtypes)
84+
self.assertIn(torch.float64, cast_dtypes)
85+
86+
def test_noop_when_no_64bit(self):
87+
gm = _edge_gm(_Add(), (torch.randn(2, 8),)) # float32 only
88+
before = len(_copy_nodes(gm))
89+
result = NarrowToCoreAIDtypesPass()(gm)
90+
self.assertFalse(result.modified)
91+
self.assertEqual(len(_copy_nodes(gm)), before) # nothing inserted
92+
93+
94+
if __name__ == "__main__":
95+
unittest.main()

0 commit comments

Comments
 (0)