Skip to content

Commit 4939b45

Browse files
authored
Normalize constant and buffer dim order in to_edge (pytorch#16681)
Summary: Add a pass in to_edge to ensure all constant tensors have contiguous or channels-last dim order. Tensors with dim orders not present in this list get converted to contiguous. This solves a class of bugs related to strided constant tensors, such as the one causing pytorch#14644. Context for this change: * Some models, such as Yolo, include strided constant tensors. * PT2E quantization can convert constant tensors to buffers. * Prior to @lucylq's fix in pytorch#16636, we would serialize these with contiguous data layout but strided metadata, causing correctness issues and/or runtime shape mismatch. * As per discussion with @Gasoonjia, EXIR already assumes that tensors have either contiguous or channels last dim order. * If we modify tensor dim order in a backend-visible manner, it has to happen before delegation. Otherwise, the backend will see a different dim order at runtime compared to lowering. * Constants and buffers can be passed into delegate subgraphs as inputs. * XNNPACK, for example, will conditionally insert transposes based on input dim order. * Correspondingly, if we're going to convert dim order in EXIR, we need to do so before delegation. Test Plan: I've added tests for constants and buffers with a variety of dim orders and arbitrary strides in test_passes.py. I've also added some end to end test coverage under the XNNPACK backend that also covers the scenario we originally encountered with Yolo and XNNPACK. Differential Revision: D90898806
1 parent d06dd80 commit 4939b45

7 files changed

Lines changed: 322 additions & 0 deletions

File tree

‎backends/xnnpack/test/TARGETS‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,28 @@ runtime.python_test(
2727
],
2828
)
2929

30+
runtime.python_test(
31+
name = "test_xnnpack_fragments",
32+
srcs = glob([
33+
"fragments/*.py",
34+
]) + [
35+
"test_xnnpack_utils.py",
36+
],
37+
deps = [
38+
"//executorch/backends/xnnpack/partition:xnnpack_partitioner",
39+
"//executorch/backends/xnnpack/quantizer:xnnpack_quantizer",
40+
"//executorch/backends/xnnpack/test/tester:tester",
41+
"//executorch/devtools:lib",
42+
"//executorch/devtools/bundled_program:config",
43+
"//executorch/devtools/bundled_program/serialize:lib",
44+
"//executorch/exir/passes:constant_prop_pass",
45+
"//pytorch/ao:torchao", # @manual
46+
],
47+
external_deps = [
48+
"libtorch",
49+
],
50+
)
51+
3052
runtime.python_test(
3153
name = "test_xnnpack_ops",
3254
srcs = glob([
Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
import unittest
8+
9+
import torch
10+
from executorch.backends.xnnpack.test.tester import Tester
11+
12+
13+
class TestDimOrder(unittest.TestCase):
14+
def setUp(self):
15+
torch._dynamo.reset()
16+
17+
def test_add_constant_transposed(self):
18+
class Model(torch.nn.Module):
19+
def __init__(self):
20+
super().__init__()
21+
self.constant = torch.randn(2, 16).transpose(1, 0)
22+
23+
def forward(self, x):
24+
return x + self.constant
25+
26+
inputs = (torch.randn(16, 2),)
27+
28+
for quantize in [False, True]:
29+
tester = Tester(Model(), inputs)
30+
31+
if quantize:
32+
tester.quantize()
33+
34+
(
35+
tester.export()
36+
.to_edge_transform_and_lower()
37+
.to_executorch()
38+
.serialize()
39+
.run_method_and_compare_outputs()
40+
)

‎exir/passes/TARGETS‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ runtime.python_library(
99
],
1010
deps = [
1111
":const_prop_pass",
12+
":convert_constant_dim_order_pass",
1213
":debug_handle_generator_pass",
1314
":external_constants_pass",
1415
":init_mutable_pass",
@@ -415,3 +416,13 @@ runtime.python_library(
415416
"//executorch/exir/dialects:lib",
416417
],
417418
)
419+
420+
runtime.python_library(
421+
name = "convert_constant_dim_order_pass",
422+
srcs = [
423+
"convert_constant_dim_order_pass.py",
424+
],
425+
deps = [
426+
"//caffe2:torch",
427+
],
428+
)
Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
import torch
2+
from torch.export import ExportedProgram
3+
from torch.export.graph_signature import InputKind
4+
5+
6+
def _should_transform(tensor: torch.Tensor) -> bool:
7+
"""
8+
Returns whether the given tensor should be transformed by the pass to
9+
contiguous dim order.
10+
"""
11+
return not tensor.is_contiguous() and not tensor.is_contiguous(
12+
memory_format=torch.channels_last
13+
)
14+
15+
16+
def _update_placeholder_meta(
17+
exported_program: ExportedProgram,
18+
target: str,
19+
kind: InputKind,
20+
):
21+
input_spec = next(
22+
(
23+
spec
24+
for spec in exported_program.graph_signature.input_specs
25+
if spec.kind == kind and spec.target == target
26+
),
27+
None,
28+
)
29+
if input_spec is None:
30+
raise RuntimeError(f"Missing input spec for lifted tensor {target}.")
31+
32+
placeholder_node = next(
33+
(
34+
n
35+
for n in exported_program.graph.nodes
36+
if n.op == "placeholder" and n.name == input_spec.arg.name
37+
),
38+
None,
39+
)
40+
if input_spec is None:
41+
raise RuntimeError(f"Missing placeholder for {input_spec.arg.name}.")
42+
43+
placeholder_node.meta["val"] = placeholder_node.meta["val"].contiguous()
44+
45+
46+
def convert_constant_dim_order_pass(
47+
exported_program: ExportedProgram,
48+
) -> ExportedProgram:
49+
"""
50+
Normalize the dim order of constant tensors, ensuring that all constant tensors
51+
have either default or channels_last dim order. Tensors with other dim orders or
52+
striding are converted to contiguous tensors. This pass acts in-place on the
53+
unlifted exported program.
54+
55+
Args:
56+
exported_program: The ExportedProgram to transform.
57+
58+
Returns:
59+
The modified ExportedProgram with normalized constant dim order.
60+
"""
61+
62+
for key, const in exported_program.constants.items():
63+
if isinstance(const, torch.Tensor) and _should_transform(const):
64+
exported_program.constants[key] = const.contiguous()
65+
66+
# Also update the corresponding placeholder node meta value. This doesn't
67+
# get automatically updated during retracing as ExportPass uses the placeholder
68+
# meta as the source of truth. TODO(?)
69+
_update_placeholder_meta(exported_program, key, InputKind.CONSTANT_TENSOR)
70+
71+
# Convert buffers.
72+
non_persistent_buffer_names = set(
73+
exported_program.graph_signature.non_persistent_buffers
74+
)
75+
for key, buffer in exported_program.named_buffers():
76+
if (
77+
key not in non_persistent_buffer_names
78+
and isinstance(buffer, torch.Tensor)
79+
and _should_transform(buffer)
80+
):
81+
# Persistent buffers are stored in the state dict.
82+
exported_program.state_dict[key] = buffer.contiguous()
83+
84+
_update_placeholder_meta(exported_program, key, InputKind.BUFFER)
85+
86+
return exported_program

‎exir/program/TARGETS‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ runtime.python_library(
4545
"//executorch/exir/passes:replace_view_copy_with_view_pass",
4646
"//executorch/exir/passes:spec_prop_pass",
4747
"//executorch/exir/passes:weights_to_outputs_pass",
48+
"//executorch/exir/passes:convert_constant_dim_order_pass",
4849
"//executorch/exir/verification:verifier",
4950
"//executorch/extension/flat_tensor/serialize:serialize",
5051
] + (["//executorch/exir/program/fb:logger"] if not runtime.is_oss else [])

‎exir/program/_program.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242
from executorch.exir.passes import (
4343
base_post_op_replace_passes,
4444
base_pre_op_replace_passes,
45+
convert_constant_dim_order_pass,
4546
dead_code_elimination_pass,
4647
EdgeToBackendOpsPass,
4748
MemoryFormatOpsPass,
@@ -910,8 +911,15 @@ def _generate_edge_program(
910911
)
911912
],
912913
)
914+
913915
# Lift the tensor constants created in ScalarToTensorPass
914916
edge_program = lift_constant_tensor_pass(edge_program)
917+
918+
# Normalize constant tensor dim order on the unlifted graph
919+
edge_program = convert_constant_dim_order_pass.convert_constant_dim_order_pass(
920+
edge_program
921+
)
922+
915923
edge_program = _transform(edge_program, *post_op_replace_passes)
916924

917925
return edge_program

‎exir/tests/test_passes.py‎

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
# pyre-strict
88
import copy
9+
import itertools
910
import os
1011
import tempfile
1112
import unittest
@@ -41,6 +42,7 @@
4142
from executorch.exir.graph_module import get_control_flow_submodules
4243
from executorch.exir.pass_base import ExportPass, PassResult
4344
from executorch.exir.passes import (
45+
convert_constant_dim_order_pass,
4446
dead_code_elimination_pass,
4547
DebugPass,
4648
HintBasedSymShapeEvalPass,
@@ -2330,3 +2332,155 @@ def forward(self, x):
23302332
prop_tensor.is_contiguous(),
23312333
f"Propagated tensor is not contiguous: {prop_tensor.stride()}",
23322334
)
2335+
2336+
class AddConstant(torch.nn.Module):
2337+
def __init__(self, constant: torch.Tensor, use_buffer: bool = False):
2338+
super().__init__()
2339+
if use_buffer:
2340+
self.register_buffer("constant", constant)
2341+
else:
2342+
self.constant = constant
2343+
2344+
def forward(self, x):
2345+
return x + self.constant
2346+
2347+
def test_convert_constant_dim_order_noop(self):
2348+
"""
2349+
Verify that the convert_constant_dim_order_pass does not modify
2350+
constant tensors in contiguous or channels last format.
2351+
"""
2352+
2353+
memory_formats = [
2354+
torch.contiguous_format,
2355+
torch.channels_last,
2356+
]
2357+
2358+
use_buffer_options = [False, True]
2359+
2360+
for memory_format, use_buffer in itertools.product(
2361+
memory_formats, use_buffer_options
2362+
):
2363+
constant = torch.randn(2, 3, 4, 5).to(memory_format=memory_format)
2364+
model = self.AddConstant(constant, use_buffer=use_buffer)
2365+
inputs = (torch.randn(2, 3, 4, 5).to(memory_format=memory_format),)
2366+
2367+
ep = torch.export.export(model, inputs)
2368+
modified_ep = (
2369+
convert_constant_dim_order_pass.convert_constant_dim_order_pass(
2370+
copy.deepcopy(ep)
2371+
)
2372+
)
2373+
2374+
# Check the outputs - they should match in both data and dim order.
2375+
original_outputs = ep.module()(*inputs)
2376+
modified_outputs = modified_ep.module()(*inputs)
2377+
2378+
torch.testing.assert_close(
2379+
original_outputs, modified_outputs, check_stride=True
2380+
)
2381+
2382+
# Verify that the constant/buffer tensor is not modified. The interface for
2383+
# constants and buffers is just different enough to make it hard to unify the
2384+
# below logic.
2385+
if use_buffer:
2386+
original_buffers = dict(ep.named_buffers())
2387+
modified_buffers = dict(modified_ep.named_buffers())
2388+
2389+
self.assertEqual(len(original_buffers), 1)
2390+
self.assertEqual(len(modified_buffers), 1)
2391+
2392+
buffer_key = next(iter(original_buffers.keys()))
2393+
original_buffer = original_buffers[buffer_key]
2394+
modified_buffer = modified_buffers[buffer_key]
2395+
2396+
torch.testing.assert_close(
2397+
original_buffer, modified_buffer, check_stride=True
2398+
)
2399+
else:
2400+
self.assertEqual(len(ep.constants), 1)
2401+
self.assertEqual(len(modified_ep.constants), 1)
2402+
2403+
const_key = next(iter(ep.constants.keys()))
2404+
original_const = ep.constants[const_key]
2405+
modified_const = modified_ep.constants[const_key]
2406+
2407+
torch.testing.assert_close(
2408+
original_const, modified_const, check_stride=True
2409+
)
2410+
2411+
def test_convert_constant_dim_order_to_contiguous(self):
2412+
"""
2413+
Verify that the convert_constant_dim_order_pass converts constant/buffer
2414+
tensors with dim orders / memory formats outside the allowed list
2415+
(contiguous and channels_last) to contiguous.
2416+
"""
2417+
2418+
# Test cases using transpose/permute to create non-contiguous tensors.
2419+
# Each case is (base_shape, permutation, description) where we create a
2420+
# contiguous tensor of base_shape and then permute/transpose it.
2421+
test_cases = [
2422+
# Rank 2 - transposed matrix (non-contiguous strides)
2423+
((5, 4), (1, 0), "2D transposed"),
2424+
# Rank 3 - permuted dimensions
2425+
((4, 3, 2), (2, 0, 1), "3D permuted"),
2426+
# Rank 3 - another permutation
2427+
((2, 4, 3), (1, 2, 0), "3D permuted v2"),
2428+
# Rank 4 - permuted (not contiguous, not channels_last)
2429+
((5, 4, 3, 2), (3, 1, 0, 2), "4D permuted"),
2430+
# Rank 5 - permuted
2431+
((6, 5, 4, 3, 2), (4, 2, 0, 3, 1), "5D permuted"),
2432+
]
2433+
2434+
use_buffer_options = [False, True]
2435+
2436+
for (base_shape, permutation, description), use_buffer in itertools.product(
2437+
test_cases, use_buffer_options
2438+
):
2439+
with self.subTest(description=description, use_buffer=use_buffer):
2440+
# Create a contiguous tensor and permute it to make it non-contiguous
2441+
base_tensor = torch.randn(base_shape)
2442+
constant = base_tensor.permute(permutation)
2443+
2444+
# Sanity check the tensor construction
2445+
self.assertFalse(
2446+
constant.is_contiguous(),
2447+
"Test setup error: tensor should be non-contiguous",
2448+
)
2449+
2450+
model = self.AddConstant(constant, use_buffer=use_buffer)
2451+
inputs = (torch.randn(constant.shape),)
2452+
2453+
ep = torch.export.export(model, inputs)
2454+
modified_ep = (
2455+
convert_constant_dim_order_pass.convert_constant_dim_order_pass(
2456+
copy.deepcopy(ep)
2457+
)
2458+
)
2459+
2460+
# Check the outputs - they should match in data.
2461+
original_outputs = ep.module()(*inputs)
2462+
modified_outputs = modified_ep.module()(*inputs)
2463+
torch.testing.assert_close(original_outputs, modified_outputs)
2464+
2465+
# Verify that the constant/buffer tensor is now contiguous.
2466+
if use_buffer:
2467+
modified_buffers = dict(modified_ep.named_buffers())
2468+
self.assertEqual(len(modified_buffers), 1)
2469+
2470+
buffer_key = next(iter(modified_buffers.keys()))
2471+
modified_buffer = modified_buffers[buffer_key]
2472+
2473+
self.assertTrue(
2474+
modified_buffer.is_contiguous(),
2475+
f"Buffer should be contiguous after pass, got strides {modified_buffer.stride()}",
2476+
)
2477+
else:
2478+
self.assertEqual(len(modified_ep.constants), 1)
2479+
2480+
const_key = next(iter(modified_ep.constants.keys()))
2481+
modified_const = modified_ep.constants[const_key]
2482+
2483+
self.assertTrue(
2484+
modified_const.is_contiguous(),
2485+
f"Constant should be contiguous after pass, got strides {modified_const.stride()}",
2486+
)

0 commit comments

Comments
 (0)