Skip to content

Commit 8362c00

Browse files
Arm backend: Add fused i8 warp+downsample example (pytorch#21972)
Adds example where int8 warp+downsample is fused into one custom shader. This pattern is present in rife model and such a fusion can reduce memory bandwidth significantly as it avoids processing pixels that are not used by downsampling. cc @digantdesai @freddan80 @per @zingo @mansnils @Sebastian-Larsson @robell @rascani Signed-off-by: Oscar Andersson <oscar.andersson@arm.com>
1 parent 2f4fb0f commit 8362c00

13 files changed

Lines changed: 1684 additions & 1 deletion

‎examples/arm/QAT_example/qat_loop.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@
3535
import numpy as np
3636
import torch
3737
import torch.nn.functional as F
38-
from PIL import Image
38+
from PIL import Image # type: ignore[import-untyped]
3939
from torch.export import export
4040
from torchao.quantization.pt2e import (
4141
move_exported_model_to_eval,
@@ -50,6 +50,10 @@
5050
EXECUTORCH_ROOT = Path(__file__).resolve().parents[3]
5151
sys.path.insert(0, str(EXECUTORCH_ROOT))
5252

53+
from examples.arm.QAT_example.rife_vgf import ( # noqa: E402
54+
configure_rife_vgf,
55+
configure_rife_vgf_quantizer,
56+
)
5357
from executorch.backends.arm.quantizer import ( # noqa: E402
5458
get_uint8_io_quantization_config,
5559
get_vgf_snorm_quantization_config,
@@ -665,6 +669,7 @@ def make_quantizer(
665669
quantizer = VgfQuantizer(compile_spec, use_composable_quantizer=True)
666670
global_config = get_vgf_snorm_quantization_config(is_qat=is_qat)
667671
quantizer.set_global(global_config)
672+
configure_rife_vgf_quantizer(quantizer)
668673

669674
if io_quantization == "int8":
670675
quantizer.set_io(global_config)
@@ -801,6 +806,7 @@ def export_vgf_artifact(
801806
artifact_dir.mkdir(parents=True, exist_ok=True)
802807
compile_spec = make_vgf_compile_spec(artifact_dir)
803808
partitioner = VgfPartitioner(compile_spec)
809+
configure_rife_vgf(partitioner)
804810
cast(Any, model).to(memory_format=torch.channels_last)
805811
inputs = (
806812
inputs[0].contiguous(memory_format=torch.channels_last),
@@ -1791,6 +1797,7 @@ def main() -> int:
17911797
print(f"Data source: {source}")
17921798

17931799
model = load_rife_model(args.model_root, args.checkpoint)
1800+
configure_rife_vgf()
17941801
eager_channels_last_model = clone_model(model, channels_last=True)
17951802
ptq_model, qat_model, quantization_reports = build_requested_quantized_models(
17961803
args,
Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
# Copyright 2026 Arm Limited and/or its affiliates.
2+
#
3+
# This source code is licensed under the BSD-style license found in the
4+
# LICENSE file in the root directory of this source tree.
5+
6+
from .extension import configure_rife_vgf, configure_rife_vgf_quantizer # noqa: F401
Lines changed: 189 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,189 @@
1+
# Copyright 2026 Arm Limited and/or its affiliates.
2+
#
3+
# This source code is licensed under the BSD-style license found in the
4+
# LICENSE file in the root directory of this source tree.
5+
6+
import torch
7+
8+
from executorch.backends.arm._passes import RewriteConvPass
9+
from executorch.backends.arm._passes.arm_pass_manager import (
10+
_registered_pass_insertions,
11+
register_pass_insertions_before,
12+
)
13+
from executorch.backends.arm.common.annotation_meta import ArmAnnotationInfo
14+
from executorch.backends.arm.vgf import VgfPartitioner
15+
from executorch.backends.cortex_m.quantizer_reporter import (
16+
QuantizerInfo,
17+
QuantizerReporterUser,
18+
)
19+
from executorch.exir.pass_base import ExportPass
20+
from torch._ops import OpOverload
21+
from torchao.quantization.pt2e.quantizer import (
22+
FixedQParamsQuantizationSpec,
23+
QuantizationAnnotation,
24+
Quantizer,
25+
)
26+
from torchao.quantization.pt2e.quantizer.quantizer import Q_ANNOTATION_KEY
27+
28+
from .passes import RewriteWarpDownsampleToTosaCustomPass
29+
30+
_RIFE_LIBRARY: torch.library.Library | None = None
31+
_RIFE_FAKE_IMPLS_REGISTERED = False
32+
33+
34+
def _warp_downsample_fake(
35+
image: torch.Tensor, flow: torch.Tensor, scale: int
36+
) -> torch.Tensor:
37+
del flow
38+
return torch.empty(
39+
(
40+
image.shape[0],
41+
image.shape[1],
42+
image.shape[2] // scale,
43+
image.shape[3] // scale,
44+
),
45+
dtype=image.dtype,
46+
device=image.device,
47+
)
48+
49+
50+
def _has_warp_downsample_op(scale: int) -> bool:
51+
try:
52+
getattr(torch.ops.rife, f"warp_downsample{scale}").default
53+
except AttributeError:
54+
return False
55+
return True
56+
57+
58+
def _register_warp_downsample_fake_impls() -> None:
59+
global _RIFE_FAKE_IMPLS_REGISTERED
60+
if _RIFE_FAKE_IMPLS_REGISTERED:
61+
return
62+
63+
for scale in (2, 4, 8):
64+
65+
def _warp_downsample_fake_impl(
66+
image: torch.Tensor,
67+
flow: torch.Tensor,
68+
scale: int = scale,
69+
) -> torch.Tensor:
70+
return _warp_downsample_fake(image, flow, scale)
71+
72+
try:
73+
torch.library.register_fake(f"rife::warp_downsample{scale}")(
74+
_warp_downsample_fake_impl
75+
)
76+
except RuntimeError as error:
77+
message = str(error)
78+
if "already" not in message and "CompositeImplicitAutograd" not in message:
79+
raise
80+
81+
_RIFE_FAKE_IMPLS_REGISTERED = True
82+
83+
84+
def _ensure_warp_downsample_ops_defined() -> None:
85+
global _RIFE_LIBRARY
86+
if _RIFE_LIBRARY is None:
87+
_RIFE_LIBRARY = torch.library.Library("rife", "FRAGMENT")
88+
missing_scales = [
89+
scale for scale in (2, 4, 8) if not _has_warp_downsample_op(scale)
90+
]
91+
for scale in missing_scales:
92+
_RIFE_LIBRARY.define(
93+
f"warp_downsample{scale}(Tensor image, Tensor flow) -> Tensor"
94+
)
95+
96+
97+
def _ensure_warp_downsample_ops_registered() -> None:
98+
_ensure_warp_downsample_ops_defined()
99+
_register_warp_downsample_fake_impls()
100+
101+
102+
def _warp_downsample_target(scale: int) -> OpOverload:
103+
_ensure_warp_downsample_ops_registered()
104+
return getattr(torch.ops.rife, f"warp_downsample{scale}").default
105+
106+
107+
def _warp_downsample_targets() -> tuple[OpOverload, ...]:
108+
return tuple(_warp_downsample_target(scale) for scale in (2, 4, 8))
109+
110+
111+
def _warp_downsample_snorm_qspec() -> FixedQParamsQuantizationSpec:
112+
return FixedQParamsQuantizationSpec(
113+
dtype=torch.int8,
114+
scale=1.0 / 127.0,
115+
zero_point=0,
116+
quant_min=-127,
117+
quant_max=127,
118+
qscheme=torch.per_tensor_symmetric,
119+
is_dynamic=False,
120+
)
121+
122+
123+
class _WarpDownsampleQuantizer(Quantizer, QuantizerReporterUser):
124+
def __init__(self) -> None:
125+
super().__init__()
126+
QuantizerReporterUser.__init__(self)
127+
self.targets = set(_warp_downsample_targets())
128+
self.snorm_qspec = _warp_downsample_snorm_qspec()
129+
130+
def get_quantizer_info(self) -> QuantizerInfo:
131+
return QuantizerInfo(
132+
self.__class__.__name__,
133+
"rife.warp_downsample{2,4,8}",
134+
"rife_warp_downsample_snorm",
135+
"examples.arm.QAT_example.rife_vgf",
136+
)
137+
138+
def annotate(self, model: torch.fx.GraphModule) -> torch.fx.GraphModule:
139+
for node in model.graph.nodes:
140+
if (
141+
node.op != "call_function"
142+
or node.target not in self.targets
143+
or len(node.args) != 2
144+
):
145+
continue
146+
image = node.args[0]
147+
if not isinstance(image, torch.fx.Node):
148+
continue
149+
node.meta[Q_ANNOTATION_KEY] = QuantizationAnnotation(
150+
input_qspec_map={image: self.snorm_qspec},
151+
output_qspec=self.snorm_qspec,
152+
_annotated=True,
153+
)
154+
meta_custom = node.meta.get("custom", {})
155+
meta_custom[ArmAnnotationInfo.CUSTOM_META_KEY] = ArmAnnotationInfo(
156+
quantized=True
157+
)
158+
node.meta["custom"] = meta_custom
159+
self.report_accept([node])
160+
return model
161+
162+
def validate(self, model: torch.fx.GraphModule) -> None:
163+
return None
164+
165+
166+
def _register_pass_before(target_pass_type: type, pass_: ExportPass) -> None:
167+
existing_insertions = _registered_pass_insertions.get(target_pass_type)
168+
if existing_insertions is not None and any(
169+
isinstance(existing_pass, type(pass_))
170+
for existing_pass in existing_insertions.before_passes
171+
):
172+
return
173+
register_pass_insertions_before(target_pass_type, [pass_])
174+
175+
176+
def configure_rife_vgf(partitioner: VgfPartitioner | None = None) -> None:
177+
"""Enable the Practical-RIFE warp-downsample VGF extension."""
178+
_ensure_warp_downsample_ops_registered()
179+
if partitioner is not None:
180+
for target in _warp_downsample_targets():
181+
partitioner.register_custom_partition_op(target)
182+
183+
_register_pass_before(RewriteConvPass, RewriteWarpDownsampleToTosaCustomPass())
184+
185+
186+
def configure_rife_vgf_quantizer(quantizer) -> None:
187+
"""Quantize RIFE warp-downsample image input/output as int8 SNORM."""
188+
_ensure_warp_downsample_ops_registered()
189+
quantizer.add_quantizer(_WarpDownsampleQuantizer())
Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
# Copyright 2026 Arm Limited and/or its affiliates.
2+
#
3+
# This source code is licensed under the BSD-style license found in the
4+
# LICENSE file in the root directory of this source tree.
5+
6+
from .rewrite_warp_downsample_to_tosa_custom import ( # noqa: F401
7+
RewriteWarpDownsampleToTosaCustomPass,
8+
)

0 commit comments

Comments
 (0)