Skip to content

Commit 2fb0334

Browse files
authored
Add FuseGQAWithSDPAPass (pytorch#22843)
Summary: Fold the HuggingFace repeat_kv expansion (unsqueeze -> expand -> clone -> view on K and V) into scaled_dot_product_attention's enable_gqa=True, feeding the un-repeated [B, n_kv, T, D] tensors directly. Removes the materialized KV copies from the delegate; result is unchanged since enable_gqa broadcasts kv heads in the same order. Reviewed By: SS-JIA Differential Revision: D111753300 Pull Request resolved: pytorch#22843
1 parent f5e1369 commit 2fb0334

7 files changed

Lines changed: 925 additions & 2 deletions

File tree

‎backends/native/BUCK‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,8 @@ fbcode_target(
3232
"fbsource//third-party/pypi/flatbuffers:flatbuffers",
3333
"//caffe2:torch",
3434
"//executorch/backends/transforms:collapse_view_copy",
35+
"//executorch/backends/transforms:fuse_gqa_with_sdpa",
36+
"//executorch/backends/transforms:normalize_sdpa_input_rank",
3537
"//executorch/exir:lib",
3638
"//executorch/exir/_serialize:lib",
3739
"//executorch/exir/backend:backend_details",

‎backends/native/passes/__init__.py‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,10 @@
3939
)
4040

4141
from executorch.backends.transforms.collapse_view_copy import CollapseViewCopyPass
42+
from executorch.backends.transforms.fuse_gqa_with_sdpa import FuseGQAWithSDPAPass
43+
from executorch.backends.transforms.normalize_sdpa_input_rank import (
44+
NormalizeSDPAInputRankPass,
45+
)
4246

4347
from executorch.exir.pass_base import ExportedProgramPassBase, ExportPass
4448
from executorch.exir.passes.cse_pass import CSEPass
@@ -47,19 +51,24 @@
4751
"backend_inplace_aten_variants",
4852
"BACKEND_INPLACE_OPS",
4953
"CollapseViewCopyPass",
54+
"FuseGQAWithSDPAPass",
5055
"get_default_passes",
5156
"NativeReinplacePass",
57+
"NormalizeSDPAInputRankPass",
5258
"ReplaceCopyWithAliasPass",
5359
]
5460

5561

5662
def get_default_passes() -> List[Union[ExportPass, ExportedProgramPassBase]]:
5763
"""Passes enabled by default for the native backend.
5864
59-
view_copy collapsing and CSE run first to settle the graph; reinplace runs
60-
last, rewriting functional ops into their in-place edge forms.
65+
GQA fusion runs before SDPA rank normalization, then view_copy collapsing
66+
and CSE settle the graph. Reinplace runs last, rewriting functional ops into
67+
their in-place edge forms.
6168
"""
6269
return [
70+
FuseGQAWithSDPAPass(),
71+
NormalizeSDPAInputRankPass(),
6372
CollapseViewCopyPass(),
6473
CSEPass(),
6574
NativeReinplacePass(),
Lines changed: 190 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,190 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
"""Fold a HuggingFace repeat_kv expansion into scaled_dot_product_attention."""
8+
9+
import torch
10+
11+
from executorch.exir.pass_base import ExportPass, PassResult
12+
from torch.fx import GraphModule, Node
13+
from torch.fx.experimental.symbolic_shapes import statically_known_true
14+
from torch.fx.passes.shape_prop import _extract_tensor_metadata
15+
16+
17+
def _resolve_aten(target: object) -> "torch._ops.OpOverload | None":
18+
"""Return the underlying aten OpOverload for an edge or plain aten target."""
19+
inner = getattr(target, "_op", None)
20+
if isinstance(inner, torch._ops.OpOverload):
21+
return inner
22+
if isinstance(target, torch._ops.OpOverload):
23+
return target
24+
return None
25+
26+
27+
def _single_user(node: object) -> bool:
28+
return isinstance(node, Node) and len(node.users) == 1
29+
30+
31+
def _matches_shape(node: Node, shape: list) -> bool:
32+
value = node.meta.get("val")
33+
return (
34+
isinstance(value, torch.Tensor)
35+
and value.dim() == len(shape)
36+
and all(statically_known_true(a == b) for a, b in zip(value.shape, shape))
37+
)
38+
39+
40+
def _unwrap_rank_normalization(node: object) -> Node | None:
41+
if (
42+
not isinstance(node, Node)
43+
or _resolve_aten(node.target) is not torch.ops.aten.unsqueeze_copy.default
44+
or not node.args
45+
):
46+
return None
47+
dim = node.args[1] if len(node.args) > 1 else node.kwargs.get("dim")
48+
inner = node.args[0]
49+
value = inner.meta.get("val") if isinstance(inner, Node) else None
50+
if (
51+
dim == 0
52+
and isinstance(value, torch.Tensor)
53+
and value.dim() == 3
54+
and _matches_shape(node, [1, *value.shape])
55+
):
56+
return inner
57+
return None
58+
59+
60+
def _unwrap_repeat_kv(node: object, rank: int, n_heads: int) -> Node | None:
61+
"""Match head repetition for [H, S, D] or [B, H, S, D] KV tensors."""
62+
if _resolve_aten(
63+
getattr(node, "target", None)
64+
) is not torch.ops.aten.view_copy.default or not _single_user(node):
65+
return None
66+
inner = node.args[0] if node.args else node.kwargs.get("self")
67+
clone = None
68+
if _resolve_aten(getattr(inner, "target", None)) is torch.ops.aten.clone.default:
69+
if not _single_user(inner):
70+
return None
71+
clone = inner
72+
inner = inner.args[0] if inner.args else inner.kwargs.get("self")
73+
74+
if _resolve_aten(
75+
getattr(inner, "target", None)
76+
) is not torch.ops.aten.expand_copy.default or not _single_user(inner):
77+
return None
78+
expand = inner
79+
unsqueeze = expand.args[0] if expand.args else expand.kwargs.get("self")
80+
if _resolve_aten(
81+
getattr(unsqueeze, "target", None)
82+
) is not torch.ops.aten.unsqueeze_copy.default or not _single_user(unsqueeze):
83+
return None
84+
base = unsqueeze.args[0] if unsqueeze.args else unsqueeze.kwargs.get("self")
85+
value = base.meta.get("val") if isinstance(base, Node) else None
86+
if not isinstance(value, torch.Tensor) or value.dim() != rank:
87+
return None
88+
89+
head_axis = rank - 3
90+
repeat_axis = head_axis + 1
91+
dim = unsqueeze.args[1] if len(unsqueeze.args) > 1 else unsqueeze.kwargs.get("dim")
92+
if dim not in (repeat_axis, -3):
93+
return None
94+
n_kv = value.shape[head_axis]
95+
if not isinstance(n_kv, int) or n_kv <= 0 or n_heads <= n_kv or n_heads % n_kv:
96+
return None
97+
98+
shape = list(value.shape)
99+
inserted = shape[:repeat_axis] + [1] + shape[repeat_axis:]
100+
expanded = shape[:repeat_axis] + [n_heads // n_kv] + shape[repeat_axis:]
101+
folded = shape[:head_axis] + [n_heads] + shape[head_axis + 1 :]
102+
# Prove that only the head-repeat dimension changed, not sequence or batch.
103+
if not (
104+
_matches_shape(unsqueeze, inserted)
105+
and _matches_shape(expand, expanded)
106+
and (clone is None or _matches_shape(clone, expanded))
107+
and _matches_shape(node, folded)
108+
):
109+
return None
110+
return base
111+
112+
113+
class FuseGQAWithSDPAPass(ExportPass):
114+
"""Fold a HuggingFace repeat_kv expansion into scaled_dot_product_attention.
115+
116+
Grouped-query attention repeats each KV head n_rep = n_heads // n_kv times so
117+
K/V match the query head count before SDPA. aten SDPA can do that broadcast
118+
itself via ``enable_gqa=True``, so when both K and V feed SDPA through a
119+
repeat_kv chain we drop the expansion and pass the un-repeated
120+
rank-3 ``[n_kv, T, D]`` or rank-4 ``[B, n_kv, T, D]`` tensors directly,
121+
setting ``enable_gqa=True``. This removes
122+
the materialized KV copies (expand + clone) from the graph. ``enable_gqa``
123+
broadcasts kv head i to query heads ``[i*n_rep, (i+1)*n_rep)``, matching
124+
repeat_kv's head ordering, so the result is unchanged. Leading singleton
125+
wrappers from SDPA rank normalization are preserved around the base tensors.
126+
"""
127+
128+
def call(self, graph_module: GraphModule) -> PassResult:
129+
graph = graph_module.graph
130+
sdpa = torch.ops.aten.scaled_dot_product_attention.default
131+
modified = False
132+
133+
for node in list(graph.nodes):
134+
if node.op != "call_function" or _resolve_aten(node.target) is not sdpa:
135+
continue
136+
if node.kwargs.get("enable_gqa"):
137+
continue
138+
q, k, v = (
139+
node.args[i] if i < len(node.args) else node.kwargs.get(name)
140+
for i, name in enumerate(("query", "key", "value"))
141+
)
142+
q_val = q.meta.get("val") if isinstance(q, Node) else None
143+
if not isinstance(q_val, torch.Tensor) or q_val.dim() not in (3, 4):
144+
continue
145+
n_heads = q_val.shape[-3]
146+
if not isinstance(n_heads, int):
147+
continue
148+
k_inner = _unwrap_rank_normalization(k)
149+
v_inner = _unwrap_rank_normalization(v)
150+
normalized = (
151+
q_val.dim() == 4 and k_inner is not None and v_inner is not None
152+
)
153+
rank = 3 if normalized else q_val.dim()
154+
k_base = _unwrap_repeat_kv(k_inner if normalized else k, rank, n_heads)
155+
v_base = _unwrap_repeat_kv(v_inner if normalized else v, rank, n_heads)
156+
if k_base is None or v_base is None:
157+
continue
158+
if k_base.meta["val"].shape[-3] != v_base.meta["val"].shape[-3]:
159+
continue
160+
161+
args, kwargs = list(node.args), dict(node.kwargs)
162+
for i, name, base, wrapper in (
163+
(1, "key", k_base, k),
164+
(2, "value", v_base, v),
165+
):
166+
if normalized:
167+
# Keep any other consumers of the original wrapper unchanged.
168+
with graph.inserting_before(node):
169+
padded = graph.call_function(wrapper.target, (base, 0))
170+
value = torch.ops.aten.unsqueeze_copy.default(base.meta["val"], 0)
171+
padded.meta = {
172+
**wrapper.meta,
173+
"val": value,
174+
"tensor_meta": _extract_tensor_metadata(value),
175+
}
176+
base = padded
177+
if i < len(args):
178+
args[i] = base
179+
else:
180+
kwargs[name] = base
181+
node.args = tuple(args)
182+
node.kwargs = {**kwargs, "enable_gqa": True}
183+
modified = True
184+
185+
if modified:
186+
graph.eliminate_dead_code()
187+
graph.lint()
188+
graph_module.recompile()
189+
190+
return PassResult(graph_module, modified)
Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
import torch
8+
9+
from executorch.exir.dialects._ops import ops as exir_ops
10+
from executorch.exir.pass_base import ExportPass, PassResult, ProxyValue
11+
12+
13+
class NormalizeSDPAInputRankPass(ExportPass):
14+
"""Pad rank-2/3 SDPA inputs to rank 4 and restore the original output rank.
15+
16+
Leading singleton axes preserve the head axis of [H, L, D] inputs. Rank-2
17+
[L, D] inputs gain both a batch and a head axis. Masks are also padded to
18+
rank 4 for fused-kernel compatibility. SDPA options are unchanged, and only
19+
the inserted output axes are squeezed.
20+
"""
21+
22+
def call_operator(self, op, args, kwargs, meta):
23+
if op not in (
24+
torch.ops.aten.scaled_dot_product_attention.default,
25+
exir_ops.edge.aten.scaled_dot_product_attention.default,
26+
):
27+
return super().call_operator(op, args, kwargs, meta)
28+
29+
inputs = [
30+
args[i] if i < len(args) else kwargs.get(name)
31+
for i, name in enumerate(("query", "key", "value"))
32+
]
33+
if not all(isinstance(x, ProxyValue) and x.is_tensor() for x in inputs):
34+
return super().call_operator(op, args, kwargs, meta)
35+
rank = inputs[0].to_tensor().dim()
36+
output = meta.data.get("val")
37+
if (
38+
rank not in (2, 3)
39+
or any(x.to_tensor().dim() != rank for x in inputs)
40+
or not isinstance(output, torch.Tensor)
41+
or output.dim() != rank
42+
):
43+
return super().call_operator(op, args, kwargs, meta)
44+
45+
aten = (
46+
torch.ops.aten
47+
if op is torch.ops.aten.scaled_dot_product_attention.default
48+
else exir_ops.edge.aten
49+
)
50+
mask = args[3] if len(args) > 3 else kwargs.get("attn_mask")
51+
if isinstance(mask, ProxyValue) and mask.is_tensor():
52+
inputs.append(mask)
53+
new_args, new_kwargs = list(args), dict(kwargs)
54+
names = ("query", "key", "value", "attn_mask")
55+
for i, (name, value) in enumerate(zip(names, inputs)):
56+
for _ in range(4 - value.to_tensor().dim()):
57+
value = super().call_operator(
58+
aten.unsqueeze_copy.default, (value, 0), {}, meta
59+
)
60+
if i < len(new_args):
61+
new_args[i] = value
62+
else:
63+
new_kwargs[name] = value
64+
result = super().call_operator(op, tuple(new_args), new_kwargs, meta)
65+
self._modified = True
66+
return super().call_operator(
67+
aten.squeeze_copy.dims, (result, list(range(4 - rank))), {}, meta
68+
)
69+
70+
def call(self, graph_module: torch.fx.GraphModule) -> PassResult:
71+
self._modified = False
72+
result = super().call(graph_module)
73+
return PassResult(result.graph_module, self._modified)

‎backends/transforms/targets.bzl‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,52 @@ def define_common_targets():
4343
],
4444
)
4545

46+
runtime.python_library(
47+
name = "fuse_gqa_with_sdpa",
48+
srcs = ["fuse_gqa_with_sdpa.py"],
49+
visibility = [
50+
"//executorch/backends/...",
51+
],
52+
deps = [
53+
"//caffe2:torch",
54+
"//executorch/exir:pass_base",
55+
],
56+
)
57+
58+
runtime.python_test(
59+
name = "test_fuse_gqa_with_sdpa",
60+
srcs = ["test/test_fuse_gqa_with_sdpa.py"],
61+
deps = [
62+
":fuse_gqa_with_sdpa",
63+
":normalize_sdpa_input_rank",
64+
"//caffe2:torch",
65+
"//executorch/exir:lib",
66+
"//executorch/exir/dialects:lib",
67+
],
68+
)
69+
70+
runtime.python_library(
71+
name = "normalize_sdpa_input_rank",
72+
srcs = ["normalize_sdpa_input_rank.py"],
73+
visibility = ["//executorch/backends/..."],
74+
deps = [
75+
"//caffe2:torch",
76+
"//executorch/exir:pass_base",
77+
"//executorch/exir/dialects:lib",
78+
],
79+
)
80+
81+
runtime.python_test(
82+
name = "test_normalize_sdpa_input_rank",
83+
srcs = ["test/test_normalize_sdpa_input_rank.py"],
84+
deps = [
85+
":normalize_sdpa_input_rank",
86+
"//caffe2:torch",
87+
"//executorch/exir:lib",
88+
"//executorch/exir/dialects:lib",
89+
],
90+
)
91+
4692
runtime.python_library(
4793
name = "fuse_batch_norm_with_conv",
4894
srcs = ["fuse_batch_norm_with_conv.py"],

0 commit comments

Comments
 (0)