|
| 1 | +# Copyright (c) Meta Platforms, Inc. and affiliates. |
| 2 | +# All rights reserved. |
| 3 | +# |
| 4 | +# This source code is licensed under the BSD-style license found in the |
| 5 | +# LICENSE file in the root directory of this source tree. |
| 6 | + |
| 7 | +"""Fold a HuggingFace repeat_kv expansion into scaled_dot_product_attention.""" |
| 8 | + |
| 9 | +import torch |
| 10 | + |
| 11 | +from executorch.exir.pass_base import ExportPass, PassResult |
| 12 | +from torch.fx import GraphModule, Node |
| 13 | +from torch.fx.experimental.symbolic_shapes import statically_known_true |
| 14 | +from torch.fx.passes.shape_prop import _extract_tensor_metadata |
| 15 | + |
| 16 | + |
| 17 | +def _resolve_aten(target: object) -> "torch._ops.OpOverload | None": |
| 18 | + """Return the underlying aten OpOverload for an edge or plain aten target.""" |
| 19 | + inner = getattr(target, "_op", None) |
| 20 | + if isinstance(inner, torch._ops.OpOverload): |
| 21 | + return inner |
| 22 | + if isinstance(target, torch._ops.OpOverload): |
| 23 | + return target |
| 24 | + return None |
| 25 | + |
| 26 | + |
| 27 | +def _single_user(node: object) -> bool: |
| 28 | + return isinstance(node, Node) and len(node.users) == 1 |
| 29 | + |
| 30 | + |
| 31 | +def _matches_shape(node: Node, shape: list) -> bool: |
| 32 | + value = node.meta.get("val") |
| 33 | + return ( |
| 34 | + isinstance(value, torch.Tensor) |
| 35 | + and value.dim() == len(shape) |
| 36 | + and all(statically_known_true(a == b) for a, b in zip(value.shape, shape)) |
| 37 | + ) |
| 38 | + |
| 39 | + |
| 40 | +def _unwrap_rank_normalization(node: object) -> Node | None: |
| 41 | + if ( |
| 42 | + not isinstance(node, Node) |
| 43 | + or _resolve_aten(node.target) is not torch.ops.aten.unsqueeze_copy.default |
| 44 | + or not node.args |
| 45 | + ): |
| 46 | + return None |
| 47 | + dim = node.args[1] if len(node.args) > 1 else node.kwargs.get("dim") |
| 48 | + inner = node.args[0] |
| 49 | + value = inner.meta.get("val") if isinstance(inner, Node) else None |
| 50 | + if ( |
| 51 | + dim == 0 |
| 52 | + and isinstance(value, torch.Tensor) |
| 53 | + and value.dim() == 3 |
| 54 | + and _matches_shape(node, [1, *value.shape]) |
| 55 | + ): |
| 56 | + return inner |
| 57 | + return None |
| 58 | + |
| 59 | + |
| 60 | +def _unwrap_repeat_kv(node: object, rank: int, n_heads: int) -> Node | None: |
| 61 | + """Match head repetition for [H, S, D] or [B, H, S, D] KV tensors.""" |
| 62 | + if _resolve_aten( |
| 63 | + getattr(node, "target", None) |
| 64 | + ) is not torch.ops.aten.view_copy.default or not _single_user(node): |
| 65 | + return None |
| 66 | + inner = node.args[0] if node.args else node.kwargs.get("self") |
| 67 | + clone = None |
| 68 | + if _resolve_aten(getattr(inner, "target", None)) is torch.ops.aten.clone.default: |
| 69 | + if not _single_user(inner): |
| 70 | + return None |
| 71 | + clone = inner |
| 72 | + inner = inner.args[0] if inner.args else inner.kwargs.get("self") |
| 73 | + |
| 74 | + if _resolve_aten( |
| 75 | + getattr(inner, "target", None) |
| 76 | + ) is not torch.ops.aten.expand_copy.default or not _single_user(inner): |
| 77 | + return None |
| 78 | + expand = inner |
| 79 | + unsqueeze = expand.args[0] if expand.args else expand.kwargs.get("self") |
| 80 | + if _resolve_aten( |
| 81 | + getattr(unsqueeze, "target", None) |
| 82 | + ) is not torch.ops.aten.unsqueeze_copy.default or not _single_user(unsqueeze): |
| 83 | + return None |
| 84 | + base = unsqueeze.args[0] if unsqueeze.args else unsqueeze.kwargs.get("self") |
| 85 | + value = base.meta.get("val") if isinstance(base, Node) else None |
| 86 | + if not isinstance(value, torch.Tensor) or value.dim() != rank: |
| 87 | + return None |
| 88 | + |
| 89 | + head_axis = rank - 3 |
| 90 | + repeat_axis = head_axis + 1 |
| 91 | + dim = unsqueeze.args[1] if len(unsqueeze.args) > 1 else unsqueeze.kwargs.get("dim") |
| 92 | + if dim not in (repeat_axis, -3): |
| 93 | + return None |
| 94 | + n_kv = value.shape[head_axis] |
| 95 | + if not isinstance(n_kv, int) or n_kv <= 0 or n_heads <= n_kv or n_heads % n_kv: |
| 96 | + return None |
| 97 | + |
| 98 | + shape = list(value.shape) |
| 99 | + inserted = shape[:repeat_axis] + [1] + shape[repeat_axis:] |
| 100 | + expanded = shape[:repeat_axis] + [n_heads // n_kv] + shape[repeat_axis:] |
| 101 | + folded = shape[:head_axis] + [n_heads] + shape[head_axis + 1 :] |
| 102 | + # Prove that only the head-repeat dimension changed, not sequence or batch. |
| 103 | + if not ( |
| 104 | + _matches_shape(unsqueeze, inserted) |
| 105 | + and _matches_shape(expand, expanded) |
| 106 | + and (clone is None or _matches_shape(clone, expanded)) |
| 107 | + and _matches_shape(node, folded) |
| 108 | + ): |
| 109 | + return None |
| 110 | + return base |
| 111 | + |
| 112 | + |
| 113 | +class FuseGQAWithSDPAPass(ExportPass): |
| 114 | + """Fold a HuggingFace repeat_kv expansion into scaled_dot_product_attention. |
| 115 | +
|
| 116 | + Grouped-query attention repeats each KV head n_rep = n_heads // n_kv times so |
| 117 | + K/V match the query head count before SDPA. aten SDPA can do that broadcast |
| 118 | + itself via ``enable_gqa=True``, so when both K and V feed SDPA through a |
| 119 | + repeat_kv chain we drop the expansion and pass the un-repeated |
| 120 | + rank-3 ``[n_kv, T, D]`` or rank-4 ``[B, n_kv, T, D]`` tensors directly, |
| 121 | + setting ``enable_gqa=True``. This removes |
| 122 | + the materialized KV copies (expand + clone) from the graph. ``enable_gqa`` |
| 123 | + broadcasts kv head i to query heads ``[i*n_rep, (i+1)*n_rep)``, matching |
| 124 | + repeat_kv's head ordering, so the result is unchanged. Leading singleton |
| 125 | + wrappers from SDPA rank normalization are preserved around the base tensors. |
| 126 | + """ |
| 127 | + |
| 128 | + def call(self, graph_module: GraphModule) -> PassResult: |
| 129 | + graph = graph_module.graph |
| 130 | + sdpa = torch.ops.aten.scaled_dot_product_attention.default |
| 131 | + modified = False |
| 132 | + |
| 133 | + for node in list(graph.nodes): |
| 134 | + if node.op != "call_function" or _resolve_aten(node.target) is not sdpa: |
| 135 | + continue |
| 136 | + if node.kwargs.get("enable_gqa"): |
| 137 | + continue |
| 138 | + q, k, v = ( |
| 139 | + node.args[i] if i < len(node.args) else node.kwargs.get(name) |
| 140 | + for i, name in enumerate(("query", "key", "value")) |
| 141 | + ) |
| 142 | + q_val = q.meta.get("val") if isinstance(q, Node) else None |
| 143 | + if not isinstance(q_val, torch.Tensor) or q_val.dim() not in (3, 4): |
| 144 | + continue |
| 145 | + n_heads = q_val.shape[-3] |
| 146 | + if not isinstance(n_heads, int): |
| 147 | + continue |
| 148 | + k_inner = _unwrap_rank_normalization(k) |
| 149 | + v_inner = _unwrap_rank_normalization(v) |
| 150 | + normalized = ( |
| 151 | + q_val.dim() == 4 and k_inner is not None and v_inner is not None |
| 152 | + ) |
| 153 | + rank = 3 if normalized else q_val.dim() |
| 154 | + k_base = _unwrap_repeat_kv(k_inner if normalized else k, rank, n_heads) |
| 155 | + v_base = _unwrap_repeat_kv(v_inner if normalized else v, rank, n_heads) |
| 156 | + if k_base is None or v_base is None: |
| 157 | + continue |
| 158 | + if k_base.meta["val"].shape[-3] != v_base.meta["val"].shape[-3]: |
| 159 | + continue |
| 160 | + |
| 161 | + args, kwargs = list(node.args), dict(node.kwargs) |
| 162 | + for i, name, base, wrapper in ( |
| 163 | + (1, "key", k_base, k), |
| 164 | + (2, "value", v_base, v), |
| 165 | + ): |
| 166 | + if normalized: |
| 167 | + # Keep any other consumers of the original wrapper unchanged. |
| 168 | + with graph.inserting_before(node): |
| 169 | + padded = graph.call_function(wrapper.target, (base, 0)) |
| 170 | + value = torch.ops.aten.unsqueeze_copy.default(base.meta["val"], 0) |
| 171 | + padded.meta = { |
| 172 | + **wrapper.meta, |
| 173 | + "val": value, |
| 174 | + "tensor_meta": _extract_tensor_metadata(value), |
| 175 | + } |
| 176 | + base = padded |
| 177 | + if i < len(args): |
| 178 | + args[i] = base |
| 179 | + else: |
| 180 | + kwargs[name] = base |
| 181 | + node.args = tuple(args) |
| 182 | + node.kwargs = {**kwargs, "enable_gqa": True} |
| 183 | + modified = True |
| 184 | + |
| 185 | + if modified: |
| 186 | + graph.eliminate_dead_code() |
| 187 | + graph.lint() |
| 188 | + graph_module.recompile() |
| 189 | + |
| 190 | + return PassResult(graph_module, modified) |
0 commit comments