Skip to content

Commit 93821ca

Browse files
authored
[ET-VK][embedding] Enable embedding weight dedup with tied linear weights (pytorch#18360)
When embedding and output linear weights are tied (same underlying tensor, as in Llama 3.2 1B), they are quantized independently with opposite nibble packing conventions, preventing SHA256-based dedup in the NamedDataStore. This adds an `is_linear_weight` flag to `embedding_q4gsw` so the embedding op can consume weights packed in the linear convention, enabling dedup and saving ~125 MB. The implementation spans Python (detection + repacking), GLSL (packed block format reading), and C++ (shared prepacking with linear): - custom_ops_lib.py: Add `is_linear_weight` param to op signature and reference impl, swapping nibble extraction order when True - patterns/quantized_embedding.py: Add `_detect_tied_linear_weight()` that unpacks the embedding weight and compares against int8 linear weight placeholders. When matched, repack using linear convention and pass `is_linear_weight=True` to the op - embedding_q4gsw.glsl: Extract weight loading into `load_embedding_weights()` returning VEC4_T, with compile-time LINEAR_WEIGHT variant that reads from the block-interleaved format produced by `pack_q4_linear_weight` - embedding_q4gsw.yaml: Add 4 new shader variants for linear_weight x {buffer, texture2d} weight storage - EmbeddingQ4gsw.cpp: When `is_linear_weight`, call `prepack_quantized_linear_weight` (shared with linear op) instead of `prepack_standard`. Pass `embed_dim` as resize_arg since the prepacked weight tensor has a different shape - test_embedding_q4gsw.cpp: Add `is_linear_weight` to EmbeddingConfig, update reference function, add 6 new test cases Authored with Claude. Differential Revision: [D97430803](https://our.internmc.facebook.com/intern/diff/D97430803/) [ghstack-poisoned]
1 parent a903977 commit 93821ca

13 files changed

Lines changed: 1212 additions & 7 deletions

File tree

‎backends/vulkan/_passes/insert_prepack_nodes.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
from torch.export import ExportedProgram
1717

1818

19-
def insert_prepack_nodes(program: ExportedProgram) -> ExportedProgram:
19+
def insert_prepack_nodes(program: ExportedProgram) -> ExportedProgram: # noqa: C901
2020
"""
2121
Insert `et_vk.prepack` nodes for constant tensors in the graph. The prepack operator
2222
is responsible for transferring the tensor data, which is serialized with the model,
@@ -54,9 +54,13 @@ def insert_prepack_nodes(program: ExportedProgram) -> ExportedProgram:
5454
# Most prepacking ops have the primary input at arg 0, but
5555
# embedding is embedding(weight, indices, ...) where the
5656
# primary input (indices) is at arg 1.
57+
# For embedding_q4gsw, args are (weight, weight_scales, group_size,
58+
# indices) so the primary input (indices) is at arg 3.
5759
primary_arg_idx = 0
5860
if user.target == exir_ops.edge.aten.embedding.default:
5961
primary_arg_idx = 1
62+
elif user.target == exir_ops.edge.et_vk.embedding_q4gsw.default:
63+
primary_arg_idx = 3
6064

6165
if node in user.args and user.args.index(node) == primary_arg_idx:
6266
nodes_to_replace_input.append(user)

‎backends/vulkan/custom_ops_lib.py‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -879,6 +879,46 @@ def q8ta_relu_impl(
879879
lib.impl(name, q8ta_relu_impl, "CompositeExplicitAutograd")
880880
q8ta_relu_op = getattr(getattr(torch.ops, namespace), name)
881881

882+
########################
883+
## embedding_q4gsw ##
884+
########################
885+
886+
887+
def embedding_q4gsw_impl(
888+
weight: torch.Tensor,
889+
weight_scales: torch.Tensor,
890+
group_size: int,
891+
indices: torch.Tensor,
892+
is_linear_weight: bool = False,
893+
) -> torch.Tensor:
894+
# Unpack 4-bit values from packed uint8 tensor
895+
# Packing convention: packed_byte = (even_val + 8) << 4 | (odd_val + 8)
896+
high = (weight >> 4).to(torch.int8) - 8
897+
low = (weight & 0xF).to(torch.int8) - 8
898+
if is_linear_weight:
899+
unpacked = torch.stack([low, high], dim=-1).reshape(weight.shape[0], -1)
900+
else:
901+
unpacked = torch.stack([high, low], dim=-1).reshape(weight.shape[0], -1)
902+
# Dequantize using per-group scales
903+
num_groups = weight_scales.shape[1] if weight_scales.dim() > 1 else 1
904+
unpacked_groups = unpacked.reshape(weight.shape[0], num_groups, group_size)
905+
scales = (
906+
weight_scales.unsqueeze(-1)
907+
if weight_scales.dim() > 1
908+
else weight_scales.reshape(1, 1, 1)
909+
)
910+
dequantized = unpacked_groups.float() * scales.float()
911+
dequantized = dequantized.reshape(weight.shape[0], -1)
912+
return torch.nn.functional.embedding(indices, dequantized)
913+
914+
915+
name = "embedding_q4gsw"
916+
lib.define(
917+
f"{name}(Tensor weight, Tensor weight_scales, int group_size, Tensor indices, bool is_linear_weight = False) -> Tensor"
918+
)
919+
lib.impl(name, embedding_q4gsw_impl, "CompositeExplicitAutograd")
920+
embedding_q4gsw_op = getattr(getattr(torch.ops, namespace), name)
921+
882922
#############################
883923
## select_as_symint ##
884924
#############################

‎backends/vulkan/op_registry.py‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1436,6 +1436,38 @@ def check_embedding_weight_size(node: torch.fx.Node) -> bool:
14361436
)
14371437

14381438

1439+
# =============================================================================
1440+
# EmbeddingQ4gsw (Quantized Embedding)
1441+
# =============================================================================
1442+
1443+
1444+
@update_features(exir_ops.edge.quantized_decomposed.embedding_4bit.dtype)
1445+
def register_quantized_decomposed_embedding_4bit():
1446+
def check_embedding_4bit_weight_size(node: torch.fx.Node) -> bool:
1447+
weight = node.args[0]
1448+
if isinstance(weight, torch.fx.Node) and utils.is_tensor_node(weight):
1449+
numel = weight.meta["val"].numel()
1450+
if numel > utils.DEFAULT_BUFFER_LIMIT:
1451+
return False
1452+
return True
1453+
1454+
return OpFeatures(
1455+
inputs_storage=utils.ANY_BUFFER,
1456+
supports_prepacking=True,
1457+
supports_resize=True,
1458+
are_node_inputs_supported_fn=check_embedding_4bit_weight_size,
1459+
)
1460+
1461+
1462+
@update_features(exir_ops.edge.et_vk.embedding_q4gsw.default)
1463+
def register_embedding_q4gsw():
1464+
return OpFeatures(
1465+
inputs_storage=utils.CONTIGUOUS_ANY,
1466+
supports_prepacking=True,
1467+
supports_resize=True,
1468+
)
1469+
1470+
14391471
# =============================================================================
14401472
# BatchNorm.cpp
14411473
# =============================================================================

‎backends/vulkan/patterns/BUCK‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ fbcode_target(_kind = runtime.python_library,
1010
"__init__.py",
1111
"pattern_registry.py",
1212
"rope.py",
13+
"quantized_embedding.py",
1314
"quantized_linear.py",
1415
"quantized_convolution.py",
1516
"quantized_binary.py",

‎backends/vulkan/patterns/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@
1010

1111
import executorch.backends.vulkan.patterns.quantized_convolution # noqa
1212

13+
import executorch.backends.vulkan.patterns.quantized_embedding # noqa
14+
1315
import executorch.backends.vulkan.patterns.quantized_linear # noqa
1416

1517
import executorch.backends.vulkan.patterns.quantized_unary # noqa
Lines changed: 177 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,177 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
from typing import Optional
8+
9+
import executorch.backends.vulkan.utils as utils
10+
import torch
11+
from executorch.backends.transforms.utils import get_param_tensor
12+
from executorch.backends.vulkan.patterns.pattern_registry import (
13+
PatternMatch,
14+
register_pattern_detector,
15+
register_pattern_replacement,
16+
)
17+
from executorch.exir import ExportedProgram
18+
from executorch.exir.dialects._ops import ops as exir_ops
19+
20+
21+
class QuantizedEmbeddingMatch(PatternMatch):
22+
def __init__(self, node: torch.fx.Node) -> None:
23+
self.anchor_node = node
24+
self.match_found = False
25+
self.all_nodes = [node]
26+
27+
# quantized_decomposed.embedding_4bit.dtype args:
28+
# (weight, weight_scales, weight_zero_points, quant_min, quant_max,
29+
# indices, *, dtype)
30+
self.weight_node = node.args[0]
31+
self.scales_node = node.args[1]
32+
self.indices_node = node.args[5]
33+
34+
# Validate quantization parameters match our shader's assumptions.
35+
# The shader hardcodes the 4-bit signed offset (subtract 8), which
36+
# corresponds to quant_min=-8, quant_max=7, zero_points=0.
37+
quant_min = node.args[3]
38+
quant_max = node.args[4]
39+
if quant_min != -8 or quant_max != 7:
40+
self.match_found = False
41+
return
42+
43+
# weight_zero_points (args[2]) should be None or all-zeros
44+
weight_zp_node = node.args[2]
45+
if weight_zp_node is not None:
46+
# If it's a constant tensor, verify it's all zeros
47+
if (
48+
isinstance(weight_zp_node, torch.fx.Node)
49+
and "val" in weight_zp_node.meta
50+
):
51+
zp_val = weight_zp_node.meta["val"]
52+
if isinstance(zp_val, torch.Tensor) and not torch.all(zp_val == 0):
53+
self.match_found = False
54+
return
55+
56+
# Trace weight to its placeholder
57+
const_node, arg_chain = utils.trace_args_until_placeholder(self.weight_node)
58+
if const_node is not None:
59+
self.weight_node = const_node
60+
self.all_nodes.extend(arg_chain)
61+
62+
# Trace scales to their placeholder
63+
scales_node, arg_chain = utils.trace_args_until_placeholder(self.scales_node)
64+
if scales_node is not None:
65+
self.scales_node = scales_node
66+
self.all_nodes.extend(arg_chain)
67+
68+
self.match_found = True
69+
70+
71+
embedding_4bit_target = exir_ops.edge.quantized_decomposed.embedding_4bit.dtype
72+
73+
74+
def _detect_tied_linear_weight(
75+
ep: ExportedProgram,
76+
weight_node: torch.fx.Node,
77+
weight_tensor: torch.Tensor,
78+
) -> bool:
79+
"""Check if this embedding weight is tied to a linear weight.
80+
81+
The embedding weight is packed uint8 [vocab_size, embed_dim/2]. The linear
82+
output weight may be stored as unpacked int8 [vocab_size, embed_dim]. If we
83+
find a placeholder whose int8 values match our unpacked embedding values,
84+
the weights are tied and we should use the linear packing to enable dedup.
85+
"""
86+
vocab_size = weight_tensor.shape[0]
87+
embed_dim = weight_tensor.shape[1] * 2
88+
89+
# Unpack embedding weight using embedding convention (high nibble first)
90+
emb_high = (weight_tensor >> 4).to(torch.int8) - 8
91+
emb_low = (weight_tensor & 0xF).to(torch.int8) - 8
92+
emb_unpacked = torch.stack([emb_high, emb_low], dim=-1).reshape(
93+
vocab_size, embed_dim
94+
)
95+
96+
for node in ep.graph_module.graph.nodes:
97+
if node.op != "placeholder" or node == weight_node:
98+
continue
99+
100+
try:
101+
candidate = get_param_tensor(ep, node)
102+
except RuntimeError:
103+
continue
104+
if candidate is None:
105+
continue
106+
if candidate.shape != (vocab_size, embed_dim) or candidate.dtype != torch.int8:
107+
continue
108+
109+
if torch.equal(emb_unpacked, candidate):
110+
return True
111+
112+
return False
113+
114+
115+
@register_pattern_detector("quantized_embedding")
116+
def find_quantized_embedding_patterns(
117+
node: torch.fx.Node,
118+
) -> Optional[QuantizedEmbeddingMatch]:
119+
if node.target != embedding_4bit_target:
120+
return None
121+
122+
matched_pattern = QuantizedEmbeddingMatch(node)
123+
if matched_pattern.match_found:
124+
return matched_pattern
125+
return None
126+
127+
128+
@register_pattern_replacement("quantized_embedding")
129+
def replace_quantized_embedding_patterns(
130+
ep: ExportedProgram,
131+
graph_module: torch.fx.GraphModule,
132+
match: QuantizedEmbeddingMatch,
133+
):
134+
weight_tensor = get_param_tensor(ep, match.weight_node)
135+
assert weight_tensor is not None
136+
137+
scales_tensor = get_param_tensor(ep, match.scales_node)
138+
assert scales_tensor is not None
139+
140+
is_linear_weight = _detect_tied_linear_weight(ep, match.weight_node, weight_tensor)
141+
142+
if is_linear_weight:
143+
# Repack using linear convention (low nibble = even, high nibble = odd)
144+
vocab_size = weight_tensor.shape[0]
145+
high = (weight_tensor >> 4).to(torch.int8) - 8
146+
low = (weight_tensor & 0xF).to(torch.int8) - 8
147+
unpacked = torch.stack([high, low], dim=-1).reshape(vocab_size, -1)
148+
repacked = unpacked.to(torch.uint8) + 8
149+
weight_tensor = repacked[:, 1::2] << 4 | repacked[:, ::2]
150+
# Update the state dict with repacked tensor
151+
original_weight = get_param_tensor(ep, match.weight_node)
152+
if original_weight is not None:
153+
for key, value in ep.state_dict.items():
154+
if value.data_ptr() == original_weight.data_ptr():
155+
ep.state_dict[key] = weight_tensor
156+
break
157+
158+
# Compute group_size from weight and scales shapes
159+
embed_dim = weight_tensor.shape[1] * 2 # packed, 2 values per byte
160+
groups_per_row = scales_tensor.shape[1] if scales_tensor.ndim > 1 else 1
161+
group_size = embed_dim // groups_per_row
162+
163+
with graph_module.graph.inserting_before(match.anchor_node):
164+
embedding_q4gsw_node = graph_module.graph.create_node(
165+
"call_function",
166+
exir_ops.edge.et_vk.embedding_q4gsw.default,
167+
args=(
168+
match.weight_node,
169+
match.scales_node,
170+
group_size,
171+
match.indices_node,
172+
is_linear_weight,
173+
),
174+
)
175+
176+
embedding_q4gsw_node.meta["val"] = match.anchor_node.meta["val"]
177+
match.anchor_node.replace_all_uses_with(embedding_q4gsw_node)

0 commit comments

Comments
 (0)