Skip to content

Commit 4891009

Browse files
authored
[ET-VK] Synthesize the layer_norm affine parameters that are absent (pytorch#22506)
Fixes pytorch#22504. `add_native_layer_norm_node` already synthesizes a zero bias for `nn.LayerNorm(bias=False)`, but still throws on a missing weight. `F.layer_norm` called with no affine parameters at all is an ordinary call, used whenever a module applies its own scale and shift afterwards, and it currently makes the model abort at prepack. This synthesizes a unit weight the same way, sizing it from `normalized_shape` when there is no weight tensor to take a shape from. A unit weight and a zero bias reproduce `out = (x - mean) * rstd` exactly. The two paths now share one helper. Verified on device: kokoro's duration predictor, whose `AdaLayerNorm` normalizes with no affine parameters, aborts on `main` and runs to completion with this change on an Adreno 840. `cases.py` gains three `native_layer_norm` cases covering weight-only, bias-only and neither. cc @SS-JIA @manuelcandales @digantdesai @cbilgin
1 parent bba7552 commit 4891009

2 files changed

Lines changed: 76 additions & 5 deletions

File tree

‎backends/vulkan/runtime/graph/ops/impl/NativeLayerNorm.cpp‎

Lines changed: 69 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,9 @@
1616

1717
#include <executorch/backends/vulkan/runtime/graph/ops/utils/ShaderNameUtils.h>
1818

19+
#include <algorithm>
20+
#include <cstdint>
21+
1922
namespace vkcompute {
2023

2124
std::vector<int64_t> calc_out_mean_sizes(
@@ -64,6 +67,50 @@ GlobalWorkGrid layer_norm_buffer_gwg(
6467
{1u, num_rows, 1u}, kTiledWorkGrid, LocalWorkGroup(64u, 1u, 1u));
6568
}
6669

70+
// Builds a TensorRef of `sizes` filled with ones, or left at zero, for
71+
// synthesizing a layer_norm affine parameter the caller did not supply.
72+
// Ownership of the buffer passes to the FreeableBuffer, then to the TensorRef,
73+
// then to the graph.
74+
//
75+
// Only those two values are ever needed, which is what lets the fp16 case be a
76+
// single bit pattern. Writing it out keeps this file clear of a float16 type;
77+
// the backend already avoids that dependency elsewhere (see the hand-rolled
78+
// conversions in api/containers/StagingBuffer.cpp).
79+
ValueRef constant_affine_tensorref(
80+
ComputeGraph& graph,
81+
const std::vector<int64_t>& sizes,
82+
const vkapi::ScalarType dtype,
83+
const bool ones) {
84+
VK_CHECK_COND(
85+
dtype == vkapi::kFloat || dtype == vkapi::kHalf,
86+
"native_layer_norm cannot synthesize an affine parameter of dtype ",
87+
static_cast<int>(dtype));
88+
89+
int64_t numel = 1;
90+
for (int64_t d : sizes) {
91+
numel *= d;
92+
}
93+
const size_t total_bytes =
94+
static_cast<size_t>(numel) * vkapi::element_size(dtype);
95+
// Zero bits are zero in both fp32 and fp16, so a zero parameter needs no
96+
// fill at all.
97+
auto* data = new uint8_t[total_bytes]();
98+
if (ones) {
99+
if (dtype == vkapi::kFloat) {
100+
std::fill_n(reinterpret_cast<float*>(data), numel, 1.0f);
101+
} else {
102+
// 1.0 in IEEE binary16: sign 0, exponent 0b01111, mantissa 0.
103+
constexpr uint16_t kHalfOne = 0x3C00;
104+
std::fill_n(reinterpret_cast<uint16_t*>(data), numel, kHalfOne);
105+
}
106+
}
107+
executorch::runtime::FreeableBuffer buffer(
108+
data, total_bytes, [](void* /*ctx*/, void* ptr, size_t /*size*/) {
109+
delete[] static_cast<uint8_t*>(ptr);
110+
});
111+
return graph.add_tensorref(sizes, dtype, std::move(buffer));
112+
}
113+
67114
void add_native_layer_norm_node(
68115
ComputeGraph& graph,
69116
const ValueRef in,
@@ -78,16 +125,33 @@ void add_native_layer_norm_node(
78125
VK_THROW("native_layer_norm only supports normalized_shape with dim == 1");
79126
}
80127

128+
// The shader reads a weight and a bias binding unconditionally, but either
129+
// affine parameter can legitimately be absent:
130+
// - nn.LayerNorm(bias=False) (e.g. HF Gemma4 audio_encoder) has no bias.
131+
// - F.layer_norm called with neither, where the caller applies its own
132+
// scale and shift afterwards (e.g. kokoro's AdaLayerNorm), has neither.
133+
// Synthesize whatever is missing: a unit weight and a zero bias reproduce
134+
// out = (x - mean) * rstd exactly.
135+
const std::vector<int64_t> affine_sizes = graph.val_is_none(weight_data)
136+
? std::vector<int64_t>{*graph.get_int_list(normalized_shape)}
137+
: graph.sizes_of(weight_data);
138+
const vkapi::ScalarType affine_dtype = graph.val_is_none(weight_data)
139+
? graph.dtype_of(in)
140+
: graph.dtype_of(weight_data);
141+
142+
ValueRef synthesized_weight = weight_data;
81143
if (graph.val_is_none(weight_data)) {
82-
VK_THROW("native_layer_norm requires weight to be non-None");
144+
synthesized_weight = constant_affine_tensorref(
145+
graph, affine_sizes, affine_dtype, /*ones=*/true);
83146
}
84-
147+
ValueRef synthesized_bias = bias_data;
85148
if (graph.val_is_none(bias_data)) {
86-
VK_THROW("native_layer_norm requires bias to be non-None");
149+
synthesized_bias = constant_affine_tensorref(
150+
graph, affine_sizes, affine_dtype, /*ones=*/false);
87151
}
88152

89-
ValueRef arg_weight = prepack_standard_like(graph, weight_data, in);
90-
ValueRef arg_bias = prepack_standard_like(graph, bias_data, in);
153+
ValueRef arg_weight = prepack_standard_like(graph, synthesized_weight, in);
154+
ValueRef arg_bias = prepack_standard_like(graph, synthesized_bias, in);
91155

92156
const auto out_val = graph.get_value_list(out);
93157
const ValueRef out_tensor = out_val->at(0);

‎backends/vulkan/test/op_tests/cases.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -761,6 +761,13 @@ def get_native_layer_norm_inputs():
761761
((S1, S2), [S2], (S2), (S2), 0.001),
762762
((M, M1, M2), [M2], (M2), (M2), 0.001),
763763
((S, XL, M1, M2), [M2], (M2), (M2), 0.001),
764+
# Either affine parameter may be absent: nn.LayerNorm(bias=False)
765+
# has no bias, and F.layer_norm called with neither (kokoro's
766+
# AdaLayerNorm applies its own scale and shift afterwards) has
767+
# neither.
768+
((M, M1, M2), [M2], (M2), None, 0.001),
769+
((M, M1, M2), [M2], None, (M2), 0.001),
770+
((M, M1, M2), [M2], None, None, 0.001),
764771
]
765772
)
766773
test_suite.layouts = [

0 commit comments

Comments
 (0)