Skip to content

Commit ba4bab8

Browse files
[executorch][native-vk] Lower portable aten.rms_norm to et_vk.rms_norm at load time (pytorch#23466)
This PR was created by the merge bot to help merge the original PR into the main branch. ghstack PR number: pytorch#23431 by @digantdesai ^ Please use this as the source of truth for the PR details, comments, and reviews ghstack PR base: https://github.com/pytorch/executorch/tree/gh/digantdesai/126/base ghstack PR head: https://github.com/pytorch/executorch/tree/gh/digantdesai/126/head Merge bot PR base: https://github.com/pytorch/executorch/tree/gh/digantdesai/125/orig Merge bot PR head: https://github.com/pytorch/executorch/tree/gh/digantdesai/126/orig Differential Revision: [D121704325](https://our.internmc.facebook.com/intern/diff/D121704325/) @diff-train-skip-merge Co-authored-by: Digant Desai <digantdesai@meta.com>
1 parent bbc9690 commit ba4bab8

6 files changed

Lines changed: 230 additions & 0 deletions

File tree

‎backends/native/runtime/vulkan/VulkanEngineExecutable.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
#include <executorch/backends/native/runtime/graph/utils/GraphUtils.h>
3636
#include <executorch/backends/native/runtime/vulkan/VulkanConstantMaterializationTracker.h>
3737
#include <executorch/backends/native/runtime/vulkan/passes/InsertPrepack.h>
38+
#include <executorch/backends/native/runtime/vulkan/passes/LowerRMSNorm.h>
3839
#include <executorch/backends/native/runtime/vulkan/passes/MaterializeViewCopies.h>
3940
#include <executorch/runtime/core/freeable_buffer.h>
4041

@@ -494,6 +495,7 @@ class VulkanEngineExecutable final : public EngineExecutable {
494495
mutable_state_claim_(mutable_state_owners, method),
495496
graph_(std::make_unique<ComputeGraph>(config)) {
496497
method_.graph.rebuild_def_use();
498+
vulkan::lower_rms_norms(method_.graph);
497499
vulkan::insert_prepack_nodes(method_);
498500
validate_graph(method_.graph);
499501
validate_user_io(method_);
Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
// Copyright (c) Meta Platforms, Inc. and affiliates.
2+
// All rights reserved.
3+
//
4+
// This source code is licensed under the BSD-style license found in the
5+
// LICENSE file in the root directory of this source tree.
6+
7+
#include <executorch/backends/native/runtime/vulkan/passes/LowerRMSNorm.h>
8+
9+
#include <algorithm>
10+
#include <cmath>
11+
#include <string_view>
12+
#include <vector>
13+
14+
#include <executorch/backends/native/runtime/graph/Argument.h>
15+
#include <executorch/backends/native/runtime/graph/Node.h>
16+
#include <executorch/backends/native/runtime/graph/ScalarType.h>
17+
#include <executorch/backends/native/runtime/graph/Value.h>
18+
#include <executorch/backends/native/runtime/graph/utils/GraphUtils.h>
19+
20+
namespace ptn::vulkan {
21+
namespace {
22+
23+
constexpr std::string_view kAtenRMSNorm = "torch.ops.aten.rms_norm.default";
24+
constexpr const char* kVulkanRMSNorm = "torch.ops.et_vk.rms_norm.default";
25+
26+
const Argument*
27+
get_arg(const Node& node, std::string_view name, size_t position) {
28+
const auto found = std::ranges::find_if(
29+
node.inputs,
30+
[name](const NamedArgument& input) { return input.name == name; });
31+
if (found != node.inputs.end()) {
32+
return &found->arg;
33+
}
34+
return position < node.inputs.size() ? &node.inputs[position].arg : nullptr;
35+
}
36+
37+
bool is_constant(const Value& value) {
38+
return value.role == ValueRole::Parameter ||
39+
value.role == ValueRole::ConstantTensor;
40+
}
41+
42+
} // namespace
43+
44+
size_t lower_rms_norms(Graph& graph) {
45+
size_t count = 0;
46+
for (const NodeId node_id : graph.schedule) {
47+
Node& node = graph.node(node_id);
48+
if (!node.is_call() || node.target != kAtenRMSNorm ||
49+
node.outputs.size() != 1 || !valid(node.outputs[0].value_id)) {
50+
continue;
51+
}
52+
53+
const Argument* input = get_arg(node, "input", 0);
54+
const Argument* normalized_shape = get_arg(node, "normalized_shape", 1);
55+
const Argument* weight = get_arg(node, "weight", 2);
56+
const Argument* eps = get_arg(node, "eps", 3);
57+
if (input == nullptr || input->kind() != ArgKind::Tensor ||
58+
normalized_shape == nullptr ||
59+
normalized_shape->kind() != ArgKind::IntList || weight == nullptr ||
60+
weight->kind() != ArgKind::Tensor || eps == nullptr ||
61+
eps->kind() != ArgKind::Float || valid(eps->as_float().id) ||
62+
!std::isfinite(eps->as_float().value) || eps->as_float().value < 0) {
63+
continue;
64+
}
65+
66+
const ValueId input_id = input->as_tensor().id;
67+
const ValueId weight_id = weight->as_tensor().id;
68+
const ValueId output_id = node.outputs[0].value_id;
69+
if (!valid(input_id) || !valid(weight_id)) {
70+
continue;
71+
}
72+
const Value& input_value = graph.value(input_id);
73+
const Value& weight_value = graph.value(weight_id);
74+
const Value& output_value = graph.value(output_id);
75+
if (!input_value.is_tensor() || !weight_value.is_tensor() ||
76+
!output_value.is_tensor() || !is_constant(weight_value)) {
77+
continue;
78+
}
79+
80+
const TensorMeta& input_meta = input_value.tensor_meta();
81+
const TensorMeta& weight_meta = weight_value.tensor_meta();
82+
const TensorMeta& output_meta = output_value.tensor_meta();
83+
const IntListArg& shape = normalized_shape->as_int_list();
84+
if (input_meta.sizes.empty() || !shape.ids.empty() ||
85+
shape.values != std::vector<int64_t>{input_meta.sizes.back()} ||
86+
weight_meta.sizes != shape.values ||
87+
(input_meta.dtype != ScalarType::Float &&
88+
input_meta.dtype != ScalarType::Half) ||
89+
weight_meta.dtype != input_meta.dtype ||
90+
output_meta.dtype != input_meta.dtype ||
91+
output_meta.sizes != input_meta.sizes) {
92+
continue;
93+
}
94+
95+
const Argument input_arg = *input;
96+
const Argument weight_arg = *weight;
97+
const Argument eps_arg = *eps;
98+
node.target = kVulkanRMSNorm;
99+
node.inputs = {
100+
{.name = "input", .arg = input_arg},
101+
{.name = "weight", .arg = weight_arg},
102+
{.name = "epsilon", .arg = eps_arg},
103+
};
104+
++count;
105+
}
106+
if (count > 0) {
107+
graph.rebuild_def_use();
108+
validate_graph(graph);
109+
}
110+
return count;
111+
}
112+
113+
} // namespace ptn::vulkan
Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
// Copyright (c) Meta Platforms, Inc. and affiliates.
2+
// All rights reserved.
3+
//
4+
// This source code is licensed under the BSD-style license found in the
5+
// LICENSE file in the root directory of this source tree.
6+
7+
#pragma once
8+
9+
#include <cstddef>
10+
11+
#include <executorch/backends/native/runtime/graph/Graph.h>
12+
13+
namespace ptn::vulkan {
14+
15+
size_t lower_rms_norms(Graph& graph);
16+
17+
} // namespace ptn::vulkan

‎backends/native/runtime/vulkan/targets.bzl‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,12 @@ def define_common_targets():
1818
name = "vulkan_passes",
1919
srcs = [
2020
"passes/InsertPrepack.cpp",
21+
"passes/LowerRMSNorm.cpp",
2122
"passes/MaterializeViewCopies.cpp",
2223
],
2324
exported_headers = [
2425
"passes/InsertPrepack.h",
26+
"passes/LowerRMSNorm.h",
2527
"passes/MaterializeViewCopies.h",
2628
],
2729
exported_deps = [

‎backends/native/test/runtime/vulkan/targets.bzl‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,12 +41,14 @@ def define_common_targets():
4141
name = "vulkan_passes_test",
4242
srcs = [
4343
"test_insert_prepack.cpp",
44+
"test_lower_rms_norm.cpp",
4445
"test_materialize_view_copies.cpp",
4546
],
4647
deps = [
4748
"//executorch/backends/native/runtime/vulkan:vulkan_passes",
4849
"//executorch/backends/native/runtime/graph:argument",
4950
"//executorch/backends/native/runtime/graph:graph",
51+
"//executorch/backends/native/runtime/graph:graph_utils",
5052
"//executorch/backends/native/runtime/graph:node",
5153
"//executorch/backends/native/runtime/graph:scalar_type",
5254
"//executorch/backends/native/runtime/graph:value",
Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
// Copyright (c) Meta Platforms, Inc. and affiliates.
2+
// All rights reserved.
3+
//
4+
// This source code is licensed under the BSD-style license found in the
5+
// LICENSE file in the root directory of this source tree.
6+
7+
// cppcheck-suppress-file syntaxError
8+
9+
#include <executorch/backends/native/runtime/vulkan/passes/LowerRMSNorm.h>
10+
11+
#include <vector>
12+
13+
#include <gtest/gtest.h>
14+
15+
#include <executorch/backends/native/runtime/Method.h>
16+
#include <executorch/backends/native/runtime/graph/Argument.h>
17+
#include <executorch/backends/native/runtime/graph/Node.h>
18+
#include <executorch/backends/native/runtime/graph/ScalarType.h>
19+
#include <executorch/backends/native/runtime/graph/Value.h>
20+
#include <executorch/backends/native/runtime/graph/utils/GraphUtils.h>
21+
22+
namespace ptn::vulkan {
23+
namespace {
24+
25+
Method make_rms_norm() {
26+
Method method;
27+
method.name = "forward";
28+
Graph& graph = method.graph;
29+
graph.values = {
30+
Value("input", ScalarType::Float, {2, 4}),
31+
Value("weight", ScalarType::Float, {4}),
32+
Value("output", ScalarType::Float, {2, 4}),
33+
};
34+
graph.value(1).role = ValueRole::Parameter;
35+
graph.nodes = {
36+
Node{
37+
.name = "input",
38+
.op_kind = OpKind::Placeholder,
39+
.outputs = {{.value_id = 0}},
40+
},
41+
Node{
42+
.name = "weight",
43+
.op_kind = OpKind::Placeholder,
44+
.outputs = {{.value_id = 1}},
45+
},
46+
Node{
47+
.name = "rms_norm",
48+
.target = "torch.ops.aten.rms_norm.default",
49+
.inputs =
50+
{
51+
{.name = "input", .arg = TensorArg{0}},
52+
{.name = "normalized_shape", .arg = IntListArg{{4}}},
53+
{.name = "weight", .arg = TensorArg{1}},
54+
{.name = "eps", .arg = FloatArg{1e-5}},
55+
},
56+
.outputs = {{.value_id = 2}},
57+
},
58+
Node{
59+
.name = "output",
60+
.op_kind = OpKind::Output,
61+
.inputs = {{.arg = TensorArg{2}}},
62+
},
63+
};
64+
graph.schedule = {0, 1, 2, 3};
65+
graph.input_ids = {0};
66+
graph.output_ids = {2};
67+
graph.rebuild_def_use();
68+
return method;
69+
}
70+
71+
TEST(LowerRMSNormTest, RewritesPortableWeightedNorm) {
72+
Method method = make_rms_norm();
73+
74+
EXPECT_EQ(lower_rms_norms(method.graph), 1);
75+
76+
const Node& norm = method.graph.node(2);
77+
EXPECT_EQ(norm.target, "torch.ops.et_vk.rms_norm.default");
78+
ASSERT_EQ(norm.inputs.size(), 3);
79+
EXPECT_EQ(norm.inputs[0].arg.as_tensor().id, 0);
80+
EXPECT_EQ(norm.inputs[1].arg.as_tensor().id, 1);
81+
EXPECT_DOUBLE_EQ(norm.inputs[2].arg.as_float().value, 1e-5);
82+
EXPECT_NO_THROW(validate_graph(method.graph));
83+
}
84+
85+
TEST(LowerRMSNormTest, RejectsMismatchedNormalizedShape) {
86+
Method method = make_rms_norm();
87+
method.graph.node(2).inputs[1].arg = IntListArg{{8}};
88+
89+
EXPECT_EQ(lower_rms_norms(method.graph), 0);
90+
EXPECT_EQ(method.graph.node(2).target, "torch.ops.aten.rms_norm.default");
91+
}
92+
93+
} // namespace
94+
} // namespace ptn::vulkan

0 commit comments

Comments
 (0)