Skip to content

Commit d6b8260

Browse files
[executorch][native-vk] Prepack immutable constants for elementwise and index ops (pytorch#23456)
This PR was created by the merge bot to help merge the original PR into the main branch. ghstack PR number: pytorch#23420 by @digantdesai ^ Please use this as the source of truth for the PR details, comments, and reviews ghstack PR base: https://github.com/pytorch/executorch/tree/gh/digantdesai/116/base ghstack PR head: https://github.com/pytorch/executorch/tree/gh/digantdesai/116/head Merge bot PR base: https://github.com/pytorch/executorch/tree/gh/digantdesai/115/orig Merge bot PR head: https://github.com/pytorch/executorch/tree/gh/digantdesai/116/orig Differential Revision: [D119601869](https://our.internmc.facebook.com/intern/diff/D119601869/) @diff-train-skip-merge Co-authored-by: Digant Desai <digantdesai@meta.com>
1 parent b3b82cd commit d6b8260

6 files changed

Lines changed: 249 additions & 0 deletions

File tree

‎backends/native/runtime/vulkan/VulkanEngineExecutable.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
#include <executorch/backends/native/runtime/graph/Value.h>
3333
#include <executorch/backends/native/runtime/graph/utils/GraphUtils.h>
3434
#include <executorch/backends/native/runtime/vulkan/VulkanConstantMaterializationTracker.h>
35+
#include <executorch/backends/native/runtime/vulkan/passes/InsertPrepack.h>
3536
#include <executorch/runtime/core/freeable_buffer.h>
3637

3738
#include <executorch/backends/vulkan/runtime/api/api.h>
@@ -277,6 +278,7 @@ class VulkanEngineExecutable final : public EngineExecutable {
277278
materializations_(materializations),
278279
graph_(std::make_unique<ComputeGraph>(config)) {
279280
method_.graph.rebuild_def_use();
281+
vulkan::insert_prepack_nodes(method_);
280282
validate_graph(method_.graph);
281283
validate_supported_method(method_);
282284
nodes_ = method_.graph.schedule;
Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
// Copyright (c) Meta Platforms, Inc. and affiliates.
2+
// All rights reserved.
3+
//
4+
// This source code is licensed under the BSD-style license found in the
5+
// LICENSE file in the root directory of this source tree.
6+
7+
#include <executorch/backends/native/runtime/vulkan/passes/InsertPrepack.h>
8+
9+
#include <algorithm>
10+
#include <array>
11+
#include <string_view>
12+
#include <vector>
13+
14+
#include <executorch/backends/native/runtime/graph/Argument.h>
15+
#include <executorch/backends/native/runtime/graph/Node.h>
16+
#include <executorch/backends/native/runtime/graph/utils/GraphUtils.h>
17+
18+
namespace ptn::vulkan {
19+
namespace {
20+
21+
bool needs_prepack(const Node& node) {
22+
constexpr std::array<std::string_view, 3> kOps{
23+
"torch.ops.aten.add.Tensor",
24+
"torch.ops.aten.index.Tensor",
25+
"torch.ops.aten.mul.Tensor",
26+
};
27+
return node.is_call() && std::ranges::find(kOps, node.target) != kOps.end();
28+
}
29+
30+
Node make_prepack_node(ValueId input, ValueId output, const std::string& name) {
31+
return Node{
32+
.name = name,
33+
.target = "torch.ops.et_vk.prepack.default",
34+
.inputs = {{.name = "self", .arg = TensorArg{input}}},
35+
.outputs = {{.value_id = output}},
36+
};
37+
}
38+
39+
} // namespace
40+
41+
// cppcheck-suppress unusedFunction
42+
size_t insert_prepack_nodes(Method& method) {
43+
Graph& graph = method.graph;
44+
size_t count = 0;
45+
for (const DataBinding& binding : method.data_bindings) {
46+
if (!binding.has_data || binding.mutated || !valid(binding.value_id)) {
47+
continue;
48+
}
49+
50+
const Value& source = graph.value(binding.value_id);
51+
if (!source.is_tensor()) {
52+
continue;
53+
}
54+
std::vector<NodeId> consumers;
55+
for (const NodeId consumer_id : source.consumer_ids) {
56+
if (needs_prepack(graph.node(consumer_id))) {
57+
consumers.push_back(consumer_id);
58+
}
59+
}
60+
if (consumers.empty()) {
61+
continue;
62+
}
63+
64+
const std::string source_name = source.name;
65+
Value prepacked(source_name + "_vulkan_prepacked", source.tensor_meta());
66+
prepacked.role = ValueRole::ConstantTensor;
67+
const ValueId prepacked_id = graph.append_value(std::move(prepacked));
68+
graph.insert_node_before(
69+
consumers.front(),
70+
make_prepack_node(
71+
binding.value_id, prepacked_id, source_name + "_vulkan_prepack"));
72+
for (const NodeId consumer_id : consumers) {
73+
graph.replace_input(consumer_id, binding.value_id, prepacked_id);
74+
}
75+
++count;
76+
}
77+
if (count > 0) {
78+
stable_topological_sort(graph);
79+
validate_graph(graph);
80+
}
81+
return count;
82+
}
83+
84+
} // namespace ptn::vulkan
Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
// Copyright (c) Meta Platforms, Inc. and affiliates.
2+
// All rights reserved.
3+
//
4+
// This source code is licensed under the BSD-style license found in the
5+
// LICENSE file in the root directory of this source tree.
6+
7+
#pragma once
8+
9+
#include <cstddef>
10+
11+
#include <executorch/backends/native/runtime/Method.h>
12+
13+
namespace ptn::vulkan {
14+
15+
size_t insert_prepack_nodes(Method& method);
16+
17+
} // namespace ptn::vulkan

‎backends/native/runtime/vulkan/targets.bzl‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,20 @@ def define_common_targets():
1414
if is_xplat():
1515
return
1616

17+
runtime.cxx_library(
18+
name = "vulkan_passes",
19+
srcs = ["passes/InsertPrepack.cpp"],
20+
exported_headers = ["passes/InsertPrepack.h"],
21+
exported_deps = [
22+
"//executorch/backends/native/runtime:method",
23+
"//executorch/backends/native/runtime/graph:graph",
24+
],
25+
deps = [
26+
"//executorch/backends/native/runtime/graph:graph_utils",
27+
],
28+
visibility = ["PUBLIC"],
29+
)
30+
1731
# The Vulkan engine: implements the EngineContext / EngineExecutable
1832
# boundary by lowering a native Method's Graph onto ET-VK's ComputeGraph,
1933
# reusing every ET-VK compute shader and prepack path via the operator
@@ -47,6 +61,7 @@ def define_common_targets():
4761
"//executorch/backends/native/runtime/graph:value",
4862
"//executorch/backends/native/runtime/graph:graph_utils",
4963
"//executorch/runtime/core:core",
64+
":vulkan_passes",
5065
# ET-VK: ComputeGraph + operator registry + shaders (link_whole, so
5166
# the static op registrations survive), and the Vulkan API layer
5267
# (Runtime / Adapter / Context; SwiftShader on a headless host).

‎backends/native/test/runtime/vulkan/targets.bzl‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,3 +36,17 @@ def define_common_targets():
3636
"fbsource//third-party/swiftshader/lib/linux-x64:libvk_swiftshader_fbcode",
3737
],
3838
)
39+
40+
runtime.cxx_test(
41+
name = "vulkan_passes_test",
42+
srcs = ["test_insert_prepack.cpp"],
43+
deps = [
44+
"//executorch/backends/native/runtime/vulkan:vulkan_passes",
45+
"//executorch/backends/native/runtime/graph:argument",
46+
"//executorch/backends/native/runtime/graph:graph",
47+
"//executorch/backends/native/runtime/graph:node",
48+
"//executorch/backends/native/runtime/graph:scalar_type",
49+
"//executorch/backends/native/runtime/graph:value",
50+
"//executorch/backends/native/runtime/graph:tensor_meta",
51+
],
52+
)
Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
// Copyright (c) Meta Platforms, Inc. and affiliates.
2+
// All rights reserved.
3+
//
4+
// This source code is licensed under the BSD-style license found in the
5+
// LICENSE file in the root directory of this source tree.
6+
7+
#include <executorch/backends/native/runtime/vulkan/passes/InsertPrepack.h>
8+
9+
#include <gtest/gtest.h>
10+
11+
namespace ptn::vulkan {
12+
namespace {
13+
14+
Node placeholder(const char* name, ValueId output) {
15+
return Node{
16+
.name = name,
17+
.op_kind = OpKind::Placeholder,
18+
.outputs = {{.value_id = output}},
19+
};
20+
}
21+
22+
Node binary(
23+
const char* name,
24+
const char* target,
25+
ValueId left,
26+
ValueId right,
27+
ValueId output) {
28+
return Node{
29+
.name = name,
30+
.target = target,
31+
.inputs =
32+
{
33+
{.name = "self", .arg = TensorArg{left}},
34+
{.name = "other", .arg = TensorArg{right}},
35+
},
36+
.outputs = {{.value_id = output}},
37+
};
38+
}
39+
40+
Method make_method() {
41+
Method method;
42+
method.name = "forward";
43+
Graph& graph = method.graph;
44+
graph.values = {
45+
Value("input", ScalarType::Half, {4}),
46+
Value("constant", ScalarType::Half, {4}),
47+
Value("mul", ScalarType::Half, {4}),
48+
Value("add", ScalarType::Half, {4}),
49+
};
50+
graph.values[0].role = ValueRole::UserInput;
51+
graph.values[1].role = ValueRole::Parameter;
52+
graph.nodes = {
53+
placeholder("input", 0),
54+
placeholder("constant", 1),
55+
binary("mul", "torch.ops.aten.mul.Tensor", 0, 1, 2),
56+
binary("add", "torch.ops.aten.add.Tensor", 2, 1, 3),
57+
Node{
58+
.name = "output",
59+
.op_kind = OpKind::Output,
60+
.inputs = {{.arg = TensorArg{3}}},
61+
},
62+
};
63+
graph.input_ids = {0};
64+
graph.output_ids = {3};
65+
graph.initialize_schedule();
66+
graph.rebuild_def_use();
67+
method.data_bindings = {{
68+
.value_id = 1,
69+
.role = ValueRole::Parameter,
70+
.key = "constant",
71+
}};
72+
return method;
73+
}
74+
75+
// cppcheck-suppress-begin syntaxError
76+
TEST(InsertPrepackTest, SharesOnePrepackAcrossConsumers) {
77+
Method method = make_method();
78+
79+
EXPECT_EQ(insert_prepack_nodes(method), 1);
80+
81+
const Graph& graph = method.graph;
82+
ASSERT_EQ(graph.values.size(), 5);
83+
ASSERT_EQ(graph.nodes.size(), 6);
84+
const Node& prepack = graph.node(5);
85+
EXPECT_EQ(prepack.target, "torch.ops.et_vk.prepack.default");
86+
EXPECT_EQ(prepack.input_value_ids(), std::vector<ValueId>({1}));
87+
EXPECT_EQ(graph.node(2).input_value_ids(), std::vector<ValueId>({0, 4}));
88+
EXPECT_EQ(graph.node(3).input_value_ids(), std::vector<ValueId>({2, 4}));
89+
EXPECT_EQ(graph.value(4).role, ValueRole::ConstantTensor);
90+
}
91+
92+
TEST(InsertPrepackTest, LeavesSelfPrepackingOpUntouched) {
93+
Method method = make_method();
94+
method.graph.node(2).target = "torch.ops.aten.linear.default";
95+
method.graph.node(3).target = "torch.ops.et_vk.embedding_q4gsw.default";
96+
97+
EXPECT_EQ(insert_prepack_nodes(method), 0);
98+
EXPECT_EQ(method.graph.values.size(), 4);
99+
EXPECT_EQ(method.graph.nodes.size(), 5);
100+
}
101+
102+
TEST(InsertPrepackTest, LeavesMutableBindingUntouched) {
103+
Method method = make_method();
104+
method.data_bindings[0].mutated = true;
105+
106+
EXPECT_EQ(insert_prepack_nodes(method), 0);
107+
EXPECT_EQ(method.graph.values.size(), 4);
108+
EXPECT_EQ(method.graph.nodes.size(), 5);
109+
EXPECT_EQ(
110+
method.graph.node(2).input_value_ids(), std::vector<ValueId>({0, 1}));
111+
EXPECT_EQ(
112+
method.graph.node(3).input_value_ids(), std::vector<ValueId>({2, 1}));
113+
}
114+
// cppcheck-suppress-end syntaxError
115+
116+
} // namespace
117+
} // namespace ptn::vulkan

0 commit comments

Comments
 (0)