Skip to content

Commit 9123df8

Browse files
[executorch][native-vk] Route in-place ReLU to Vulkan (pytorch#23458)
This PR was created by the merge bot to help merge the original PR into the main branch. ghstack PR number: pytorch#23422 by @digantdesai ^ Please use this as the source of truth for the PR details, comments, and reviews ghstack PR base: https://github.com/pytorch/executorch/tree/gh/digantdesai/118/base ghstack PR head: https://github.com/pytorch/executorch/tree/gh/digantdesai/118/head Merge bot PR base: https://github.com/pytorch/executorch/tree/gh/digantdesai/117/orig Merge bot PR head: https://github.com/pytorch/executorch/tree/gh/digantdesai/118/orig Differential Revision: [D121140202](https://our.internmc.facebook.com/intern/diff/D121140202/) @diff-train-skip-merge cc @SS-JIA @manuelcandales @digantdesai @cbilgin Co-authored-by: Digant Desai <digantdesai@meta.com>
1 parent 176560f commit 9123df8

2 files changed

Lines changed: 162 additions & 5 deletions

File tree

‎backends/native/runtime/vulkan/VulkanEngineExecutable.cpp‎

Lines changed: 50 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -110,14 +110,23 @@ size_t logical_nbytes(const TensorMeta& meta) {
110110
return numel * element_bytes;
111111
}
112112

113-
// Strip the leading "torch.ops." namespace so a native fx target becomes an
114-
// ET-VK operator-registry key (torch.ops.aten.add.Tensor -> aten.add.Tensor).
113+
// ET-VK operator-registry key for a native fx target: strips the leading
114+
// "torch.ops." namespace (torch.ops.aten.add.Tensor -> aten.add.Tensor) and
115+
// renames targets whose ET-VK kernel is registered under another key.
115116
std::string registry_key(const std::string& target) {
116117
constexpr std::string_view kPrefix = "torch.ops.";
117-
if (target.starts_with(kPrefix)) {
118-
return target.substr(kPrefix.size());
118+
std::string key =
119+
target.starts_with(kPrefix) ? target.substr(kPrefix.size()) : target;
120+
if (key == "aten.relu_.default") {
121+
key = "aten.relu.default";
119122
}
120-
return target;
123+
return key;
124+
}
125+
126+
// In-place targets that registry_key maps to a functional ET-VK kernel.
127+
bool is_functional_in_place(const std::string& target) {
128+
return target == "torch.ops.aten.relu_.default" ||
129+
target == "aten.relu_.default";
121130
}
122131

123132
// Ops whose ET-VK implementations require width-packed operands. This list
@@ -959,6 +968,41 @@ class VulkanEngineExecutable final : public EngineExecutable {
959968
}
960969
}
961970

971+
// A functional kernel whose output views its input binds one storage for
972+
// both read and write. ET-VK records only the last binding's access, so the
973+
// write is forgotten and the next reader of the output gets no barrier. Such
974+
// ops run out of place instead, which is only equivalent while nothing else
975+
// reads the mutated tensor. Runs after layout assignment so the detached
976+
// output keeps its input's layout and storage.
977+
void detach_in_place_outputs() {
978+
for (const NodeId nid : nodes_) {
979+
const Node& n = g().node(nid);
980+
if (!n.is_call() || !is_functional_in_place(n.target) ||
981+
n.inputs.empty() || n.inputs[0].arg.kind() != ArgKind::Tensor ||
982+
n.outputs.size() != 1) {
983+
continue;
984+
}
985+
const ValueId self_id = n.inputs[0].arg.as_tensor().id;
986+
const ValueId out_id = n.outputs[0].value_id;
987+
if (!valid(self_id) || !valid(out_id) ||
988+
g().value(out_id).alias_id != self_id) {
989+
continue;
990+
}
991+
const Value& self = g().value(self_id);
992+
const bool observed = (self.role != ValueRole::Intermediate &&
993+
self.role != ValueRole::UserInput) ||
994+
valid(self.alias_id) || self.consumer_ids.size() != 1 ||
995+
std::ranges::find(outputs_, self_id) != outputs_.end();
996+
if (observed) {
997+
throw std::runtime_error(
998+
"vulkan: node '" + n.name + "' (" + n.target +
999+
"): in-place update of '" + self.name +
1000+
"' is read elsewhere; only an unshared input can run out of place");
1001+
}
1002+
method_.graph.value(out_id).alias_id = kInvalid;
1003+
}
1004+
}
1005+
9621006
void plan_allocations() {
9631007
std::vector<AllocationRequest> requests;
9641008
requests.reserve(g().values.size());
@@ -1108,6 +1152,7 @@ class VulkanEngineExecutable final : public EngineExecutable {
11081152
void build(const Package& package) {
11091153
check_mutation_outputs();
11101154
assign_layouts();
1155+
detach_in_place_outputs();
11111156
index_output_kinds();
11121157
index_bindings();
11131158
plan_allocations();

‎backends/native/test/runtime/vulkan/test_vulkan_engine.cpp‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -398,6 +398,77 @@ std::vector<uint8_t> make_state_program(
398398
builder.GetBufferPointer() + builder.GetSize()};
399399
}
400400

401+
// relu_ mutates `input`, or with `from_sum` the intermediate input + input.
402+
// With `output_mutated`, the mutated tensor is also a graph output.
403+
std::vector<uint8_t> make_inplace_relu_program(
404+
bool from_sum = false,
405+
bool output_mutated = false) {
406+
flatbuffers::FlatBufferBuilder builder;
407+
const auto meta = create_tensor_meta(builder, fbs::ScalarType::FLOAT);
408+
const char* mutated = from_sum ? "sum" : "input";
409+
std::vector<flatbuffers::Offset<fbs::TensorValue>> tensor_values = {
410+
fbs::CreateTensorValueDirect(builder, "input", meta),
411+
fbs::CreateTensorValueDirect(builder, "relu", meta),
412+
};
413+
const std::vector<flatbuffers::Offset<fbs::Output>> input_outputs = {
414+
fbs::CreateOutputDirect(builder, "input")};
415+
const std::vector<flatbuffers::Offset<fbs::Output>> sum_outputs = {
416+
fbs::CreateOutputDirect(builder, "sum")};
417+
const std::vector<flatbuffers::Offset<fbs::Output>> relu_outputs = {
418+
fbs::CreateOutputDirect(builder, "relu", mutated)};
419+
const std::vector<flatbuffers::Offset<fbs::NamedArgument>> add_inputs = {
420+
fbs::CreateNamedArgumentDirect(
421+
builder, "self", create_tensor_arg(builder, "input")),
422+
fbs::CreateNamedArgumentDirect(
423+
builder, "other", create_tensor_arg(builder, "input")),
424+
fbs::CreateNamedArgumentDirect(
425+
builder, "alpha", create_int_arg(builder, 1)),
426+
};
427+
const std::vector<flatbuffers::Offset<fbs::NamedArgument>> relu_inputs = {
428+
fbs::CreateNamedArgumentDirect(
429+
builder, "self", create_tensor_arg(builder, mutated), true)};
430+
std::vector<flatbuffers::Offset<fbs::NamedArgument>> output_inputs = {
431+
fbs::CreateNamedArgumentDirect(
432+
builder, "", create_tensor_arg(builder, "relu"))};
433+
std::vector<std::string> graph_outputs = {"relu"};
434+
std::vector<flatbuffers::Offset<fbs::OutputSpec>> output_specs = {
435+
fbs::CreateOutputSpecDirect(builder, "relu")};
436+
if (output_mutated) {
437+
output_inputs.push_back(fbs::CreateNamedArgumentDirect(
438+
builder, "", create_tensor_arg(builder, mutated)));
439+
graph_outputs.emplace_back(mutated);
440+
output_specs.push_back(fbs::CreateOutputSpecDirect(builder, mutated));
441+
}
442+
std::vector<flatbuffers::Offset<fbs::Node>> nodes = {fbs::CreateNodeDirect(
443+
builder, "input", fbs::OpKind::PLACEHOLDER, "", nullptr, &input_outputs)};
444+
if (from_sum) {
445+
tensor_values.push_back(fbs::CreateTensorValueDirect(builder, "sum", meta));
446+
nodes.push_back(fbs::CreateNodeDirect(
447+
builder,
448+
"sum",
449+
fbs::OpKind::CALL_FUNCTION,
450+
"torch.ops.aten.add.Tensor",
451+
&add_inputs,
452+
&sum_outputs));
453+
}
454+
nodes.push_back(fbs::CreateNodeDirect(
455+
builder,
456+
"relu",
457+
fbs::OpKind::CALL_FUNCTION,
458+
"torch.ops.aten.relu_.default",
459+
&relu_inputs,
460+
&relu_outputs));
461+
nodes.push_back(fbs::CreateNodeDirect(
462+
builder, "output", fbs::OpKind::OUTPUT, "", &output_inputs));
463+
const auto graph = fbs::CreateGraph(
464+
builder,
465+
builder.CreateVector(nodes),
466+
create_strings(builder, {"input"}),
467+
create_strings(builder, graph_outputs),
468+
builder.CreateVector(tensor_values));
469+
return finish_program(builder, graph, output_specs);
470+
}
471+
401472
struct CompiledProgram {
402473
std::unique_ptr<VulkanEngineHost> host;
403474
std::shared_ptr<const Program> program;
@@ -623,6 +694,47 @@ TEST(VulkanEngineTest, RejectsMutationOutputNotWrittenInPlace) {
623694
<< e.what();
624695
}
625696
}
697+
698+
TEST(VulkanEngineTest, ExecutesSerializedInPlaceRelu) {
699+
CompiledProgram compiled = compile_program(make_inplace_relu_program());
700+
const std::vector<float> input = {-2.0f, -0.5f, 0.0f, 3.0f};
701+
compiled.executable->set_input(
702+
0, input.data(), input.size(), ScalarType::Float);
703+
compiled.executable->execute();
704+
705+
std::vector<float> output(4);
706+
compiled.executable->get_output(
707+
0, output.data(), output.size(), ScalarType::Float);
708+
EXPECT_EQ(output, (std::vector<float>{0.0f, 0.0f, 0.0f, 3.0f}));
709+
}
710+
711+
TEST(VulkanEngineTest, ExecutesInPlaceReluOfIntermediate) {
712+
CompiledProgram compiled =
713+
compile_program(make_inplace_relu_program(/*from_sum=*/true));
714+
const std::vector<float> input = {-2.0f, -0.5f, 0.0f, 3.0f};
715+
compiled.executable->set_input(
716+
0, input.data(), input.size(), ScalarType::Float);
717+
compiled.executable->execute();
718+
719+
std::vector<float> output(4);
720+
compiled.executable->get_output(
721+
0, output.data(), output.size(), ScalarType::Float);
722+
EXPECT_EQ(output, (std::vector<float>{0.0f, 0.0f, 0.0f, 6.0f}));
723+
}
724+
725+
TEST(VulkanEngineTest, RejectsInPlaceReluWhoseInputIsReadElsewhere) {
726+
try {
727+
compile_program(make_inplace_relu_program(
728+
/*from_sum=*/true, /*output_mutated=*/true));
729+
FAIL() << "expected the observed in-place relu to be rejected";
730+
} catch (const std::runtime_error& e) {
731+
EXPECT_NE(
732+
std::string(e.what()).find(
733+
"in-place update of 'sum' is read elsewhere"),
734+
std::string::npos)
735+
<< e.what();
736+
}
737+
}
626738
// cppcheck-suppress-end syntaxError
627739

628740
} // namespace

0 commit comments

Comments
 (0)