Skip to content

Commit 71edbbc

Browse files
[executorch][native-vk] Resize tensor views with their source (pytorch#23475)
This PR was created by the merge bot to help merge the original PR into the main branch. ghstack PR number: pytorch#23442 by @digantdesai ^ Please use this as the source of truth for the PR details, comments, and reviews ghstack PR base: https://github.com/pytorch/executorch/tree/gh/digantdesai/135/base ghstack PR head: https://github.com/pytorch/executorch/tree/gh/digantdesai/135/head Merge bot PR base: https://github.com/pytorch/executorch/tree/gh/digantdesai/134/orig Merge bot PR head: https://github.com/pytorch/executorch/tree/gh/digantdesai/135/orig Differential Revision: [D121704333](https://our.internmc.facebook.com/intern/diff/D121704333/) @diff-train-skip-merge Co-authored-by: Digant Desai <digantdesai@meta.com>
1 parent fd82979 commit 71edbbc

2 files changed

Lines changed: 191 additions & 4 deletions

File tree

‎backends/native/runtime/vulkan/VulkanEngineExecutable.cpp‎

Lines changed: 52 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -321,6 +321,37 @@ bool is_metadata_only_view(std::string_view key) {
321321
std::ranges::find(kOps, key) != kOps.end();
322322
}
323323

324+
// ET-VK tensor views keep the sizes they were created with, so a view of a
325+
// dynamically sized source must be resized whenever that source is.
326+
void resize_view_alias(
327+
ComputeGraph* graph,
328+
const std::vector<vkcompute::ArgGroup>& /*args*/,
329+
const std::vector<VkRef>& resize_args) {
330+
const int64_t numel =
331+
utils::multiply_integers(graph->sizes_of(resize_args.at(1)));
332+
std::vector<int64_t> sizes =
333+
graph->extract_int_or_symint_list(resize_args.at(2));
334+
int64_t known_numel = 1;
335+
auto inferred = sizes.end();
336+
for (auto it = sizes.begin(); it != sizes.end(); ++it) {
337+
if (*it == -1 && inferred == sizes.end()) {
338+
inferred = it;
339+
} else if (
340+
*it < 0 || __builtin_mul_overflow(known_numel, *it, &known_numel)) {
341+
throw std::runtime_error("vulkan: invalid view size list");
342+
}
343+
}
344+
if (inferred != sizes.end()) {
345+
if (known_numel != 0 && numel % known_numel != 0) {
346+
throw std::runtime_error("vulkan: view size list does not divide input");
347+
}
348+
*inferred = known_numel == 0 ? 0 : numel / known_numel;
349+
} else if (known_numel != numel) {
350+
throw std::runtime_error("vulkan: view size list does not match input");
351+
}
352+
graph->virtual_resize(resize_args.at(0), sizes);
353+
}
354+
324355
MemoryKind memory_kind(utils::StorageType storage) {
325356
switch (storage) {
326357
case utils::StorageType::BUFFER:
@@ -1586,6 +1617,18 @@ class VulkanEngineExecutable final : public EngineExecutable {
15861617
VK_GET_OP_FN(key)(*graph_, args);
15871618
}
15881619

1620+
bool is_dynamic(ValueId id) const {
1621+
return !meta_of(id).lower_bounds.empty();
1622+
}
1623+
1624+
void add_view_resize(const Node& n) {
1625+
graph_->execute_nodes().emplace_back(new vkcompute::ExecuteNode(
1626+
resize_view_alias,
1627+
{vref_at(n.outputs.at(0).value_id),
1628+
resolve_arg(n.inputs.at(0).arg),
1629+
resolve_arg(n.inputs.at(1).arg)}));
1630+
}
1631+
15891632
void index_output_kinds() {
15901633
symint_.assign(g().values.size(), 0);
15911634
for (const NodeId node_id : nodes_) {
@@ -1814,12 +1857,20 @@ class VulkanEngineExecutable final : public EngineExecutable {
18141857
if (!n.is_call()) {
18151858
continue;
18161859
}
1817-
if (is_metadata_only_view(registry_key(n.target)) &&
1860+
const std::string key = registry_key(n.target);
1861+
if (is_metadata_only_view(key) &&
18181862
std::ranges::all_of(n.outputs, [&](const Output& output) {
18191863
return valid(output.value_id) &&
18201864
valid(g().value(output.value_id).alias_id);
18211865
})) {
18221866
check_view_offset(n);
1867+
if (key == "aten.view.default") {
1868+
add_view_resize(n);
1869+
} else if (is_dynamic(g().value(n.outputs.at(0).value_id).alias_id)) {
1870+
throw std::runtime_error(
1871+
"vulkan: node '" + n.name + "' (" + n.target +
1872+
"): only aten.view can alias a dynamically shaped source");
1873+
}
18231874
continue;
18241875
}
18251876
try {

‎backends/native/test/runtime/vulkan/test_vulkan_engine.cpp‎

Lines changed: 139 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -212,10 +212,13 @@ std::vector<uint8_t> make_add_program(
212212
// With `alias_first`, the alias value is serialized ahead of its source.
213213
std::vector<uint8_t> make_alias_program(
214214
fbs::ScalarType alias_dtype = fbs::ScalarType::FLOAT,
215+
const std::vector<int64_t>& lower_bounds = {},
215216
bool alias_first = false) {
216217
flatbuffers::FlatBufferBuilder builder;
217-
const auto meta = create_tensor_meta(builder, fbs::ScalarType::FLOAT);
218-
const auto alias_meta = create_tensor_meta(builder, alias_dtype);
218+
const auto meta =
219+
create_tensor_meta(builder, fbs::ScalarType::FLOAT, {4}, lower_bounds);
220+
const auto alias_meta =
221+
create_tensor_meta(builder, alias_dtype, {4}, lower_bounds);
219222
std::vector<flatbuffers::Offset<fbs::TensorValue>> tensor_values = {
220223
fbs::CreateTensorValueDirect(builder, "input", meta),
221224
fbs::CreateTensorValueDirect(builder, "alias", alias_meta),
@@ -744,6 +747,99 @@ std::vector<uint8_t> make_dynamic_view_mm_program() {
744747
builder, graph, {fbs::CreateOutputSpecDirect(builder, "product")});
745748
}
746749

750+
// index.Tensor keeps its operands in buffer storage, so `rows` is a zero-copy
751+
// buffer view of `input`, and gathering along dim 1 sizes the output by it.
752+
std::vector<uint8_t> make_dynamic_buffer_view_index_program() {
753+
flatbuffers::FlatBufferBuilder builder;
754+
const auto input_meta =
755+
create_tensor_meta(builder, fbs::ScalarType::FLOAT, {4, 2}, {1, 2});
756+
const auto rows_meta =
757+
create_tensor_meta(builder, fbs::ScalarType::FLOAT, {2, 4}, {1, 4});
758+
const auto index_meta =
759+
create_tensor_meta(builder, fbs::ScalarType::LONG, {2});
760+
const auto gathered_meta =
761+
create_tensor_meta(builder, fbs::ScalarType::FLOAT, {2, 2}, {1, 2});
762+
const std::vector<flatbuffers::Offset<fbs::TensorValue>> tensor_values = {
763+
fbs::CreateTensorValueDirect(builder, "input", input_meta),
764+
fbs::CreateTensorValueDirect(builder, "index", index_meta),
765+
fbs::CreateTensorValueDirect(builder, "rows", rows_meta),
766+
fbs::CreateTensorValueDirect(builder, "gathered", gathered_meta),
767+
};
768+
769+
const std::vector<flatbuffers::Offset<fbs::Output>> input_outputs = {
770+
fbs::CreateOutputDirect(builder, "input")};
771+
const std::vector<flatbuffers::Offset<fbs::Output>> index_outputs = {
772+
fbs::CreateOutputDirect(builder, "index")};
773+
const std::vector<flatbuffers::Offset<fbs::Output>> view_outputs = {
774+
fbs::CreateOutputDirect(builder, "rows", "input")};
775+
const std::vector<flatbuffers::Offset<fbs::Output>> gathered_outputs = {
776+
fbs::CreateOutputDirect(builder, "gathered")};
777+
const std::vector<flatbuffers::Offset<fbs::NamedArgument>> view_inputs = {
778+
fbs::CreateNamedArgumentDirect(
779+
builder, "self", create_tensor_arg(builder, "input")),
780+
fbs::CreateNamedArgumentDirect(
781+
builder, "size", create_int_list_arg(builder, {-1, 4})),
782+
};
783+
const auto indices = fbs::CreateOptionalTensorListArg(
784+
builder,
785+
create_strings(builder, {"", "index"}),
786+
builder.CreateVector(std::vector<uint8_t>{0, 1}));
787+
const std::vector<flatbuffers::Offset<fbs::NamedArgument>> index_inputs = {
788+
fbs::CreateNamedArgumentDirect(
789+
builder, "self", create_tensor_arg(builder, "rows")),
790+
fbs::CreateNamedArgumentDirect(
791+
builder,
792+
"indices",
793+
fbs::CreateArgument(
794+
builder,
795+
fbs::ArgumentValue::OptionalTensorListArg,
796+
indices.Union())),
797+
};
798+
const std::vector<flatbuffers::Offset<fbs::NamedArgument>> output_inputs = {
799+
fbs::CreateNamedArgumentDirect(
800+
builder, "", create_tensor_arg(builder, "gathered"))};
801+
const std::vector<flatbuffers::Offset<fbs::Node>> nodes = {
802+
fbs::CreateNodeDirect(
803+
builder,
804+
"input",
805+
fbs::OpKind::PLACEHOLDER,
806+
"",
807+
nullptr,
808+
&input_outputs),
809+
fbs::CreateNodeDirect(
810+
builder,
811+
"index",
812+
fbs::OpKind::PLACEHOLDER,
813+
"",
814+
nullptr,
815+
&index_outputs),
816+
fbs::CreateNodeDirect(
817+
builder,
818+
"view",
819+
fbs::OpKind::CALL_FUNCTION,
820+
"torch.ops.aten.view.default",
821+
&view_inputs,
822+
&view_outputs),
823+
fbs::CreateNodeDirect(
824+
builder,
825+
"gather",
826+
fbs::OpKind::CALL_FUNCTION,
827+
"torch.ops.aten.index.Tensor",
828+
&index_inputs,
829+
&gathered_outputs),
830+
fbs::CreateNodeDirect(
831+
builder, "output", fbs::OpKind::OUTPUT, "", &output_inputs),
832+
};
833+
const auto graph = fbs::CreateGraph(
834+
builder,
835+
builder.CreateVector(nodes),
836+
create_strings(builder, {"input", "index"}),
837+
create_strings(builder, {"gathered"}),
838+
builder.CreateVector(tensor_values));
839+
return finish_program(
840+
builder, graph, {fbs::CreateOutputSpecDirect(builder, "gathered")});
841+
}
842+
747843
// Values of the int4 [4, 8] weight in groups of 4 used by the program below.
748844
constexpr int64_t kQ4Out = 4;
749845
constexpr int64_t kQ4In = 8;
@@ -1066,7 +1162,7 @@ TEST(VulkanEngineTest, ExecutesMetadataOnlyAliasAfterAliasLowering) {
10661162

10671163
TEST(VulkanEngineTest, ExecutesAliasSerializedBeforeItsSource) {
10681164
CompiledProgram compiled = compile_program(
1069-
make_alias_program(fbs::ScalarType::FLOAT, /*alias_first=*/true));
1165+
make_alias_program(fbs::ScalarType::FLOAT, {}, /*alias_first=*/true));
10701166
const std::vector<float> input = {1.0f, 2.0f, 3.0f, 4.0f};
10711167
compiled.executable->set_input(
10721168
0, input.data(), input.size(), ScalarType::Float);
@@ -1129,6 +1225,18 @@ TEST(VulkanEngineTest, RejectsAliasWithDifferentDtypeFromSource) {
11291225
}
11301226
}
11311227

1228+
TEST(VulkanEngineTest, RejectsNonViewAliasOfDynamicSource) {
1229+
try {
1230+
compile_program(make_alias_program(fbs::ScalarType::FLOAT, {1}));
1231+
FAIL() << "expected the dynamic alias to be rejected";
1232+
} catch (const std::runtime_error& e) {
1233+
EXPECT_NE(
1234+
std::string(e.what()).find("only aten.view can alias"),
1235+
std::string::npos)
1236+
<< e.what();
1237+
}
1238+
}
1239+
11321240
TEST(VulkanEngineTest, RejectsMutableStateHeldByAnotherLiveExecutable) {
11331241
CompiledProgram compiled = compile_program(
11341242
make_state_program({{"prefill", "cache"}, {"decode", "cache"}}),
@@ -1245,6 +1353,34 @@ TEST(VulkanEngineTest, ResizesWidthPackedViewOfWidthPackedInput) {
12451353
0, output.data(), output.size(), ScalarType::Float);
12461354
EXPECT_EQ(output, (std::vector<float>{12.0f, 34.0f}));
12471355
}
1356+
1357+
TEST(VulkanEngineTest, ResizesBufferViewWithItsSource) {
1358+
CompiledProgram compiled =
1359+
compile_program(make_dynamic_buffer_view_index_program());
1360+
compiled.executable->resize_input(0, {2, 2});
1361+
1362+
const std::vector<float> input = {1.0f, 2.0f, 3.0f, 4.0f};
1363+
const std::vector<int64_t> index = {3, 0};
1364+
compiled.executable->set_input(
1365+
0, input.data(), input.size(), ScalarType::Float);
1366+
compiled.executable->set_input(
1367+
1, index.data(), index.size(), ScalarType::Long);
1368+
compiled.executable->execute();
1369+
1370+
EXPECT_EQ(compiled.executable->output_sizes(0), (std::vector<int64_t>{1, 2}));
1371+
std::vector<float> output(2);
1372+
compiled.executable->get_output(
1373+
0, output.data(), output.size(), ScalarType::Float);
1374+
EXPECT_EQ(output, (std::vector<float>{4.0f, 1.0f}));
1375+
}
1376+
1377+
TEST(VulkanEngineTest, RejectsBufferViewResizeThatDoesNotDivide) {
1378+
CompiledProgram compiled =
1379+
compile_program(make_dynamic_buffer_view_index_program());
1380+
compiled.executable->resize_input(0, {3, 2});
1381+
EXPECT_THROW(compiled.executable->execute(), std::runtime_error);
1382+
}
1383+
12481384
TEST(VulkanEngineTest, ExecutesLinearOnDirectlyQuantizedWeight) {
12491385
CompiledProgram compiled = compile_program(
12501386
make_direct_q4_linear_program(), make_direct_q4_linear_tensors());

0 commit comments

Comments
 (0)