Skip to content

Commit 56594e4

Browse files
[executorch][native-vk] Resize Vulkan inputs within their bounds (pytorch#23463)
This PR was created by the merge bot to help merge the original PR into the main branch. ghstack PR number: pytorch#23428 by @digantdesai ^ Please use this as the source of truth for the PR details, comments, and reviews ghstack PR base: https://github.com/pytorch/executorch/tree/gh/digantdesai/123/base ghstack PR head: https://github.com/pytorch/executorch/tree/gh/digantdesai/123/head Merge bot PR base: https://github.com/pytorch/executorch/tree/gh/digantdesai/122/orig Merge bot PR head: https://github.com/pytorch/executorch/tree/gh/digantdesai/123/orig Differential Revision: [D121704322](https://our.internmc.facebook.com/intern/diff/D121704322/) @diff-train-skip-merge cc @SS-JIA @manuelcandales @digantdesai @cbilgin Co-authored-by: Digant Desai <digantdesai@meta.com>
1 parent 25e167f commit 56594e4

4 files changed

Lines changed: 167 additions & 16 deletions

File tree

‎backends/native/runtime/vulkan/VulkanEngineExecutable.cpp‎

Lines changed: 65 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,12 @@ size_t logical_numel(const TensorMeta& meta) {
100100
return static_cast<size_t>(numel);
101101
}
102102

103+
size_t logical_numel(const std::vector<int64_t>& sizes) {
104+
TensorMeta meta;
105+
meta.sizes = sizes;
106+
return logical_numel(meta);
107+
}
108+
103109
size_t logical_nbytes(const TensorMeta& meta) {
104110
const size_t element_bytes = element_size(meta.dtype);
105111
if (element_bytes == 0) {
@@ -123,6 +129,19 @@ std::string registry_key(const std::string& target) {
123129
key = "aten.relu.default";
124130
} else if (key == "llama.update_cache.default") {
125131
key = "update_cache.default";
132+
} else if (key == "aten.sym_size.int") {
133+
key = "sym_size.int";
134+
} else {
135+
constexpr std::array<std::string_view, 4> kSymbolicOps{
136+
"add", "floordiv", "mul", "sub"};
137+
constexpr std::string_view kOperatorPrefix = "_operator.";
138+
if (key.starts_with(kOperatorPrefix)) {
139+
const std::string_view candidate =
140+
std::string_view(key).substr(kOperatorPrefix.size());
141+
if (std::ranges::find(kSymbolicOps, candidate) != kSymbolicOps.end()) {
142+
key = candidate;
143+
}
144+
}
126145
}
127146
return key;
128147
}
@@ -317,6 +336,15 @@ void validate_user_io(const Method& method) {
317336
throw std::runtime_error(
318337
"vulkan: quantized graph inputs and outputs are not supported");
319338
}
339+
// set_input and get_output copy contiguous bytes.
340+
const auto is_contiguous = [&method](ValueId id) {
341+
return method.graph.value(id).tensor_meta().is_contiguous();
342+
};
343+
if (!std::ranges::all_of(method.graph.input_ids, is_contiguous) ||
344+
!std::ranges::all_of(method.graph.output_ids, is_contiguous)) {
345+
throw std::runtime_error(
346+
"vulkan: graph inputs and outputs must be contiguous");
347+
}
320348
}
321349
// cppcheck-suppress-end useStlAlgorithm
322350

@@ -441,6 +469,9 @@ class VulkanEngineExecutable final : public EngineExecutable {
441469
VkRef& vref_at(ValueId id) {
442470
return checked_slot(vref_, id, "vref");
443471
}
472+
const VkRef& vref_at(ValueId id) const {
473+
return checked_slot(vref_, id, "vref");
474+
}
444475
utils::GPUMemoryLayout& layout_at(ValueId id) {
445476
return checked_slot(layout_, id, "layout");
446477
}
@@ -492,11 +523,11 @@ class VulkanEngineExecutable final : public EngineExecutable {
492523
}
493524

494525
std::vector<int64_t> input_sizes(size_t i) const override {
495-
return meta_of(inputs_.at(i)).sizes;
526+
return graph_->sizes_of(vref_at(inputs_.at(i)));
496527
}
497528

498529
std::vector<int64_t> output_sizes(size_t i) const override {
499-
return meta_of(outputs_.at(user_output_positions_.at(i))).sizes;
530+
return graph_->sizes_of(vref_at(outputs_.at(user_output_positions_.at(i))));
500531
}
501532

502533
ScalarType input_dtype(size_t i) const override {
@@ -507,12 +538,21 @@ class VulkanEngineExecutable final : public EngineExecutable {
507538
return meta_of(outputs_.at(user_output_positions_.at(i))).dtype;
508539
}
509540

541+
void resize_input(size_t i, const std::vector<int64_t>& sizes) override {
542+
const ValueId input = inputs_.at(i);
543+
if (!meta_of(input).accepts_sizes(sizes)) {
544+
throw std::runtime_error(
545+
"vulkan: input shape is outside serialized bounds");
546+
}
547+
graph_->resize_input(static_cast<int64_t>(i), sizes);
548+
}
549+
510550
void set_input(size_t i, const void* data, size_t numel, ScalarType src_dtype)
511551
override {
512552
const TensorMeta& meta = meta_of(inputs_.at(i));
513-
if (numel != logical_numel(meta)) {
553+
if (numel != logical_numel(input_sizes(i))) {
514554
throw std::runtime_error(
515-
"vulkan: input element count does not match its shape");
555+
"vulkan: input element count does not match its current shape");
516556
}
517557
if (numel == 0) {
518558
return;
@@ -536,9 +576,9 @@ class VulkanEngineExecutable final : public EngineExecutable {
536576
void get_output(size_t i, void* data, size_t numel, ScalarType dst_dtype)
537577
override {
538578
const size_t output_position = user_output_positions_.at(i);
539-
if (numel != logical_numel(meta_of(outputs_.at(output_position)))) {
579+
if (numel != logical_numel(output_sizes(i))) {
540580
throw std::runtime_error(
541-
"vulkan: output element count does not match its shape");
581+
"vulkan: output element count does not match its current shape");
542582
}
543583
if (numel == 0) {
544584
return;
@@ -894,9 +934,25 @@ class VulkanEngineExecutable final : public EngineExecutable {
894934
const BoolArg& a = arg.as_bool();
895935
return valid(a.id) ? vref_at(a.id) : graph_->add_scalar<bool>(a.value);
896936
}
897-
case ArgKind::IntList:
898-
return graph_->add_scalar_list(
899-
std::vector<int64_t>(arg.as_int_list().values));
937+
case ArgKind::IntList: {
938+
const IntListArg& list = arg.as_int_list();
939+
if (list.ids.empty()) {
940+
return graph_->add_scalar_list(std::vector<int64_t>(list.values));
941+
}
942+
if (list.ids.size() != list.values.size()) {
943+
throw std::runtime_error(
944+
"vulkan: symbolic IntList ids and values must have equal size");
945+
}
946+
std::vector<VkRef> refs;
947+
refs.reserve(list.values.size());
948+
for (size_t i = 0; i < list.values.size(); ++i) {
949+
refs.push_back(
950+
valid(list.ids.at(i))
951+
? vref_at(list.ids.at(i))
952+
: graph_->add_scalar<int64_t>(list.values[i]));
953+
}
954+
return graph_->add_value_list(std::move(refs));
955+
}
900956
case ArgKind::FloatList:
901957
return graph_->add_scalar_list(
902958
std::vector<double>(arg.as_float_list().values));

‎backends/native/runtime/vulkan/passes/InsertPrepack.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,11 @@ namespace ptn::vulkan {
1919
namespace {
2020

2121
bool needs_prepack(const Node& node) {
22-
constexpr std::array<std::string_view, 3> kOps{
22+
constexpr std::array<std::string_view, 4> kOps{
2323
"torch.ops.aten.add.Tensor",
2424
"torch.ops.aten.index.Tensor",
2525
"torch.ops.aten.mul.Tensor",
26+
"torch.ops.aten.slice_copy.Tensor",
2627
};
2728
return node.is_call() && std::ranges::find(kOps, node.target) != kOps.end();
2829
}

‎backends/native/test/runtime/vulkan/test_insert_prepack.cpp‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,15 @@ TEST(InsertPrepackTest, SharesOnePrepackAcrossConsumers) {
8989
EXPECT_EQ(graph.value(4).role, ValueRole::ConstantTensor);
9090
}
9191

92+
TEST(InsertPrepackTest, PrepacksConstantSliceInput) {
93+
Method method = make_method();
94+
method.graph.node(2).target = "torch.ops.aten.slice_copy.Tensor";
95+
96+
EXPECT_EQ(insert_prepack_nodes(method), 1);
97+
EXPECT_EQ(
98+
method.graph.node(2).input_value_ids(), std::vector<ValueId>({0, 4}));
99+
}
100+
92101
TEST(InsertPrepackTest, LeavesSelfPrepackingOpUntouched) {
93102
Method method = make_method();
94103
method.graph.node(2).target = "torch.ops.aten.linear.default";

‎backends/native/test/runtime/vulkan/test_vulkan_engine.cpp‎

Lines changed: 91 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -44,13 +44,20 @@ create_strings(
4444
flatbuffers::Offset<fbs::TensorMeta> create_tensor_meta(
4545
flatbuffers::FlatBufferBuilder& builder,
4646
fbs::ScalarType dtype,
47-
const std::vector<int64_t>& shape) {
47+
const std::vector<int64_t>& shape,
48+
const std::vector<int64_t>& lower_bounds = {},
49+
const std::vector<int32_t>& dim_order = {}) {
4850
std::vector<flatbuffers::Offset<fbs::Dim>> sizes;
4951
sizes.reserve(shape.size());
50-
for (const int64_t size : shape) {
51-
sizes.push_back(fbs::CreateDim(builder, size, size));
52+
for (size_t i = 0; i < shape.size(); ++i) {
53+
const int64_t lower = lower_bounds.empty() ? shape[i] : lower_bounds.at(i);
54+
sizes.push_back(fbs::CreateDim(builder, lower, shape[i]));
5255
}
53-
return fbs::CreateTensorMeta(builder, dtype, builder.CreateVector(sizes));
56+
return fbs::CreateTensorMeta(
57+
builder,
58+
dtype,
59+
builder.CreateVector(sizes),
60+
builder.CreateVector(dim_order));
5461
}
5562

5663
flatbuffers::Offset<fbs::TensorMeta> create_tensor_meta(
@@ -121,9 +128,11 @@ std::vector<uint8_t> finish_program(
121128

122129
std::vector<uint8_t> make_add_program(
123130
fbs::ScalarType dtype,
131+
const std::vector<int64_t>& shape = {4},
132+
const std::vector<int64_t>& lower_bounds = {},
124133
bool include_mutation_output = false) {
125134
flatbuffers::FlatBufferBuilder builder;
126-
const auto meta = create_tensor_meta(builder, dtype);
135+
const auto meta = create_tensor_meta(builder, dtype, shape, lower_bounds);
127136
const std::vector<flatbuffers::Offset<fbs::TensorValue>> tensor_values = {
128137
fbs::CreateTensorValueDirect(builder, "left", meta),
129138
fbs::CreateTensorValueDirect(builder, "right", meta),
@@ -601,6 +610,52 @@ std::vector<uint8_t> make_mean_view_mm_program() {
601610
builder, graph, {fbs::CreateOutputSpecDirect(builder, "product")});
602611
}
603612

613+
std::vector<uint8_t> make_relu_program(const std::vector<int32_t>& dim_order) {
614+
flatbuffers::FlatBufferBuilder builder;
615+
const auto meta = create_tensor_meta(
616+
builder, fbs::ScalarType::FLOAT, {2, 2}, /*lower_bounds=*/{}, dim_order);
617+
const std::vector<flatbuffers::Offset<fbs::TensorValue>> tensor_values = {
618+
fbs::CreateTensorValueDirect(builder, "input", meta),
619+
fbs::CreateTensorValueDirect(builder, "relu", meta),
620+
};
621+
const std::vector<flatbuffers::Offset<fbs::Output>> input_outputs = {
622+
fbs::CreateOutputDirect(builder, "input")};
623+
const std::vector<flatbuffers::Offset<fbs::Output>> relu_outputs = {
624+
fbs::CreateOutputDirect(builder, "relu")};
625+
const std::vector<flatbuffers::Offset<fbs::NamedArgument>> relu_inputs = {
626+
fbs::CreateNamedArgumentDirect(
627+
builder, "self", create_tensor_arg(builder, "input"))};
628+
const std::vector<flatbuffers::Offset<fbs::NamedArgument>> output_inputs = {
629+
fbs::CreateNamedArgumentDirect(
630+
builder, "", create_tensor_arg(builder, "relu"))};
631+
const std::vector<flatbuffers::Offset<fbs::Node>> nodes = {
632+
fbs::CreateNodeDirect(
633+
builder,
634+
"input",
635+
fbs::OpKind::PLACEHOLDER,
636+
"",
637+
nullptr,
638+
&input_outputs),
639+
fbs::CreateNodeDirect(
640+
builder,
641+
"relu",
642+
fbs::OpKind::CALL_FUNCTION,
643+
"torch.ops.aten.relu.default",
644+
&relu_inputs,
645+
&relu_outputs),
646+
fbs::CreateNodeDirect(
647+
builder, "output", fbs::OpKind::OUTPUT, "", &output_inputs),
648+
};
649+
const auto graph = fbs::CreateGraph(
650+
builder,
651+
builder.CreateVector(nodes),
652+
create_strings(builder, {"input"}),
653+
create_strings(builder, {"relu"}),
654+
builder.CreateVector(tensor_values));
655+
return finish_program(
656+
builder, graph, {fbs::CreateOutputSpecDirect(builder, "relu")});
657+
}
658+
604659
struct CompiledProgram {
605660
std::unique_ptr<VulkanEngineHost> host;
606661
std::shared_ptr<const Program> program;
@@ -683,7 +738,7 @@ TEST(VulkanEngineTest, ExposesOnlyUserOutputsWhenMutationOutputsArePresent) {
683738

684739
TEST(VulkanEngineTest, RejectsUserInputMutation) {
685740
try {
686-
compile_program(make_add_program(fbs::ScalarType::FLOAT, true));
741+
compile_program(make_add_program(fbs::ScalarType::FLOAT, {4}, {}, true));
687742
FAIL() << "expected the user input mutation to be rejected";
688743
} catch (const std::runtime_error& e) {
689744
EXPECT_NE(
@@ -693,6 +748,36 @@ TEST(VulkanEngineTest, RejectsUserInputMutation) {
693748
}
694749
}
695750

751+
TEST(VulkanEngineTest, ResizesInputsWithinSerializedBounds) {
752+
CompiledProgram compiled =
753+
compile_program(make_add_program(fbs::ScalarType::FLOAT, {4}, {1}));
754+
755+
EXPECT_THROW(compiled.executable->resize_input(0, {0}), std::runtime_error);
756+
EXPECT_THROW(compiled.executable->resize_input(0, {5}), std::runtime_error);
757+
compiled.executable->resize_input(0, {2});
758+
compiled.executable->resize_input(1, {2});
759+
EXPECT_EQ(compiled.executable->input_sizes(0), (std::vector<int64_t>{2}));
760+
761+
const std::vector<float> left = {1.0f, 2.0f};
762+
const std::vector<float> right = {10.0f, 20.0f};
763+
compiled.executable->set_input(
764+
0, left.data(), left.size(), ScalarType::Float);
765+
compiled.executable->set_input(
766+
1, right.data(), right.size(), ScalarType::Float);
767+
compiled.executable->execute();
768+
769+
EXPECT_EQ(compiled.executable->output_sizes(0), (std::vector<int64_t>{2}));
770+
std::vector<float> output(2);
771+
compiled.executable->get_output(
772+
0, output.data(), output.size(), ScalarType::Float);
773+
EXPECT_EQ(output, (std::vector<float>{11.0f, 22.0f}));
774+
}
775+
776+
TEST(VulkanEngineTest, RejectsNonContiguousGraphInputsAndOutputs) {
777+
EXPECT_NO_THROW(compile_program(make_relu_program({0, 1})));
778+
EXPECT_THROW(compile_program(make_relu_program({1, 0})), std::runtime_error);
779+
}
780+
696781
TEST(VulkanEngineTest, PreservesLogicalLongDtypeAndChecksNarrowing) {
697782
CompiledProgram compiled =
698783
compile_program(make_add_program(fbs::ScalarType::LONG));

0 commit comments

Comments
 (0)