|
6 | 6 |
|
7 | 7 | #include <executorch/backends/native/runtime/vulkan/VulkanEngine.h> |
8 | 8 |
|
| 9 | +#include <cmath> |
9 | 10 | #include <cstdint> |
10 | 11 | #include <cstring> |
11 | 12 | #include <limits> |
@@ -840,6 +841,148 @@ std::vector<uint8_t> make_dynamic_buffer_view_index_program() { |
840 | 841 | builder, graph, {fbs::CreateOutputSpecDirect(builder, "gathered")}); |
841 | 842 | } |
842 | 843 |
|
| 844 | +// Grouped-query causal attention of a query with a dynamic sequence length, |
| 845 | +// starting at the position equal to that length, over caches of kSdpaCache. |
| 846 | +constexpr int64_t kSdpaSeq = 4; |
| 847 | +constexpr int64_t kSdpaCache = 8; |
| 848 | +constexpr int64_t kSdpaHeads = 2; |
| 849 | +constexpr int64_t kSdpaKvHeads = 1; |
| 850 | +constexpr int64_t kSdpaDim = 8; |
| 851 | + |
| 852 | +std::vector<uint8_t> make_dynamic_sdpa_program() { |
| 853 | + flatbuffers::FlatBufferBuilder builder; |
| 854 | + const auto query_meta = create_tensor_meta( |
| 855 | + builder, |
| 856 | + fbs::ScalarType::FLOAT, |
| 857 | + {1, kSdpaSeq, kSdpaHeads, kSdpaDim}, |
| 858 | + {1, 1, kSdpaHeads, kSdpaDim}); |
| 859 | + const auto cache_meta = create_tensor_meta( |
| 860 | + builder, fbs::ScalarType::FLOAT, {1, kSdpaCache, kSdpaKvHeads, kSdpaDim}); |
| 861 | + const auto flat_meta = create_tensor_meta( |
| 862 | + builder, |
| 863 | + fbs::ScalarType::FLOAT, |
| 864 | + {1, kSdpaSeq, kSdpaHeads * kSdpaDim}, |
| 865 | + {1, 1, kSdpaHeads * kSdpaDim}); |
| 866 | + const std::vector<flatbuffers::Offset<fbs::TensorValue>> tensor_values = { |
| 867 | + fbs::CreateTensorValueDirect(builder, "query", query_meta), |
| 868 | + fbs::CreateTensorValueDirect(builder, "key", cache_meta), |
| 869 | + fbs::CreateTensorValueDirect(builder, "value", cache_meta), |
| 870 | + fbs::CreateTensorValueDirect(builder, "attention", query_meta), |
| 871 | + fbs::CreateTensorValueDirect(builder, "flat", flat_meta), |
| 872 | + }; |
| 873 | + |
| 874 | + const std::vector<flatbuffers::Offset<fbs::Output>> query_outputs = { |
| 875 | + fbs::CreateOutputDirect(builder, "query")}; |
| 876 | + const std::vector<flatbuffers::Offset<fbs::Output>> key_outputs = { |
| 877 | + fbs::CreateOutputDirect(builder, "key")}; |
| 878 | + const std::vector<flatbuffers::Offset<fbs::Output>> value_outputs = { |
| 879 | + fbs::CreateOutputDirect(builder, "value")}; |
| 880 | + const std::vector<flatbuffers::Offset<fbs::Output>> pos_outputs = { |
| 881 | + fbs::CreateOutputDirect( |
| 882 | + builder, "pos", nullptr, fbs::OutputValueKind::INT)}; |
| 883 | + const std::vector<flatbuffers::Offset<fbs::Output>> attention_outputs = { |
| 884 | + fbs::CreateOutputDirect(builder, "attention")}; |
| 885 | + const std::vector<flatbuffers::Offset<fbs::Output>> view_outputs = { |
| 886 | + fbs::CreateOutputDirect(builder, "flat", "attention")}; |
| 887 | + const std::vector<flatbuffers::Offset<fbs::NamedArgument>> sym_size_inputs = { |
| 888 | + fbs::CreateNamedArgumentDirect( |
| 889 | + builder, "self", create_tensor_arg(builder, "query")), |
| 890 | + fbs::CreateNamedArgumentDirect( |
| 891 | + builder, "dim", create_int_arg(builder, 1)), |
| 892 | + }; |
| 893 | + const auto start_pos = fbs::CreateIntArgDirect(builder, 0, "pos"); |
| 894 | + const auto dropout = fbs::CreateFloatArg(builder, 0.0); |
| 895 | + const std::vector<flatbuffers::Offset<fbs::NamedArgument>> sdpa_inputs = { |
| 896 | + fbs::CreateNamedArgumentDirect( |
| 897 | + builder, "query", create_tensor_arg(builder, "query")), |
| 898 | + fbs::CreateNamedArgumentDirect( |
| 899 | + builder, "key", create_tensor_arg(builder, "key")), |
| 900 | + fbs::CreateNamedArgumentDirect( |
| 901 | + builder, "value", create_tensor_arg(builder, "value")), |
| 902 | + fbs::CreateNamedArgumentDirect( |
| 903 | + builder, |
| 904 | + "start_pos", |
| 905 | + fbs::CreateArgument( |
| 906 | + builder, fbs::ArgumentValue::IntArg, start_pos.Union())), |
| 907 | + fbs::CreateNamedArgumentDirect( |
| 908 | + builder, "attn_mask", create_none_arg(builder)), |
| 909 | + fbs::CreateNamedArgumentDirect( |
| 910 | + builder, |
| 911 | + "drpout_p", |
| 912 | + fbs::CreateArgument( |
| 913 | + builder, fbs::ArgumentValue::FloatArg, dropout.Union())), |
| 914 | + fbs::CreateNamedArgumentDirect( |
| 915 | + builder, "is_causal", create_bool_arg(builder, true)), |
| 916 | + fbs::CreateNamedArgumentDirect( |
| 917 | + builder, "scale", create_none_arg(builder)), |
| 918 | + }; |
| 919 | + const std::vector<flatbuffers::Offset<fbs::NamedArgument>> view_inputs = { |
| 920 | + fbs::CreateNamedArgumentDirect( |
| 921 | + builder, "self", create_tensor_arg(builder, "attention")), |
| 922 | + fbs::CreateNamedArgumentDirect( |
| 923 | + builder, |
| 924 | + "size", |
| 925 | + create_int_list_arg(builder, {1, -1, kSdpaHeads * kSdpaDim})), |
| 926 | + }; |
| 927 | + const std::vector<flatbuffers::Offset<fbs::NamedArgument>> output_inputs = { |
| 928 | + fbs::CreateNamedArgumentDirect( |
| 929 | + builder, "", create_tensor_arg(builder, "flat"))}; |
| 930 | + const std::vector<flatbuffers::Offset<fbs::Node>> nodes = { |
| 931 | + fbs::CreateNodeDirect( |
| 932 | + builder, |
| 933 | + "query", |
| 934 | + fbs::OpKind::PLACEHOLDER, |
| 935 | + "", |
| 936 | + nullptr, |
| 937 | + &query_outputs), |
| 938 | + fbs::CreateNodeDirect( |
| 939 | + builder, "key", fbs::OpKind::PLACEHOLDER, "", nullptr, &key_outputs), |
| 940 | + fbs::CreateNodeDirect( |
| 941 | + builder, |
| 942 | + "value", |
| 943 | + fbs::OpKind::PLACEHOLDER, |
| 944 | + "", |
| 945 | + nullptr, |
| 946 | + &value_outputs), |
| 947 | + fbs::CreateNodeDirect( |
| 948 | + builder, |
| 949 | + "pos", |
| 950 | + fbs::OpKind::CALL_FUNCTION, |
| 951 | + "torch.ops.aten.sym_size.int", |
| 952 | + &sym_size_inputs, |
| 953 | + &pos_outputs), |
| 954 | + fbs::CreateNodeDirect( |
| 955 | + builder, |
| 956 | + "sdpa", |
| 957 | + fbs::OpKind::CALL_FUNCTION, |
| 958 | + "torch.ops.llama.custom_sdpa.default", |
| 959 | + &sdpa_inputs, |
| 960 | + &attention_outputs), |
| 961 | + fbs::CreateNodeDirect( |
| 962 | + builder, |
| 963 | + "view", |
| 964 | + fbs::OpKind::CALL_FUNCTION, |
| 965 | + "torch.ops.aten.view.default", |
| 966 | + &view_inputs, |
| 967 | + &view_outputs), |
| 968 | + fbs::CreateNodeDirect( |
| 969 | + builder, "output", fbs::OpKind::OUTPUT, "", &output_inputs), |
| 970 | + }; |
| 971 | + const auto graph = fbs::CreateGraph( |
| 972 | + builder, |
| 973 | + builder.CreateVector(nodes), |
| 974 | + create_strings(builder, {"query", "key", "value"}), |
| 975 | + create_strings(builder, {"flat"}), |
| 976 | + builder.CreateVector(tensor_values)); |
| 977 | + return finish_program( |
| 978 | + builder, graph, {fbs::CreateOutputSpecDirect(builder, "flat")}); |
| 979 | +} |
| 980 | + |
| 981 | +float sdpa_input_value(size_t index, int64_t salt) { |
| 982 | + return 0.125f * static_cast<float>((static_cast<int64_t>(index) * salt) % 9) - |
| 983 | + 0.5f; |
| 984 | +} |
| 985 | + |
843 | 986 | // Values of the int4 [4, 8] weight in groups of 4 used by the program below. |
844 | 987 | constexpr int64_t kQ4Out = 4; |
845 | 988 | constexpr int64_t kQ4In = 8; |
@@ -1381,6 +1524,66 @@ TEST(VulkanEngineTest, RejectsBufferViewResizeThatDoesNotDivide) { |
1381 | 1524 | EXPECT_THROW(compiled.executable->execute(), std::runtime_error); |
1382 | 1525 | } |
1383 | 1526 |
|
| 1527 | +TEST(VulkanEngineTest, ExecutesCausalSdpaOnResizedQuery) { |
| 1528 | + CompiledProgram compiled = compile_program(make_dynamic_sdpa_program()); |
| 1529 | + constexpr int64_t kSeq = 3; |
| 1530 | + compiled.executable->resize_input(0, {1, kSeq, kSdpaHeads, kSdpaDim}); |
| 1531 | + |
| 1532 | + std::vector<float> query(kSeq * kSdpaHeads * kSdpaDim); |
| 1533 | + std::vector<float> key(kSdpaCache * kSdpaKvHeads * kSdpaDim); |
| 1534 | + std::vector<float> value(key.size()); |
| 1535 | + for (size_t i = 0; i < query.size(); ++i) { |
| 1536 | + query[i] = sdpa_input_value(i, 5); |
| 1537 | + } |
| 1538 | + for (size_t i = 0; i < key.size(); ++i) { |
| 1539 | + key[i] = sdpa_input_value(i, 7); |
| 1540 | + value[i] = sdpa_input_value(i, 4); |
| 1541 | + } |
| 1542 | + compiled.executable->set_input( |
| 1543 | + 0, query.data(), query.size(), ScalarType::Float); |
| 1544 | + compiled.executable->set_input(1, key.data(), key.size(), ScalarType::Float); |
| 1545 | + compiled.executable->set_input( |
| 1546 | + 2, value.data(), value.size(), ScalarType::Float); |
| 1547 | + compiled.executable->execute(); |
| 1548 | + |
| 1549 | + EXPECT_EQ( |
| 1550 | + compiled.executable->output_sizes(0), |
| 1551 | + (std::vector<int64_t>{1, kSeq, kSdpaHeads * kSdpaDim})); |
| 1552 | + std::vector<float> output(query.size()); |
| 1553 | + compiled.executable->get_output( |
| 1554 | + 0, output.data(), output.size(), ScalarType::Float); |
| 1555 | + |
| 1556 | + const float scale = 1.0f / std::sqrt(static_cast<float>(kSdpaDim)); |
| 1557 | + for (int64_t s = 0; s < kSeq; ++s) { |
| 1558 | + const int64_t context = kSeq + s + 1; |
| 1559 | + for (int64_t h = 0; h < kSdpaHeads; ++h) { |
| 1560 | + const int64_t kv_head = h / (kSdpaHeads / kSdpaKvHeads); |
| 1561 | + const float* q = &query[(s * kSdpaHeads + h) * kSdpaDim]; |
| 1562 | + std::vector<float> weights(context); |
| 1563 | + float total = 0.0f; |
| 1564 | + for (int64_t c = 0; c < context; ++c) { |
| 1565 | + const float* k = &key[(c * kSdpaKvHeads + kv_head) * kSdpaDim]; |
| 1566 | + float dot = 0.0f; |
| 1567 | + for (int64_t d = 0; d < kSdpaDim; ++d) { |
| 1568 | + dot += q[d] * k[d]; |
| 1569 | + } |
| 1570 | + weights[c] = std::exp(dot * scale); |
| 1571 | + total += weights[c]; |
| 1572 | + } |
| 1573 | + for (int64_t d = 0; d < kSdpaDim; ++d) { |
| 1574 | + float expected = 0.0f; |
| 1575 | + for (int64_t c = 0; c < context; ++c) { |
| 1576 | + expected += weights[c] / total * |
| 1577 | + value[(c * kSdpaKvHeads + kv_head) * kSdpaDim + d]; |
| 1578 | + } |
| 1579 | + EXPECT_NEAR( |
| 1580 | + output[(s * kSdpaHeads + h) * kSdpaDim + d], expected, 1e-4f) |
| 1581 | + << "position " << s << " head " << h << " dim " << d; |
| 1582 | + } |
| 1583 | + } |
| 1584 | + } |
| 1585 | +} |
| 1586 | + |
1384 | 1587 | TEST(VulkanEngineTest, ExecutesLinearOnDirectlyQuantizedWeight) { |
1385 | 1588 | CompiledProgram compiled = compile_program( |
1386 | 1589 | make_direct_q4_linear_program(), make_direct_q4_linear_tensors()); |
|
0 commit comments