Skip to content

Commit 5d22fd9

Browse files
authored
Arm backend: Fixes to Arm neural statistics (pytorch#22074)
Change-Id: Ie34c3e240f22aa7f0f26fa4b469e1e9bab77770f cc @digantdesai @freddan80 @per @zingo @oscarandersson8218 @mansnils @Sebastian-Larsson @robell @rascani Signed-off-by: Elena Zhelezina <elena.zhelezina@arm.com>
1 parent 47a22c5 commit 5d22fd9

8 files changed

Lines changed: 338 additions & 29 deletions

File tree

‎backends/arm/runtime/VGFBackend.cpp‎

Lines changed: 91 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -98,15 +98,14 @@ VkResult vkml_allocate_basics(
9898
VkDevice* device,
9999
VkQueue* queue,
100100
VkCommandPool* command_pool,
101-
uint32_t* queue_family_index);
101+
uint32_t* queue_family_index,
102+
bool request_neural_statistics,
103+
bool* neural_statistics_device_enabled);
102104

103105
// Helper functions to dump VGF Delegate Boundary Inputs
104106
constexpr const char* kVgfDumpInputsDirEnv = "EXECUTORCH_VGF_DUMP_INPUTS_DIR";
105107
constexpr const char* kVgfDumpInputsAndExitEnv =
106108
"EXECUTORCH_VGF_DUMP_INPUTS_AND_EXIT";
107-
constexpr const char* kVgfNeuralStatisticsEnv =
108-
"EXECUTORCH_VGF_ENABLE_NEURAL_STATISTICS";
109-
110109
std::atomic<uint64_t> g_vgf_dump_invocation{0};
111110

112111
bool env_flag_enabled(const char* name) {
@@ -414,6 +413,7 @@ class VGFBackend final : public ::executorch::runtime::BackendInterface {
414413
}
415414

416415
VkResult result;
416+
neural_statistics_config_ = get_vgf_neural_statistics_runtime_config();
417417

418418
// Fetch basic vulkan objects once
419419
result = vkml_allocate_basics(
@@ -422,7 +422,9 @@ class VGFBackend final : public ::executorch::runtime::BackendInterface {
422422
&vk_device,
423423
&vk_queue,
424424
&vk_command_pool,
425-
&vk_queue_family_index);
425+
&vk_queue_family_index,
426+
neural_statistics_config_.requested,
427+
&neural_statistics_device_enabled_);
426428
if (result != VK_SUCCESS) {
427429
ET_LOG(
428430
Error, "Failed to initialize the Vulkan device error 0x%08X", result);
@@ -508,7 +510,10 @@ class VGFBackend final : public ::executorch::runtime::BackendInterface {
508510
vk_device,
509511
vk_queue,
510512
vk_command_pool,
511-
vk_queue_family_index);
513+
vk_queue_family_index,
514+
neural_statistics_config_.requested,
515+
neural_statistics_device_enabled_,
516+
neural_statistics_config_.mode_index);
512517

513518
#ifdef ET_EVENT_TRACER_ENABLED
514519
event_tracer_end_profiling_delegate(event_tracer, allocate_repr_event);
@@ -683,7 +688,7 @@ class VGFBackend final : public ::executorch::runtime::BackendInterface {
683688
#ifdef ET_EVENT_TRACER_ENABLED
684689
event_tracer_end_profiling_delegate(event_tracer, dispatch_event);
685690

686-
if (event_tracer != nullptr && env_flag_enabled(kVgfNeuralStatisticsEnv)) {
691+
if (event_tracer != nullptr && repr->neural_statistics_requested()) {
687692
// We attach the neural statistics JSON blob to ETDump as delegate
688693
// metadata, when event tracer is active.
689694

@@ -787,6 +792,8 @@ class VGFBackend final : public ::executorch::runtime::BackendInterface {
787792
VkQueue vk_queue = VK_NULL_HANDLE;
788793
VkCommandPool vk_command_pool = VK_NULL_HANDLE;
789794
uint32_t vk_queue_family_index = UINT32_MAX;
795+
VgfNeuralStatisticsRuntimeConfig neural_statistics_config_{};
796+
bool neural_statistics_device_enabled_ = false;
790797
bool is_initialized_ = false;
791798
};
792799

@@ -802,9 +809,15 @@ VkResult vkml_allocate_basics(
802809
VkDevice* device,
803810
VkQueue* queue,
804811
VkCommandPool* command_pool,
805-
uint32_t* queue_family_index) {
812+
uint32_t* queue_family_index,
813+
bool request_neural_statistics,
814+
bool* neural_statistics_device_enabled) {
806815
VkResult result;
807816

817+
if (neural_statistics_device_enabled != nullptr) {
818+
*neural_statistics_device_enabled = false;
819+
}
820+
808821
if (VK_SUCCESS != volkInitialize()) {
809822
ET_LOG(Error, "Volk failed to initialize");
810823
return VK_ERROR_INITIALIZATION_FAILED;
@@ -948,9 +961,21 @@ VkResult vkml_allocate_basics(
948961
.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_1_FEATURES,
949962
.pNext = &available_12,
950963
};
964+
#if defined(VK_ARM_data_graph_neural_accelerator_statistics)
965+
VkPhysicalDeviceDataGraphNeuralAcceleratorStatisticsFeaturesARM
966+
available_neural_statistics{
967+
.sType =
968+
VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DATA_GRAPH_NEURAL_ACCELERATOR_STATISTICS_FEATURES_ARM,
969+
.pNext = &available_11,
970+
.dataGraphNeuralAcceleratorStatistics = VK_FALSE,
971+
};
972+
void* available_features_pnext = &available_neural_statistics;
973+
#else
974+
void* available_features_pnext = &available_11;
975+
#endif
951976
VkPhysicalDeviceFeatures2 available_2 = {
952977
.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2,
953-
.pNext = &available_11,
978+
.pNext = available_features_pnext,
954979
};
955980
vkGetPhysicalDeviceFeatures2(*physical_device, &available_2);
956981

@@ -992,6 +1017,15 @@ VkResult vkml_allocate_basics(
9921017
VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DATA_GRAPH_FEATURES_ARM, nullptr};
9931018
features_graph.dataGraph = true;
9941019
features_graph.pNext = &features_tensor;
1020+
#if defined(VK_ARM_data_graph_neural_accelerator_statistics)
1021+
VkPhysicalDeviceDataGraphNeuralAcceleratorStatisticsFeaturesARM
1022+
features_neural_statistics{
1023+
.sType =
1024+
VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DATA_GRAPH_NEURAL_ACCELERATOR_STATISTICS_FEATURES_ARM,
1025+
.pNext = &features_tensor,
1026+
.dataGraphNeuralAcceleratorStatistics = VK_FALSE,
1027+
};
1028+
#endif
9951029

9961030
VkPhysicalDeviceFeatures device_features = {};
9971031
device_features.shaderInt16 = VK_TRUE;
@@ -1014,6 +1048,54 @@ VkResult vkml_allocate_basics(
10141048
*physical_device, nullptr, &exts, available.data());
10151049

10161050
vector<const char*> requested_exts;
1051+
1052+
#if defined(VK_ARM_data_graph_neural_accelerator_statistics)
1053+
const bool neural_statistics_extension_available = std::any_of(
1054+
available.begin(), available.end(), [](const auto& ext_avail) {
1055+
return std::strcmp(
1056+
VK_ARM_DATA_GRAPH_NEURAL_ACCELERATOR_STATISTICS_EXTENSION_NAME,
1057+
ext_avail.extensionName) == 0;
1058+
});
1059+
const bool neural_statistics_feature_available =
1060+
available_neural_statistics.dataGraphNeuralAcceleratorStatistics ==
1061+
VK_TRUE;
1062+
const bool enable_neural_statistics_device = request_neural_statistics &&
1063+
neural_statistics_extension_available &&
1064+
neural_statistics_feature_available;
1065+
1066+
if (request_neural_statistics && !neural_statistics_extension_available) {
1067+
ET_LOG(
1068+
Info,
1069+
"%s was requested but the Vulkan device does not expose %s",
1070+
kVgfNeuralStatisticsEnableEnv,
1071+
VK_ARM_DATA_GRAPH_NEURAL_ACCELERATOR_STATISTICS_EXTENSION_NAME);
1072+
} else if (
1073+
request_neural_statistics && !neural_statistics_feature_available) {
1074+
ET_LOG(
1075+
Info,
1076+
"%s was requested but dataGraphNeuralAcceleratorStatistics is not supported",
1077+
kVgfNeuralStatisticsEnableEnv);
1078+
}
1079+
1080+
if (enable_neural_statistics_device) {
1081+
requested_exts.push_back(
1082+
VK_ARM_DATA_GRAPH_NEURAL_ACCELERATOR_STATISTICS_EXTENSION_NAME);
1083+
features_neural_statistics.dataGraphNeuralAcceleratorStatistics = VK_TRUE;
1084+
features_graph.pNext = &features_neural_statistics;
1085+
if (neural_statistics_device_enabled != nullptr) {
1086+
*neural_statistics_device_enabled = true;
1087+
}
1088+
}
1089+
#else
1090+
if (request_neural_statistics) {
1091+
ET_LOG(
1092+
Info,
1093+
"%s was requested but Vulkan headers do not expose "
1094+
"VK_ARM_data_graph_neural_accelerator_statistics",
1095+
kVgfNeuralStatisticsEnableEnv);
1096+
}
1097+
#endif
1098+
10171099
for (auto& ext : dev_exts) {
10181100
bool found = false;
10191101
for (auto const& ext_avail : available) {

‎backends/arm/runtime/VGFNeuralStatistics.cpp‎

Lines changed: 50 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@
88
#include <executorch/backends/arm/runtime/VGFNeuralStatistics.h>
99

1010
#include <algorithm>
11+
#include <cctype>
12+
#include <cstdlib>
1113
#include <cstring>
1214
#include <iomanip>
1315
#include <sstream>
@@ -23,6 +25,25 @@ namespace backends {
2325
namespace vgf {
2426
namespace {
2527

28+
std::string ascii_lower(const char* value) {
29+
std::string normalized = value == nullptr ? "" : value;
30+
std::transform(
31+
normalized.begin(),
32+
normalized.end(),
33+
normalized.begin(),
34+
[](unsigned char c) { return static_cast<char>(std::tolower(c)); });
35+
return normalized;
36+
}
37+
38+
bool env_flag_requested(const char* value) {
39+
if (value == nullptr || value[0] == '\0') {
40+
return false;
41+
}
42+
const std::string normalized = ascii_lower(value);
43+
return normalized != "0" && normalized != "false" && normalized != "off" &&
44+
normalized != "no";
45+
}
46+
2647
// Converts a C++ string into a valid JSON string literal.
2748
std::string json_escape(const std::string& value) {
2849
std::ostringstream out;
@@ -142,11 +163,7 @@ VgfNeuralStatisticsBlob query_pipeline_property(
142163
return make_unavailable_blob("VkPipeline is null");
143164
}
144165

145-
#if defined(VK_ARM_data_graph) && \
146-
defined( \
147-
VK_DATA_GRAPH_PIPELINE_PROPERTY_NEURAL_ACCELERATOR_DEBUG_DATABASE_ARM) && \
148-
defined( \
149-
VK_DATA_GRAPH_PIPELINE_PROPERTY_NEURAL_ACCELERATOR_STATISTICS_INFO_ARM)
166+
#if defined(VK_ARM_data_graph_neural_accelerator_statistics)
150167

151168
if (!vkGetDataGraphPipelineAvailablePropertiesARM ||
152169
!vkGetDataGraphPipelinePropertiesARM) {
@@ -330,12 +347,34 @@ VgfNeuralStatisticsCollectorForTest& test_collector_storage() {
330347

331348
} // namespace
332349

350+
VgfNeuralStatisticsRuntimeConfig get_vgf_neural_statistics_runtime_config() {
351+
VgfNeuralStatisticsRuntimeConfig config;
352+
config.requested =
353+
env_flag_requested(std::getenv(kVgfNeuralStatisticsEnableEnv));
354+
355+
const char* mode_value = std::getenv(kVgfNeuralStatisticsModeEnv);
356+
if (mode_value == nullptr || mode_value[0] == '\0' ||
357+
std::strcmp(mode_value, "1") == 0) {
358+
config.mode_index = 1;
359+
return config;
360+
}
361+
362+
if (std::strcmp(mode_value, "0") == 0) {
363+
config.mode_index = 0;
364+
return config;
365+
}
366+
367+
ET_LOG(
368+
Info,
369+
"Invalid %s=%s; expected 0 or 1. Falling back to mode 1.",
370+
kVgfNeuralStatisticsModeEnv,
371+
mode_value);
372+
config.mode_index = 1;
373+
return config;
374+
}
375+
333376
bool vgf_neural_statistics_api_available() {
334-
#if defined(VK_ARM_data_graph) && \
335-
defined( \
336-
VK_DATA_GRAPH_PIPELINE_PROPERTY_NEURAL_ACCELERATOR_DEBUG_DATABASE_ARM) && \
337-
defined( \
338-
VK_DATA_GRAPH_PIPELINE_PROPERTY_NEURAL_ACCELERATOR_STATISTICS_INFO_ARM)
377+
#if defined(VK_ARM_data_graph_neural_accelerator_statistics)
339378
return vkGetDataGraphPipelineAvailablePropertiesARM != nullptr &&
340379
vkGetDataGraphPipelinePropertiesARM != nullptr;
341380
#else
@@ -348,11 +387,7 @@ VgfNeuralStatisticsCollection collect_vgf_neural_statistics(
348387
const std::vector<VgfNeuralStatisticsSegmentContext>& segments) {
349388
VgfNeuralStatisticsCollection collection;
350389

351-
#if defined(VK_ARM_data_graph) && \
352-
defined( \
353-
VK_DATA_GRAPH_PIPELINE_PROPERTY_NEURAL_ACCELERATOR_DEBUG_DATABASE_ARM) && \
354-
defined( \
355-
VK_DATA_GRAPH_PIPELINE_PROPERTY_NEURAL_ACCELERATOR_STATISTICS_INFO_ARM)
390+
#if defined(VK_ARM_data_graph_neural_accelerator_statistics)
356391

357392
collection.api_available = vgf_neural_statistics_api_available();
358393

‎backends/arm/runtime/VGFNeuralStatistics.h‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,23 @@ constexpr const char* kVgfNeuralStatisticsSchema =
2525
"executorch.vgf.neural_statistics";
2626
constexpr int kVgfNeuralStatisticsSchemaVersion = 1;
2727

28+
constexpr const char* kVgfNeuralStatisticsEnableEnv =
29+
"EXECUTORCH_VGF_ENABLE_NEURAL_STATISTICS";
30+
constexpr const char* kVgfNeuralStatisticsModeEnv =
31+
"EXECUTORCH_VGF_NEURAL_STATISTICS_MODE";
32+
33+
// Runtime neural-statistics configuration. The public mode is 0/1, while the
34+
// Vulkan enum values are STATISTICS0/STATISTICS1 (1/2 respectively).
35+
struct VgfNeuralStatisticsRuntimeConfig {
36+
bool requested = false;
37+
38+
// ExecuTorch mode indices 0 and 1 select Vulkan modes STATISTICS0 and
39+
// STATISTICS1, whose underlying enum values are 1 and 2, respectively.
40+
int mode_index = 1;
41+
};
42+
43+
VgfNeuralStatisticsRuntimeConfig get_vgf_neural_statistics_runtime_config();
44+
2845
// One binary payload from neural statistics API
2946
struct VgfNeuralStatisticsBlob {
3047
// Whether we got it successfully

‎backends/arm/runtime/VGFSetup.cpp‎

Lines changed: 49 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3045,9 +3045,23 @@ bool VgfRepr::process_vgf(
30453045
.pConstants = constants.data(),
30463046
};
30473047

3048+
#if defined(VK_ARM_data_graph_neural_accelerator_statistics)
3049+
VkDataGraphPipelineNeuralStatisticsCreateInfoARM neural_statistics_info{
3050+
.sType =
3051+
VK_STRUCTURE_TYPE_DATA_GRAPH_PIPELINE_NEURAL_STATISTICS_CREATE_INFO_ARM,
3052+
.pNext = &shader_info,
3053+
.allowNeuralStatistics = VK_TRUE,
3054+
};
3055+
const void* graph_pipeline_pnext = neural_statistics_device_enabled_
3056+
? static_cast<const void*>(&neural_statistics_info)
3057+
: static_cast<const void*>(&shader_info);
3058+
#else
3059+
const void* graph_pipeline_pnext = &shader_info;
3060+
#endif
3061+
30483062
VkDataGraphPipelineCreateInfoARM graph_pipeline_info{
30493063
.sType = VK_STRUCTURE_TYPE_DATA_GRAPH_PIPELINE_CREATE_INFO_ARM,
3050-
.pNext = &shader_info,
3064+
.pNext = graph_pipeline_pnext,
30513065
.flags = 0,
30523066
.layout = segment.vk_pipeline_layout,
30533067
.resourceInfoCount =
@@ -3074,10 +3088,30 @@ bool VgfRepr::process_vgf(
30743088
return false;
30753089
}
30763090

3091+
#if defined(VK_ARM_data_graph_neural_accelerator_statistics)
3092+
const VkNeuralAcceleratorStatisticsModeARM neural_statistics_mode =
3093+
neural_statistics_mode_index_ == 0
3094+
? VK_NEURAL_ACCELERATOR_STATISTICS_MODE_STATISTICS0_ARM
3095+
: VK_NEURAL_ACCELERATOR_STATISTICS_MODE_STATISTICS1_ARM;
3096+
3097+
VkDataGraphPipelineSessionNeuralStatisticsCreateInfoARM
3098+
neural_statistics_session_info{
3099+
.sType =
3100+
VK_STRUCTURE_TYPE_DATA_GRAPH_PIPELINE_SESSION_NEURAL_STATISTICS_CREATE_INFO_ARM,
3101+
.pNext = nullptr,
3102+
.mode = neural_statistics_mode,
3103+
};
3104+
const void* pipeline_session_pnext = neural_statistics_device_enabled_
3105+
? static_cast<const void*>(&neural_statistics_session_info)
3106+
: nullptr;
3107+
#else
3108+
const void* pipeline_session_pnext = nullptr;
3109+
#endif
3110+
30773111
VkDataGraphPipelineSessionCreateInfoARM pipeline_session_info{
30783112
.sType =
30793113
VK_STRUCTURE_TYPE_DATA_GRAPH_PIPELINE_SESSION_CREATE_INFO_ARM,
3080-
.pNext = nullptr,
3114+
.pNext = pipeline_session_pnext,
30813115
.flags = 0,
30823116
.dataGraphPipeline = segment.vk_pipeline,
30833117
};
@@ -3137,7 +3171,7 @@ bool VgfRepr::process_vgf(
31373171
VK_DATA_GRAPH_PIPELINE_SESSION_BIND_POINT_TRANSIENT_ARM;
31383172

31393173
bool is_neural_statistics_bind_point = false;
3140-
#ifdef VK_DATA_GRAPH_PIPELINE_SESSION_BIND_POINT_NEURAL_ACCELERATOR_STATISTICS_ARM
3174+
#if defined(VK_ARM_data_graph_neural_accelerator_statistics)
31413175
is_neural_statistics_bind_point = bind_point_requirement.bindPoint ==
31423176
VK_DATA_GRAPH_PIPELINE_SESSION_BIND_POINT_NEURAL_ACCELERATOR_STATISTICS_ARM;
31433177
#endif
@@ -3161,6 +3195,13 @@ bool VgfRepr::process_vgf(
31613195
ET_LOG(Info, "%s", message.c_str());
31623196
};
31633197

3198+
if (is_neural_statistics_bind_point &&
3199+
!neural_statistics_device_enabled_) {
3200+
mark_neural_statistics_unavailable(
3201+
"Neural accelerator statistics were not enabled for this device");
3202+
continue;
3203+
}
3204+
31643205
if (bind_point_requirement.bindPointType !=
31653206
VK_DATA_GRAPH_PIPELINE_SESSION_BIND_POINT_TYPE_MEMORY_ARM) {
31663207
const std::string message =
@@ -3962,6 +4003,11 @@ VgfRepr::get_neural_statistics_segment_contexts() const {
39624003
}
39634004

39644005
std::string VgfRepr::collect_neural_statistics_metadata() const {
4006+
if (neural_statistics_requested_ && !neural_statistics_device_enabled_) {
4007+
return make_vgf_neural_statistics_unavailable_metadata(
4008+
"VK_ARM_data_graph_neural_accelerator_statistics is unavailable or its feature is disabled");
4009+
}
4010+
39654011
return collect_vgf_neural_statistics_metadata(
39664012
vk_device, get_neural_statistics_segment_contexts());
39674013
}

0 commit comments

Comments
 (0)