Skip to content

Commit 5f72739

Browse files
authored
Arm backend: Add zero-copy boundary metadata to the VGF runtime. (pytorch#23487)
In this PR I build complete description of each model-boundary IO resource, so that later we can safely decide whether models IO can be backed directly by host memory instead of copying into VGF-owned Vulkan memory. This is stage 3 of the zero copy IO project. cc @digantdesai @freddan80 @per @zingo @oscarandersson8218 @mansnils @Sebastian-Larsson @robell @rascani Signed-off-by: Elena Zhelezina <elena.zhelezina@arm.com>
1 parent 059fa80 commit 5f72739

5 files changed

Lines changed: 577 additions & 7 deletions

File tree

‎backends/arm/runtime/VGFSetup.cpp‎

Lines changed: 161 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -897,13 +897,8 @@ VkResult create_tensor_unbound(
897897
VkTensorDescriptionARM* description,
898898
VkTensorARM* tensor,
899899
VkMemoryRequirements2* memory_requirements) {
900-
VkTensorUsageFlagsARM tensor_usage = VK_TENSOR_USAGE_SHADER_BIT_ARM |
901-
VK_TENSOR_USAGE_TRANSFER_SRC_BIT_ARM |
902-
VK_TENSOR_USAGE_TRANSFER_DST_BIT_ARM | VK_TENSOR_USAGE_DATA_GRAPH_BIT_ARM;
903-
904-
if (image_aliasing) {
905-
tensor_usage |= VK_TENSOR_USAGE_IMAGE_ALIASING_BIT_ARM;
906-
}
900+
const VkTensorUsageFlagsARM tensor_usage =
901+
vgf_tensor_usage_flags(image_aliasing);
907902

908903
*description = VkTensorDescriptionARM{
909904
.sType = VK_STRUCTURE_TYPE_TENSOR_DESCRIPTION_ARM,
@@ -951,6 +946,64 @@ VkResult create_tensor_unbound(
951946
return VK_SUCCESS;
952947
}
953948

949+
static void cache_external_tensor_host_allocation_capabilities(
950+
VkPhysicalDevice physical_device,
951+
uint32_t resource_index,
952+
const VkTensorDescriptionARM& tensor_description,
953+
VgfZeroCopyIoMetadata* metadata) {
954+
// In this fuction we try to decide:
955+
// for this exact VGF tensor, does the physical GPU support
956+
// importing host memory as external memory.
957+
// We don't import memory here and we don't change execution.
958+
// We only fills feilds in VgfZeroCopyIoMetadata for later.
959+
if (metadata == nullptr ||
960+
!vgf_zero_copy_io_structurally_eligible(*metadata) ||
961+
!metadata->host_memory_import_advertised) {
962+
return;
963+
}
964+
965+
#if defined(VK_EXT_external_memory_host)
966+
// This is a physical-device capability query.
967+
if (vkGetPhysicalDeviceExternalTensorPropertiesARM == nullptr) {
968+
ET_LOG(
969+
Info,
970+
"VGF resource %u cannot query external tensor properties; marking "
971+
"zero-copy metadata ineligible",
972+
resource_index);
973+
return;
974+
}
975+
976+
const VkPhysicalDeviceExternalTensorInfoARM external_tensor_info{
977+
.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_TENSOR_INFO_ARM,
978+
.pNext = nullptr,
979+
.flags = metadata->tensor_create_flags,
980+
.pDescription = &tensor_description,
981+
.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT,
982+
};
983+
VkExternalTensorPropertiesARM external_tensor_properties{
984+
.sType = VK_STRUCTURE_TYPE_EXTERNAL_TENSOR_PROPERTIES_ARM,
985+
.pNext = nullptr,
986+
};
987+
988+
vkGetPhysicalDeviceExternalTensorPropertiesARM(
989+
physical_device, &external_tensor_info, &external_tensor_properties);
990+
vgf_cache_external_tensor_properties(
991+
metadata, external_tensor_properties.externalMemoryProperties);
992+
993+
if (metadata->external_tensor_requires_dedicated_allocation) {
994+
ET_LOG(
995+
Info,
996+
"VGF resource %u reports dedicated-only external tensor allocation; "
997+
"zero-copy host import is ineligible",
998+
resource_index);
999+
}
1000+
#else
1001+
(void)physical_device;
1002+
(void)resource_index;
1003+
(void)tensor_description;
1004+
#endif
1005+
}
1006+
9541007
VkResult bind_tensor_memory_and_create_view(
9551008
VkDevice device,
9561009
VkFormat format,
@@ -2027,6 +2080,24 @@ bool VgfRepr::process_vgf(
20272080
owns_memory,
20282081
true,
20292082
is_in});
2083+
2084+
auto zero_copy_metadata = make_vgf_zero_copy_io_metadata(
2085+
/*is_model_boundary_resource=*/true,
2086+
is_in,
2087+
VK_DESCRIPTOR_TYPE_TENSOR_ARM,
2088+
resource_format,
2089+
/*tensor_backed=*/true,
2090+
uses_alias_group,
2091+
image_aliasing,
2092+
&tensor_description,
2093+
host_memory_import_capabilities_);
2094+
cache_external_tensor_host_allocation_capabilities(
2095+
vk_physical,
2096+
static_cast<uint32_t>(i),
2097+
tensor_description,
2098+
&zero_copy_metadata);
2099+
zero_copy_io_metadata.push_back(std::move(zero_copy_metadata));
2100+
20302101
resource_index_to_io_index[i] = static_cast<int>(IOs.size() - 1);
20312102

20322103
resource_bindings[i] = ResourceBinding{
@@ -2120,6 +2191,16 @@ bool VgfRepr::process_vgf(
21202191
owns_memory,
21212192
true,
21222193
is_in});
2194+
zero_copy_io_metadata.push_back(make_vgf_zero_copy_io_metadata(
2195+
/*is_model_boundary_resource=*/true,
2196+
is_in,
2197+
VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
2198+
resource_format,
2199+
/*tensor_backed=*/false,
2200+
uses_alias_group,
2201+
image_aliasing,
2202+
nullptr,
2203+
host_memory_import_capabilities_));
21232204
resource_index_to_io_index[i] = static_cast<int>(IOs.size() - 1);
21242205

21252206
resource_bindings[i] = ResourceBinding{
@@ -2336,6 +2417,16 @@ bool VgfRepr::process_vgf(
23362417
true,
23372418
owns_image_memory,
23382419
is_in});
2420+
zero_copy_io_metadata.push_back(make_vgf_zero_copy_io_metadata(
2421+
/*is_model_boundary_resource=*/true,
2422+
is_in,
2423+
resource_type,
2424+
resource_format,
2425+
/*tensor_backed=*/false,
2426+
uses_alias_group,
2427+
needs_tensor_aliasing,
2428+
nullptr,
2429+
host_memory_import_capabilities_));
23392430
resource_index_to_io_index[i] = static_cast<int>(IOs.size() - 1);
23402431

23412432
resource_bindings[i] = ResourceBinding{
@@ -3077,6 +3168,33 @@ bool VgfRepr::process_vgf(
30773168
auto mrt_i =
30783169
sequence_decoder->getBindingSlotMrtIndex(descriptor_slots, i);
30793170
const auto& binding_info = resource_bindings[mrt_i];
3171+
3172+
if (mrt_i < resource_index_to_io_index.size()) {
3173+
const int boundary_io_index = resource_index_to_io_index[mrt_i];
3174+
if (boundary_io_index >= 0 &&
3175+
static_cast<size_t>(boundary_io_index) <
3176+
zero_copy_io_metadata.size()) {
3177+
const VgfBoundaryBindingRef binding_ref{
3178+
.segment_index = static_cast<uint32_t>(segment_id),
3179+
.set_index = set_index,
3180+
.binding = binding,
3181+
};
3182+
if (!vgf_record_boundary_binding(
3183+
&zero_copy_io_metadata,
3184+
static_cast<size_t>(boundary_io_index),
3185+
binding_ref)) {
3186+
ET_LOG(
3187+
Info,
3188+
"VGF boundary descriptor mapping is ambiguous: "
3189+
"segment=%u set=%u binding=%u mrt=%u; zero-copy "
3190+
"metadata is ineligible",
3191+
static_cast<uint32_t>(segment_id),
3192+
set_index,
3193+
binding,
3194+
mrt_i);
3195+
}
3196+
}
3197+
}
30803198
if (binding_info.descriptor_type == VK_DESCRIPTOR_TYPE_TENSOR_ARM) {
30813199
ET_LOG(
30823200
Debug,
@@ -3674,6 +3792,12 @@ bool VgfRepr::process_vgf(
36743792
}
36753793

36763794
model_input_io_index[model_input_idx] = io_idx;
3795+
if (static_cast<size_t>(io_idx) < zero_copy_io_metadata.size()) {
3796+
auto& metadata = zero_copy_io_metadata[io_idx];
3797+
metadata.mapped_to_model_boundary = true;
3798+
metadata.executorch_argument_index =
3799+
static_cast<int64_t>(model_input_idx);
3800+
}
36773801

36783802
ET_LOG(
36793803
Debug,
@@ -3720,6 +3844,12 @@ bool VgfRepr::process_vgf(
37203844
// sequence output binding slots in model-output order, matching the output
37213845
// names vector, so map outputs by binding-slot position.
37223846
model_output_io_index[output_idx] = io_idx;
3847+
if (static_cast<size_t>(io_idx) < zero_copy_io_metadata.size()) {
3848+
auto& metadata = zero_copy_io_metadata[io_idx];
3849+
metadata.mapped_to_model_boundary = true;
3850+
metadata.executorch_argument_index =
3851+
static_cast<int64_t>(model_input_count + output_idx);
3852+
}
37233853

37243854
ET_LOG(
37253855
Debug,
@@ -3730,6 +3860,29 @@ bool VgfRepr::process_vgf(
37303860
io_idx);
37313861
}
37323862

3863+
for (size_t io_idx = 0; io_idx < zero_copy_io_metadata.size(); ++io_idx) {
3864+
auto& metadata = zero_copy_io_metadata[io_idx];
3865+
vgf_finalize_zero_copy_io_metadata(&metadata);
3866+
ET_LOG(
3867+
Info,
3868+
"VGF zero-copy boundary metadata: io=%zu arg=%lld input=%d "
3869+
"eligible=%d type=%u tiling=%u alias=%d bindings=%zu "
3870+
"host_import_enabled=%d external_tensor_supported=%d "
3871+
"dedicated_only=%d",
3872+
io_idx,
3873+
static_cast<long long>(metadata.executorch_argument_index),
3874+
static_cast<int>(metadata.is_input),
3875+
static_cast<int>(metadata.eligible),
3876+
static_cast<uint32_t>(metadata.descriptor_type),
3877+
static_cast<uint32_t>(metadata.tiling),
3878+
static_cast<int>(metadata.has_alias_group),
3879+
metadata.bindings.size(),
3880+
static_cast<int>(metadata.host_memory_import_enabled),
3881+
static_cast<int>(metadata.external_tensor_host_allocation_supported),
3882+
static_cast<int>(
3883+
metadata.external_tensor_requires_dedicated_allocation));
3884+
}
3885+
37333886
{
37343887
VGF_PROFILE_SCOPE(event_tracer, "VGF_INIT_ALLOCATE_COMMAND_BUFFER");
37353888

@@ -4296,6 +4449,7 @@ void VgfRepr::free_vgf() {
42964449
}
42974450
}
42984451
IOs.clear();
4452+
zero_copy_io_metadata.clear();
42994453
for (const auto& alloc : extra_allocs) {
43004454
if (alloc.descriptor_type == VK_DESCRIPTOR_TYPE_TENSOR_ARM) {
43014455
if (alloc.owns_memory) {

‎backends/arm/runtime/VGFSetup.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ using executorch::runtime::CompileSpec;
2323

2424
#include <executorch/backends/arm/runtime/VGFNeuralStatistics.h>
2525
#include <executorch/backends/arm/runtime/VGFVulkanFeatures.h>
26+
#include <executorch/backends/arm/runtime/VGFZeroCopy.h>
2627

2728
namespace executorch {
2829
namespace backends {
@@ -153,6 +154,11 @@ class VgfRepr {
153154
std::vector<SegmentState> segments;
154155
std::vector<ResourceAlloc> extra_allocs;
155156

157+
// Metadata for each IO entry. The ExecuTorch argument index is assigned
158+
// later from the model boundary mapping and
159+
// must not be assumed from the IO index.
160+
std::vector<VgfZeroCopyIoMetadata> zero_copy_io_metadata;
161+
156162
// Mapping to persistent IO memory
157163
static bool map_io(IO* io, void** handle) {
158164
if (io->persistent_memory == nullptr) {

0 commit comments

Comments
 (0)