Skip to content

Commit 067c74b

Browse files
authored
Arm backend: Fix bad init performance in VGF for multi segments (pytorch#22588)
Improvements to init performance in VGF. cc @digantdesai @freddan80 @per @zingo @oscarandersson8218 @mansnils @Sebastian-Larsson @robell @rascani Signed-off-by: Elena Zhelezina <elena.zhelezina@arm.com>
1 parent 55a969e commit 067c74b

2 files changed

Lines changed: 63 additions & 2 deletions

File tree

‎backends/arm/runtime/VGFSetup.cpp‎

Lines changed: 51 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2668,9 +2668,44 @@ bool VgfRepr::process_vgf(
26682668
}
26692669
}
26702670

2671+
// Keep the pipeline cache local to this VgfRepr. This preserves reuse across
2672+
// the many segment pipelines in one VGF without sharing mutable cache state
2673+
// or cache lifetime across independently initialized delegate handles.
2674+
if (vk_pipeline_cache == VK_NULL_HANDLE) {
2675+
VkPipelineCacheCreateInfo pipeline_cache_info{
2676+
.sType = VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO,
2677+
.pNext = nullptr,
2678+
// Default mode: Vulkan internally synchronizes concurrent
2679+
// pipeline-cache access. The cache is per-VgfRepr anyway, so
2680+
// independent delegates do not contend on it.
2681+
.flags = 0,
2682+
.initialDataSize = 0,
2683+
.pInitialData = nullptr,
2684+
};
2685+
2686+
{
2687+
VGF_PROFILE_SCOPE(event_tracer, "VGF_INIT_CREATE_PIPELINE_CACHE");
2688+
result = vkCreatePipelineCache(
2689+
vk_device, &pipeline_cache_info, nullptr, &vk_pipeline_cache);
2690+
}
2691+
2692+
if (result != VK_SUCCESS) {
2693+
ET_LOG(
2694+
Info,
2695+
"Failed to create optional per-VgfRepr Vulkan pipeline cache, "
2696+
"error 0x%08X; continuing without pipeline caching",
2697+
result);
2698+
vk_pipeline_cache = VK_NULL_HANDLE;
2699+
} else {
2700+
ET_LOG(Info, "VGF per-VgfRepr Vulkan pipeline cache enabled");
2701+
}
2702+
}
2703+
26712704
// Build per-segment pipelines and descriptor sets.
26722705
segments.clear();
26732706
segments.reserve(segment_count);
2707+
size_t graph_segment_count = 0;
2708+
size_t compute_segment_count = 0;
26742709
{
26752710
VGF_PROFILE_SCOPE(event_tracer, "VGF_INIT_BUILD_SEGMENTS");
26762711

@@ -2683,6 +2718,12 @@ bool VgfRepr::process_vgf(
26832718
return false;
26842719
}
26852720

2721+
if (segment_type == vgflib::ModuleType::GRAPH) {
2722+
++graph_segment_count;
2723+
} else {
2724+
++compute_segment_count;
2725+
}
2726+
26862727
SegmentState segment;
26872728
segment.segment_id = segment_id;
26882729
segment.use_data_graph_pipeline =
@@ -3137,7 +3178,7 @@ bool VgfRepr::process_vgf(
31373178
result = vkCreateDataGraphPipelinesARM(
31383179
vk_device,
31393180
VK_NULL_HANDLE,
3140-
VK_NULL_HANDLE,
3181+
vk_pipeline_cache,
31413182
1,
31423183
&graph_pipeline_info,
31433184
nullptr,
@@ -3429,7 +3470,7 @@ bool VgfRepr::process_vgf(
34293470
VGF_PROFILE_SCOPE(event_tracer, "VGF_INIT_CREATE_COMPUTE_PIPELINE");
34303471
result = vkCreateComputePipelines(
34313472
vk_device,
3432-
VK_NULL_HANDLE,
3473+
vk_pipeline_cache,
34333474
1,
34343475
&compute_info,
34353476
nullptr,
@@ -3445,6 +3486,14 @@ bool VgfRepr::process_vgf(
34453486
}
34463487
}
34473488

3489+
ET_LOG(
3490+
Info,
3491+
"VGF segment counts: total=%d graph=%zu compute=%zu pipeline_cache=%s",
3492+
segment_count,
3493+
graph_segment_count,
3494+
compute_segment_count,
3495+
vk_pipeline_cache != VK_NULL_HANDLE ? "enabled" : "disabled");
3496+
34483497
// Map model sequence inputs/outputs to IO indices
34493498
auto input_handle =
34503499
sequence_decoder->getModelSequenceInputBindingSlotsHandle();

‎backends/arm/runtime/VGFSetup.h‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -177,6 +177,12 @@ class VgfRepr {
177177

178178
~VgfRepr() {
179179
free_vgf();
180+
if (vk_pipeline_cache != VK_NULL_HANDLE) {
181+
// The cache is private to this VgfRepr, so no other delegate instance can
182+
// be accessing it while this object is being destroyed.
183+
vkDestroyPipelineCache(vk_device, vk_pipeline_cache, nullptr);
184+
vk_pipeline_cache = VK_NULL_HANDLE;
185+
}
180186
}
181187

182188
private:
@@ -188,6 +194,12 @@ class VgfRepr {
188194
VkCommandPool vk_command_pool;
189195
uint32_t vk_queue_family_index = UINT32_MAX;
190196

197+
// Owned by this VgfRepr. One cache is reused across all graph and compute
198+
// segments in this loaded VGF, but is not shared with independent VgfRepr
199+
// instances. flags=0 uses Vulkan's default internally synchronized cache
200+
// mode.
201+
VkPipelineCache vk_pipeline_cache = VK_NULL_HANDLE;
202+
191203
bool neural_statistics_requested_ = false;
192204
bool neural_statistics_device_enabled_ = false;
193205
int neural_statistics_mode_index_ = 1;

0 commit comments

Comments
 (0)