Skip to content

Commit 74d024c

Browse files
committed
[ET-VK][runtime][3/3] Enforce device-safe dispatch geometry
Pull Request resolved: pytorch#22225 Migrate dispatch, prepack, profiling, and workgroup picker interfaces to `GlobalWorkGrid` and `LocalWorkGroup`. Wrap oversized linear grids using device limits and validate final geometry before `vkCmdDispatch`. Carry exact local geometry as `GlobalWorkGrid` metadata. Cooperative and reduction grid pickers set it explicitly, while `pick_required_lwg` validates and returns it. Preserve elongated Conv1d depthwise workgroups where the default shape would regress GPU latency. Add end-to-end oversized linear dispatch coverage. Authored with Codex. ghstack-source-id: 421281492 @exported-using-ghexport Differential Revision: [D117706355](https://our.internmc.facebook.com/intern/diff/D117706355/)
1 parent c99fcd4 commit 74d024c

110 files changed

Lines changed: 1500 additions & 1520 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎backends/vulkan/runtime/api/Context.cpp‎

Lines changed: 30 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -73,16 +73,11 @@ void Context::cmd_reset_querypool() {
7373

7474
void Context::report_shader_dispatch_start(
7575
const std::string& shader_name,
76-
const utils::uvec3& global_wg_size,
77-
const LocalWorkGroup& local_wg_size,
76+
const GlobalWorkGrid& gwg,
77+
const LocalWorkGroup& lwg,
7878
const uint32_t dispatch_id) {
7979
if (querypool_) {
80-
querypool_.shader_profile_begin(
81-
cmd_,
82-
dispatch_id,
83-
shader_name,
84-
vkapi::create_extent3d(global_wg_size),
85-
vkapi::create_extent3d((utils::uvec3)local_wg_size));
80+
querypool_.shader_profile_begin(cmd_, dispatch_id, shader_name, gwg, lwg);
8681
}
8782
}
8883

@@ -133,7 +128,7 @@ void Context::check_device_capabilities(const vkapi::ShaderInfo& shader) {
133128

134129
vkapi::DescriptorSet Context::get_descriptor_set(
135130
const vkapi::ShaderInfo& shader_descriptor,
136-
const LocalWorkGroup& local_workgroup_size,
131+
const LocalWorkGroup& lwg,
137132
const vkapi::SpecVarList& additional_constants,
138133
const uint32_t push_constants_size) {
139134
VkDescriptorSetLayout shader_layout =
@@ -142,10 +137,7 @@ vkapi::DescriptorSet Context::get_descriptor_set(
142137
VkPipelineLayout pipeline_layout =
143138
pipeline_layout_cache().retrieve(shader_layout, push_constants_size);
144139

145-
vkapi::SpecVarList spec_constants = {
146-
SV(local_workgroup_size[0u]),
147-
SV(local_workgroup_size[1u]),
148-
SV(local_workgroup_size[2u])};
140+
vkapi::SpecVarList spec_constants = {SV(lwg[0u]), SV(lwg[1u]), SV(lwg[2u])};
149141

150142
spec_constants.append(additional_constants);
151143

@@ -158,7 +150,7 @@ vkapi::DescriptorSet Context::get_descriptor_set(
158150
spec_constants,
159151
resolved_required_subgroup_size});
160152

161-
cmd_.bind_pipeline(pipeline, pipeline_layout, local_workgroup_size);
153+
cmd_.bind_pipeline(pipeline, pipeline_layout, lwg);
162154

163155
return descriptor_pool().get_descriptor_set(
164156
shader_layout, shader_descriptor.kernel_layout);
@@ -168,30 +160,35 @@ void Context::register_shader_dispatch(
168160
const vkapi::DescriptorSet& descriptors,
169161
vkapi::PipelineBarrier& pipeline_barrier,
170162
const vkapi::ShaderInfo& shader_descriptor,
171-
const utils::uvec3& global_workgroup_size,
163+
const GlobalWorkGrid& gwg,
164+
const LocalWorkGroup& lwg,
172165
const void* push_constants_data,
173166
const uint32_t push_constants_size) {
167+
vkapi::Adapter* const adapter = adapter_ptr();
168+
const LocalWorkGroup& dispatch_lwg =
169+
gwg.required_lwg_size().is_valid() ? gwg.required_lwg_size() : lwg;
170+
dispatch_lwg.validate(
171+
adapter->max_compute_workgroup_size(),
172+
adapter->max_compute_workgroup_invocations());
173+
gwg.validate(
174+
dispatch_lwg,
175+
adapter->max_compute_workgroup_count(),
176+
shader_descriptor.out_tile_size);
177+
174178
// Adjust the global workgroup size based on the output tile size
175-
uint32_t global_wg_w = utils::div_up(
176-
global_workgroup_size[0u], shader_descriptor.out_tile_size[0u]);
177-
uint32_t global_wg_h = utils::div_up(
178-
global_workgroup_size[1u], shader_descriptor.out_tile_size[1u]);
179-
uint32_t global_wg_d = utils::div_up(
180-
global_workgroup_size[2u], shader_descriptor.out_tile_size[2u]);
179+
uint32_t gwg_w = utils::div_up(gwg[0u], shader_descriptor.out_tile_size[0u]);
180+
uint32_t gwg_h = utils::div_up(gwg[1u], shader_descriptor.out_tile_size[1u]);
181+
uint32_t gwg_d = utils::div_up(gwg[2u], shader_descriptor.out_tile_size[2u]);
181182

182183
// Submitting a global work group size of 0 is undefined behaviour. If this is
183184
// detected then submit a single workgroup instead.
184-
if (global_wg_w == 0u || global_wg_h == 0u || global_wg_d == 0u) {
185-
global_wg_w = 1u;
186-
global_wg_h = 1u;
187-
global_wg_d = 1u;
185+
if (gwg_w == 0u || gwg_h == 0u || gwg_d == 0u) {
186+
gwg_w = 1u;
187+
gwg_h = 1u;
188+
gwg_d = 1u;
188189
}
189190

190-
const utils::uvec3 effective_global_wg = {
191-
global_wg_w,
192-
global_wg_h,
193-
global_wg_d,
194-
};
191+
const GlobalWorkGrid effective_gwg({gwg_w, gwg_h, gwg_d}, kExplicitWorkGrid);
195192

196193
cmd_.bind_descriptors(descriptors.get_bind_handle());
197194
cmd_.insert_barrier(pipeline_barrier);
@@ -205,7 +202,7 @@ void Context::register_shader_dispatch(
205202
pipeline_layout, push_constants_data, push_constants_size);
206203
}
207204

208-
cmd_.dispatch(effective_global_wg);
205+
cmd_.dispatch(effective_gwg, dispatch_lwg);
209206
}
210207

211208
void Context::register_barrier(vkapi::PipelineBarrier& pipeline_barrier) {
@@ -324,11 +321,8 @@ VkPipeline Context::get_shader_pipeline(
324321
VkPipelineLayout pipeline_layout =
325322
pipeline_layout_cache().retrieve(shader_layout, push_constants_size);
326323

327-
const LocalWorkGroup local_workgroup_size(4u, 4u, 1u);
328-
vkapi::SpecVarList spec_constants = {
329-
SV(local_workgroup_size[0u]),
330-
SV(local_workgroup_size[1u]),
331-
SV(local_workgroup_size[2u])};
324+
const LocalWorkGroup lwg(4u, 4u, 1u);
325+
vkapi::SpecVarList spec_constants = {SV(lwg[0u]), SV(lwg[1u]), SV(lwg[2u])};
332326

333327
spec_constants.append(additional_constants);
334328

‎backends/vulkan/runtime/api/Context.h‎

Lines changed: 17 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -151,8 +151,8 @@ class Context final {
151151
*/
152152
void report_shader_dispatch_start(
153153
const std::string& shader_name,
154-
const utils::uvec3& global_wg_size,
155-
const LocalWorkGroup& local_wg_size,
154+
const GlobalWorkGrid& gwg,
155+
const LocalWorkGroup& lwg,
156156
const uint32_t dispatch_id = UINT32_MAX);
157157

158158
/*
@@ -197,15 +197,16 @@ class Context final {
197197

198198
inline vkapi::DescriptorSet get_descriptor_set(
199199
const vkapi::ShaderInfo& shader_descriptor,
200-
const LocalWorkGroup& local_work_group_size) {
201-
return get_descriptor_set(shader_descriptor, local_work_group_size, {}, 0u);
200+
const LocalWorkGroup& lwg) {
201+
return get_descriptor_set(shader_descriptor, lwg, {}, 0u);
202202
}
203203

204204
void register_shader_dispatch(
205205
const vkapi::DescriptorSet&,
206206
vkapi::PipelineBarrier&,
207207
const vkapi::ShaderInfo&,
208-
const utils::uvec3&,
208+
const GlobalWorkGrid&,
209+
const LocalWorkGroup&,
209210
const void* = nullptr,
210211
const uint32_t = 0);
211212

@@ -220,8 +221,8 @@ class Context final {
220221
bool submit_compute_job(
221222
const vkapi::ShaderInfo&,
222223
vkapi::PipelineBarrier&,
223-
const utils::uvec3&,
224-
const utils::uvec3&,
224+
const GlobalWorkGrid&,
225+
const LocalWorkGroup&,
225226
const vkapi::SpecVarList&,
226227
VkFence fence_handle,
227228
const uint32_t dispatch_id,
@@ -334,8 +335,8 @@ template <typename... Arguments>
334335
inline bool Context::submit_compute_job(
335336
const vkapi::ShaderInfo& shader,
336337
vkapi::PipelineBarrier& pipeline_barrier,
337-
const utils::uvec3& global_work_group,
338-
const utils::uvec3& local_work_group_size,
338+
const GlobalWorkGrid& gwg,
339+
const LocalWorkGroup& lwg,
339340
const vkapi::SpecVarList& specialization_constants,
340341
VkFence fence_handle,
341342
const uint32_t dispatch_id,
@@ -368,20 +369,17 @@ inline bool Context::submit_compute_job(
368369

369370
set_cmd();
370371

372+
const LocalWorkGroup& dispatch_lwg =
373+
gwg.required_lwg_size().is_valid() ? gwg.required_lwg_size() : lwg;
374+
371375
report_shader_dispatch_start(
372-
shader.kernel_name,
373-
global_work_group,
374-
LocalWorkGroup(local_work_group_size),
375-
dispatch_id);
376+
shader.kernel_name, gwg, dispatch_lwg, dispatch_id);
376377

377378
// Factor out template parameter independent code to minimize code bloat.
378379
// Note that push constants are not exposed yet via this API, therefore the
379380
// push constants size is assumed to be 0.
380-
vkapi::DescriptorSet descriptor_set = get_descriptor_set(
381-
shader,
382-
LocalWorkGroup(local_work_group_size),
383-
specialization_constants,
384-
0u);
381+
vkapi::DescriptorSet descriptor_set =
382+
get_descriptor_set(shader, dispatch_lwg, specialization_constants, 0u);
385383

386384
detail::bind(
387385
descriptor_set,
@@ -390,7 +388,7 @@ inline bool Context::submit_compute_job(
390388

391389
// Factor out template parameter independent code to minimize code bloat.
392390
register_shader_dispatch(
393-
descriptor_set, pipeline_barrier, shader, global_work_group);
391+
descriptor_set, pipeline_barrier, shader, gwg, dispatch_lwg);
394392

395393
report_shader_dispatch_end();
396394

‎backends/vulkan/runtime/graph/ComputeGraph.cpp‎

Lines changed: 30 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -845,7 +845,7 @@ void ComputeGraph::update_descriptor_counts(
845845

846846
void ComputeGraph::register_pipeline_to_create(
847847
const vkapi::ShaderInfo& shader_info,
848-
const LocalWorkGroup& local_workgroup_size,
848+
const LocalWorkGroup& lwg,
849849
const vkapi::SpecVarList& spec_vars,
850850
const std::vector<PushConstantDataInfo>& push_constants) {
851851
VkDescriptorSetLayout shader_layout =
@@ -857,10 +857,7 @@ void ComputeGraph::register_pipeline_to_create(
857857
pc_offset += pc.write(pc_data.data(), pc_offset, kMaxPushConstantSize);
858858
}
859859

860-
vkapi::SpecVarList spec_constants = {
861-
SV(local_workgroup_size[0u]),
862-
SV(local_workgroup_size[1u]),
863-
SV(local_workgroup_size[2u])};
860+
vkapi::SpecVarList spec_constants = {SV(lwg[0u]), SV(lwg[1u]), SV(lwg[2u])};
864861

865862
spec_constants.append(spec_vars);
866863

@@ -889,64 +886,46 @@ void ComputeGraph::register_pipeline_to_create(
889886
pipeline_descriptors_.insert(desc);
890887
}
891888

892-
utils::uvec3 ComputeGraph::create_global_wg_size(const ValueRef idx) {
889+
GlobalWorkGrid ComputeGraph::create_gwg(const ValueRef idx) {
893890
if (is_buffer_storage(idx)) {
894-
return {uint32_t(numel_of(idx)), 1u, 1u};
891+
return create_linear_gwg(utils::safe_downcast<uint64_t>(numel_of(idx)));
895892
}
896-
return logical_limits_of(idx);
897-
}
898893

899-
utils::uvec3 ComputeGraph::create_local_wg_size(
900-
const utils::uvec3 global_wg_size) {
901-
if (config_.enable_local_wg_size_override) {
902-
return config_.local_wg_size_override;
903-
}
894+
return GlobalWorkGrid(
895+
utils::make_uvec3(logical_limits_of(idx)), kTextureExtentsWorkGrid);
896+
}
904897

905-
// array containing axis index and global workgroup size
906-
std::pair<uint32_t, uint32_t> global_wg_size_desc[] = {
907-
{0u, global_wg_size[0]},
908-
{1u, global_wg_size[1]},
909-
{2u, global_wg_size[2]}};
898+
GlobalWorkGrid ComputeGraph::create_linear_gwg(const uint64_t numel) {
899+
vkapi::Adapter* const adapter = context()->adapter_ptr();
900+
GlobalWorkGrid gwg(
901+
{utils::safe_downcast<uint32_t>(numel), 1u, 1u}, kLinearWorkGrid);
902+
gwg.wrap_linear_dispatch(
903+
adapter->max_compute_workgroup_count(),
904+
adapter->recommended_lwg_nthreads());
905+
return gwg;
906+
}
910907

911-
// sort the global workgroup size in descending order
912-
if (global_wg_size_desc[0].second < global_wg_size_desc[1].second) {
913-
std::swap(global_wg_size_desc[0], global_wg_size_desc[1]);
914-
}
915-
if (global_wg_size_desc[1].second < global_wg_size_desc[2].second) {
916-
std::swap(global_wg_size_desc[1], global_wg_size_desc[2]);
917-
}
918-
if (global_wg_size_desc[0].second < global_wg_size_desc[1].second) {
919-
std::swap(global_wg_size_desc[0], global_wg_size_desc[1]);
908+
LocalWorkGroup ComputeGraph::create_lwg(const GlobalWorkGrid& gwg) {
909+
if (gwg.required_lwg_size().is_valid()) {
910+
return gwg.required_lwg_size();
920911
}
921912

922-
utils::uvec3 local_group_size = {
923-
8,
924-
global_wg_size_desc[1].second >= 4u ? 4u
925-
: global_wg_size_desc[1].second >= 2u ? 2u
926-
: 1u,
927-
global_wg_size_desc[2].second >= 2u ? 2u : 1u};
913+
vkapi::Adapter* const adapter = context()->adapter_ptr();
928914

929-
if (global_wg_size_desc[2u].second == 1) {
930-
if (global_wg_size_desc[1u].second == 1) {
931-
local_group_size[0u] = 64;
932-
local_group_size[1u] = 1;
933-
} else if (global_wg_size_desc[1u].second % 4 == 0) {
934-
local_group_size[0u] = 16;
935-
local_group_size[1u] = 4;
936-
} else {
937-
local_group_size[0u] = 32;
938-
local_group_size[1u] = 2;
939-
}
915+
utils::uvec3 shape_weights{
916+
gwg[0] > 1u ? 1u : 0u, gwg[1] > 1u ? 1u : 0u, gwg[2] > 1u ? 1u : 0u};
917+
if (shape_weights == utils::uvec3{0u, 0u, 0u}) {
918+
shape_weights[0] = 1u;
940919
}
941920

942-
return {
943-
local_group_size[global_wg_size_desc[0].first],
944-
local_group_size[global_wg_size_desc[1].first],
945-
local_group_size[global_wg_size_desc[2].first]};
921+
LocalWorkGroup lwg(
922+
LwgShape(shape_weights), adapter->recommended_lwg_nthreads());
923+
lwg.fit_to_global(gwg);
924+
return lwg;
946925
}
947926

948-
utils::uvec3 ComputeGraph::create_local_wg_size(const ValueRef idx) {
949-
return create_local_wg_size(create_global_wg_size(idx));
927+
LocalWorkGroup ComputeGraph::create_lwg(const ValueRef idx) {
928+
return create_lwg(create_gwg(idx));
950929
}
951930

952931
void ComputeGraph::bind_tensor_to_descriptor_set(

‎backends/vulkan/runtime/graph/ComputeGraph.h‎

Lines changed: 13 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1012,7 +1012,7 @@ class ComputeGraph final {
10121012

10131013
void register_pipeline_to_create(
10141014
const vkapi::ShaderInfo& shader_info,
1015-
const LocalWorkGroup& local_workgroup_size,
1015+
const LocalWorkGroup& lwg,
10161016
const vkapi::SpecVarList& spec_vars,
10171017
const std::vector<PushConstantDataInfo>& push_constants);
10181018

@@ -1025,37 +1025,36 @@ class ComputeGraph final {
10251025
//
10261026

10271027
/*
1028-
* Create a global workgroup size for a given `api::vTensor` value assuming
1029-
* that every shader invocation calculates one texel element of the output
1030-
* tensor.
1028+
* Create a global invocation size for a given `api::vTensor` value assuming
1029+
* that every shader invocation calculates one element of the output tensor.
10311030
*
10321031
* For tensors that use texture storage, the image extents of the
10331032
* `api::vTensor` will be used to set the global workgroup size.
10341033
*
1035-
* For tensor that use buffer storage, the number of texels in the texel
1036-
* buffer will be used to set the x component of the global workgroup size.
1037-
* All other components will be set to 1 (i.e. {ntexels, 1, 1} will be
1038-
* returned).
1034+
* Buffer tensors use linear dispatch intent. Oversized X dimensions are
1035+
* wrapped across X and Y according to device workgroup-count limits.
10391036
*/
1040-
utils::uvec3 create_global_wg_size(const ValueRef idx);
1037+
GlobalWorkGrid create_gwg(const ValueRef idx);
1038+
1039+
GlobalWorkGrid create_linear_gwg(const uint64_t numel);
10411040

10421041
/*
1043-
* Suggest a local workgroup size for a given global workgroup size.
1042+
* Suggest a local workgroup size for a given global invocation size.
10441043
*
10451044
* The local workgroup size will be formed to try and minimize the number of
10461045
* inactive invocations.
10471046
*
1048-
* Currently, the local workgroup size is hard-coded to contain a total of 64
1049-
* shader invocations. In the future, this value can be configured.
1047+
* Linear dispatches return their binding hint. Other dispatches use a shape
1048+
* heuristic targeting the adapter's recommended thread count.
10501049
*/
1051-
utils::uvec3 create_local_wg_size(const utils::uvec3 global_wg_size);
1050+
LocalWorkGroup create_lwg(const GlobalWorkGrid& gwg);
10521051

10531052
/*
10541053
* Convenience function to suggest a local workgroup size for a given
10551054
* `api::vTensor` value, assuming that every shader invocation calculates one
10561055
* texel element of the output tensor.
10571056
*/
1058-
utils::uvec3 create_local_wg_size(const ValueRef idx);
1057+
LocalWorkGroup create_lwg(const ValueRef idx);
10591058

10601059
void bind_tensor_to_descriptor_set(
10611060
const ValueRef ref,

0 commit comments

Comments
 (0)