Skip to content

Commit 2f964f8

Browse files
authored
[ET-VK] Scale softmax workers with the row length and reduce as a tree (pytorch#22349)
`softmax_buffer.glsl` hardcodes ```glsl #define NWORKERS 4 #define MAX_NTHREADS 16 ``` and `pick_softmax_gwg()` matches it with `lwg_extents[reduce_dim] = 4u`, so four threads normalise a row no matter how long it is. On `all-MiniLM-L6-v2` at its published 254-token shape that is six dispatches of 633 us each: **21.2% of GPU time** on an Adreno 840. Widening alone only recovered a quarter of it. The aggregation after each barrier was a serial walk of all `NWORKERS` shared entries executed by **every thread**, not just thread 0, so its cost grew with the worker count and cancelled most of the benefit. Replacing both walks (max, then sum) with a shared-memory tree is the larger half of the win. The loop bound is uniform and `barrier()` sits outside the guard, so every thread still reaches every barrier. ### Measurements `all-MiniLM-L6-v2` @254 on an Adreno 840, four interleaved order-reversed rounds, best of 20 executions each: | | best | | --- | --- | | baseline | 18.30 ms | | + workers scaled | 17.40 ms | | + tree aggregation | **15.53 ms** (-15.1%) | `softmax_buffer` itself goes 3.80 -> 2.91 ms. Output matches the XNNPACK build to cosine 0.9999975 and is 60/60 bit-identical across runs. ### Notes `reduce_dim` is a WHCN/xyz index (0 = x = last dim) while `size_at()` counts back from the end, hence the `-(reduce_dim + 1)` in `softmax_nworkers()`. Buffer storage only. The texture path uses a different shader and grouping scheme and is left alone; it has the same hardcoded constants and is worth a follow-up. Fixes pytorch#22351 cc @SS-JIA @manuelcandales @digantdesai @cbilgin
1 parent 5fc01ad commit 2f964f8

2 files changed

Lines changed: 95 additions & 22 deletions

File tree

‎backends/vulkan/runtime/graph/ops/glsl/softmax_buffer.glsl‎

Lines changed: 25 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -33,11 +33,14 @@ layout(local_size_x_id = 0, local_size_y_id = 1, local_size_z_id = 2) in;
3333

3434
layout(constant_id = 3) const int reduce_dim = 0;
3535

36-
#define NWORKERS 4
37-
#define MAX_NTHREADS 16
36+
// Supplied by the dispatch so it can scale with the length of the row being
37+
// normalised. Four threads walking a 254-element row left the GPU idle.
38+
layout(constant_id = 4) const int NWORKERS = 4;
39+
// Upper bound on NWORKERS, and the size of the shared arrays below.
40+
#define MAX_NTHREADS 256
3841

39-
shared T shared_max[NWORKERS];
40-
shared T shared_sum[NWORKERS];
42+
shared T shared_max[MAX_NTHREADS];
43+
shared T shared_sum[MAX_NTHREADS];
4144

4245
/*
4346
* Buffer-based softmax. Each workgroup processes one "row" along the reduction
@@ -81,11 +84,18 @@ void main() {
8184
shared_max[tid] = local_max;
8285
barrier();
8386

84-
// Reduce partial maximums across workers
85-
T max_val = shared_max[0];
86-
for (int i = 1; i < NWORKERS; ++i) {
87-
max_val = max(max_val, shared_max[i]);
87+
// Combine the partials as a tree. This was a serial walk of all NWORKERS
88+
// entries run by EVERY thread, so its cost grew with the worker count and
89+
// cancelled most of the benefit of widening the group. The loop bound is
90+
// uniform and barrier() sits outside the guard, so every thread reaches
91+
// every barrier.
92+
for (uint s = uint(NWORKERS) >> 1; s > 0u; s >>= 1) {
93+
if (tid < s) {
94+
shared_max[tid] = max(shared_max[tid], shared_max[tid + s]);
95+
}
96+
barrier();
8897
}
98+
const T max_val = shared_max[0];
8999

90100
// Phase 2: Compute sum of exp(x - max_val)
91101
T local_sum = T(0);
@@ -97,13 +107,15 @@ void main() {
97107
shared_sum[tid] = local_sum;
98108
barrier();
99109

100-
// Reduce partial sums across workers
101-
T sum_val = shared_sum[0];
102-
for (int i = 1; i < NWORKERS; ++i) {
103-
sum_val += shared_sum[i];
110+
// Same tree reduction for the sum.
111+
for (uint s = uint(NWORKERS) >> 1; s > 0u; s >>= 1) {
112+
if (tid < s) {
113+
shared_sum[tid] = shared_sum[tid] + shared_sum[tid + s];
114+
}
115+
barrier();
104116
}
105117
// Clamp denominator to avoid 0/0 = NaN when all exp values underflow.
106-
sum_val = max(sum_val, T(1e-37));
118+
const T sum_val = max(shared_sum[0], T(1e-37));
107119

108120
// Phase 3: Write outputs
109121
for (int i = int(tid); i < R; i += NWORKERS) {

‎backends/vulkan/runtime/graph/ops/impl/Softmax.cpp‎

Lines changed: 70 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,56 @@
1414
#include <executorch/backends/vulkan/runtime/graph/ops/impl/utils/TensorUtils.h>
1515
#include <executorch/backends/vulkan/runtime/graph/ops/utils/ShaderNameUtils.h>
1616

17+
#include <algorithm>
18+
1719
namespace vkcompute {
1820

1921
using namespace utils;
2022

23+
// This should match the value of MAX_NTHREADS in softmax_buffer.
24+
constexpr uint32_t kSoftmaxBufferMaxNThreads = 256u;
25+
26+
// The largest worker count the buffer dispatch may ask for. The shared arrays
27+
// in the shader are one ceiling, but they are not the only one: a device
28+
// bounds the invocations in a work group (maxComputeWorkGroupInvocations is
29+
// only guaranteed to be 128) and bounds each axis separately. The buffer
30+
// launch puts every worker on the reduction axis and leaves the other two at
31+
// one, so both device limits apply to the worker count directly, and
32+
// overrunning either aborts the dispatch in LocalWorkGroup::validate.
33+
uint32_t softmax_nworkers_cap(ComputeGraph* graph, const int32_t reduce_dim) {
34+
const vkapi::Adapter* const adapter = graph->context()->adapter_ptr();
35+
uint32_t cap = kSoftmaxBufferMaxNThreads;
36+
cap = std::min(cap, adapter->max_compute_workgroup_invocations());
37+
cap = std::min(cap, adapter->max_compute_workgroup_size()[reduce_dim]);
38+
// The shader folds the partials as a tree that halves the worker count each
39+
// step, so the count has to be a power of two for the last step to land on
40+
// slot 0. Round the cap down to one.
41+
uint32_t pow2 = 1u;
42+
while (pow2 * 2u <= cap) {
43+
pow2 *= 2u;
44+
}
45+
return pow2;
46+
}
47+
48+
// Threads co-operating on one softmax row, scaled with the row length. Buffer
49+
// storage only: the texture path uses a different shader and grouping scheme.
50+
uint32_t softmax_nworkers(
51+
ComputeGraph* graph,
52+
const ValueRef in,
53+
const int32_t reduce_dim) {
54+
const uint32_t cap = softmax_nworkers_cap(graph, reduce_dim);
55+
// reduce_dim is a WHCN/xyz index (0 = x = last dim) while size_at counts back
56+
// from the end, so xyz 0 -> -1, 1 -> -2, 2 -> -3.
57+
const uint32_t extent = graph->size_at<uint32_t>(-(reduce_dim + 1), in);
58+
// 4 is what this used to be unconditionally; keep it as the floor so short
59+
// rows dispatch exactly as they did before.
60+
uint32_t nworkers = std::min(4u, cap);
61+
while (nworkers * 2u <= cap && nworkers < extent) {
62+
nworkers *= 2u;
63+
}
64+
return nworkers;
65+
}
66+
2167
GlobalWorkGrid pick_softmax_gwg(
2268
ComputeGraph* graph,
2369
const vkapi::ShaderInfo& shader,
@@ -36,6 +82,13 @@ GlobalWorkGrid pick_softmax_gwg(
3682
utils::uvec3 lwg_extents{1u, 1u, 1u};
3783
lwg_extents[reduce_dim] = 4u;
3884
if (graph->is_buffer_storage(out)) {
85+
// NWORKERS is baked into softmax_buffer as a specialization constant when
86+
// the node is built, so it reflects the dynamic upper bound. Recomputing it
87+
// here from the resized extent would launch fewer threads than the shader's
88+
// tree reduction indexes over, leaving it to fold in shared memory slots
89+
// that no thread wrote. Read the count the node was built with instead.
90+
lwg_extents[reduce_dim] = static_cast<uint32_t>(
91+
graph->extract_scalar<int32_t>(resize_args.at(1)));
3992
utils::uvec3 extents = {
4093
graph->size_at<uint32_t>(-1, out),
4194
graph->size_at<uint32_t>(-2, out),
@@ -95,18 +148,26 @@ void add_softmax_node(
95148
kernel_name = "log_" + kernel_name;
96149
}
97150

98-
// This should match the value of MAX_NTHREADS in the softmax shader.
99-
constexpr uint32_t max_nthreads = 16;
100-
const uint32_t nworkers_per_group = 4;
101-
const uint32_t ngroups = 4;
102-
VK_CHECK_COND(nworkers_per_group * ngroups <= max_nthreads);
103-
104151
const int dim_val = graph.extract_scalar<int>(dim_ref);
105152

106-
vkapi::SpecVarList spec_constants = {reduce_dim_xyz};
107-
std::vector<ValueRef> resize_args = {dim_val};
153+
vkapi::SpecVarList spec_constants;
154+
std::vector<ValueRef> resize_args;
155+
156+
if (graph.is_buffer_storage(out)) {
157+
// Computed once here so the launch in pick_softmax_gwg() uses the same
158+
// count that is baked into the shader.
159+
const int32_t nworkers = utils::safe_downcast<int32_t>(
160+
softmax_nworkers(&graph, in, reduce_dim_xyz));
161+
spec_constants = {reduce_dim_xyz, nworkers};
162+
resize_args = {dim_val, graph.get_or_add_value_for_int(nworkers)};
163+
} else {
164+
// The texture shader still fixes its worker count, and this should match
165+
// the value of MAX_NTHREADS in it.
166+
constexpr uint32_t max_nthreads = 16;
167+
const uint32_t nworkers_per_group = 4;
168+
const uint32_t ngroups = 4;
169+
VK_CHECK_COND(nworkers_per_group * ngroups <= max_nthreads);
108170

109-
if (!graph.is_buffer_storage(out)) {
110171
const int other_dim_1 = (reduce_dim_xyz + 1) % 3;
111172
const int other_dim_2 = (reduce_dim_xyz + 2) % 3;
112173
int32_t group_dim;

0 commit comments

Comments
 (0)