Skip to content

Commit 1d95a2a

Browse files
authored
[CUDA] Tune hardware-aware split-K decode scheduling (pytorch#22190)
## Stack dependency Depends on pytorch#22189. ## Summary Combine the D113 SM-sized split-K work with the final scheduling policy for the numerically stable online-softmax implementation. - Derive split counts from the target GPU SM count, `L_kv`, and `B * H_kv`. - Keep the launch size static while reading runtime `kv_len` on device, preserving CUDA graph compatibility. - Retune the occupancy target from D113's two waves to 16/9 waves. - Cap splits by both KV-buffer size and a maximum of 128 partitions to avoid excessive reduction overhead on high-SM GPUs. - Use contiguous KV partitions instead of D113's persistent strided traversal; profiling showed that contiguous partitions are faster for the online-softmax kernel. - Apply the same hardware-aware policy to both decode and small-query split-K paths. - Keep scale arithmetic explicitly in FP32. The policy contains no model-specific or GPU-specific shape hard-coding. The SM count is specialized during native export. Under the supported deployment flow, each target architecture is exported independently and those variants may then be merged into one PTE; moving a native artifact to a same-architecture GPU with a different SM count remains correct, although its scheduling can be suboptimal. ## Operator-level performance RTX 5090 CUDA Graph replay, BF16 GQA decode (`B=1, Hq=32, Hkv=2, D=128`), compared with pytorch#22189: | KV buffer | Runtime KV | pytorch#22189 | This PR | Change | |---:|---:|---:|---:|---:| | 4,096 | 512 | 10.86 us | 9.99 us | +8.6% | | 4,096 | 2,048 | 12.66 us | 10.66 us | +18.8% | | 4,096 | 4,096 | 13.49 us | 12.51 us | +7.8% | | 130,048 | 32,768 | 33.21 us | 33.13 us | +0.2% | | 130,048 | 130,048 | 95.50 us | 96.50 us | -1.0% | The optimization primarily improves the sliding-window decode family. The 128-split cap keeps the long global-attention path within approximately 1% of the preceding PR instead of over-splitting it. ## Test plan - `lintrunner init && lintrunner -a` (clean). - Triton split-K and FP8 dispatch suite: 47 passed. - Standalone D113 decode correctness smoke: 3 passed. - Covered runtime KV-length tail exclusion, non-power-of-two split counts, and large-logit stability. - `git diff --check`.
1 parent 645ec92 commit 1d95a2a

2 files changed

Lines changed: 105 additions & 14 deletions

File tree

‎backends/cuda/tests/test_triton_sdpa_splitk.py‎

Lines changed: 35 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -314,17 +314,18 @@ def test_qwen35_config(self):
314314
def test_large_positive_logits_stable(self):
315315
"""Large logits stay finite in both split-K kernel families."""
316316
B, H_q, H_kv, Lk, D = 1, 32, 8, 512, 128
317-
num_splits, chunk_size = self.splitk_config(Lk)
317+
num_splits, chunk_size = self.splitk_config(Lk, B * H_kv, torch.device("cuda"))
318318
self.assertGreater(num_splits, 1)
319-
self.assertNotEqual(0 // chunk_size, 300 // chunk_size)
319+
second_key = chunk_size
320+
self.assertLess(second_key, Lk)
320321

321322
k = torch.zeros(B, H_kv, Lk, D, dtype=torch.bfloat16, device="cuda")
322323

323-
# Put nearly equal high scores in separate 256-token splits. The old
324-
# fixed-phi path overflowed both partial softmaxes, and their proximity
325-
# makes the result sensitive to correct cross-split rescaling.
324+
# Put nearly equal high scores in separate splits. The old fixed-phi
325+
# path overflowed both partial softmaxes, and their proximity makes the
326+
# result sensitive to correct cross-split rescaling.
326327
k[:, :, 0, :] = 3.0
327-
k[:, :, 300, :] = 2.96875
328+
k[:, :, second_key, :] = 2.96875
328329
torch.manual_seed(42)
329330
v = torch.randn(B, H_kv, Lk, D, dtype=torch.bfloat16, device="cuda")
330331

@@ -343,7 +344,7 @@ def test_large_positive_logits_stable(self):
343344
high_scores = torch.stack(
344345
[
345346
(q[0, 0, 0].float() * k[0, 0, pos].float()).sum() / D**0.5
346-
for pos in (0, 300)
347+
for pos in (0, second_key)
347348
]
348349
)
349350
self.assertTrue(
@@ -389,10 +390,15 @@ def test_large_negative_logits_do_not_underflow_to_zero(self):
389390
self.assertLess(_max_abs_error(out, ref), MAX_ABS_TOL)
390391

391392
def test_non_power_of_two_split_count(self):
392-
"""Reduction masks lanes beyond the runtime split count."""
393+
"""Reduction masks lanes beyond a non-power-of-two split count."""
393394
B, H_q, H_kv, Lk, D = 1, 8, 2, 768, 128
394-
num_splits, _ = self.splitk_config(Lk)
395-
self.assertEqual(num_splits, 3)
395+
sm_count_patch = mock.patch.object(
396+
self.sdpa_module, "_device_sm_count", return_value=80
397+
)
398+
sm_count_patch.start()
399+
self.addCleanup(sm_count_patch.stop)
400+
num_splits, _ = self.splitk_config(Lk, B * H_kv, torch.device("cuda"))
401+
self.assertEqual(num_splits, 6)
396402

397403
torch.manual_seed(42)
398404
k = torch.randn(B, H_kv, Lk, D, dtype=torch.bfloat16, device="cuda")
@@ -410,6 +416,25 @@ def test_non_power_of_two_split_count(self):
410416
self.assertTrue(torch.isfinite(out).all())
411417
self.assertLess(_max_abs_error(out, ref), MAX_ABS_TOL)
412418

419+
def test_split_count_scales_with_sm_count_and_buffer_size(self):
420+
"""The scheduling policy adapts without depending on the test GPU."""
421+
device = torch.device("cuda")
422+
cases = [
423+
# (SM count, L_kv, grid_y, expected splits)
424+
(80, 128, 2, 1),
425+
(80, 2048, 2, 16),
426+
(108, 131072, 4, 48),
427+
(170, 131072, 4, 76),
428+
(1024, 131072, 1, 128),
429+
]
430+
for n_sm, L_kv, grid_y, expected in cases:
431+
with self.subTest(n_sm=n_sm, L_kv=L_kv, grid_y=grid_y), mock.patch.object(
432+
self.sdpa_module, "_device_sm_count", return_value=n_sm
433+
):
434+
num_splits, chunk_size = self.splitk_config(L_kv, grid_y, device)
435+
self.assertEqual(num_splits, expected)
436+
self.assertEqual(chunk_size, math.ceil(L_kv / expected))
437+
413438
def test_custom_scale(self):
414439
"""Non-default attention scale."""
415440
B, H_q, H_kv, Lq, Lk, D = 1, 8, 2, 1, 256, 128

‎backends/cuda/triton/kernels/sdpa.py‎

Lines changed: 70 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
and improves tile utilization, especially during decode (seqlen_q=1).
3232
"""
3333

34+
import functools
3435
import math
3536
from typing import Optional
3637

@@ -51,8 +52,71 @@ def _is_power_of_2(n: int) -> bool:
5152
_SPLITK_LKV_THRESHOLD = 256
5253

5354

54-
def _decode_splitk_config(L_kv: int) -> tuple[int, int]:
55-
num_splits = min(max(triton.cdiv(L_kv, _SPLITK_LKV_THRESHOLD), 1), 128)
55+
# Decode split-K occupancy target. A sweep across both production attention
56+
# families showed that targeting 16/9 waves gives a good balance between split
57+
# kernel occupancy and reduction/empty-CTA overhead. Keep the ratio integral so
58+
# the launch policy is deterministic and does not depend on floating-point
59+
# rounding.
60+
_SPLITK_TARGET_WAVES_NUMERATOR = 16
61+
_SPLITK_TARGET_WAVES_DENOMINATOR = 9
62+
_SPLITK_MAX_SPLITS = 128
63+
64+
# Do not create more static split CTAs than one per 128 elements in the KV
65+
# buffer. This permits 16 splits for the 2048-element sliding-window family.
66+
# A 40-state, >L2 working-set A/B showed gains at the production prompt lengths;
67+
# single-state measurements are cache-hot and are not representative here.
68+
# For the 128K global family with grid_y=4 on a 170-SM device, the occupancy
69+
# target selects 76 splits instead of reaching the merge-base buffer cap of 128.
70+
_SPLITK_BUFFER_ELEMENTS_PER_SPLIT = 128
71+
72+
73+
@functools.lru_cache(maxsize=None)
74+
def _device_sm_count(device_index: int) -> int:
75+
"""SM (multiprocessor) count of a CUDA device.
76+
77+
Read once per device from torch device properties and bake it into the
78+
exported artifact before cuda-graph capture. The supported deployment flow
79+
exports a native artifact per target GPU and may merge those variants into
80+
one PTE; moving one native artifact to a same-architecture GPU with a
81+
different SM count remains correct, but may not retain optimal scheduling.
82+
"""
83+
return torch.cuda.get_device_properties(device_index).multi_processor_count
84+
85+
86+
def _decode_splitk_config(
87+
L_kv: int, grid_y: int, device: torch.device
88+
) -> tuple[int, int]:
89+
"""Hardware-derived split count for the flash-decoding decode path.
90+
91+
The split kernel launches a (num_splits, grid_y) grid where
92+
grid_y = B * H_kv. We size num_splits so the total launched CTAs
93+
(num_splits * grid_y) target 16/9 waves of this device's SM array, capped by
94+
cdiv(L_kv, 128) so a short KV buffer is not needlessly over-split and
95+
clamped to >= 1.
96+
97+
num_splits depends only on host-side constants (L_kv buffer size, the
98+
static grid_y, and the device SM count), so it is fixed before cuda-graph
99+
capture and identical on every replay.
100+
"""
101+
device_index = (
102+
device.index if device.index is not None else torch.cuda.current_device()
103+
)
104+
n_sm = _device_sm_count(device_index)
105+
# Cap so num_splits * grid_y ~= 16/9 * n_sm (>= 1). Use ceil division: a
106+
# partial final wave is preferable to leaving an SM idle.
107+
sm_cap = max(
108+
triton.cdiv(
109+
_SPLITK_TARGET_WAVES_NUMERATOR * n_sm,
110+
_SPLITK_TARGET_WAVES_DENOMINATOR * max(grid_y, 1),
111+
),
112+
1,
113+
)
114+
buffer_cap = max(triton.cdiv(L_kv, _SPLITK_BUFFER_ELEMENTS_PER_SPLIT), 1)
115+
# Avoid letting very high-SM GPUs over-split long global-attention buffers:
116+
# beyond 128 partitions the extra reduction and launch work outweighed the
117+
# occupancy gain in CUDA-graph measurements. Lower-SM devices continue to
118+
# use their hardware-derived cap.
119+
num_splits = min(buffer_cap, sm_cap, _SPLITK_MAX_SPLITS)
56120
return num_splits, triton.cdiv(L_kv, num_splits)
57121

58122

@@ -1256,6 +1320,7 @@ def _sdpa_decode_splitk_kernel(
12561320
NUM_GROUPS: tl.constexpr,
12571321
BLOCK_G: tl.constexpr,
12581322
):
1323+
sm_scale = sm_scale.to(tl.float32)
12591324
split_id = tl.program_id(axis=0)
12601325
pid_bh = tl.program_id(axis=1)
12611326
b = pid_bh // H_kv
@@ -1437,7 +1502,7 @@ def _launch_decode_splitk(
14371502
kv_len_ptr: Optional[torch.Tensor] = None,
14381503
HAS_KV_LEN: bool = False,
14391504
) -> None:
1440-
num_splits, chunk_size = _decode_splitk_config(L_kv)
1505+
num_splits, chunk_size = _decode_splitk_config(L_kv, B * H_kv, query.device)
14411506

14421507
# Each valid (split, batch, query head) slot belongs to exactly one program,
14431508
# and empty KV ranges still reach the stores with m=-inf and l=acc=0.
@@ -1717,6 +1782,7 @@ def _sdpa_small_query_splitk_kernel(
17171782
BLOCK_G: tl.constexpr,
17181783
BLOCK_M: tl.constexpr,
17191784
):
1785+
sm_scale = sm_scale.to(tl.float32)
17201786
split_id = tl.program_id(axis=0)
17211787
pid_bh = tl.program_id(axis=1)
17221788
b = pid_bh // H_kv
@@ -1955,7 +2021,7 @@ def _launch_small_query_splitk(
19552021
HAS_KV_LEN: bool = False,
19562022
mask_is_causal: bool = False,
19572023
) -> None:
1958-
num_splits, chunk_size = _decode_splitk_config(L_kv)
2024+
num_splits, chunk_size = _decode_splitk_config(L_kv, B * H_kv, query.device)
19592025

19602026
# Each valid (split, batch, query head, query row) slot belongs to exactly
19612027
# one program, and empty KV ranges still store m=-inf and l=acc=0.

0 commit comments

Comments
 (0)