Skip to content

Commit 645ec92

Browse files
authored
[CUDA] Elide dense causal masks in long-context prefill (pytorch#22189)
## Summary Elide causal-mask-elision work on top of `split_k_nan`. - Generate bottom-right causal bounds inside Triton SDPA instead of materializing dense masks. - Cover standard SDPA plus the Gemma4 and Muse-Glimmer source transformations; TQ4 SDPA and its operator schema remain unchanged. - Preserve explicit arbitrary masks by composing them with the reconstructed causal bound. - Apply bottom-right causal masking in the verifier-sized (`L_q=2..4`) split-K path. - Preserve the numerically stable online-softmax implementation from `split_k_nan`; this does not restore the previous `phi=5` approximation. ## Operator-level performance RTX 5090, BF16 GQA (`B=1, Hq=32, Hkv=2, D=128`), explicit dense causal mask versus in-kernel causal reconstruction: | Lq | Lkv | Dense mask | In-kernel causal | Speedup | |---:|---:|---:|---:|---:| | 128 | 16,384 | 0.643 ms | 0.505 ms | 1.27x | | 512 | 16,384 | 1.190 ms | 1.008 ms | 1.18x | | 512 | 32,768 | 2.397 ms | 2.004 ms | 1.20x | ## Test plan - `lintrunner init && lintrunner -a` (clean). - Standard Triton SDPA: 27 passed. - TQ4 SDPA schema and implementation match `split_k_nan`; focused long-context checks: 2 passed. - Full TQ4 kernel suite: 35 passed, 2 skipped; the export-only case requires the generated `program.fbs` resource unavailable in the ad-hoc GPU worktree. - `git diff --check`.
1 parent 08ce4df commit 645ec92

4 files changed

Lines changed: 248 additions & 15 deletions

File tree

‎backends/cuda/tests/test_triton_sdpa.py‎

Lines changed: 169 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -517,6 +517,175 @@ def test_non_pow2_no_mask(self):
517517
self.assertFalse(torch.isnan(out).any())
518518
self.assertLess(_max_abs_error(out, ref), MAX_ABS_TOL)
519519

520+
# ------------------------------------------------------------------
521+
# is_causal + kv_len: in-kernel bottom-right causal reconstruction
522+
#
523+
# This mirrors the global-attention path: instead of passing a
524+
# dense [B, 1, L_q, L_kv] causal bool mask, the caller passes
525+
# attn_mask=None, is_causal=True, kv_len=<filled positions>. The kernel
526+
# reconstructs a standard causal mask with BOTTOM-RIGHT alignment (query row
527+
# i sits at absolute position (kv_len - L_q) + i and attends to keys
528+
# [0, (kv_len - L_q) + i]) — the correct semantics for chunked prefill /
529+
# decode over a KV cache. These tests assert numerically equivalent output
530+
# to the explicit dense bottom-right causal mask.
531+
# ------------------------------------------------------------------
532+
533+
@staticmethod
534+
def _dense_bottom_right_causal_mask(B, Lq, kv_len, Lk, device):
535+
"""Dense [B, 1, Lq, Lk] causal mask, bottom-right aligned + kv_len bound.
536+
537+
Row i (absolute position (kv_len - Lq) + i) attends to keys
538+
[0, (kv_len - Lq) + i]; positions >= kv_len are never attended (matches
539+
the FlatKVCache empty-tail semantics the kv_len bound enforces).
540+
"""
541+
q_abs = (kv_len - Lq) + torch.arange(Lq, device=device).view(Lq, 1)
542+
cache_pos = torch.arange(Lk, device=device).view(1, Lk)
543+
keep = (cache_pos <= q_abs) & (cache_pos < kv_len)
544+
return keep.view(1, 1, Lq, Lk).expand(B, 1, Lq, Lk).contiguous()
545+
546+
def test_mask_is_causal_matches_dense_prefill(self):
547+
"""is_causal + kv_len reconstruction == explicit dense causal, prefill shapes.
548+
549+
Sweeps GQA ratios and chunked-prefill shapes (L_q < L_kv, the global
550+
layer case: a prefill chunk of L_q new queries over a kv_len-long
551+
context in a large flat buffer).
552+
"""
553+
D = 128 # head_dim (pow2)
554+
B = 1
555+
# (H_q, H_kv, L_q, kv_len, L_kv_buffer)
556+
shapes = [
557+
(16, 2, 128, 128, 256), # first chunk: kv_len == L_q
558+
(16, 2, 64, 200, 256), # later chunk: L_q < kv_len < buffer
559+
(8, 2, 32, 96, 512), # smaller chunk, larger buffer
560+
(4, 4, 64, 64, 128), # MHA square
561+
]
562+
for H_q, H_kv, Lq, kv_len, Lk in shapes:
563+
with self.subTest(H_q=H_q, H_kv=H_kv, Lq=Lq, kv_len=kv_len, Lk=Lk):
564+
torch.manual_seed(0)
565+
q = torch.randn(B, H_q, Lq, D, dtype=torch.bfloat16, device="cuda")
566+
k = torch.randn(B, H_kv, Lk, D, dtype=torch.bfloat16, device="cuda")
567+
v = torch.randn(B, H_kv, Lk, D, dtype=torch.bfloat16, device="cuda")
568+
kv_len_t = torch.tensor([kv_len], dtype=torch.int32, device="cuda")
569+
570+
dense = self._dense_bottom_right_causal_mask(B, Lq, kv_len, Lk, "cuda")
571+
out_dense = self.sdpa(
572+
q, k, v, attn_mask=dense, enable_gqa=True, kv_len=kv_len_t
573+
)
574+
out_inkernel = self.sdpa(
575+
q,
576+
k,
577+
v,
578+
attn_mask=None,
579+
enable_gqa=True,
580+
kv_len=kv_len_t,
581+
is_causal=True,
582+
)
583+
# Same masking semantics; the two paths may autotune to different
584+
# tiles (different bf16 reduction order), so compare with a tight
585+
# tolerance rather than bit-exact.
586+
self.assertLess(
587+
_max_abs_error(out_inkernel, out_dense),
588+
MAX_ABS_TOL,
589+
"is_causal + kv_len != dense causal",
590+
)
591+
# And both must match the fp32 reference within tolerance.
592+
ref = _reference_sdpa(q, k, v, attn_mask=dense)
593+
self.assertLess(_max_abs_error(out_inkernel, ref), MAX_ABS_TOL)
594+
self.assertLess(_max_abs_error(out_dense, ref), MAX_ABS_TOL)
595+
596+
def test_mask_is_causal_small_query_splitk(self):
597+
"""Verifier-sized query blocks retain bottom-right causal masking."""
598+
B, H_q, H_kv, D = 1, 8, 2, 128
599+
kv_len, Lk = 131, 256
600+
for Lq in (2, 3, 4):
601+
with self.subTest(Lq=Lq):
602+
torch.manual_seed(Lq)
603+
q = torch.randn(B, H_q, Lq, D, dtype=torch.bfloat16, device="cuda")
604+
k = torch.randn(B, H_kv, Lk, D, dtype=torch.bfloat16, device="cuda")
605+
v = torch.randn(B, H_kv, Lk, D, dtype=torch.bfloat16, device="cuda")
606+
kv_len_t = torch.tensor([kv_len], dtype=torch.int32, device="cuda")
607+
dense = self._dense_bottom_right_causal_mask(B, Lq, kv_len, Lk, "cuda")
608+
609+
out = self.sdpa(
610+
q,
611+
k,
612+
v,
613+
enable_gqa=True,
614+
kv_len=kv_len_t,
615+
is_causal=True,
616+
)
617+
ref = _reference_sdpa(q, k, v, attn_mask=dense)
618+
619+
self.assertFalse(torch.isnan(out).any())
620+
self.assertLess(_max_abs_error(out, ref), MAX_ABS_TOL)
621+
622+
def test_explicit_mask_composes_with_causal_kv_len(self):
623+
"""An explicit mask is intersected with the reconstructed causal mask."""
624+
B, H_q, H_kv, Lq, Lk, D = 1, 8, 2, 4, 256, 128
625+
kv_len = 131
626+
torch.manual_seed(5)
627+
q = torch.randn(B, H_q, Lq, D, dtype=torch.bfloat16, device="cuda")
628+
k = torch.randn(B, H_kv, Lk, D, dtype=torch.bfloat16, device="cuda")
629+
v = torch.randn(B, H_kv, Lk, D, dtype=torch.bfloat16, device="cuda")
630+
kv_len_t = torch.tensor([kv_len], dtype=torch.int32, device="cuda")
631+
extra = torch.ones(B, 1, Lq, Lk, dtype=torch.bool, device="cuda")
632+
extra[:, :, :, 1::3] = False
633+
causal = self._dense_bottom_right_causal_mask(B, Lq, kv_len, Lk, "cuda")
634+
635+
out = self.sdpa(
636+
q,
637+
k,
638+
v,
639+
attn_mask=extra,
640+
enable_gqa=True,
641+
kv_len=kv_len_t,
642+
is_causal=True,
643+
)
644+
ref = _reference_sdpa(q, k, v, attn_mask=extra & causal)
645+
646+
self.assertFalse(torch.isnan(out).any())
647+
self.assertLess(_max_abs_error(out, ref), MAX_ABS_TOL)
648+
649+
def test_mask_is_causal_matches_dense_decode(self):
650+
"""is_causal + kv_len is a no-op vs dense for L_q==1 decode over a KV cache."""
651+
D, B, H_q, H_kv = 128, 1, 16, 2
652+
for kv_len, Lk in [(64, 512), (300, 512), (511, 512)]:
653+
with self.subTest(kv_len=kv_len, Lk=Lk):
654+
torch.manual_seed(1)
655+
q = torch.randn(B, H_q, 1, D, dtype=torch.bfloat16, device="cuda")
656+
k = torch.randn(B, H_kv, Lk, D, dtype=torch.bfloat16, device="cuda")
657+
v = torch.randn(B, H_kv, Lk, D, dtype=torch.bfloat16, device="cuda")
658+
kv_len_t = torch.tensor([kv_len], dtype=torch.int32, device="cuda")
659+
660+
dense = self._dense_bottom_right_causal_mask(B, 1, kv_len, Lk, "cuda")
661+
out_dense = self.sdpa(
662+
q, k, v, attn_mask=dense, enable_gqa=True, kv_len=kv_len_t
663+
)
664+
out_inkernel = self.sdpa(
665+
q,
666+
k,
667+
v,
668+
attn_mask=None,
669+
enable_gqa=True,
670+
kv_len=kv_len_t,
671+
is_causal=True,
672+
)
673+
self.assertLess(
674+
_max_abs_error(out_inkernel, out_dense),
675+
MAX_ABS_TOL,
676+
"decode is_causal + kv_len != dense",
677+
)
678+
679+
def test_mask_is_causal_requires_kv_len(self):
680+
"""is_causal=True with L_q != L_kv and no kv_len should raise."""
681+
B, H, Lq, Lk, D = 1, 4, 32, 64, 128
682+
torch.manual_seed(0)
683+
q = torch.randn(B, H, Lq, D, dtype=torch.bfloat16, device="cuda")
684+
k = torch.randn(B, H, Lk, D, dtype=torch.bfloat16, device="cuda")
685+
v = torch.randn(B, H, Lk, D, dtype=torch.bfloat16, device="cuda")
686+
with self.assertRaises(RuntimeError):
687+
self.sdpa(q, k, v, is_causal=True)
688+
520689

521690
if __name__ == "__main__":
522691
unittest.main()

0 commit comments

Comments
 (0)