@@ -517,6 +517,175 @@ def test_non_pow2_no_mask(self):
517517 self .assertFalse (torch .isnan (out ).any ())
518518 self .assertLess (_max_abs_error (out , ref ), MAX_ABS_TOL )
519519
520+ # ------------------------------------------------------------------
521+ # is_causal + kv_len: in-kernel bottom-right causal reconstruction
522+ #
523+ # This mirrors the global-attention path: instead of passing a
524+ # dense [B, 1, L_q, L_kv] causal bool mask, the caller passes
525+ # attn_mask=None, is_causal=True, kv_len=<filled positions>. The kernel
526+ # reconstructs a standard causal mask with BOTTOM-RIGHT alignment (query row
527+ # i sits at absolute position (kv_len - L_q) + i and attends to keys
528+ # [0, (kv_len - L_q) + i]) — the correct semantics for chunked prefill /
529+ # decode over a KV cache. These tests assert numerically equivalent output
530+ # to the explicit dense bottom-right causal mask.
531+ # ------------------------------------------------------------------
532+
533+ @staticmethod
534+ def _dense_bottom_right_causal_mask (B , Lq , kv_len , Lk , device ):
535+ """Dense [B, 1, Lq, Lk] causal mask, bottom-right aligned + kv_len bound.
536+
537+ Row i (absolute position (kv_len - Lq) + i) attends to keys
538+ [0, (kv_len - Lq) + i]; positions >= kv_len are never attended (matches
539+ the FlatKVCache empty-tail semantics the kv_len bound enforces).
540+ """
541+ q_abs = (kv_len - Lq ) + torch .arange (Lq , device = device ).view (Lq , 1 )
542+ cache_pos = torch .arange (Lk , device = device ).view (1 , Lk )
543+ keep = (cache_pos <= q_abs ) & (cache_pos < kv_len )
544+ return keep .view (1 , 1 , Lq , Lk ).expand (B , 1 , Lq , Lk ).contiguous ()
545+
546+ def test_mask_is_causal_matches_dense_prefill (self ):
547+ """is_causal + kv_len reconstruction == explicit dense causal, prefill shapes.
548+
549+ Sweeps GQA ratios and chunked-prefill shapes (L_q < L_kv, the global
550+ layer case: a prefill chunk of L_q new queries over a kv_len-long
551+ context in a large flat buffer).
552+ """
553+ D = 128 # head_dim (pow2)
554+ B = 1
555+ # (H_q, H_kv, L_q, kv_len, L_kv_buffer)
556+ shapes = [
557+ (16 , 2 , 128 , 128 , 256 ), # first chunk: kv_len == L_q
558+ (16 , 2 , 64 , 200 , 256 ), # later chunk: L_q < kv_len < buffer
559+ (8 , 2 , 32 , 96 , 512 ), # smaller chunk, larger buffer
560+ (4 , 4 , 64 , 64 , 128 ), # MHA square
561+ ]
562+ for H_q , H_kv , Lq , kv_len , Lk in shapes :
563+ with self .subTest (H_q = H_q , H_kv = H_kv , Lq = Lq , kv_len = kv_len , Lk = Lk ):
564+ torch .manual_seed (0 )
565+ q = torch .randn (B , H_q , Lq , D , dtype = torch .bfloat16 , device = "cuda" )
566+ k = torch .randn (B , H_kv , Lk , D , dtype = torch .bfloat16 , device = "cuda" )
567+ v = torch .randn (B , H_kv , Lk , D , dtype = torch .bfloat16 , device = "cuda" )
568+ kv_len_t = torch .tensor ([kv_len ], dtype = torch .int32 , device = "cuda" )
569+
570+ dense = self ._dense_bottom_right_causal_mask (B , Lq , kv_len , Lk , "cuda" )
571+ out_dense = self .sdpa (
572+ q , k , v , attn_mask = dense , enable_gqa = True , kv_len = kv_len_t
573+ )
574+ out_inkernel = self .sdpa (
575+ q ,
576+ k ,
577+ v ,
578+ attn_mask = None ,
579+ enable_gqa = True ,
580+ kv_len = kv_len_t ,
581+ is_causal = True ,
582+ )
583+ # Same masking semantics; the two paths may autotune to different
584+ # tiles (different bf16 reduction order), so compare with a tight
585+ # tolerance rather than bit-exact.
586+ self .assertLess (
587+ _max_abs_error (out_inkernel , out_dense ),
588+ MAX_ABS_TOL ,
589+ "is_causal + kv_len != dense causal" ,
590+ )
591+ # And both must match the fp32 reference within tolerance.
592+ ref = _reference_sdpa (q , k , v , attn_mask = dense )
593+ self .assertLess (_max_abs_error (out_inkernel , ref ), MAX_ABS_TOL )
594+ self .assertLess (_max_abs_error (out_dense , ref ), MAX_ABS_TOL )
595+
596+ def test_mask_is_causal_small_query_splitk (self ):
597+ """Verifier-sized query blocks retain bottom-right causal masking."""
598+ B , H_q , H_kv , D = 1 , 8 , 2 , 128
599+ kv_len , Lk = 131 , 256
600+ for Lq in (2 , 3 , 4 ):
601+ with self .subTest (Lq = Lq ):
602+ torch .manual_seed (Lq )
603+ q = torch .randn (B , H_q , Lq , D , dtype = torch .bfloat16 , device = "cuda" )
604+ k = torch .randn (B , H_kv , Lk , D , dtype = torch .bfloat16 , device = "cuda" )
605+ v = torch .randn (B , H_kv , Lk , D , dtype = torch .bfloat16 , device = "cuda" )
606+ kv_len_t = torch .tensor ([kv_len ], dtype = torch .int32 , device = "cuda" )
607+ dense = self ._dense_bottom_right_causal_mask (B , Lq , kv_len , Lk , "cuda" )
608+
609+ out = self .sdpa (
610+ q ,
611+ k ,
612+ v ,
613+ enable_gqa = True ,
614+ kv_len = kv_len_t ,
615+ is_causal = True ,
616+ )
617+ ref = _reference_sdpa (q , k , v , attn_mask = dense )
618+
619+ self .assertFalse (torch .isnan (out ).any ())
620+ self .assertLess (_max_abs_error (out , ref ), MAX_ABS_TOL )
621+
622+ def test_explicit_mask_composes_with_causal_kv_len (self ):
623+ """An explicit mask is intersected with the reconstructed causal mask."""
624+ B , H_q , H_kv , Lq , Lk , D = 1 , 8 , 2 , 4 , 256 , 128
625+ kv_len = 131
626+ torch .manual_seed (5 )
627+ q = torch .randn (B , H_q , Lq , D , dtype = torch .bfloat16 , device = "cuda" )
628+ k = torch .randn (B , H_kv , Lk , D , dtype = torch .bfloat16 , device = "cuda" )
629+ v = torch .randn (B , H_kv , Lk , D , dtype = torch .bfloat16 , device = "cuda" )
630+ kv_len_t = torch .tensor ([kv_len ], dtype = torch .int32 , device = "cuda" )
631+ extra = torch .ones (B , 1 , Lq , Lk , dtype = torch .bool , device = "cuda" )
632+ extra [:, :, :, 1 ::3 ] = False
633+ causal = self ._dense_bottom_right_causal_mask (B , Lq , kv_len , Lk , "cuda" )
634+
635+ out = self .sdpa (
636+ q ,
637+ k ,
638+ v ,
639+ attn_mask = extra ,
640+ enable_gqa = True ,
641+ kv_len = kv_len_t ,
642+ is_causal = True ,
643+ )
644+ ref = _reference_sdpa (q , k , v , attn_mask = extra & causal )
645+
646+ self .assertFalse (torch .isnan (out ).any ())
647+ self .assertLess (_max_abs_error (out , ref ), MAX_ABS_TOL )
648+
649+ def test_mask_is_causal_matches_dense_decode (self ):
650+ """is_causal + kv_len is a no-op vs dense for L_q==1 decode over a KV cache."""
651+ D , B , H_q , H_kv = 128 , 1 , 16 , 2
652+ for kv_len , Lk in [(64 , 512 ), (300 , 512 ), (511 , 512 )]:
653+ with self .subTest (kv_len = kv_len , Lk = Lk ):
654+ torch .manual_seed (1 )
655+ q = torch .randn (B , H_q , 1 , D , dtype = torch .bfloat16 , device = "cuda" )
656+ k = torch .randn (B , H_kv , Lk , D , dtype = torch .bfloat16 , device = "cuda" )
657+ v = torch .randn (B , H_kv , Lk , D , dtype = torch .bfloat16 , device = "cuda" )
658+ kv_len_t = torch .tensor ([kv_len ], dtype = torch .int32 , device = "cuda" )
659+
660+ dense = self ._dense_bottom_right_causal_mask (B , 1 , kv_len , Lk , "cuda" )
661+ out_dense = self .sdpa (
662+ q , k , v , attn_mask = dense , enable_gqa = True , kv_len = kv_len_t
663+ )
664+ out_inkernel = self .sdpa (
665+ q ,
666+ k ,
667+ v ,
668+ attn_mask = None ,
669+ enable_gqa = True ,
670+ kv_len = kv_len_t ,
671+ is_causal = True ,
672+ )
673+ self .assertLess (
674+ _max_abs_error (out_inkernel , out_dense ),
675+ MAX_ABS_TOL ,
676+ "decode is_causal + kv_len != dense" ,
677+ )
678+
679+ def test_mask_is_causal_requires_kv_len (self ):
680+ """is_causal=True with L_q != L_kv and no kv_len should raise."""
681+ B , H , Lq , Lk , D = 1 , 4 , 32 , 64 , 128
682+ torch .manual_seed (0 )
683+ q = torch .randn (B , H , Lq , D , dtype = torch .bfloat16 , device = "cuda" )
684+ k = torch .randn (B , H , Lk , D , dtype = torch .bfloat16 , device = "cuda" )
685+ v = torch .randn (B , H , Lk , D , dtype = torch .bfloat16 , device = "cuda" )
686+ with self .assertRaises (RuntimeError ):
687+ self .sdpa (q , k , v , is_causal = True )
688+
520689
521690if __name__ == "__main__" :
522691 unittest .main ()
0 commit comments