Skip to content

Commit 2a64e2d

Browse files
authored
Fix GDN channelwise_gated_delta_rule chunked-prefill NaN for tiny decay (pytorch#21417)
Differential Revision: D113866825 Pull Request resolved: pytorch#21417
1 parent e47f373 commit 2a64e2d

1 file changed

Lines changed: 40 additions & 9 deletions

File tree

‎extension/llm/custom_ops/op_sdpa.cpp‎

Lines changed: 40 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1177,12 +1177,41 @@ void channelwise_gated_delta_rule_chunked(
11771177
// output. Detect that so callers can route to the recurrence, which handles
11781178
// decay <= 0 exactly. NaN is treated as non-positive so NaN decay also takes
11791179
// the faithful path instead of contaminating every output element.
1180-
bool channelwise_gated_delta_rule_decay_all_positive(const Tensor& decay) {
1180+
// The chunked prefill kernel resets gc = cumsum(log decay) every CHUNK_SIZE
1181+
// tokens and forms eg_inv = exp(-gc). For tiny positive decay the within-chunk
1182+
// cumulative -log(decay) can exceed the float expf overflow limit (~88.7),
1183+
// producing +inf that turns eg * eg_inv into 0 * inf = NaN and poisons the
1184+
// whole head's output. A non-positive decay is likewise undefined (std::log).
1185+
// Detect both so callers can route to the exact recurrence, which is stable for
1186+
// tiny decay and handles decay <= 0. NaN decay is treated as unsafe as well.
1187+
bool channelwise_gated_delta_rule_chunked_is_safe(const Tensor& decay) {
1188+
// Margin below the float32 expf overflow threshold (std::exp overflows around
1189+
// 88.7); 80 leaves headroom for the downstream eg * eg_inv product.
1190+
constexpr float kMaxNegLogSum = 80.0f;
11811191
const auto* decay_data = decay.const_data_ptr<float>();
1182-
const auto numel = decay.numel();
1183-
for (int64_t idx = 0; idx < numel; ++idx) {
1184-
if (!(decay_data[idx] > 0.0f)) {
1185-
return false;
1192+
const auto batch_heads = decay.size(0) * decay.size(1);
1193+
const auto sequence_length = decay.size(2);
1194+
const auto k_head_dim = decay.size(3);
1195+
const auto head_stride = sequence_length * k_head_dim;
1196+
for (int64_t bh = 0; bh < batch_heads; ++bh) {
1197+
const float* decay_head = decay_data + bh * head_stride;
1198+
for (int64_t base = 0; base < sequence_length; base += CHUNK_SIZE) {
1199+
const int64_t cur = std::min<int64_t>(CHUNK_SIZE, sequence_length - base);
1200+
// gc resets per chunk, so overflow is bounded by the within-chunk
1201+
// cumulative -log(decay) per channel.
1202+
for (int64_t k = 0; k < k_head_dim; ++k) {
1203+
float neg_log_sum = 0.0f;
1204+
for (int64_t r = 0; r < cur; ++r) {
1205+
const float val = decay_head[(base + r) * k_head_dim + k];
1206+
if (!(val > 0.0f)) {
1207+
return false; // decay <= 0 or NaN: log undefined -> recurrence
1208+
}
1209+
neg_log_sum -= std::log(val);
1210+
if (neg_log_sum > kMaxNegLogSum) {
1211+
return false; // exp(-gc) would overflow -> recurrence
1212+
}
1213+
}
1214+
}
11861215
}
11871216
}
11881217
return true;
@@ -1245,9 +1274,11 @@ std::tuple<Tensor&, Tensor&> channelwise_gated_delta_rule_out(
12451274
"Failed to resize channelwise_gated_delta_rule final_state tensor.");
12461275

12471276
// Route on sequence length: T == 1 is autoregressive decode (token-by-token
1248-
// recurrence), T != 1 is prefill (chunkwise WY kernel). Prefill with any
1249-
// non-positive decay falls back to the recurrence, which is exact for
1250-
// decay <= 0 (the chunked kernel takes log(decay) and would produce NaN).
1277+
// recurrence), T != 1 is prefill (chunkwise WY kernel). Prefill falls back to
1278+
// the exact recurrence whenever the chunked kernel would be numerically
1279+
// unsafe
1280+
// -- non-positive decay (log undefined) or tiny positive decay whose
1281+
// within-chunk cumulative -log(decay) would overflow exp(-gc) to inf -> NaN.
12511282
if (query.size(2) == 1) {
12521283
channelwise_gated_delta_rule_decode(
12531284
ctx,
@@ -1259,7 +1290,7 @@ std::tuple<Tensor&, Tensor&> channelwise_gated_delta_rule_out(
12591290
initial_state,
12601291
out,
12611292
final_state_out);
1262-
} else if (channelwise_gated_delta_rule_decay_all_positive(decay)) {
1293+
} else if (channelwise_gated_delta_rule_chunked_is_safe(decay)) {
12631294
channelwise_gated_delta_rule_chunked(
12641295
ctx,
12651296
query,

0 commit comments

Comments
 (0)