Skip to content

Commit b87b417

Browse files
authored
Rename channelwise_gated_delta_rule to gated_delta_rule (pytorch#21920)
Differential Revision: D114737925 Pull Request resolved: pytorch#21920
1 parent 8362c00 commit b87b417

10 files changed

Lines changed: 179 additions & 130 deletions

File tree

‎examples/models/llama/attention.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -74,13 +74,13 @@ def decorator(cls: Type[Attention]):
7474
def _get_gated_delta_rule_op() -> Optional[Any]:
7575
"""Return the fused gated delta rule op, or None if it is unavailable.
7676
77-
``channelwise_gated_delta_rule`` accepts a per-head scalar decay as a 3D
77+
``gated_delta_rule`` accepts a per-head scalar decay as a 3D
7878
``[B, H, T]`` tensor, which is the layout GatedDeltaNet produces. Resolve it
7979
lazily (importing the custom ops library on first use) and memoize so
8080
environments without the custom op fall back to the Python recurrence.
8181
"""
8282
try:
83-
return torch.ops.llama.channelwise_gated_delta_rule.default
83+
return torch.ops.llama.gated_delta_rule.default
8484
except (AttributeError, RuntimeError):
8585
pass
8686

@@ -91,7 +91,7 @@ def _get_gated_delta_rule_op() -> Optional[Any]:
9191
return None
9292

9393
try:
94-
return torch.ops.llama.channelwise_gated_delta_rule.default
94+
return torch.ops.llama.gated_delta_rule.default
9595
except (AttributeError, RuntimeError):
9696
return None
9797

‎examples/models/llama/tests/test_qwen3_5_attention.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,7 @@ def test_gated_deltanet_fused_op_matches_python_recurrence(self):
131131
# the decode (T == 1) and prefill (T > 1) routes.
132132
from executorch.extension.llm.custom_ops import custom_ops # noqa: F401
133133

134-
op = torch.ops.llama.channelwise_gated_delta_rule.default
134+
op = torch.ops.llama.gated_delta_rule.default
135135

136136
for seq_len in (1, 5, 40):
137137
with self.subTest(seq_len=seq_len):

‎extension/llm/custom_ops/BUCK‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -114,20 +114,20 @@ fbcode_target(_kind = runtime.python_test,
114114
)
115115

116116
fbcode_target(_kind = runtime.python_library,
117-
name = "bench_channelwise_gated_delta_rule_lib",
118-
srcs = ["bench_channelwise_gated_delta_rule.py"],
117+
name = "bench_gated_delta_rule_lib",
118+
srcs = ["bench_gated_delta_rule.py"],
119119
base_module = "executorch.extension.llm.custom_ops",
120120
deps = ["//caffe2:torch"],
121121
)
122122

123123
fbcode_target(_kind = runtime.python_binary,
124-
name = "bench_channelwise_gated_delta_rule",
125-
main_module = "executorch.extension.llm.custom_ops.bench_channelwise_gated_delta_rule",
124+
name = "bench_gated_delta_rule",
125+
main_module = "executorch.extension.llm.custom_ops.bench_gated_delta_rule",
126126
preload_deps = [
127127
":custom_ops_aot_lib_mkl_noomp",
128128
":custom_ops_aot_py",
129129
],
130-
deps = [":bench_channelwise_gated_delta_rule_lib"],
130+
deps = [":bench_gated_delta_rule_lib"],
131131
)
132132

133133
fbcode_target(_kind = runtime.python_test,

extension/llm/custom_ops/bench_channelwise_gated_delta_rule.py renamed to extension/llm/custom_ops/bench_gated_delta_rule.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,15 @@
66

77
# pyre-unsafe
88

9-
"""Microbenchmark for the ``channelwise_gated_delta_rule`` custom op.
9+
"""Microbenchmark for the ``gated_delta_rule`` custom op.
1010
1111
Times the fused recurrent kernel in isolation (no model) at representative
1212
GatedDeltaNet sizes, for both the decode (T=1) and prefill (T>1) regimes. Use it
1313
to compare kernel variants (e.g. naive vs. fused) — rebuild the custom op, run
1414
this, and diff the numbers.
1515
1616
Run (in an env where the custom op is built):
17-
python -m executorch.extension.llm.custom_ops.bench_channelwise_gated_delta_rule
17+
python -m executorch.extension.llm.custom_ops.bench_gated_delta_rule
1818
"""
1919

2020
import itertools
@@ -24,7 +24,7 @@
2424

2525
from executorch.extension.llm.custom_ops import custom_ops # noqa: F401
2626

27-
_OP = torch.ops.llama.channelwise_gated_delta_rule.default
27+
_OP = torch.ops.llama.gated_delta_rule.default
2828

2929

3030
def _make_inputs(b: int, h: int, t: int, k: int, v: int, scalar_decay: bool = False):
@@ -74,7 +74,7 @@ def main() -> None:
7474
]
7575

7676
print(
77-
f"channelwise_gated_delta_rule microbenchmark "
77+
f"gated_delta_rule microbenchmark "
7878
f"(K=V={k}, fp32, 1 thread, state={32 * k * v * 4 // 1024}KB)"
7979
)
8080
print(

‎extension/llm/custom_ops/custom_ops.py‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -379,7 +379,7 @@ def update_cache_with_indices_meta(
379379
return torch.empty((1,), dtype=value.dtype, device="meta")
380380

381381

382-
def _validate_channelwise_gated_delta_rule_params(
382+
def _validate_gated_delta_rule_params(
383383
query,
384384
key,
385385
value,
@@ -436,16 +436,16 @@ def _validate_channelwise_gated_delta_rule_params(
436436
)
437437

438438

439-
@impl(custom_ops_lib, "channelwise_gated_delta_rule", "Meta")
440-
def channelwise_gated_delta_rule_meta(
439+
@impl(custom_ops_lib, "gated_delta_rule", "Meta")
440+
def gated_delta_rule_meta(
441441
query,
442442
key,
443443
value,
444444
decay,
445445
beta,
446446
initial_state,
447447
):
448-
_validate_channelwise_gated_delta_rule_params(
448+
_validate_gated_delta_rule_params(
449449
query,
450450
key,
451451
value,
@@ -457,6 +457,20 @@ def channelwise_gated_delta_rule_meta(
457457
return query.new_empty(output_shape), torch.empty_like(initial_state)
458458

459459

460+
# Deprecated alias of gated_delta_rule, kept so graphs built against the
461+
# pre-scalar-decay name keep tracing. Remove once those have been re-exported.
462+
@impl(custom_ops_lib, "channelwise_gated_delta_rule", "Meta")
463+
def channelwise_gated_delta_rule_meta(
464+
query,
465+
key,
466+
value,
467+
decay,
468+
beta,
469+
initial_state,
470+
):
471+
return gated_delta_rule_meta(query, key, value, decay, beta, initial_state)
472+
473+
460474
def _validate_quantized_sdpa_params(
461475
query,
462476
key,

‎extension/llm/custom_ops/op_sdpa.cpp‎

Lines changed: 19 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -193,7 +193,7 @@ bool validate_cache_params(
193193
return true;
194194
}
195195

196-
bool validate_channelwise_gated_delta_rule_args(
196+
bool validate_gated_delta_rule_args(
197197
const Tensor& query,
198198
const Tensor& key,
199199
const Tensor& value,
@@ -1531,7 +1531,7 @@ void run_gated_delta_rule_chunked(
15311531
}
15321532
}
15331533

1534-
std::tuple<Tensor&, Tensor&> channelwise_gated_delta_rule_out(
1534+
std::tuple<Tensor&, Tensor&> gated_delta_rule_out(
15351535
RuntimeContext& ctx,
15361536
const Tensor& query,
15371537
const Tensor& key,
@@ -1546,7 +1546,7 @@ std::tuple<Tensor&, Tensor&> channelwise_gated_delta_rule_out(
15461546
// caller's out/final_state_out tensors untouched.
15471547
ET_KERNEL_CHECK(
15481548
ctx,
1549-
validate_channelwise_gated_delta_rule_args(
1549+
validate_gated_delta_rule_args(
15501550
query, key, value, decay, beta, initial_state),
15511551
InvalidArgument,
15521552
ret);
@@ -1555,7 +1555,7 @@ std::tuple<Tensor&, Tensor&> channelwise_gated_delta_rule_out(
15551555
!tensor_memory_ranges_overlap(initial_state, final_state_out),
15561556
InvalidArgument,
15571557
ret,
1558-
"channelwise_gated_delta_rule final_state_out must not alias initial_state.");
1558+
"gated_delta_rule final_state_out must not alias initial_state.");
15591559
ET_KERNEL_CHECK(
15601560
ctx, out.scalar_type() == ScalarType::Float, InvalidArgument, ret);
15611561
ET_KERNEL_CHECK(
@@ -1587,13 +1587,13 @@ std::tuple<Tensor&, Tensor&> channelwise_gated_delta_rule_out(
15871587
output_sizes, 4)) == Error::Ok,
15881588
InvalidArgument,
15891589
ret,
1590-
"Failed to resize channelwise_gated_delta_rule output tensor.");
1590+
"Failed to resize gated_delta_rule output tensor.");
15911591
ET_KERNEL_CHECK_MSG(
15921592
ctx,
15931593
resize_tensor(final_state_out, initial_state.sizes()) == Error::Ok,
15941594
InvalidArgument,
15951595
ret,
1596-
"Failed to resize channelwise_gated_delta_rule final_state tensor.");
1596+
"Failed to resize gated_delta_rule final_state tensor.");
15971597

15981598
// Route on sequence length: T == 1 is autoregressive decode (token-by-token
15991599
// recurrence), T != 1 is prefill (chunkwise WY kernel). Prefill falls back to
@@ -1651,13 +1651,13 @@ EXECUTORCH_LIBRARY(
16511651

16521652
namespace {
16531653

1654-
void channelwise_gated_delta_rule_out_boxed(
1654+
void gated_delta_rule_out_boxed(
16551655
executorch::runtime::KernelRuntimeContext& ctx,
16561656
executorch::runtime::Span<executorch::runtime::EValue*> stack) {
16571657
executorch::runtime::internal::EventTracerProfileOpScope
16581658
event_tracer_op_scope(
16591659
ctx.internal_event_tracer(),
1660-
"native_call_llama::channelwise_gated_delta_rule.out");
1660+
"native_call_llama::gated_delta_rule.out");
16611661
// Multi-output out variants get a trailing TensorList aggregating the two
16621662
// outputs appended by the emitter, so the boxed stack has 9 entries: 6 inputs
16631663
// + out + final_state_out + [out, final_state_out]. The aggregate (stack[8])
@@ -1680,13 +1680,22 @@ void channelwise_gated_delta_rule_out_boxed(
16801680
auto& out = stack[6]->toTensor();
16811681
auto& final_state_out = stack[7]->toTensor();
16821682

1683-
(void)torch::executor::native::channelwise_gated_delta_rule_out(
1683+
(void)torch::executor::native::gated_delta_rule_out(
16841684
ctx, query, key, value, decay, beta, initial_state, out, final_state_out);
16851685
}
16861686

1687+
const auto gated_delta_rule_out_registration =
1688+
executorch::runtime::register_kernel(executorch::runtime::Kernel(
1689+
"llama::gated_delta_rule.out",
1690+
gated_delta_rule_out_boxed));
1691+
1692+
// Deprecated alias: the op was named channelwise_gated_delta_rule before it
1693+
// learned the scalar decay layout. Kept so .pte files emitted against the old
1694+
// name still resolve a kernel; remove once those have been re-exported. Traces
1695+
// report the new name for both, since the boxed function is shared.
16871696
const auto channelwise_gated_delta_rule_out_registration =
16881697
executorch::runtime::register_kernel(executorch::runtime::Kernel(
16891698
"llama::channelwise_gated_delta_rule.out",
1690-
channelwise_gated_delta_rule_out_boxed));
1699+
gated_delta_rule_out_boxed));
16911700

16921701
} // namespace

‎extension/llm/custom_ops/op_sdpa.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ Tensor& custom_quantized_sdpa_out(
7878
const bool is_seq_at_dim_1,
7979
Tensor& output);
8080

81-
std::tuple<Tensor&, Tensor&> channelwise_gated_delta_rule_out(
81+
std::tuple<Tensor&, Tensor&> gated_delta_rule_out(
8282
RuntimeContext& ctx,
8383
const Tensor& query,
8484
const Tensor& key,

‎extension/llm/custom_ops/op_sdpa_aot.cpp‎

Lines changed: 26 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -377,7 +377,7 @@ at::Tensor update_cache_with_indices_aten(
377377
return output;
378378
}
379379

380-
std::tuple<Tensor&, Tensor&> channelwise_gated_delta_rule_out_no_context(
380+
std::tuple<Tensor&, Tensor&> gated_delta_rule_out_no_context(
381381
const Tensor& query,
382382
const Tensor& key,
383383
const Tensor& value,
@@ -387,15 +387,15 @@ std::tuple<Tensor&, Tensor&> channelwise_gated_delta_rule_out_no_context(
387387
Tensor& out,
388388
Tensor& final_state_out);
389389

390-
std::tuple<at::Tensor, at::Tensor> channelwise_gated_delta_rule_aten(
390+
std::tuple<at::Tensor, at::Tensor> gated_delta_rule_aten(
391391
const at::Tensor& query,
392392
const at::Tensor& key,
393393
const at::Tensor& value,
394394
const at::Tensor& decay,
395395
const at::Tensor& beta,
396396
const at::Tensor& initial_state);
397397

398-
std::tuple<at::Tensor&, at::Tensor&> channelwise_gated_delta_rule_out_aten(
398+
std::tuple<at::Tensor&, at::Tensor&> gated_delta_rule_out_aten(
399399
const at::Tensor& query,
400400
const at::Tensor& key,
401401
const at::Tensor& value,
@@ -405,7 +405,7 @@ std::tuple<at::Tensor&, at::Tensor&> channelwise_gated_delta_rule_out_aten(
405405
at::Tensor& out,
406406
at::Tensor& final_state_out);
407407

408-
std::tuple<Tensor&, Tensor&> channelwise_gated_delta_rule_out_no_context(
408+
std::tuple<Tensor&, Tensor&> gated_delta_rule_out_no_context(
409409
const Tensor& query,
410410
const Tensor& key,
411411
const Tensor& value,
@@ -415,7 +415,7 @@ std::tuple<Tensor&, Tensor&> channelwise_gated_delta_rule_out_no_context(
415415
Tensor& out,
416416
Tensor& final_state_out) {
417417
executorch::aten::RuntimeContext context{};
418-
auto result = torch::executor::native::channelwise_gated_delta_rule_out(
418+
auto result = torch::executor::native::gated_delta_rule_out(
419419
context,
420420
query,
421421
key,
@@ -427,11 +427,11 @@ std::tuple<Tensor&, Tensor&> channelwise_gated_delta_rule_out_no_context(
427427
final_state_out);
428428
TORCH_CHECK(
429429
context.failure_state() == executorch::runtime::Error::Ok,
430-
"channelwise_gated_delta_rule failed");
430+
"gated_delta_rule failed");
431431
return result;
432432
}
433433

434-
std::tuple<at::Tensor, at::Tensor> channelwise_gated_delta_rule_aten(
434+
std::tuple<at::Tensor, at::Tensor> gated_delta_rule_aten(
435435
const at::Tensor& query,
436436
const at::Tensor& key,
437437
const at::Tensor& value,
@@ -442,12 +442,12 @@ std::tuple<at::Tensor, at::Tensor> channelwise_gated_delta_rule_aten(
442442
{query.size(0), query.size(1), query.size(2), value.size(3)},
443443
query.options());
444444
auto final_state = at::empty_like(initial_state);
445-
channelwise_gated_delta_rule_out_aten(
445+
gated_delta_rule_out_aten(
446446
query, key, value, decay, beta, initial_state, output, final_state);
447447
return {output, final_state};
448448
}
449449

450-
std::tuple<at::Tensor&, at::Tensor&> channelwise_gated_delta_rule_out_aten(
450+
std::tuple<at::Tensor&, at::Tensor&> gated_delta_rule_out_aten(
451451
const at::Tensor& query,
452452
const at::Tensor& key,
453453
const at::Tensor& value,
@@ -458,8 +458,8 @@ std::tuple<at::Tensor&, at::Tensor&> channelwise_gated_delta_rule_out_aten(
458458
at::Tensor& final_state_out) {
459459
TORCH_CHECK(
460460
!initial_state.is_alias_of(final_state_out),
461-
"channelwise_gated_delta_rule final_state_out must not alias initial_state.");
462-
return WRAP_TO_ATEN(channelwise_gated_delta_rule_out_no_context, 6)(
461+
"gated_delta_rule final_state_out must not alias initial_state.");
462+
return WRAP_TO_ATEN(gated_delta_rule_out_no_context, 6)(
463463
query, key, value, decay, beta, initial_state, out, final_state_out);
464464
}
465465

@@ -508,6 +508,15 @@ TORCH_LIBRARY_FRAGMENT(llama, m) {
508508
"float? scale=None, Tensor? q_zero_points=None, Tensor? q_scales=None, "
509509
"Tensor? k_zero_points=None, Tensor? k_scales=None, Tensor? v_zero_points=None, "
510510
"Tensor? v_scales=None, bool is_seq_at_dim_2=False, *, Tensor(a!) out) -> Tensor(a!)");
511+
m.def(
512+
"gated_delta_rule(Tensor query, Tensor key, Tensor value, Tensor decay, "
513+
"Tensor beta, Tensor initial_state) -> (Tensor, Tensor)");
514+
m.def(
515+
"gated_delta_rule.out(Tensor query, Tensor key, Tensor value, Tensor decay, "
516+
"Tensor beta, Tensor initial_state, *, Tensor(a!) out, Tensor(b!) final_state_out) "
517+
"-> (Tensor(a!), Tensor(b!))");
518+
// Deprecated alias of gated_delta_rule, kept so graphs and .pte files built
519+
// against the pre-scalar-decay name keep working. Remove once re-exported.
511520
m.def(
512521
"channelwise_gated_delta_rule(Tensor query, Tensor key, Tensor value, Tensor decay, "
513522
"Tensor beta, Tensor initial_state) -> (Tensor, Tensor)");
@@ -548,10 +557,14 @@ TORCH_LIBRARY_IMPL(llama, CompositeExplicitAutograd, m) {
548557
"custom_quantized_sdpa.out",
549558
WRAP_TO_ATEN(
550559
torch::executor::native::custom_quantized_sdpa_out_no_context, 15));
560+
m.impl("gated_delta_rule", torch::executor::native::gated_delta_rule_aten);
561+
m.impl(
562+
"gated_delta_rule.out",
563+
torch::executor::native::gated_delta_rule_out_aten);
551564
m.impl(
552565
"channelwise_gated_delta_rule",
553-
torch::executor::native::channelwise_gated_delta_rule_aten);
566+
torch::executor::native::gated_delta_rule_aten);
554567
m.impl(
555568
"channelwise_gated_delta_rule.out",
556-
torch::executor::native::channelwise_gated_delta_rule_out_aten);
569+
torch::executor::native::gated_delta_rule_out_aten);
557570
}

0 commit comments

Comments
 (0)