Skip to content

Commit 7454b89

Browse files
authored
Fix Qwen3.5 CUDA autotuning with released KV storage (pytorch#21803)
### Summary Qwen3.5-MoE low-memory CUDA export releases zero-filled KV-cache storage before AOTInductor compilation. PyTorch 2.13's Triton autotuner later clones, resets, and launches kernels with those original tensors, so main fails with `setStorage ... out of bounds for storage of size 0`; releasing restored storage before the final winner launch instead produces an asynchronous illegal-memory access. Also scopes reduction loads to their individual Triton loops in the opt-in low-memory path, avoiding invalid inductor generated code that referenced an undefined CSE temporary; A upstream issue is here: pytorch/pytorch#193988 cc @digantdesai
1 parent 7c3cdbb commit 7454b89

3 files changed

Lines changed: 157 additions & 4 deletions

File tree

‎backends/cuda/cuda_backend.py‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,28 @@ def _is_cpu_clone_active() -> bool:
6060
return getattr(_CPU_CLONE_GUARD, "active", False)
6161

6262

63+
@contextlib.contextmanager
64+
def _keep_triton_reduction_loads_loop_scoped():
65+
"""Conservatively treating loads as reduction-masked keeps each definition in its
66+
own loop to bypass the undefined variable bug in pytorch/pytorch.
67+
(https://github.com/pytorch/pytorch/issues/193988)
68+
69+
TODO(gasoonjia): remove this setting once bug fixed in upstream
70+
"""
71+
from torch._inductor.codegen.triton import IndexingOptions
72+
73+
orig_has_rmask = IndexingOptions.has_rmask
74+
75+
def _has_rmask(_self) -> bool:
76+
return True
77+
78+
IndexingOptions.has_rmask = _has_rmask
79+
try:
80+
yield
81+
finally:
82+
IndexingOptions.has_rmask = orig_has_rmask
83+
84+
6385
def _full_zeros_preserving_strides(x: torch.Tensor, device) -> torch.Tensor:
6486
"""Allocate a zero-filled tensor matching ``x``'s size/stride/dtype on ``device``.
6587
@@ -82,18 +104,71 @@ def _is_emptied(x) -> bool:
82104
)
83105

84106

107+
def _required_storage_nbytes(x: torch.Tensor) -> int:
108+
"""Return the backing storage required by ``x``'s logical view."""
109+
if x.numel() == 0:
110+
return 0
111+
last_element = x.storage_offset()
112+
for size, stride in zip(x.size(), x.stride()):
113+
last_element += (size - 1) * stride
114+
return int(last_element + 1) * x.element_size()
115+
116+
117+
@contextlib.contextmanager
118+
def _rehydrate_emptied_tensors(tensors):
119+
"""Temporarily restore zero storage while preserving tensor aliases.
120+
121+
Low-memory CUDA export keeps KV tensors' sizes and strides but releases
122+
their backing storage. Inductor autotuning needs those original tensor
123+
objects throughout cloning, reset-to-zero, and kernel launch. Restoring the
124+
shared storages in place preserves views/aliases that replacing individual
125+
arguments with fresh tensors would break.
126+
"""
127+
emptied = [tensor for tensor in tensors if _is_emptied(tensor)]
128+
storages = {}
129+
for tensor in emptied:
130+
storage = tensor.untyped_storage()
131+
key = storage._cdata
132+
required = _required_storage_nbytes(tensor)
133+
if key not in storages or required > storages[key][1]:
134+
storages[key] = (storage, required)
135+
136+
restored = []
137+
try:
138+
for storage, required in storages.values():
139+
storage.resize_(required)
140+
restored.append(storage)
141+
for tensor in emptied:
142+
tensor.zero_()
143+
yield
144+
finally:
145+
# The autotuner finishes with reset_to_zero_args(), whose CUDA zero_
146+
# launches asynchronously. Releasing the storage before that work has
147+
# completed leaves the kernel writing through a freed pointer; the
148+
# resulting illegal access is then reported by some later CUDA API
149+
# (often preserve_rng_state's set_rng_state). All users of storage that
150+
# is about to be resized away must be complete first.
151+
cuda_devices = {tensor.device for tensor in emptied if tensor.is_cuda}
152+
for device in cuda_devices:
153+
torch.cuda.synchronize(device)
154+
for storage in reversed(restored):
155+
storage.resize_(0)
156+
157+
85158
@contextlib.contextmanager
86159
def _compile_time_cpu_clones(target_device: torch.device):
87160
"""Force AOTI's mutated-buffer clones onto CPU while preserving the
88161
serialized constants' target device."""
89162
from torch._inductor import compile_fx as _cfx, graph as _graph
90163
from torch._inductor.codegen.cpp_wrapper_cpu import CppWrapperCpu as _Cpp
91164
from torch._inductor.graph import GraphLowering as _GL
165+
from torch._inductor.runtime.triton_heuristics import CachingAutotuner as _Autotuner
92166

93167
orig_clone = _cfx.clone_preserve_strides
94168
orig_codegen_device = _Cpp.codegen_device
95169
orig_get_const = _GL.get_original_value_of_constant
96170
orig_is_same = _graph.is_same_tensor
171+
orig_autotuner_run = _Autotuner.run
97172

98173
def _is_same_skip_emptied(data, value):
99174
# KV buffers freed via resize_(0) all have data_ptr 0, so the stock
@@ -124,6 +199,20 @@ def _cpu_clone_preserve_strides(x: torch.Tensor) -> torch.Tensor:
124199
return orig_clone(x).cpu()
125200
return orig_clone(x)
126201

202+
def _autotuner_run_with_rehydrated_emptied_args(self, *args, **kwargs):
203+
# CachingAutotuner.run first benchmarks configurations (where cloning and
204+
# reset_to_zero_args touch the inputs), then launches the winning kernel
205+
# once more on the original arguments. Keep their storage valid through
206+
# both phases and the final launch; wrapping benchmark_all_configs alone
207+
# would release it too early for that last call.
208+
tensors = (
209+
value
210+
for value in (*args, *kwargs.values())
211+
if isinstance(value, torch.Tensor)
212+
)
213+
with _rehydrate_emptied_tensors(tensors):
214+
return orig_autotuner_run(self, *args, **kwargs)
215+
127216
def _get_const_synthesize_zeros(self, name):
128217
# AOTI serializes each constant via get_original_value_of_constant ->
129218
# _to_bytes. For KV buffers we freed with resize_(0) this would otherwise
@@ -152,6 +241,7 @@ def _codegen_device_target_aware(self, device):
152241
_Cpp.codegen_device = _codegen_device_target_aware
153242
_GL.get_original_value_of_constant = _get_const_synthesize_zeros
154243
_graph.is_same_tensor = _is_same_skip_emptied
244+
_Autotuner.run = _autotuner_run_with_rehydrated_emptied_args
155245
prev_active = getattr(_CPU_CLONE_GUARD, "active", False)
156246
_CPU_CLONE_GUARD.active = True
157247
try:
@@ -162,6 +252,7 @@ def _codegen_device_target_aware(self, device):
162252
_Cpp.codegen_device = orig_codegen_device
163253
_GL.get_original_value_of_constant = orig_get_const
164254
_graph.is_same_tensor = orig_is_same
255+
_Autotuner.run = orig_autotuner_run
165256

166257

167258
def _is_kv_buffer(name, v) -> bool:
@@ -601,6 +692,7 @@ def _combined():
601692
# `low_memory_mode="ON"` compile spec, since the
602693
# monkey-patch can interact poorly with other models'
603694
# AOTI compile pipelines.
695+
stack.enter_context(_keep_triton_reduction_loads_loop_scoped())
604696
stack.enter_context(
605697
_compile_time_cpu_clones(torch.device(cls.get_device_name()))
606698
)

‎backends/cuda/tests/test_cuda_export.py‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
import unittest
88
from typing import Tuple
9+
from unittest import mock
910

1011
import torch
1112
from executorch.backends.cuda.cuda_backend import CudaBackend
@@ -17,6 +18,50 @@
1718

1819

1920
class TestCudaBackendCompileOptions(unittest.TestCase):
21+
def test_low_memory_triton_reduction_loads_stay_loop_scoped(self):
22+
from executorch.backends.cuda.cuda_backend import (
23+
_keep_triton_reduction_loads_loop_scoped,
24+
)
25+
from torch._inductor.codegen.triton import IndexingOptions
26+
27+
original_has_rmask = IndexingOptions.has_rmask
28+
indexing = object.__new__(IndexingOptions)
29+
with _keep_triton_reduction_loads_loop_scoped():
30+
self.assertTrue(indexing.has_rmask())
31+
self.assertIs(IndexingOptions.has_rmask, original_has_rmask)
32+
33+
def test_low_memory_autotune_rehydrates_aliased_storage_for_full_run(self):
34+
from executorch.backends.cuda.cuda_backend import _compile_time_cpu_clones
35+
from torch._inductor.runtime.triton_heuristics import CachingAutotuner
36+
37+
base = torch.empty_strided((4, 4), (4, 1))
38+
view = base[:, 1:]
39+
base.untyped_storage().resize_(0)
40+
original_run = CachingAutotuner.run
41+
test_case = self
42+
43+
def fake_run(_autotuner, base_arg, view_arg, *, stream):
44+
test_case.assertGreater(base_arg.untyped_storage().nbytes(), 0)
45+
test_case.assertEqual(
46+
base_arg.untyped_storage()._cdata,
47+
view_arg.untyped_storage()._cdata,
48+
)
49+
test_case.assertEqual(view_arg.storage_offset(), 1)
50+
test_case.assertEqual(torch.count_nonzero(base_arg), 0)
51+
test_case.assertEqual(stream, 123)
52+
return "run-result"
53+
54+
CachingAutotuner.run = fake_run
55+
try:
56+
autotuner = object.__new__(CachingAutotuner)
57+
with _compile_time_cpu_clones(torch.device("cuda")):
58+
result = autotuner.run(base, view, stream=123)
59+
finally:
60+
CachingAutotuner.run = original_run
61+
62+
self.assertEqual(result, "run-result")
63+
self.assertEqual(base.untyped_storage().nbytes(), 0)
64+
2065
def test_emulate_precision_casts_compile_spec(self):
2166
options = CudaBackend.get_aoti_compile_options(
2267
[CompileSpec(key="emulate_precision_casts", value=b"OFF")]
@@ -66,6 +111,21 @@ def setUp(self):
66111
if not torch.cuda.is_available():
67112
self.skipTest("CUDA is not available")
68113

114+
def test_rehydrated_storage_is_synchronized_before_release(self):
115+
from executorch.backends.cuda.cuda_backend import _rehydrate_emptied_tensors
116+
117+
tensor = torch.empty(16, device="cuda")
118+
tensor.untyped_storage().resize_(0)
119+
120+
with mock.patch.object(
121+
torch.cuda, "synchronize", wraps=torch.cuda.synchronize
122+
) as synchronize:
123+
with _rehydrate_emptied_tensors([tensor]):
124+
tensor.zero_()
125+
126+
synchronize.assert_called_once_with(tensor.device)
127+
self.assertEqual(tensor.untyped_storage().nbytes(), 0)
128+
69129
def _export_to_cuda_with_lower(
70130
self,
71131
module: torch.nn.Module,

‎examples/models/qwen3_5_moe/export.py‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1138,8 +1138,9 @@ def _export_cuda(model, config, args):
11381138
"enable_dynamic_shape": True,
11391139
"get_mutable_buffer_metadata": _mutable_buffer_metadata_json(model),
11401140
}
1141-
# Avoid a PyTorch 2.13 Triton codegen bug in Qwen's fused cast/reduction
1142-
# compile-time autotune path while keeping the CUDA backend defaults unchanged.
1141+
# Avoid PyTorch 2.13's fused cast/reduction compile-time autotune path. Keep
1142+
# max autotuning enabled so prefill GEMMs use the CUDA backend's Triton-only
1143+
# GEMM lowering instead of emitting libtorch fallback kernels.
11431144
et_prog = to_edge_transform_and_lower(
11441145
{"decode": decode_ep, "prefill": prefill_ep},
11451146
partitioner={
@@ -1149,7 +1150,7 @@ def _export_cuda(model, config, args):
11491150
CudaBackend.generate_method_name_compile_spec("decode"),
11501151
CompileSpec("low_memory_mode", b"ON"),
11511152
CompileSpec("emulate_precision_casts", b"OFF"),
1152-
CompileSpec("max_autotune", b"OFF"),
1153+
CompileSpec("max_autotune", b"ON"),
11531154
CompileSpec("autotune_at_compile_time", b"OFF"),
11541155
]
11551156
)
@@ -1160,7 +1161,7 @@ def _export_cuda(model, config, args):
11601161
CudaBackend.generate_method_name_compile_spec("prefill"),
11611162
CompileSpec("low_memory_mode", b"ON"),
11621163
CompileSpec("emulate_precision_casts", b"OFF"),
1163-
CompileSpec("max_autotune", b"OFF"),
1164+
CompileSpec("max_autotune", b"ON"),
11641165
CompileSpec("autotune_at_compile_time", b"OFF"),
11651166
]
11661167
)

0 commit comments

Comments
 (0)