@@ -60,6 +60,28 @@ def _is_cpu_clone_active() -> bool:
6060 return getattr (_CPU_CLONE_GUARD , "active" , False )
6161
6262
63+ @contextlib .contextmanager
64+ def _keep_triton_reduction_loads_loop_scoped ():
65+ """Conservatively treating loads as reduction-masked keeps each definition in its
66+ own loop to bypass the undefined variable bug in pytorch/pytorch.
67+ (https://github.com/pytorch/pytorch/issues/193988)
68+
69+ TODO(gasoonjia): remove this setting once bug fixed in upstream
70+ """
71+ from torch ._inductor .codegen .triton import IndexingOptions
72+
73+ orig_has_rmask = IndexingOptions .has_rmask
74+
75+ def _has_rmask (_self ) -> bool :
76+ return True
77+
78+ IndexingOptions .has_rmask = _has_rmask
79+ try :
80+ yield
81+ finally :
82+ IndexingOptions .has_rmask = orig_has_rmask
83+
84+
6385def _full_zeros_preserving_strides (x : torch .Tensor , device ) -> torch .Tensor :
6486 """Allocate a zero-filled tensor matching ``x``'s size/stride/dtype on ``device``.
6587
@@ -82,18 +104,71 @@ def _is_emptied(x) -> bool:
82104 )
83105
84106
107+ def _required_storage_nbytes (x : torch .Tensor ) -> int :
108+ """Return the backing storage required by ``x``'s logical view."""
109+ if x .numel () == 0 :
110+ return 0
111+ last_element = x .storage_offset ()
112+ for size , stride in zip (x .size (), x .stride ()):
113+ last_element += (size - 1 ) * stride
114+ return int (last_element + 1 ) * x .element_size ()
115+
116+
117+ @contextlib .contextmanager
118+ def _rehydrate_emptied_tensors (tensors ):
119+ """Temporarily restore zero storage while preserving tensor aliases.
120+
121+ Low-memory CUDA export keeps KV tensors' sizes and strides but releases
122+ their backing storage. Inductor autotuning needs those original tensor
123+ objects throughout cloning, reset-to-zero, and kernel launch. Restoring the
124+ shared storages in place preserves views/aliases that replacing individual
125+ arguments with fresh tensors would break.
126+ """
127+ emptied = [tensor for tensor in tensors if _is_emptied (tensor )]
128+ storages = {}
129+ for tensor in emptied :
130+ storage = tensor .untyped_storage ()
131+ key = storage ._cdata
132+ required = _required_storage_nbytes (tensor )
133+ if key not in storages or required > storages [key ][1 ]:
134+ storages [key ] = (storage , required )
135+
136+ restored = []
137+ try :
138+ for storage , required in storages .values ():
139+ storage .resize_ (required )
140+ restored .append (storage )
141+ for tensor in emptied :
142+ tensor .zero_ ()
143+ yield
144+ finally :
145+ # The autotuner finishes with reset_to_zero_args(), whose CUDA zero_
146+ # launches asynchronously. Releasing the storage before that work has
147+ # completed leaves the kernel writing through a freed pointer; the
148+ # resulting illegal access is then reported by some later CUDA API
149+ # (often preserve_rng_state's set_rng_state). All users of storage that
150+ # is about to be resized away must be complete first.
151+ cuda_devices = {tensor .device for tensor in emptied if tensor .is_cuda }
152+ for device in cuda_devices :
153+ torch .cuda .synchronize (device )
154+ for storage in reversed (restored ):
155+ storage .resize_ (0 )
156+
157+
85158@contextlib .contextmanager
86159def _compile_time_cpu_clones (target_device : torch .device ):
87160 """Force AOTI's mutated-buffer clones onto CPU while preserving the
88161 serialized constants' target device."""
89162 from torch ._inductor import compile_fx as _cfx , graph as _graph
90163 from torch ._inductor .codegen .cpp_wrapper_cpu import CppWrapperCpu as _Cpp
91164 from torch ._inductor .graph import GraphLowering as _GL
165+ from torch ._inductor .runtime .triton_heuristics import CachingAutotuner as _Autotuner
92166
93167 orig_clone = _cfx .clone_preserve_strides
94168 orig_codegen_device = _Cpp .codegen_device
95169 orig_get_const = _GL .get_original_value_of_constant
96170 orig_is_same = _graph .is_same_tensor
171+ orig_autotuner_run = _Autotuner .run
97172
98173 def _is_same_skip_emptied (data , value ):
99174 # KV buffers freed via resize_(0) all have data_ptr 0, so the stock
@@ -124,6 +199,20 @@ def _cpu_clone_preserve_strides(x: torch.Tensor) -> torch.Tensor:
124199 return orig_clone (x ).cpu ()
125200 return orig_clone (x )
126201
202+ def _autotuner_run_with_rehydrated_emptied_args (self , * args , ** kwargs ):
203+ # CachingAutotuner.run first benchmarks configurations (where cloning and
204+ # reset_to_zero_args touch the inputs), then launches the winning kernel
205+ # once more on the original arguments. Keep their storage valid through
206+ # both phases and the final launch; wrapping benchmark_all_configs alone
207+ # would release it too early for that last call.
208+ tensors = (
209+ value
210+ for value in (* args , * kwargs .values ())
211+ if isinstance (value , torch .Tensor )
212+ )
213+ with _rehydrate_emptied_tensors (tensors ):
214+ return orig_autotuner_run (self , * args , ** kwargs )
215+
127216 def _get_const_synthesize_zeros (self , name ):
128217 # AOTI serializes each constant via get_original_value_of_constant ->
129218 # _to_bytes. For KV buffers we freed with resize_(0) this would otherwise
@@ -152,6 +241,7 @@ def _codegen_device_target_aware(self, device):
152241 _Cpp .codegen_device = _codegen_device_target_aware
153242 _GL .get_original_value_of_constant = _get_const_synthesize_zeros
154243 _graph .is_same_tensor = _is_same_skip_emptied
244+ _Autotuner .run = _autotuner_run_with_rehydrated_emptied_args
155245 prev_active = getattr (_CPU_CLONE_GUARD , "active" , False )
156246 _CPU_CLONE_GUARD .active = True
157247 try :
@@ -162,6 +252,7 @@ def _codegen_device_target_aware(self, device):
162252 _Cpp .codegen_device = orig_codegen_device
163253 _GL .get_original_value_of_constant = orig_get_const
164254 _graph .is_same_tensor = orig_is_same
255+ _Autotuner .run = orig_autotuner_run
165256
166257
167258def _is_kv_buffer (name , v ) -> bool :
@@ -601,6 +692,7 @@ def _combined():
601692 # `low_memory_mode="ON"` compile spec, since the
602693 # monkey-patch can interact poorly with other models'
603694 # AOTI compile pipelines.
695+ stack .enter_context (_keep_triton_reduction_loads_loop_scoped ())
604696 stack .enter_context (
605697 _compile_time_cpu_clones (torch .device (cls .get_device_name ()))
606698 )
0 commit comments