Skip to content

Commit 71a713c

Browse files
authored
[executorch][cuda] Check FQN weight coverage without AOTI's per-buffer warnings (#24052)
Stacked on #24050. Every CUDA method load that binds FQN weights next to an off-graph KV cache printed one AOTI warning line per KV buffer: ``` [WARNING] Found constant or module state buffer or empty module state parameter __et_offgraph_kv_layer_0_k in model, but not provided by user! ``` The `unittest-cuda-muse-glimmer-batching` log contains 1672 of these lines, out of 11650 lines total. **Why it happens.** `CudaWeightCache::load` binds the weights with `validate_full_update=true` and intentionally leaves the off-graph KV buffers unbound, because the KV pool binds them later. AOTI's full-update check (`assert_all_constants`) allows absent buffers, but it writes a warning to stderr for each absent buffer on every load. **Fix.** - When the AOTI library exports `AOTInductorModelContainerGetConstantType` and `AOTInductorModelContainerGetConstantFromFolded`, ExecuTorch runs the same completeness rule itself. Folded constants, buffers, tensor constants, and empty parameters may be absent. Every other constant must be bound, otherwise `load` fails with `InvalidProgram`. - Binding then runs with `validate_full_update=false`. In AOTI that flag only gates `assert_all_constants`, so nothing else about binding changes. - If the library does not export those two APIs, `load` keeps `validate_full_update=true` as before. Test plan: - `g++ -fsyntax-only` on `cuda_weight_cache.cpp` and `cuda_backend.cpp` against the CUDA 12.6 headers. - CI: the CUDA unittest jobs and the muse-glimmer batching job should pass, and their logs should no longer contain the `__et_offgraph_kv_` warnings.
1 parent 08e033f commit 71a713c

3 files changed

Lines changed: 85 additions & 4 deletions

File tree

‎backends/cuda/runtime/cuda_backend.cpp‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -288,6 +288,11 @@ class ET_EXPERIMENTAL CudaBackend final
288288
get_constant_dtype, AOTInductorModelContainerGetConstantDtype);
289289
LOAD_OPTIONAL_SYMBOL(
290290
get_constant_data_size, AOTInductorModelContainerGetConstantDataSize);
291+
LOAD_OPTIONAL_SYMBOL(
292+
get_constant_type, AOTInductorModelContainerGetConstantType);
293+
LOAD_OPTIONAL_SYMBOL(
294+
get_constant_from_folded,
295+
AOTInductorModelContainerGetConstantFromFolded);
291296
LOAD_OPTIONAL_SYMBOL(
292297
extract_constants_map, AOTInductorModelContainerExtractConstantsMap);
293298
LOAD_OPTIONAL_SYMBOL(

‎backends/cuda/runtime/cuda_delegate_handle.h‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,15 @@ using AOTInductorModelContainerGetConstantDataSizeFunc =
5656
aoti::AOTInductorModelContainerHandle container_handle,
5757
size_t idx,
5858
size_t* data_size);
59+
using AOTInductorModelContainerGetConstantTypeFunc = aoti::AOTIRuntimeError (*)(
60+
aoti::AOTInductorModelContainerHandle container_handle,
61+
size_t idx,
62+
int32_t* type);
63+
using AOTInductorModelContainerGetConstantFromFoldedFunc =
64+
aoti::AOTIRuntimeError (*)(
65+
aoti::AOTInductorModelContainerHandle container_handle,
66+
size_t idx,
67+
bool* from_folded);
5968
struct CudaWeightStorage {
6069
void* data{nullptr};
6170
size_t nbytes{0};
@@ -286,6 +295,11 @@ struct CudaDelegateHandle : public aoti::AOTIDelegateHandle {
286295
// storage against it before binding.
287296
AOTInductorModelContainerGetConstantDataSizeFunc get_constant_data_size{
288297
nullptr};
298+
// With these, FQN weight binding checks completeness itself instead of
299+
// AOTI's full update, which warns once per off-graph KV buffer on every load.
300+
AOTInductorModelContainerGetConstantTypeFunc get_constant_type{nullptr};
301+
AOTInductorModelContainerGetConstantFromFoldedFunc get_constant_from_folded{
302+
nullptr};
289303

290304
// The per-thread stream. Nothing owns it: the value is a fixed sentinel the
291305
// driver resolves to a different stream on each host thread, so releasing the

‎backends/cuda/runtime/cuda_weight_cache.cpp‎

Lines changed: 66 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,59 @@ bool is_offgraph_kv_fqn(const std::string& fqn) {
140140
return fqn.rfind(kOffGraphKVFqnPrefix, 0) == 0;
141141
}
142142

143+
// torch::aot_inductor::ConstantType.
144+
constexpr int32_t kAotiParameter = 1;
145+
constexpr int32_t kAotiBuffer = 2;
146+
constexpr int32_t kAotiTensorConstant = 3;
147+
148+
bool can_check_constants_bound(const CudaDelegateHandle& handle) {
149+
return handle.get_constant_type && handle.get_constant_from_folded &&
150+
handle.get_constant_data_size;
151+
}
152+
153+
// The same rule as AOTI's validate_full_update, without its warning for each
154+
// absent buffer: folded constants, buffers, tensor constants and empty
155+
// parameters may be absent, every other constant must be bound.
156+
Error check_constants_bound(
157+
const CudaDelegateHandle& handle,
158+
size_t num_constants,
159+
const std::unordered_set<std::string>& bound) {
160+
for (size_t index = 0; index < num_constants; ++index) {
161+
bool from_folded = false;
162+
int32_t type = 0;
163+
size_t data_size = 0;
164+
ET_CHECK_OK_OR_RETURN_ERROR(
165+
handle.get_constant_from_folded(
166+
handle.container_handle, index, &from_folded),
167+
"Failed to read CUDA AOTI constant folding at index %zu",
168+
index);
169+
ET_CHECK_OK_OR_RETURN_ERROR(
170+
handle.get_constant_type(handle.container_handle, index, &type),
171+
"Failed to read CUDA AOTI constant type at index %zu",
172+
index);
173+
ET_CHECK_OK_OR_RETURN_ERROR(
174+
handle.get_constant_data_size(
175+
handle.container_handle, index, &data_size),
176+
"Failed to read CUDA AOTI constant size at index %zu",
177+
index);
178+
if (from_folded || type == kAotiBuffer || type == kAotiTensorConstant ||
179+
(type == kAotiParameter && data_size == 0)) {
180+
continue;
181+
}
182+
const char* name = nullptr;
183+
ET_CHECK_OK_OR_RETURN_ERROR(
184+
handle.get_constant_name(handle.container_handle, index, &name),
185+
"Failed to read CUDA AOTI constant name at index %zu",
186+
index);
187+
ET_CHECK_OR_RETURN_ERROR(
188+
name != nullptr && bound.count(name) != 0,
189+
InvalidProgram,
190+
"CUDA AOTI constant '%s' has no bound FQN weight",
191+
name == nullptr ? "<unnamed>" : name);
192+
}
193+
return Error::Ok;
194+
}
195+
143196
} // namespace
144197

145198
bool CudaWeightCache::is_serialized(const void* data, size_t size) {
@@ -650,16 +703,25 @@ Error CudaWeightCache::load(
650703
}
651704

652705
if (!pairs.empty()) {
653-
// Full validation stays on with off-graph KV present: those constants are
654-
// AOTI buffers, which the check permits to be absent, while every weight
655-
// must still be supplied.
706+
// Off-graph KV constants are AOTI buffers bound later by the KV pool, so
707+
// they are allowed to be absent here, while every weight must be supplied.
708+
const bool check_here = can_check_constants_bound(*handle);
709+
if (check_here) {
710+
std::unordered_set<std::string> bound_names;
711+
for (const auto& pair : pairs) {
712+
bound_names.emplace(pair.name);
713+
}
714+
ET_CHECK_OK_OR_RETURN_ERROR(
715+
check_constants_bound(*handle, num_constants, bound_names),
716+
"CUDA FQN weights do not cover the AOTI library");
717+
}
656718
ET_CHECK_OK_OR_RETURN_ERROR(
657719
handle->update_user_managed_constant_buffer_pairs(
658720
handle->container_handle,
659721
pairs.data(),
660722
pairs.size(),
661723
/*use_inactive=*/false,
662-
/*validate_full_update=*/true),
724+
/*validate_full_update=*/!check_here),
663725
"Failed to bind CUDA FQN weights");
664726
}
665727
ET_LOG(

0 commit comments

Comments
 (0)