Skip to content

Commit 4199643

Browse files
MLX: release a temp slot once its chain is done with it (pytorch#23109)
`ExecutionState::tensors` is never pruned, so a chain ends holding every intermediate it produced. Whether that costs anything depends on the AOT memory plan: whisper's reuses a handful of slots, but a gemma4-e2b prefill plan assigns roughly one slot per instruction, so its 3243 instructions finish with 1918 live tensors whose sizes scale with sequence length. Measured on an iPhone 16, gemma4-e2b int4, 2392-character prompt: | | live slots | liveMB | peak phys_footprint | outcome | | --- | --- | --- | --- | --- | | stock | 1918 | 3102.9 MB | 3401 MB | killed by jetsam in prefill | | this PR | 3 | 1.2 MB | 1379 MB | completes normally | A 52-character prompt completes on both. Reported downstream as software-mansion/react-native-executorch#1489. Only temps are eligible: ids run `Constant -> Input -> Output -> MutableBuffer -> Temp`, so `>= mutable_buffer_end` is neither a method output nor a caller-visible buffer. A chain containing SCAN or IF opts out, since `for_each_tid` does not descend into nested chains. Independent of pytorch#22932's eval threshold: with the threshold off the intermediates stay lazy and the growth reappears in the final `async_eval`; with it on they are materialized and retained. This release is what makes the bounded path bounded. cc @nil-is-all @metascroy --------- Co-authored-by: Scott Roy <161522778+metascroy@users.noreply.github.com>
1 parent fa61c2b commit 4199643

6 files changed

Lines changed: 446 additions & 32 deletions

File tree

‎backends/mlx/runtime/MLXExecutor.h‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,13 @@ struct ExecutionState {
176176
uint32_t output_end{0};
177177
uint32_t mutable_buffer_end{0};
178178

179+
// Per chain, the index of the last instruction naming each temp slot, or
180+
// kNoTempLastUse. An empty table opts that chain out of release. Computed
181+
// once here rather than per run_chain, which would cost a walk of every
182+
// chain on each execute() and each scan iteration.
183+
static constexpr uint32_t kNoTempLastUse = 0xFFFFFFFFu;
184+
std::vector<std::vector<uint32_t>> temp_last_use;
185+
179186
void bind(
180187
const MLXProgram& prog,
181188
const ConstantData& const_data,
@@ -215,6 +222,66 @@ struct ExecutionState {
215222
input_end = static_cast<uint32_t>(ie);
216223
output_end = static_cast<uint32_t>(oe);
217224
mutable_buffer_end = static_cast<uint32_t>(me);
225+
226+
compute_temp_last_use(prog);
227+
}
228+
229+
// A temp any OTHER chain names is never dropped: a branch or scan body is run
230+
// through run_chain of its own, so a temp it writes for its caller, or a
231+
// carry it only reads, would otherwise die at its last use inside the body.
232+
// A chain holding a SCAN or IF opts out entirely, since for_each_tid does not
233+
// walk nested chains.
234+
void compute_temp_last_use(const MLXProgram& prog) {
235+
const size_t num_chains = prog.instruction_chains.size();
236+
temp_last_use.assign(num_chains, {});
237+
238+
// The one chain naming each temp slot, or kNoOwner / kShared.
239+
constexpr uint32_t kNoOwner = 0xFFFFFFFFu;
240+
constexpr uint32_t kShared = 0xFFFFFFFEu;
241+
std::vector<uint32_t> owner(tensors.size(), kNoOwner);
242+
for (size_t c = 0; c < num_chains; ++c) {
243+
for (const auto& instr : prog.instruction_chains[c]) {
244+
for_each_tid(instr, [&](Tid id) {
245+
if (id.idx >= mutable_buffer_end) {
246+
uint32_t& o = owner[tensor_index(id)];
247+
if (o == kNoOwner) {
248+
o = static_cast<uint32_t>(c);
249+
} else if (o != c) {
250+
o = kShared;
251+
}
252+
}
253+
});
254+
}
255+
}
256+
257+
for (size_t c = 0; c < num_chains; ++c) {
258+
const auto& chain = prog.instruction_chains[c];
259+
bool nested = false;
260+
for (const auto& instr : chain) {
261+
if (instr.op == OpCode::SCAN || instr.op == OpCode::IF) {
262+
nested = true;
263+
break;
264+
}
265+
}
266+
if (nested) {
267+
continue;
268+
}
269+
270+
std::vector<uint32_t> last_use(tensors.size(), kNoTempLastUse);
271+
uint32_t idx = 0;
272+
for (const auto& instr : chain) {
273+
for_each_tid(instr, [&](Tid id) {
274+
if (id.idx >= mutable_buffer_end) {
275+
const uint32_t slot = tensor_index(id);
276+
if (owner[slot] == c) {
277+
last_use[slot] = idx;
278+
}
279+
}
280+
});
281+
++idx;
282+
}
283+
temp_last_use[c] = std::move(last_use);
284+
}
218285
}
219286

220287
// Check if a tensor ID is a mutable buffer

‎backends/mlx/runtime/MLXInterpreter.h‎

Lines changed: 48 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2021,6 +2021,17 @@ class Interpreter {
20212021
}
20222022
const auto& chain = prog.instruction_chains[chain_idx];
20232023
const size_t threshold = eval_threshold_bytes_;
2024+
2025+
// Drop a temp as soon as this chain is done with it: ExecutionState holds
2026+
// every value a chain produces, and an AOT plan that assigns one slot per
2027+
// instruction leaves a long chain holding all of them. The table is built
2028+
// once at load; an empty one means this chain opts out.
2029+
const std::vector<uint32_t>* last_use = nullptr;
2030+
if (chain_idx < st.temp_last_use.size() &&
2031+
!st.temp_last_use[chain_idx].empty()) {
2032+
last_use = &st.temp_last_use[chain_idx];
2033+
}
2034+
20242035
size_t idx = 0;
20252036
for (const auto& instr : chain) {
20262037
st.begin_op(idx, op_name(instr.op));
@@ -2033,24 +2044,50 @@ class Interpreter {
20332044
dispatch(instr, st, stream);
20342045
}
20352046
st.end_op();
2047+
2048+
// Account before releasing: a shrinking op's widest tensor is often an
2049+
// input it was the last to read. SCAN and IF already accumulated their
2050+
// own child instructions through the shared counter; charging the parent
2051+
// for them again would double count.
2052+
if (threshold != 0 && instr.op != OpCode::SCAN &&
2053+
instr.op != OpCode::IF) {
2054+
accumulate_instruction_bytes(instr, st, pending_bytes);
2055+
}
2056+
2057+
// Release before evaluating, so the barrier skips dead temps.
2058+
if (last_use != nullptr) {
2059+
release_temp_slots(instr, st, *last_use, static_cast<uint32_t>(idx));
2060+
}
2061+
20362062
++idx;
20372063

2038-
if (threshold != 0) {
2039-
// SCAN and IF already accumulated their own child instructions through
2040-
// the shared counter; charging the parent for them again would double
2041-
// count.
2042-
if (instr.op != OpCode::SCAN && instr.op != OpCode::IF) {
2043-
accumulate_instruction_bytes(instr, st, pending_bytes);
2044-
}
2045-
if (!accumulate_only && pending_bytes >= threshold) {
2046-
evaluate_state_tensors(st);
2047-
pending_bytes = 0;
2048-
}
2064+
if (threshold != 0 && !accumulate_only && pending_bytes >= threshold) {
2065+
evaluate_state_tensors(st);
2066+
pending_bytes = 0;
20492067
}
20502068
}
20512069
}
20522070

20532071
private:
2072+
// Drop every temp whose last use in this chain is the instruction just run.
2073+
// Ids run Constant -> Input -> Output -> MutableBuffer -> Temp, so
2074+
// `>= mutable_buffer_end` is neither a method output nor a caller-visible
2075+
// buffer and dropping one cannot be observed from outside the chain.
2076+
static void release_temp_slots(
2077+
const Instruction& instr,
2078+
ExecutionState& st,
2079+
const std::vector<uint32_t>& last_use,
2080+
uint32_t idx) {
2081+
for_each_tid(instr, [&](Tid id) {
2082+
if (id.idx >= st.mutable_buffer_end) {
2083+
const uint32_t slot = st.tensor_index(id);
2084+
if (last_use[slot] == idx) {
2085+
st.tensors[slot].reset();
2086+
}
2087+
}
2088+
});
2089+
}
2090+
20542091
// Charge `pending_bytes` for one instruction. The estimate is the largest
20552092
// per-execution tensor the instruction touches, which tracks the size of what
20562093
// it just produced without needing to know which tid is its output.

‎backends/mlx/serialization/generate.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1023,6 +1023,8 @@ def generate_cpp_loader_h(schema: FBSSchema) -> str:
10231023

10241024
# for_each_tid emitters: invoke cb(Tid) for each Tid-bearing field. Field kinds
10251025
# that carry no Tid (vid, int_or_vid, scalars, strings, int lists) emit nothing.
1026+
# A new Tid-bearing kind MUST be handled here: compute_temp_last_use decides
1027+
# when a slot is freed from this walk, so a missed field frees a live tensor.
10261028
def _emit_cpp_for_each_tid(kind: str, name: str) -> List[str]:
10271029
"""Emit for_each_tid callback lines for a field kind ([] if it has no Tid)."""
10281030
if kind == "tid":

‎backends/mlx/test/CMakeLists.txt‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,30 @@ if(EXECUTORCH_MLX_ENABLE_SANITIZERS)
139139
)
140140
endif()
141141

142+
# Temp-slot release. Drives the interpreter over hand-built programs; no
143+
# model/tokenizer needed.
144+
et_cxx_test(
145+
mlx_temp_release_test
146+
SOURCES
147+
${CMAKE_CURRENT_LIST_DIR}/mlx_temp_release_test.cpp
148+
EXTRA_LIBS
149+
mlxdelegate
150+
mlx_schema
151+
mlx
152+
)
153+
target_include_directories(
154+
mlx_temp_release_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../runtime
155+
)
156+
if(EXECUTORCH_MLX_ENABLE_SANITIZERS)
157+
target_compile_options(
158+
mlx_temp_release_test PRIVATE -fsanitize=address,undefined
159+
-fno-omit-frame-pointer
160+
)
161+
target_link_options(
162+
mlx_temp_release_test PRIVATE ${_mlx_sanitizer_link_options}
163+
)
164+
endif()
165+
142166
# Off-graph KV cache op-level test (no model/tokenizer needed). et_cxx_test
143167
# links GTest + executorch_core and registers the ctest target.
144168
et_cxx_test(

‎backends/mlx/test/mlx_eval_threshold_test.cpp‎

Lines changed: 39 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,10 @@
1414
// double-counted, that crossing the threshold actually materializes the live
1515
// tensors mid-chain, and that results are unchanged by any of it.
1616
//
17+
// A mid-chain value is observed through a second OUTPUT (`kProbe`) rather than
18+
// through a temp, because the interpreter drops a temp slot as soon as the
19+
// chain has no further use for it. Outputs are caller-visible and are kept.
20+
//
1721
// Must run on Apple Silicon: MLX needs the Metal backend.
1822

1923
#include "MLXInterpreter.h"
@@ -31,7 +35,12 @@ namespace {
3135

3236
constexpr uint32_t kIn = 0; // input tid
3337
constexpr uint32_t kOut = 1; // output tid
34-
constexpr uint32_t kTemp0 = 2; // first temp tid
38+
constexpr uint32_t kProbe = 2; // second output tid, used to observe a
39+
// mid-chain value after the run
40+
constexpr uint32_t kTemp0 = 3; // first temp tid
41+
42+
// "no instruction writes to kProbe"
43+
constexpr uint32_t kNoProbe = std::numeric_limits<uint32_t>::max();
3544

3645
Instruction make_add(uint32_t a, uint32_t b, uint32_t out) {
3746
Instruction instr;
@@ -44,26 +53,37 @@ Instruction make_add(uint32_t a, uint32_t b, uint32_t out) {
4453
return instr;
4554
}
4655

47-
// A chain of `n` ADDs: in -> temp0 -> temp1 -> ... -> out.
48-
std::vector<Instruction> make_add_chain(uint32_t n) {
56+
// A chain of `n` ADDs: in -> temp0 -> temp1 -> ... -> out. Instruction
57+
// `probe_at` writes to kProbe instead of a temp, so its result outlives the
58+
// chain and a test can inspect it.
59+
std::vector<Instruction> make_add_chain(
60+
uint32_t n,
61+
uint32_t probe_at = kNoProbe) {
4962
std::vector<Instruction> chain;
5063
uint32_t prev = kIn;
5164
for (uint32_t i = 0; i < n; ++i) {
5265
const bool last = (i + 1 == n);
53-
const uint32_t out = last ? kOut : (kTemp0 + i);
66+
uint32_t out;
67+
if (last) {
68+
out = kOut;
69+
} else if (i == probe_at) {
70+
out = kProbe;
71+
} else {
72+
out = kTemp0 + i;
73+
}
5474
chain.push_back(make_add(prev, prev, out));
5575
prev = out;
5676
}
5777
return chain;
5878
}
5979

6080
// Program whose main chain is `n` ADDs and nothing else.
61-
MLXProgram make_flat_program(uint32_t n) {
81+
MLXProgram make_flat_program(uint32_t n, uint32_t probe_at = kNoProbe) {
6282
MLXProgram program;
6383
program.num_input_tensors = 1;
64-
program.num_output_tensors = 1;
84+
program.num_output_tensors = 2; // kOut and kProbe
6585
program.num_temp_tensors = n; // generous; unused slots stay nullopt
66-
program.instruction_chains.push_back(make_add_chain(n));
86+
program.instruction_chains.push_back(make_add_chain(n, probe_at));
6787
program.main_chain_idx = 0;
6888
return program;
6989
}
@@ -74,7 +94,7 @@ MLXProgram make_flat_program(uint32_t n) {
7494
MLXProgram make_if_program(uint32_t n) {
7595
MLXProgram program;
7696
program.num_input_tensors = 1;
77-
program.num_output_tensors = 1;
97+
program.num_output_tensors = 2; // kOut and kProbe
7898
program.num_temp_tensors = n;
7999

80100
Instruction if_instr;
@@ -140,7 +160,7 @@ TEST(MLXEvalThreshold, DisabledDoesNoAccountingAtAll) {
140160
// accounted, and nothing is evaluated early.
141161
TEST(MLXEvalThreshold, EnabledAccountsEveryInstruction) {
142162
const uint32_t kN = 8;
143-
MLXProgram program = make_flat_program(kN);
163+
MLXProgram program = make_flat_program(kN, /*probe_at=*/3);
144164
ConstantData constants;
145165
MutableBufferData bufs;
146166
ExecutionState st;
@@ -153,10 +173,9 @@ TEST(MLXEvalThreshold, EnabledAccountsEveryInstruction) {
153173
Interpreter::reset_accounting_calls();
154174
interp.run(program, st);
155175
EXPECT_EQ(Interpreter::accounting_calls(), kN);
156-
for (uint32_t i = 0; i < kN - 1; ++i) {
157-
const array& temp = st.tensors[st.tensor_index(Tid{kTemp0 + i})].value();
158-
EXPECT_FALSE(temp.is_available()) << "temp=" << i;
159-
}
176+
// Nothing was evaluated: neither the mid-chain value nor the output.
177+
const array& probe = st.tensors[st.tensor_index(Tid{kProbe})].value();
178+
EXPECT_FALSE(probe.is_available());
160179
const array& out = st.tensors[st.tensor_index(Tid{kOut})].value();
161180
EXPECT_FALSE(out.is_available());
162181
}
@@ -185,7 +204,8 @@ TEST(MLXEvalThreshold, NestedIfAccumulatesWithoutDoubleCounting) {
185204
TEST(MLXEvalThreshold, CrossingThresholdEvaluatesLiveTensors) {
186205
const int kFloats = 4096;
187206
const uint32_t kN = 9;
188-
MLXProgram program = make_flat_program(kN);
207+
// Probe the first ADD: any barrier at all must have materialized it.
208+
MLXProgram program = make_flat_program(kN, /*probe_at=*/0);
189209
ConstantData constants;
190210
MutableBufferData bufs;
191211

@@ -196,11 +216,9 @@ TEST(MLXEvalThreshold, CrossingThresholdEvaluatesLiveTensors) {
196216
Interpreter interp;
197217
interp.set_eval_threshold_bytes(2 * input_bytes(kFloats));
198218
interp.run(program, st);
199-
// The eighth ADD triggers a barrier; the ninth stays below the threshold.
200-
for (uint32_t i = 0; i < kN - 1; ++i) {
201-
const array& temp = st.tensors[st.tensor_index(Tid{kTemp0 + i})].value();
202-
EXPECT_TRUE(temp.is_available()) << "temp=" << i;
203-
}
219+
const array& probe = st.tensors[st.tensor_index(Tid{kProbe})].value();
220+
EXPECT_TRUE(probe.is_available());
221+
// The last ADD stays below the threshold, so the output is still lazy.
204222
const array& out = st.tensors[st.tensor_index(Tid{kOut})].value();
205223
EXPECT_FALSE(out.is_available());
206224
}
@@ -211,8 +229,8 @@ TEST(MLXEvalThreshold, CrossingThresholdEvaluatesLiveTensors) {
211229
bind_state(st, program, constants, bufs, kFloats);
212230
Interpreter interp;
213231
interp.run(program, st);
214-
const array& first_temp = st.tensors[st.tensor_index(Tid{kTemp0})].value();
215-
EXPECT_FALSE(first_temp.is_available());
232+
const array& probe = st.tensors[st.tensor_index(Tid{kProbe})].value();
233+
EXPECT_FALSE(probe.is_available());
216234
}
217235
}
218236

0 commit comments

Comments
 (0)