Skip to content

Commit ee45eb8

Browse files
Add off-graph KV-cache ring policy (pytorch#21447)
Summary Adds the sliding-window (ring) layout policy. RingPolicy keeps a layer's last window tokens in a fixed ring of window + max_write − 1 slots — oversized so a multi-token prefill step fits without clobbering context earlier queries in the step still need — and evicts the oldest. SequenceCache dispatches windowed layers to it, so a mixed model (gemma4's alternating full/sliding-window layers) shares one logical length. Backend- and tensor-free: it emits integer write/read runs; the per-query window mask stays in the backend. Files - extension/llm/cache/cache.h — LayerPolicy::Kind::Ring; max_write on CacheConfig (max tokens/step; default 1 = decode-only ring). - extension/llm/cache/sequence_cache.h — split_ring_runs (wrap into ≤2 runs), RingPolicy over the oversized ring (retained_from still window-based), the make_policy ring branch, and a ring-specific T ≤ max_write guard. - extension/llm/cache/test/cache_test.cpp — ring wrap/evict, mixed flat/ring shared length, rewind bounded by the window, and step-over-max_write rejected. Testing C++ unit tests for the ring path (wrap, eviction, mixed flat/ring, window-bounded rewind, max_write guard). Built and ran via the standard flow, all pass: cmake -B cmake-out -DEXECUTORCH_BUILD_EXTENSION_LLM=ON -DEXECUTORCH_BUILD_TESTS=ON cmake --build cmake-out --target extension_llm_cache_test -j ctest --test-dir cmake-out -R extension_llm_cache --output-on-failure
1 parent 42a178f commit ee45eb8

3 files changed

Lines changed: 159 additions & 24 deletions

File tree

‎extension/llm/cache/cache.h‎

Lines changed: 23 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,10 @@
1313
// as_control()/as_planner() (static upcasts -- no dynamic_cast/RTTI, no
1414
// diamond): a runner-facing control face (SequenceControl) and a backend-facing
1515
// planner face (SequencePlanner). One controller (SequenceCache) drives all
16-
// layers and dispatches per-layer layout to a LayoutPolicy (full-history flat
17-
// today), so a multi-layer model shares one logical length. Errors are plain
18-
// C++ (bool / std::optional); cache_et.h adapts them to Error/Result for ET
16+
// layers and dispatches per-layer layout to a LayoutPolicy (flat = full
17+
// history, ring = sliding window), so a mixed model (e.g. gemma4's alternating
18+
// full/sliding-window layers) shares one logical length. Errors are plain C++
19+
// (bool / std::optional); cache_et.h adapts them to Error/Result for ET
1920
// consumers.
2021

2122
#include <optional>
@@ -58,10 +59,10 @@ struct Run {
5859
};
5960

6061
// Integer-only handoff from the planner to the backend byte layer. Runs are in
61-
// logical order (oldest -> newest); a flat layer uses one run, a windowing
62-
// layer up to two (a write/read that wraps its buffer splits in two).
63-
// read_base_pos is the logical position of read[0].start (0 for flat), so the
64-
// backend can align RoPE / the attention mask.
62+
// logical order (oldest -> newest); a flat layer uses one run, a ring layer up
63+
// to two (a write/read that wraps the buffer splits in two). read_base_pos is
64+
// the logical position of read[0].start (0 for flat; the window start for
65+
// ring), so the backend can align RoPE / the attention mask.
6566
struct SeqStepPlan {
6667
Run write[2];
6768
int n_write;
@@ -84,26 +85,27 @@ class SequencePlanner {
8485
virtual void commit(const SeqStepPlan& plan) = 0;
8586
};
8687

87-
// Per-layer layout behavior (e.g. full-history flat). Pure: plan() has no side
88-
// effects, so the controller (SequenceCache) owns length.
88+
// Per-layer layout behavior (flat = full history; ring = sliding window). Pure:
89+
// plan() has no side effects, so the controller (SequenceCache) owns length.
8990
class LayoutPolicy {
9091
public:
9192
virtual ~LayoutPolicy() = default;
9293
// Write/read runs for T cells at logical `position`. Precondition: T fits the
9394
// policy's window (the runner chunks prefill so a step fits).
9495
virtual SeqStepPlan plan(int position, int T) const = 0;
95-
// Oldest logical position still retained given the current length (0 for a
96-
// full-history policy; a windowing policy retains only its last window). Used
97-
// to bound rewind.
96+
// Oldest logical position still retained given the current length (0 for
97+
// flat; length - window for ring). Used to bound rewind.
9898
virtual int retained_from(int length) const = 0;
9999
};
100100

101101
// Per-layer cache kind and its parameters.
102102
struct LayerPolicy {
103-
enum class Kind : int { Flat = 0 }; // serialized values: append-only
103+
enum class Kind : int {
104+
Flat = 0,
105+
Ring = 1
106+
}; // serialized values: append-only
104107
Kind kind = Kind::Flat;
105-
int window =
106-
0; // sliding-window size for windowing policies; 0 = full history
108+
int window = 0; // Ring: sliding-window size; must be 0 when Flat
107109
};
108110

109111
// Per-layer architecture facts + cache policy.
@@ -114,13 +116,17 @@ struct LayerConfig {
114116
};
115117

116118
// Model facts + runtime policy the byte layer sizes its pools from. capacity is
117-
// the logical cap; initial_capacity tunes the byte layer's lazy-doubling pool.
118-
// `layers` is per-layer: size 1 == uniform across all layers, else == n_layers.
119+
// the logical cap; initial_capacity tunes the byte layer's lazy-doubling pool;
120+
// max_write is the max tokens written per step (a ring layer sizes its slots to
121+
// window + max_write - 1 so a multi-token step fits); unset means each ring
122+
// layer uses its own window. `layers` is per-layer: size 1 == uniform across
123+
// all layers, else == n_layers.
119124
struct CacheConfig {
120125
int capacity;
121126
int n_layers;
122127
std::vector<LayerConfig> layers;
123128
int initial_capacity = 512;
129+
std::optional<int> max_write;
124130
};
125131

126132
} // namespace cache

‎extension/llm/cache/sequence_cache.h‎

Lines changed: 66 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,10 @@
88

99
#pragma once
1010

11-
// The neutral single-sequence controller (SequenceCache) and its flat layout
12-
// policy (FlatPolicy). SequenceCache owns the one logical length for the whole
13-
// model and dispatches per-layer layout to a policy, so a multi-layer model
14-
// stays coherent. Tensor-free / ET-independent.
11+
// The neutral single-sequence controller (SequenceCache) and its layout
12+
// policies (FlatPolicy / RingPolicy). SequenceCache owns the one logical length
13+
// for the whole model and dispatches per-layer layout to a policy, so a mixed
14+
// flat/ring model (gemma4) stays coherent. Tensor-free / ET-independent.
1515

1616
#include <algorithm>
1717
#include <memory>
@@ -43,14 +43,60 @@ class FlatPolicy final : public LayoutPolicy {
4343
}
4444
};
4545

46+
// Sliding window of `window` cells over a ring of `window + max_write - 1`
47+
// slots. The ring is oversized so a step of up to max_write tokens fits without
48+
// overwriting cells earlier queries in the same step still attend to; the
49+
// backend masks each query to its own window within the read span.
50+
class RingPolicy final : public LayoutPolicy {
51+
public:
52+
RingPolicy(int window, int max_write)
53+
: window_(window), ring_size_(window + max_write - 1) {}
54+
55+
int retained_from(int length) const override {
56+
return length > window_ ? length - window_ : 0;
57+
}
58+
SeqStepPlan plan(int position, int T) const override {
59+
const int end = position + T;
60+
SeqStepPlan p{};
61+
// Write this step's T cells (T <= max_write), wrapping the ring if needed.
62+
p.n_write = split_runs(position, T, p.write);
63+
// Read the union of the step's per-query windows, [position - window + 1,
64+
// end), in logical (oldest -> newest) order.
65+
const int rstart = position - window_ + 1 > 0 ? position - window_ + 1 : 0;
66+
const int rlen = end - rstart;
67+
p.n_read = split_runs(rstart, rlen, p.read);
68+
p.read_base_pos = rstart;
69+
return p;
70+
}
71+
72+
private:
73+
// Split a logical run [start, start+len) into up to two physical runs in the
74+
// ring (wrapping at ring_size_). Precondition: len <= ring_size_.
75+
int split_runs(int start, int len, Run out[2]) const {
76+
const int phys_start = start % ring_size_;
77+
const int first_len = std::min(len, ring_size_ - phys_start);
78+
out[0] = Run{phys_start, first_len};
79+
if (first_len < len) {
80+
out[1] = Run{0, len - first_len};
81+
return 2;
82+
}
83+
return 1;
84+
}
85+
86+
int window_;
87+
int ring_size_;
88+
};
89+
4690
// One controller for all layers: owns the single logical length, admission, and
4791
// rewind; dispatches per-layer layout to a shared LayoutPolicy. Policies are
48-
// deduped by (kind, window), so a uniform model holds a single policy object.
92+
// deduped by (kind, window), so a uniform or two-kind (gemma4) model holds one
93+
// or two policy objects.
4994
class SequenceCache : public CacheBase,
5095
public SequenceControl,
5196
public SequencePlanner {
5297
public:
53-
explicit SequenceCache(const CacheConfig& cfg) : capacity_(cfg.capacity) {
98+
explicit SequenceCache(const CacheConfig& cfg)
99+
: capacity_(cfg.capacity), max_write_(cfg.max_write) {
54100
layer_to_policy_.reserve(cfg.n_layers);
55101
for (int l = 0; l < cfg.n_layers; ++l) {
56102
// layers size 1 = one config broadcast to every layer, else per-layer.
@@ -105,6 +151,13 @@ class SequenceCache : public CacheBase,
105151
if (position + T > capacity_) {
106152
return std::nullopt;
107153
}
154+
// A ring layer's slots are sized window + max_write - 1 (max_write defaults
155+
// to the window), so a step larger than that would overrun it.
156+
const LayerPolicy& spec = specs_[layer_to_policy_[layer]];
157+
if (spec.kind == LayerPolicy::Kind::Ring &&
158+
T > (max_write_ ? *max_write_ : spec.window)) {
159+
return std::nullopt;
160+
}
108161
return policies_[layer_to_policy_[layer]]->plan(position, T);
109162
}
110163
void commit(const SeqStepPlan& plan) override {
@@ -129,11 +182,17 @@ class SequenceCache : public CacheBase,
129182
policies_.push_back(make_policy(lp));
130183
return static_cast<int>(policies_.size() - 1);
131184
}
132-
std::unique_ptr<LayoutPolicy> make_policy(const LayerPolicy&) const {
185+
std::unique_ptr<LayoutPolicy> make_policy(const LayerPolicy& lp) const {
186+
if (lp.kind == LayerPolicy::Kind::Ring) {
187+
// Unset max_write -> default a ring layer to its own window (ring 2w-1).
188+
const int mw = max_write_ ? *max_write_ : lp.window;
189+
return std::make_unique<RingPolicy>(lp.window, mw);
190+
}
133191
return std::make_unique<FlatPolicy>();
134192
}
135193

136194
int capacity_;
195+
std::optional<int> max_write_;
137196
int length_ = 0;
138197
std::vector<LayerPolicy> specs_; // parallel to policies_, for dedup
139198
std::vector<std::unique_ptr<LayoutPolicy>> policies_;

‎extension/llm/cache/test/cache_test.cpp‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,9 @@ namespace {
3333
LayerConfig flat_layer() {
3434
return LayerConfig{LayerPolicy{LayerPolicy::Kind::Flat, 0}, 2, 8};
3535
}
36+
LayerConfig ring_layer(int window) {
37+
return LayerConfig{LayerPolicy{LayerPolicy::Kind::Ring, window}, 2, 8};
38+
}
3639
} // namespace
3740

3841
// Initializes the ExecuTorch PAL so the ET adapter's error paths (which ET_LOG)
@@ -66,6 +69,58 @@ TEST_F(CacheTest, FlatPlanAppendsAndReadsAllHistory) {
6669
EXPECT_EQ(p1->read[0].len, 5);
6770
}
6871

72+
// ---- Ring policy: wrap, eviction, read base position -----------------------
73+
74+
TEST_F(CacheTest, RingPlanWrapsAndEvicts) {
75+
// window 4, max_write unset -> defaults to window -> ring of 2*4 - 1 = 7.
76+
SequenceCache cache(CacheConfig{100, 1, {ring_layer(4)}});
77+
78+
// Prefill chunk of 4 at position 0: one write run, reads [0,4), no wrap.
79+
auto p0 = cache.plan(0, 0, 4);
80+
ASSERT_TRUE(p0.has_value());
81+
EXPECT_EQ(p0->n_write, 1);
82+
EXPECT_EQ(p0->write[0].start, 0);
83+
EXPECT_EQ(p0->write[0].len, 4);
84+
EXPECT_EQ(p0->n_read, 1);
85+
EXPECT_EQ(p0->read[0].start, 0);
86+
EXPECT_EQ(p0->read[0].len, 4);
87+
EXPECT_EQ(p0->read_base_pos, 0);
88+
cache.commit(*p0);
89+
90+
// Decode at position 7: write wraps to slot 0 (7 % 7); the read window
91+
// [4,8) wraps the ring -> phys [4,6] then [0,0]. Positions 0-3 evicted.
92+
auto p1 = cache.plan(0, 7, 1);
93+
ASSERT_TRUE(p1.has_value());
94+
EXPECT_EQ(p1->n_write, 1);
95+
EXPECT_EQ(p1->write[0].start, 0);
96+
EXPECT_EQ(p1->write[0].len, 1);
97+
EXPECT_EQ(p1->n_read, 2);
98+
EXPECT_EQ(p1->read[0].start, 4);
99+
EXPECT_EQ(p1->read[0].len, 3);
100+
EXPECT_EQ(p1->read[1].start, 0);
101+
EXPECT_EQ(p1->read[1].len, 1);
102+
EXPECT_EQ(p1->read_base_pos, 4); // oldest retained is logical position 4
103+
104+
// A step larger than max_write would overrun the ring -> rejected.
105+
EXPECT_FALSE(cache.plan(0, 0, 5).has_value());
106+
}
107+
108+
// ---- Mixed flat/ring: one shared length across layers ----------------------
109+
110+
TEST_F(CacheTest, MixedFlatRingShareOneLength) {
111+
SequenceCache cache(CacheConfig{100, 2, {flat_layer(), ring_layer(4)}});
112+
113+
// Same step drives both layers (T <= window); commit once.
114+
auto p = cache.plan(0, 0, 3);
115+
ASSERT_TRUE(p.has_value());
116+
ASSERT_TRUE(cache.plan(1, 0, 3).has_value());
117+
cache.commit(*p);
118+
119+
// Length advanced once (not per layer): length == 3.
120+
EXPECT_TRUE(cache.can_extend(97));
121+
EXPECT_FALSE(cache.can_extend(98));
122+
}
123+
69124
// ---- Admission / rewind ----------------------------------------------------
70125

71126
TEST_F(CacheTest, CanExtendBoundedByCapacity) {
@@ -89,6 +144,21 @@ TEST_F(CacheTest, FlatRewindsFreelyToZero) {
89144
EXPECT_TRUE(cache.can_extend(8));
90145
}
91146

147+
TEST_F(CacheTest, RewindBoundedByRingWindow) {
148+
SequenceCache cache(CacheConfig{100, 2, {flat_layer(), ring_layer(4)}});
149+
// Advance length to 10 in chunks of 2 (<= window).
150+
for (int pos = 0; pos < 10; pos += 2) {
151+
auto p = cache.plan(0, pos, 2);
152+
ASSERT_TRUE(p.has_value());
153+
ASSERT_TRUE(cache.plan(1, pos, 2).has_value());
154+
cache.commit(*p);
155+
}
156+
// ring(4) has evicted everything older than 10 - 4 = 6.
157+
EXPECT_FALSE(cache.rewind(5)); // older than the ring retains
158+
EXPECT_TRUE(cache.rewind(6)); // exactly the floor
159+
EXPECT_FALSE(cache.rewind(11)); // cannot grow
160+
}
161+
92162
// ---- Faces / registry / session --------------------------------------------
93163

94164
TEST_F(CacheTest, FaceRecoveryReturnsSameObject) {

0 commit comments

Comments
 (0)