Skip to content

Commit 739fe55

Browse files
Add off-graph KV-cache registry (pytorch#21383)
Summary Adds a shared, backend-agnostic registry for the off-graph KV cache: a small set of C++ pieces that let a runtime own a cache and look it up by key. Files - extension/llm/cache/cache.h — the opaque cache handle (CacheBase) and the CacheConfig struct. - extension/llm/cache/cache_registry.h / cache_registry.cpp — a process-global registry that stores caches by key, a per-(backend_id, kind) builder registry, unique-key generation, and a RAII session that installs on construction and erases on destruction. - extension/llm/cache/CMakeLists.txt — builds the library and pulls in the test folder (cmake). - extension/llm/cache/test/cache_test.cpp — unit tests for the above. - extension/llm/cache/test/CMakeLists.txt — builds the test (cmake). - CMakeLists.txt — includes the new folder in the build. Testing Added C++ unit tests (test/cache_test.cpp) covering the registry (install/get/erase, unique keys), the builder registry (build + missing-kind NotFound error), and the session's install-on-create / erase-on-destroy. Built and ran through the standard cmake/ctest flow, all pass: cmake -B cmake-out -DEXECUTORCH_BUILD_EXTENSION_LLM=ON -DEXECUTORCH_BUILD_TESTS=ON cmake --build cmake-out --target extension_llm_cache_test -j ctest --test-dir cmake-out -R extension_llm_cache --output-on-failure
1 parent 1f0b2dc commit 739fe55

7 files changed

Lines changed: 379 additions & 0 deletions

File tree

‎CMakeLists.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -903,6 +903,8 @@ if(EXECUTORCH_BUILD_EXTENSION_LLM)
903903
)
904904
endif()
905905
list(APPEND _executorch_extensions tokenizers)
906+
add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extension/llm/cache)
907+
list(APPEND _executorch_extensions extension_llm_cache)
906908
endif()
907909

908910
if(EXECUTORCH_BUILD_EXTENSION_RUNNER_UTIL)

‎extension/llm/cache/CMakeLists.txt‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
# Install / rendezvous registry for the off-graph KV cache, shared across
8+
# backends. The neutral cache handle (cache.h) is ET-independent, so it is
9+
# usable outside an ET runner; the delegate-specific registry
10+
# (cache_registry.cpp) uses ExecuTorch Error/Result, hence the executorch_core
11+
# link.
12+
13+
if(NOT EXECUTORCH_ROOT)
14+
set(EXECUTORCH_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../../..)
15+
endif()
16+
17+
add_library(extension_llm_cache cache_registry.cpp)
18+
target_link_libraries(extension_llm_cache executorch_core)
19+
target_include_directories(
20+
extension_llm_cache PUBLIC ${_common_include_directories}
21+
)
22+
target_compile_options(extension_llm_cache PUBLIC ${_common_compile_options})
23+
24+
install(
25+
TARGETS extension_llm_cache
26+
EXPORT ExecuTorchTargets
27+
DESTINATION ${CMAKE_INSTALL_LIBDIR}
28+
INCLUDES
29+
DESTINATION ${_common_include_directories}
30+
)
31+
install(
32+
DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/
33+
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/executorch/extension/llm/cache
34+
FILES_MATCHING
35+
PATTERN "*.h"
36+
)
37+
38+
if(BUILD_TESTING)
39+
add_subdirectory(test)
40+
endif()

‎extension/llm/cache/cache.h‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
/*
2+
* Copyright (c) Meta Platforms, Inc. and affiliates.
3+
* All rights reserved.
4+
*
5+
* This source code is licensed under the BSD-style license found in the
6+
* LICENSE file in the root directory of this source tree.
7+
*/
8+
9+
#pragma once
10+
11+
// Neutral, ET-independent ownership handle for an off-graph KV cache. The
12+
// registry owns a cache as a CacheBase* -- an opaque, cache-agnostic anchor --
13+
// and hands it back by key. The typed faces a runner and backend use to drive
14+
// the cache are added by the concrete cache implementation.
15+
16+
namespace executorch {
17+
namespace extension {
18+
namespace llm {
19+
namespace cache {
20+
21+
// Ownership / erasure anchor: the registry owns and deletes a cache through
22+
// this base, staying agnostic to the concrete cache type.
23+
class CacheBase {
24+
public:
25+
virtual ~CacheBase() = default;
26+
};
27+
28+
// Model facts a cache factory builds from. capacity is the logical cap;
29+
// n_layers is the number of attention layers; initial_capacity is the byte
30+
// layer's starting pool size before it grows (by doubling) toward capacity.
31+
struct CacheConfig {
32+
int capacity;
33+
int n_layers;
34+
int initial_capacity = 512;
35+
};
36+
37+
} // namespace cache
38+
} // namespace llm
39+
} // namespace extension
40+
} // namespace executorch
Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
/*
2+
* Copyright (c) Meta Platforms, Inc. and affiliates.
3+
* All rights reserved.
4+
*
5+
* This source code is licensed under the BSD-style license found in the
6+
* LICENSE file in the root directory of this source tree.
7+
*/
8+
9+
#include <executorch/extension/llm/cache/cache_registry.h>
10+
11+
#include <atomic>
12+
13+
#include <executorch/runtime/core/error.h>
14+
15+
namespace executorch {
16+
namespace extension {
17+
namespace llm {
18+
namespace cache {
19+
20+
CacheRegistry& CacheRegistry::global() {
21+
static CacheRegistry registry;
22+
return registry;
23+
}
24+
25+
void CacheRegistry::install(
26+
const std::string& key,
27+
std::shared_ptr<CacheBase> cache) {
28+
std::lock_guard<std::mutex> lock(mu_);
29+
caches_[key] = std::move(cache);
30+
}
31+
32+
std::shared_ptr<CacheBase> CacheRegistry::get(const std::string& key) const {
33+
std::lock_guard<std::mutex> lock(mu_);
34+
const auto it = caches_.find(key);
35+
return it == caches_.end() ? nullptr : it->second;
36+
}
37+
38+
void CacheRegistry::erase(const std::string& key) {
39+
std::lock_guard<std::mutex> lock(mu_);
40+
caches_.erase(key);
41+
}
42+
43+
CacheBuilderRegistry& CacheBuilderRegistry::global() {
44+
static CacheBuilderRegistry registry;
45+
return registry;
46+
}
47+
48+
void CacheBuilderRegistry::register_builder(
49+
const std::string& backend_id,
50+
const std::string& kind,
51+
CacheBuilder builder) {
52+
std::lock_guard<std::mutex> lock(mu_);
53+
builders_[backend_id + ":" + kind] = std::move(builder);
54+
}
55+
56+
Result<std::shared_ptr<CacheBase>> CacheBuilderRegistry::build(
57+
const std::string& backend_id,
58+
const std::string& kind,
59+
const CacheConfig& cfg) const {
60+
CacheBuilder builder;
61+
{
62+
std::lock_guard<std::mutex> lock(mu_);
63+
const auto it = builders_.find(backend_id + ":" + kind);
64+
ET_CHECK_OR_RETURN_ERROR(
65+
it != builders_.end(),
66+
NotFound,
67+
"no cache builder registered for %s:%s",
68+
backend_id.c_str(),
69+
kind.c_str());
70+
builder = it->second;
71+
}
72+
return builder(cfg);
73+
}
74+
75+
std::string make_unique_key() {
76+
static std::atomic<uint64_t> counter{0};
77+
return "cache-" + std::to_string(counter.fetch_add(1));
78+
}
79+
80+
} // namespace cache
81+
} // namespace llm
82+
} // namespace extension
83+
} // namespace executorch
Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
/*
2+
* Copyright (c) Meta Platforms, Inc. and affiliates.
3+
* All rights reserved.
4+
*
5+
* This source code is licensed under the BSD-style license found in the
6+
* LICENSE file in the root directory of this source tree.
7+
*/
8+
9+
#pragma once
10+
11+
// Install / rendezvous machinery for the off-graph KV cache. The DelegateHandle
12+
// is opaque to the host, so the runner (which knows the cache kind) creates the
13+
// cache and binds it to the delegate through a process-global registry; the two
14+
// sides rendezvous on a cache_key passed as a runtime backend-load option.
15+
// Caches are owned as CacheBase*, keeping the registry agnostic to the concrete
16+
// cache type. This layer is delegate-specific and may use ExecuTorch
17+
// Error/Result directly; the cache handle (cache.h) stays ET-free.
18+
19+
#include <functional>
20+
#include <memory>
21+
#include <mutex>
22+
#include <string>
23+
#include <unordered_map>
24+
25+
#include <executorch/extension/llm/cache/cache.h>
26+
#include <executorch/runtime/core/error.h>
27+
#include <executorch/runtime/core/result.h>
28+
29+
namespace executorch {
30+
namespace extension {
31+
namespace llm {
32+
namespace cache {
33+
34+
using ::executorch::runtime::Error;
35+
using ::executorch::runtime::Result;
36+
37+
// Process-global map<cache_key, shared_ptr<CacheBase>>. Ownership is shared:
38+
// the registry entry, the runner's session guard, and the delegate handle all
39+
// hold the cache, so erasing the entry mid-method is safe.
40+
class CacheRegistry {
41+
public:
42+
static CacheRegistry& global();
43+
44+
void install(const std::string& key, std::shared_ptr<CacheBase> cache);
45+
std::shared_ptr<CacheBase> get(const std::string& key) const;
46+
void erase(const std::string& key);
47+
48+
private:
49+
CacheRegistry() = default;
50+
51+
mutable std::mutex mu_;
52+
std::unordered_map<std::string, std::shared_ptr<CacheBase>> caches_;
53+
};
54+
55+
// Cache kind is expressed by which factory you call: backends register a
56+
// builder per (backend_id, kind) and the kind survives only as an internal
57+
// lookup tag.
58+
using CacheBuilder =
59+
std::function<std::shared_ptr<CacheBase>(const CacheConfig&)>;
60+
61+
class CacheBuilderRegistry {
62+
public:
63+
static CacheBuilderRegistry& global();
64+
65+
void register_builder(
66+
const std::string& backend_id,
67+
const std::string& kind,
68+
CacheBuilder builder);
69+
// Returns Error::NotFound if no builder is registered for (backend_id, kind).
70+
Result<std::shared_ptr<CacheBase>> build(
71+
const std::string& backend_id,
72+
const std::string& kind,
73+
const CacheConfig& cfg) const;
74+
75+
private:
76+
CacheBuilderRegistry() = default;
77+
78+
mutable std::mutex mu_;
79+
std::unordered_map<std::string, CacheBuilder>
80+
builders_; // backend_id + ":" + kind
81+
};
82+
83+
// Process-global atomic counter -> "cache-N"; centralizes key generation so
84+
// keys never collide.
85+
std::string make_unique_key();
86+
87+
// RAII: installs the cache into the global registry under a unique key on
88+
// construction and erases it on destruction (no leak on any exit path). Holds
89+
// the runner's shared_ptr for the lifetime of the generation loop.
90+
class CacheSession {
91+
public:
92+
CacheSession(std::string key, std::shared_ptr<CacheBase> cache)
93+
: key_(std::move(key)), cache_(std::move(cache)) {
94+
CacheRegistry::global().install(key_, cache_);
95+
}
96+
~CacheSession() {
97+
CacheRegistry::global().erase(key_);
98+
}
99+
100+
CacheSession(const CacheSession&) = delete;
101+
CacheSession& operator=(const CacheSession&) = delete;
102+
103+
const std::shared_ptr<CacheBase>& cache() const {
104+
return cache_;
105+
}
106+
const std::string& key() const {
107+
return key_;
108+
}
109+
110+
private:
111+
std::string key_;
112+
std::shared_ptr<CacheBase> cache_;
113+
};
114+
115+
} // namespace cache
116+
} // namespace llm
117+
} // namespace extension
118+
} // namespace executorch
Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
cmake_minimum_required(VERSION 3.19)
8+
9+
set(EXECUTORCH_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../../../..)
10+
11+
include(${EXECUTORCH_ROOT}/tools/cmake/Test.cmake)
12+
13+
set(_test_srcs cache_test.cpp)
14+
15+
et_cxx_test(
16+
extension_llm_cache_test SOURCES ${_test_srcs} EXTRA_LIBS extension_llm_cache
17+
)
Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
/*
2+
* Copyright (c) Meta Platforms, Inc. and affiliates.
3+
* All rights reserved.
4+
*
5+
* This source code is licensed under the BSD-style license found in the
6+
* LICENSE file in the root directory of this source tree.
7+
*/
8+
9+
#include <executorch/extension/llm/cache/cache.h>
10+
#include <executorch/extension/llm/cache/cache_registry.h>
11+
12+
#include <memory>
13+
14+
#include <executorch/runtime/core/error.h>
15+
#include <executorch/runtime/platform/runtime.h>
16+
#include <gtest/gtest.h>
17+
18+
using executorch::extension::llm::cache::CacheBase;
19+
using executorch::extension::llm::cache::CacheBuilderRegistry;
20+
using executorch::extension::llm::cache::CacheConfig;
21+
using executorch::extension::llm::cache::CacheRegistry;
22+
using executorch::extension::llm::cache::CacheSession;
23+
using executorch::extension::llm::cache::make_unique_key;
24+
using executorch::runtime::Error;
25+
26+
namespace {
27+
// Opaque CacheBase used to exercise the registry without a concrete cache
28+
// implementation (SequenceCache and its faces arrive in a later change).
29+
class StubCache : public CacheBase {};
30+
} // namespace
31+
32+
// Initializes the ExecuTorch PAL so registry error paths (which ET_LOG) can
33+
// run.
34+
class CacheTest : public ::testing::Test {
35+
protected:
36+
void SetUp() override {
37+
executorch::runtime::runtime_init();
38+
}
39+
};
40+
41+
TEST_F(CacheTest, RegistryInstallGetErase) {
42+
auto& reg = CacheRegistry::global();
43+
const std::string key = make_unique_key();
44+
EXPECT_EQ(reg.get(key), nullptr);
45+
46+
std::shared_ptr<CacheBase> cache = std::make_shared<StubCache>();
47+
reg.install(key, cache);
48+
EXPECT_EQ(reg.get(key), cache);
49+
50+
reg.erase(key);
51+
EXPECT_EQ(reg.get(key), nullptr);
52+
}
53+
54+
TEST_F(CacheTest, UniqueKeysDoNotCollide) {
55+
EXPECT_NE(make_unique_key(), make_unique_key());
56+
}
57+
58+
TEST_F(CacheTest, BuilderRegistryBuildsRegisteredKindElseError) {
59+
auto& reg = CacheBuilderRegistry::global();
60+
reg.register_builder("TestBackend", "stub", [](const CacheConfig&) {
61+
return std::static_pointer_cast<CacheBase>(std::make_shared<StubCache>());
62+
});
63+
64+
CacheConfig cfg{32, 1};
65+
auto cache = reg.build("TestBackend", "stub", cfg);
66+
ASSERT_TRUE(cache.ok());
67+
EXPECT_NE(cache.get(), nullptr);
68+
69+
EXPECT_EQ(reg.build("TestBackend", "missing", cfg).error(), Error::NotFound);
70+
}
71+
72+
TEST_F(CacheTest, SessionInstallsOnCtorErasesOnDtor) {
73+
const std::string key = make_unique_key();
74+
{
75+
CacheSession session(key, std::make_shared<StubCache>());
76+
EXPECT_EQ(CacheRegistry::global().get(key), session.cache());
77+
}
78+
EXPECT_EQ(CacheRegistry::global().get(key), nullptr);
79+
}

0 commit comments

Comments
 (0)