|
| 1 | +/* |
| 2 | + * Copyright (c) Meta Platforms, Inc. and affiliates. |
| 3 | + * All rights reserved. |
| 4 | + * |
| 5 | + * This source code is licensed under the BSD-style license found in the |
| 6 | + * LICENSE file in the root directory of this source tree. |
| 7 | + */ |
| 8 | + |
| 9 | +#pragma once |
| 10 | + |
| 11 | +// Install / rendezvous machinery for the off-graph KV cache. The DelegateHandle |
| 12 | +// is opaque to the host, so the runner (which knows the cache kind) creates the |
| 13 | +// cache and binds it to the delegate through a process-global registry; the two |
| 14 | +// sides rendezvous on a cache_key passed as a runtime backend-load option. |
| 15 | +// Caches are owned as CacheBase*, keeping the registry agnostic to the concrete |
| 16 | +// cache type. This layer is delegate-specific and may use ExecuTorch |
| 17 | +// Error/Result directly; the cache handle (cache.h) stays ET-free. |
| 18 | + |
| 19 | +#include <functional> |
| 20 | +#include <memory> |
| 21 | +#include <mutex> |
| 22 | +#include <string> |
| 23 | +#include <unordered_map> |
| 24 | + |
| 25 | +#include <executorch/extension/llm/cache/cache.h> |
| 26 | +#include <executorch/runtime/core/error.h> |
| 27 | +#include <executorch/runtime/core/result.h> |
| 28 | + |
| 29 | +namespace executorch { |
| 30 | +namespace extension { |
| 31 | +namespace llm { |
| 32 | +namespace cache { |
| 33 | + |
| 34 | +using ::executorch::runtime::Error; |
| 35 | +using ::executorch::runtime::Result; |
| 36 | + |
| 37 | +// Process-global map<cache_key, shared_ptr<CacheBase>>. Ownership is shared: |
| 38 | +// the registry entry, the runner's session guard, and the delegate handle all |
| 39 | +// hold the cache, so erasing the entry mid-method is safe. |
| 40 | +class CacheRegistry { |
| 41 | + public: |
| 42 | + static CacheRegistry& global(); |
| 43 | + |
| 44 | + void install(const std::string& key, std::shared_ptr<CacheBase> cache); |
| 45 | + std::shared_ptr<CacheBase> get(const std::string& key) const; |
| 46 | + void erase(const std::string& key); |
| 47 | + |
| 48 | + private: |
| 49 | + CacheRegistry() = default; |
| 50 | + |
| 51 | + mutable std::mutex mu_; |
| 52 | + std::unordered_map<std::string, std::shared_ptr<CacheBase>> caches_; |
| 53 | +}; |
| 54 | + |
| 55 | +// Cache kind is expressed by which factory you call: backends register a |
| 56 | +// builder per (backend_id, kind) and the kind survives only as an internal |
| 57 | +// lookup tag. |
| 58 | +using CacheBuilder = |
| 59 | + std::function<std::shared_ptr<CacheBase>(const CacheConfig&)>; |
| 60 | + |
| 61 | +class CacheBuilderRegistry { |
| 62 | + public: |
| 63 | + static CacheBuilderRegistry& global(); |
| 64 | + |
| 65 | + void register_builder( |
| 66 | + const std::string& backend_id, |
| 67 | + const std::string& kind, |
| 68 | + CacheBuilder builder); |
| 69 | + // Returns Error::NotFound if no builder is registered for (backend_id, kind). |
| 70 | + Result<std::shared_ptr<CacheBase>> build( |
| 71 | + const std::string& backend_id, |
| 72 | + const std::string& kind, |
| 73 | + const CacheConfig& cfg) const; |
| 74 | + |
| 75 | + private: |
| 76 | + CacheBuilderRegistry() = default; |
| 77 | + |
| 78 | + mutable std::mutex mu_; |
| 79 | + std::unordered_map<std::string, CacheBuilder> |
| 80 | + builders_; // backend_id + ":" + kind |
| 81 | +}; |
| 82 | + |
| 83 | +// Process-global atomic counter -> "cache-N"; centralizes key generation so |
| 84 | +// keys never collide. |
| 85 | +std::string make_unique_key(); |
| 86 | + |
| 87 | +// RAII: installs the cache into the global registry under a unique key on |
| 88 | +// construction and erases it on destruction (no leak on any exit path). Holds |
| 89 | +// the runner's shared_ptr for the lifetime of the generation loop. |
| 90 | +class CacheSession { |
| 91 | + public: |
| 92 | + CacheSession(std::string key, std::shared_ptr<CacheBase> cache) |
| 93 | + : key_(std::move(key)), cache_(std::move(cache)) { |
| 94 | + CacheRegistry::global().install(key_, cache_); |
| 95 | + } |
| 96 | + ~CacheSession() { |
| 97 | + CacheRegistry::global().erase(key_); |
| 98 | + } |
| 99 | + |
| 100 | + CacheSession(const CacheSession&) = delete; |
| 101 | + CacheSession& operator=(const CacheSession&) = delete; |
| 102 | + |
| 103 | + const std::shared_ptr<CacheBase>& cache() const { |
| 104 | + return cache_; |
| 105 | + } |
| 106 | + const std::string& key() const { |
| 107 | + return key_; |
| 108 | + } |
| 109 | + |
| 110 | + private: |
| 111 | + std::string key_; |
| 112 | + std::shared_ptr<CacheBase> cache_; |
| 113 | +}; |
| 114 | + |
| 115 | +} // namespace cache |
| 116 | +} // namespace llm |
| 117 | +} // namespace extension |
| 118 | +} // namespace executorch |
0 commit comments