Skip to content

Commit d8cddbb

Browse files
anhnh2002claude
andcommitted
Add prompt caching for agentic calls; stop pinning temperature
Agentic/multi-turn runs now inject Anthropic-style cache_control breakpoints (via the new CachingOpenAIModel) on the system message and the final message of every request, so providers that support prompt caching reuse the growing conversation prefix instead of re-processing it each turn. OpenAI-compatible proxies such as LiteLLM forward the markers; providers that reject them get one uncached retry and the (base_url, model) pair is remembered so later calls skip injection. Controlled by a new prompt_caching config option (default on, --prompt-caching/--no-prompt-caching on generate and config set). Requests no longer send a temperature parameter anywhere (agentic settings, call_llm, litellm/azure helpers, LLMBackend.complete): some models only accept their default temperature and reject explicit values, including 0.0. Measured on a full self-documentation run through a LiteLLM proxy in front of claude-sonnet: 73.8% of input tokens served as cache reads, 47% cost reduction. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
1 parent 050dd26 commit d8cddbb

11 files changed

Lines changed: 384 additions & 24 deletions

File tree

‎codewiki/cli/adapters/doc_generator.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,7 @@ def generate(self) -> DocumentationJob:
148148
max_depth=self.config.get('max_depth', 2),
149149
agent_instructions=self.config.get('agent_instructions'),
150150
use_gitignore=self.config.get('use_gitignore', True),
151+
prompt_caching=self.config.get('prompt_caching', True),
151152
)
152153

153154
# Run backend documentation generation

‎codewiki/cli/commands/config.py‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,12 @@ def config_group():
116116
default=None,
117117
help="Apply Git ignore rules during repository analysis (default: enabled)",
118118
)
119+
@click.option(
120+
"--prompt-caching/--no-prompt-caching",
121+
default=None,
122+
help="Add prompt-cache breakpoints to agentic LLM calls; auto-falls back to "
123+
"normal calls if the provider rejects them (default: enabled)",
124+
)
119125
def config_set(
120126
api_key: Optional[str],
121127
base_url: Optional[str],
@@ -131,6 +137,7 @@ def config_set(
131137
api_version: Optional[str] = None,
132138
azure_deployment: Optional[str] = None,
133139
use_gitignore: Optional[bool] = None,
140+
prompt_caching: Optional[bool] = None,
134141
):
135142
"""
136143
Set configuration values for CodeWiki.
@@ -183,7 +190,7 @@ def config_set(
183190
"""
184191
try:
185192
# Check if at least one option is provided
186-
if not any([api_key, base_url, main_model, cluster_model, fallback_model, max_tokens, max_token_per_module, max_token_per_leaf_module, max_depth, provider, aws_region, api_version, azure_deployment, use_gitignore is not None]):
193+
if not any([api_key, base_url, main_model, cluster_model, fallback_model, max_tokens, max_token_per_module, max_token_per_leaf_module, max_depth, provider, aws_region, api_version, azure_deployment, use_gitignore is not None, prompt_caching is not None]):
187194
click.echo("No options provided. Use --help for usage information.")
188195
sys.exit(EXIT_CONFIG_ERROR)
189196

@@ -250,6 +257,9 @@ def config_set(
250257
if use_gitignore is not None:
251258
validated_data['use_gitignore'] = use_gitignore
252259

260+
if prompt_caching is not None:
261+
validated_data['prompt_caching'] = prompt_caching
262+
253263
# Create config manager and save
254264
manager = ConfigManager()
255265
manager.load() # Load existing config if present
@@ -269,6 +279,7 @@ def config_set(
269279
api_version=validated_data.get('api_version'),
270280
azure_deployment=validated_data.get('azure_deployment'),
271281
use_gitignore=validated_data.get('use_gitignore'),
282+
prompt_caching=validated_data.get('prompt_caching'),
272283
)
273284

274285
# Display success messages
@@ -332,6 +343,9 @@ def config_set(
332343
if use_gitignore is not None:
333344
click.secho(f"✓ Use gitignore: {use_gitignore}", fg="green")
334345

346+
if prompt_caching is not None:
347+
click.secho(f"✓ Prompt caching: {prompt_caching}", fg="green")
348+
335349
click.echo("\n" + click.style("Configuration updated successfully.", fg="green", bold=True))
336350

337351
except ConfigurationError as e:
@@ -393,6 +407,7 @@ def config_show(output_json: bool):
393407
"max_token_per_leaf_module": config.max_token_per_leaf_module if config else 16000,
394408
"max_depth": config.max_depth if config else 2,
395409
"use_gitignore": config.use_gitignore if config else True,
410+
"prompt_caching": config.prompt_caching if config else True,
396411
"agent_instructions": config.agent_instructions.to_dict() if config and config.agent_instructions else {},
397412
"config_file": str(manager.config_file_path)
398413
}
@@ -448,6 +463,7 @@ def config_show(output_json: bool):
448463
click.echo(f" Max Tokens: {config.max_tokens}")
449464
click.echo(f" Max Token/Module: {config.max_token_per_module}")
450465
click.echo(f" Max Token/Leaf Module: {config.max_token_per_leaf_module}")
466+
click.echo(f" Prompt Caching: {config.prompt_caching}")
451467

452468
click.echo()
453469
click.secho("Decomposition Settings", fg="cyan", bold=True)

‎codewiki/cli/commands/generate.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -301,6 +301,12 @@ def _find_affected(tree, parent_names=None):
301301
default=None,
302302
help="Maximum depth for hierarchical decomposition (overrides config)",
303303
)
304+
@click.option(
305+
"--prompt-caching/--no-prompt-caching",
306+
default=None,
307+
help="Add prompt-cache breakpoints to agentic LLM calls; auto-falls back to "
308+
"normal calls if the provider rejects them (default: enabled)",
309+
)
304310
@click.option(
305311
"--update",
306312
is_flag=True,
@@ -330,6 +336,7 @@ def generate_command(
330336
max_token_per_module: Optional[int],
331337
max_token_per_leaf_module: Optional[int],
332338
max_depth: Optional[int],
339+
prompt_caching: Optional[bool],
333340
update: bool = False,
334341
compare_to: Optional[str] = None
335342
):
@@ -532,11 +539,13 @@ def generate_command(
532539
effective_max_token_per_leaf = max_token_per_leaf_module if max_token_per_leaf_module is not None else config.max_token_per_leaf_module
533540
effective_max_depth = max_depth if max_depth is not None else config.max_depth
534541
effective_use_gitignore = use_gitignore if use_gitignore is not None else config.use_gitignore
542+
effective_prompt_caching = prompt_caching if prompt_caching is not None else config.prompt_caching
535543
logger.debug(f"Max tokens: {effective_max_tokens}")
536544
logger.debug(f"Max token/module: {effective_max_token_per_module}")
537545
logger.debug(f"Max token/leaf module: {effective_max_token_per_leaf}")
538546
logger.debug(f"Max depth: {effective_max_depth}")
539547
logger.debug(f"Use gitignore: {effective_use_gitignore}")
548+
logger.debug(f"Prompt caching: {effective_prompt_caching}")
540549

541550
# Get agent instructions (merge runtime with persistent)
542551
agent_instructions_dict = None
@@ -576,6 +585,8 @@ def generate_command(
576585
'max_depth': max_depth if max_depth is not None else config.max_depth,
577586
# Gitignore setting (runtime override takes precedence)
578587
'use_gitignore': use_gitignore if use_gitignore is not None else config.use_gitignore,
588+
# Prompt caching setting (runtime override takes precedence)
589+
'prompt_caching': prompt_caching if prompt_caching is not None else config.prompt_caching,
579590
},
580591
verbose=verbose,
581592
generate_html=github_pages,

‎codewiki/cli/config_manager.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,7 @@ def save(
137137
api_version: Optional[str] = None,
138138
azure_deployment: Optional[str] = None,
139139
use_gitignore: Optional[bool] = None,
140+
prompt_caching: Optional[bool] = None,
140141
):
141142
"""
142143
Save configuration to file and keyring.
@@ -157,6 +158,7 @@ def save(
157158
api_version: Azure OpenAI API version
158159
azure_deployment: Azure OpenAI deployment name
159160
use_gitignore: Apply Git ignore rules during repository analysis
161+
prompt_caching: Add prompt-cache breakpoints to agentic LLM calls
160162
"""
161163
# Ensure config directory exists
162164
try:
@@ -208,6 +210,8 @@ def save(
208210
self._config.azure_deployment = azure_deployment
209211
if use_gitignore is not None:
210212
self._config.use_gitignore = use_gitignore
213+
if prompt_caching is not None:
214+
self._config.prompt_caching = prompt_caching
211215

212216
# Validate configuration whenever the minimum required fields are set.
213217
# Caw providers only need main_model; API providers need base_url +

‎codewiki/cli/models/config.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,7 @@ class Configuration:
122122
max_token_per_leaf_module: Maximum tokens per leaf module (default: 16000)
123123
max_depth: Maximum depth for hierarchical decomposition (default: 2)
124124
use_gitignore: Apply Git ignore rules during repository analysis
125+
prompt_caching: Add prompt-cache breakpoints to agentic LLM calls (default: True)
125126
agent_instructions: Custom agent instructions for documentation generation
126127
"""
127128
base_url: str
@@ -138,6 +139,7 @@ class Configuration:
138139
max_token_per_leaf_module: int = 16000
139140
max_depth: int = 2
140141
use_gitignore: bool = True
142+
prompt_caching: bool = True
141143
agent_instructions: AgentInstructions = field(default_factory=AgentInstructions)
142144

143145
def validate(self):
@@ -175,6 +177,7 @@ def to_dict(self) -> dict:
175177
'max_token_per_leaf_module': self.max_token_per_leaf_module,
176178
'max_depth': self.max_depth,
177179
'use_gitignore': self.use_gitignore,
180+
'prompt_caching': self.prompt_caching,
178181
'fallback_model': self.fallback_model,
179182
}
180183
if self.agent_instructions and not self.agent_instructions.is_empty():
@@ -211,6 +214,7 @@ def from_dict(cls, data: dict) -> 'Configuration':
211214
max_token_per_leaf_module=data.get('max_token_per_leaf_module', 16000),
212215
max_depth=data.get('max_depth', 2),
213216
use_gitignore=data.get('use_gitignore', True),
217+
prompt_caching=data.get('prompt_caching', True),
214218
agent_instructions=agent_instructions,
215219
)
216220

@@ -279,4 +283,5 @@ def to_backend_config(self, repo_path: str, output_dir: str, api_key: str, runti
279283
max_depth=self.max_depth,
280284
agent_instructions=final_instructions.to_dict() if final_instructions else None,
281285
use_gitignore=self.use_gitignore,
286+
prompt_caching=self.prompt_caching,
282287
)

‎codewiki/src/be/backend.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,6 @@ def complete(
4141
prompt: str,
4242
*,
4343
model: str | None = None,
44-
temperature: float = 0.0,
4544
) -> str:
4645
"""Single-shot text completion."""
4746

‎codewiki/src/be/caw_backend.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -245,7 +245,6 @@ def complete(
245245
prompt: str,
246246
*,
247247
model: str | None = None,
248-
temperature: float = 0.0, # unused: subscription CLIs don't expose temperature
249248
) -> str:
250249
# Blocks the calling thread for the lifetime of the claude/codex
251250
# subprocess. Callers running this from an async context (e.g. the

0 commit comments

Comments
 (0)