Skip to content

Commit 3c4b4c2

Browse files
authored
Merge pull request #81 from FSoft-AI4Code/feat/prompt-caching
Add prompt caching for agentic calls; stop pinning temperature
2 parents 050dd26 + d8cddbb commit 3c4b4c2

11 files changed

Lines changed: 384 additions & 24 deletions

File tree

‎codewiki/cli/adapters/doc_generator.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,7 @@ def generate(self) -> DocumentationJob:
148148
max_depth=self.config.get('max_depth', 2),
149149
agent_instructions=self.config.get('agent_instructions'),
150150
use_gitignore=self.config.get('use_gitignore', True),
151+
prompt_caching=self.config.get('prompt_caching', True),
151152
)
152153

153154
# Run backend documentation generation

‎codewiki/cli/commands/config.py‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,12 @@ def config_group():
116116
default=None,
117117
help="Apply Git ignore rules during repository analysis (default: enabled)",
118118
)
119+
@click.option(
120+
"--prompt-caching/--no-prompt-caching",
121+
default=None,
122+
help="Add prompt-cache breakpoints to agentic LLM calls; auto-falls back to "
123+
"normal calls if the provider rejects them (default: enabled)",
124+
)
119125
def config_set(
120126
api_key: Optional[str],
121127
base_url: Optional[str],
@@ -131,6 +137,7 @@ def config_set(
131137
api_version: Optional[str] = None,
132138
azure_deployment: Optional[str] = None,
133139
use_gitignore: Optional[bool] = None,
140+
prompt_caching: Optional[bool] = None,
134141
):
135142
"""
136143
Set configuration values for CodeWiki.
@@ -183,7 +190,7 @@ def config_set(
183190
"""
184191
try:
185192
# Check if at least one option is provided
186-
if not any([api_key, base_url, main_model, cluster_model, fallback_model, max_tokens, max_token_per_module, max_token_per_leaf_module, max_depth, provider, aws_region, api_version, azure_deployment, use_gitignore is not None]):
193+
if not any([api_key, base_url, main_model, cluster_model, fallback_model, max_tokens, max_token_per_module, max_token_per_leaf_module, max_depth, provider, aws_region, api_version, azure_deployment, use_gitignore is not None, prompt_caching is not None]):
187194
click.echo("No options provided. Use --help for usage information.")
188195
sys.exit(EXIT_CONFIG_ERROR)
189196

@@ -250,6 +257,9 @@ def config_set(
250257
if use_gitignore is not None:
251258
validated_data['use_gitignore'] = use_gitignore
252259

260+
if prompt_caching is not None:
261+
validated_data['prompt_caching'] = prompt_caching
262+
253263
# Create config manager and save
254264
manager = ConfigManager()
255265
manager.load() # Load existing config if present
@@ -269,6 +279,7 @@ def config_set(
269279
api_version=validated_data.get('api_version'),
270280
azure_deployment=validated_data.get('azure_deployment'),
271281
use_gitignore=validated_data.get('use_gitignore'),
282+
prompt_caching=validated_data.get('prompt_caching'),
272283
)
273284

274285
# Display success messages
@@ -332,6 +343,9 @@ def config_set(
332343
if use_gitignore is not None:
333344
click.secho(f"✓ Use gitignore: {use_gitignore}", fg="green")
334345

346+
if prompt_caching is not None:
347+
click.secho(f"✓ Prompt caching: {prompt_caching}", fg="green")
348+
335349
click.echo("\n" + click.style("Configuration updated successfully.", fg="green", bold=True))
336350

337351
except ConfigurationError as e:
@@ -393,6 +407,7 @@ def config_show(output_json: bool):
393407
"max_token_per_leaf_module": config.max_token_per_leaf_module if config else 16000,
394408
"max_depth": config.max_depth if config else 2,
395409
"use_gitignore": config.use_gitignore if config else True,
410+
"prompt_caching": config.prompt_caching if config else True,
396411
"agent_instructions": config.agent_instructions.to_dict() if config and config.agent_instructions else {},
397412
"config_file": str(manager.config_file_path)
398413
}
@@ -448,6 +463,7 @@ def config_show(output_json: bool):
448463
click.echo(f" Max Tokens: {config.max_tokens}")
449464
click.echo(f" Max Token/Module: {config.max_token_per_module}")
450465
click.echo(f" Max Token/Leaf Module: {config.max_token_per_leaf_module}")
466+
click.echo(f" Prompt Caching: {config.prompt_caching}")
451467

452468
click.echo()
453469
click.secho("Decomposition Settings", fg="cyan", bold=True)

‎codewiki/cli/commands/generate.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -301,6 +301,12 @@ def _find_affected(tree, parent_names=None):
301301
default=None,
302302
help="Maximum depth for hierarchical decomposition (overrides config)",
303303
)
304+
@click.option(
305+
"--prompt-caching/--no-prompt-caching",
306+
default=None,
307+
help="Add prompt-cache breakpoints to agentic LLM calls; auto-falls back to "
308+
"normal calls if the provider rejects them (default: enabled)",
309+
)
304310
@click.option(
305311
"--update",
306312
is_flag=True,
@@ -330,6 +336,7 @@ def generate_command(
330336
max_token_per_module: Optional[int],
331337
max_token_per_leaf_module: Optional[int],
332338
max_depth: Optional[int],
339+
prompt_caching: Optional[bool],
333340
update: bool = False,
334341
compare_to: Optional[str] = None
335342
):
@@ -532,11 +539,13 @@ def generate_command(
532539
effective_max_token_per_leaf = max_token_per_leaf_module if max_token_per_leaf_module is not None else config.max_token_per_leaf_module
533540
effective_max_depth = max_depth if max_depth is not None else config.max_depth
534541
effective_use_gitignore = use_gitignore if use_gitignore is not None else config.use_gitignore
542+
effective_prompt_caching = prompt_caching if prompt_caching is not None else config.prompt_caching
535543
logger.debug(f"Max tokens: {effective_max_tokens}")
536544
logger.debug(f"Max token/module: {effective_max_token_per_module}")
537545
logger.debug(f"Max token/leaf module: {effective_max_token_per_leaf}")
538546
logger.debug(f"Max depth: {effective_max_depth}")
539547
logger.debug(f"Use gitignore: {effective_use_gitignore}")
548+
logger.debug(f"Prompt caching: {effective_prompt_caching}")
540549

541550
# Get agent instructions (merge runtime with persistent)
542551
agent_instructions_dict = None
@@ -576,6 +585,8 @@ def generate_command(
576585
'max_depth': max_depth if max_depth is not None else config.max_depth,
577586
# Gitignore setting (runtime override takes precedence)
578587
'use_gitignore': use_gitignore if use_gitignore is not None else config.use_gitignore,
588+
# Prompt caching setting (runtime override takes precedence)
589+
'prompt_caching': prompt_caching if prompt_caching is not None else config.prompt_caching,
579590
},
580591
verbose=verbose,
581592
generate_html=github_pages,

‎codewiki/cli/config_manager.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,7 @@ def save(
137137
api_version: Optional[str] = None,
138138
azure_deployment: Optional[str] = None,
139139
use_gitignore: Optional[bool] = None,
140+
prompt_caching: Optional[bool] = None,
140141
):
141142
"""
142143
Save configuration to file and keyring.
@@ -157,6 +158,7 @@ def save(
157158
api_version: Azure OpenAI API version
158159
azure_deployment: Azure OpenAI deployment name
159160
use_gitignore: Apply Git ignore rules during repository analysis
161+
prompt_caching: Add prompt-cache breakpoints to agentic LLM calls
160162
"""
161163
# Ensure config directory exists
162164
try:
@@ -208,6 +210,8 @@ def save(
208210
self._config.azure_deployment = azure_deployment
209211
if use_gitignore is not None:
210212
self._config.use_gitignore = use_gitignore
213+
if prompt_caching is not None:
214+
self._config.prompt_caching = prompt_caching
211215

212216
# Validate configuration whenever the minimum required fields are set.
213217
# Caw providers only need main_model; API providers need base_url +

‎codewiki/cli/models/config.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,7 @@ class Configuration:
122122
max_token_per_leaf_module: Maximum tokens per leaf module (default: 16000)
123123
max_depth: Maximum depth for hierarchical decomposition (default: 2)
124124
use_gitignore: Apply Git ignore rules during repository analysis
125+
prompt_caching: Add prompt-cache breakpoints to agentic LLM calls (default: True)
125126
agent_instructions: Custom agent instructions for documentation generation
126127
"""
127128
base_url: str
@@ -138,6 +139,7 @@ class Configuration:
138139
max_token_per_leaf_module: int = 16000
139140
max_depth: int = 2
140141
use_gitignore: bool = True
142+
prompt_caching: bool = True
141143
agent_instructions: AgentInstructions = field(default_factory=AgentInstructions)
142144

143145
def validate(self):
@@ -175,6 +177,7 @@ def to_dict(self) -> dict:
175177
'max_token_per_leaf_module': self.max_token_per_leaf_module,
176178
'max_depth': self.max_depth,
177179
'use_gitignore': self.use_gitignore,
180+
'prompt_caching': self.prompt_caching,
178181
'fallback_model': self.fallback_model,
179182
}
180183
if self.agent_instructions and not self.agent_instructions.is_empty():
@@ -211,6 +214,7 @@ def from_dict(cls, data: dict) -> 'Configuration':
211214
max_token_per_leaf_module=data.get('max_token_per_leaf_module', 16000),
212215
max_depth=data.get('max_depth', 2),
213216
use_gitignore=data.get('use_gitignore', True),
217+
prompt_caching=data.get('prompt_caching', True),
214218
agent_instructions=agent_instructions,
215219
)
216220

@@ -279,4 +283,5 @@ def to_backend_config(self, repo_path: str, output_dir: str, api_key: str, runti
279283
max_depth=self.max_depth,
280284
agent_instructions=final_instructions.to_dict() if final_instructions else None,
281285
use_gitignore=self.use_gitignore,
286+
prompt_caching=self.prompt_caching,
282287
)

‎codewiki/src/be/backend.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,6 @@ def complete(
4141
prompt: str,
4242
*,
4343
model: str | None = None,
44-
temperature: float = 0.0,
4544
) -> str:
4645
"""Single-shot text completion."""
4746

‎codewiki/src/be/caw_backend.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -245,7 +245,6 @@ def complete(
245245
prompt: str,
246246
*,
247247
model: str | None = None,
248-
temperature: float = 0.0, # unused: subscription CLIs don't expose temperature
249248
) -> str:
250249
# Blocks the calling thread for the lifetime of the claude/codex
251250
# subprocess. Callers running this from an async context (e.g. the

0 commit comments

Comments
 (0)