From ec4fc04e9f4d018a6babbb6999289c0c92b24f32 Mon Sep 17 00:00:00 2001 From: waldekmastykarz Date: Sat, 31 Oct 2026 10:36:45 +0100 Subject: [PATCH 1/4] Support bring-your-own LLM for language model integration. Closes #2034 - Add `apiKey` to `languageModel` config with `@env:VAR_NAME` support. Sent as a Bearer token (OpenAI-compatible, Ollama) or `api-key` header (Azure OpenAI). The key is never logged. - Add `AzureOpenAI` client targeting the Azure OpenAI v1 API. - Check availability with `GET /models` instead of a billable completion, falling back to the completion probe when /models is unsupported or doesn't list the configured model. - Update v4.0.0 rc.schema.json and docs. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../OpenAILanguageModelClientTests.cs | 188 ++++++++++++++++++ .../LanguageModel/BaseLanguageModelClient.cs | 53 +++++ .../LanguageModelClientFactory.cs | 1 + .../LanguageModelConfiguration.cs | 13 +- .../OllamaLanguageModelClient.cs | 25 ++- .../OpenAILanguageModelClient.cs | 105 +++++++++- schemas/v4.0.0/rc.schema.json | 15 +- skills/dev-proxy/references/configuration.md | 16 +- skills/dev-proxy/references/test-llm-apps.md | 7 +- 9 files changed, 402 insertions(+), 21 deletions(-) create mode 100644 DevProxy.Abstractions.Tests/LanguageModel/OpenAILanguageModelClientTests.cs diff --git a/DevProxy.Abstractions.Tests/LanguageModel/OpenAILanguageModelClientTests.cs b/DevProxy.Abstractions.Tests/LanguageModel/OpenAILanguageModelClientTests.cs new file mode 100644 index 000000000..3cf5f18f1 --- /dev/null +++ b/DevProxy.Abstractions.Tests/LanguageModel/OpenAILanguageModelClientTests.cs @@ -0,0 +1,188 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using DevProxy.Abstractions.LanguageModel; +using Microsoft.Extensions.Logging.Abstractions; +using System.Net; +using System.Text; +using Xunit; + +namespace DevProxy.Abstractions.Tests.LanguageModel; + +public class OpenAILanguageModelClientTests +{ + private const string ChatCompletionResponse = """ + {"id":"1","object":"chat.completion","created":0,"model":"m","choices":[{"index":0,"finish_reason":"stop","message":{"role":"assistant","content":"yes"}}]} + """; + + private sealed class FakeHandler(Func respond) : HttpMessageHandler + { + public List Requests { get; } = []; + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + Requests.Add(request); + return Task.FromResult(respond(request)); + } + } + + private static HttpResponseMessage Json(HttpStatusCode status, string body) => + new(status) { Content = new StringContent(body, Encoding.UTF8, "application/json") }; + + private static (OpenAILanguageModelClient client, FakeHandler handler) CreateClient( + LanguageModelConfiguration config, + Func respond) + { + var handler = new FakeHandler(respond); +#pragma warning disable CA2000 // HttpClient lifetime is tied to the test + var httpClient = new HttpClient(handler); +#pragma warning restore CA2000 + return (new OpenAILanguageModelClient(httpClient, config, NullLogger.Instance), handler); + } + + private static LanguageModelConfiguration Config(LanguageModelClient client = LanguageModelClient.OpenAI, string? apiKey = null) => new() + { + Enabled = true, + Client = client, + Model = "gpt-4.1-mini", + Url = "https://example.test/v1/", + ApiKey = apiKey + }; + + [Fact] + public async Task IsEnabled_ModelListed_DoesNotSendCompletion() + { + var (client, handler) = CreateClient(Config(apiKey: "secret"), _ => + Json(HttpStatusCode.OK, """{"data":[{"id":"gpt-4.1-mini"}]}""")); + + Assert.True(await client.IsEnabledAsync(CancellationToken.None)); + + var request = Assert.Single(handler.Requests); + Assert.Equal(HttpMethod.Get, request.Method); + Assert.Equal("https://example.test/v1/models", request.RequestUri!.ToString()); + Assert.Equal("Bearer", request.Headers.Authorization?.Scheme); + Assert.Equal("secret", request.Headers.Authorization?.Parameter); + } + + [Fact] + public async Task IsEnabled_OllamaTaggedModelListed_DoesNotSendCompletion() + { + var config = Config(); + config.Model = "llama3.2"; + var (client, handler) = CreateClient(config, _ => + Json(HttpStatusCode.OK, """{"data":[{"id":"llama3.2:latest"}]}""")); + + Assert.True(await client.IsEnabledAsync(CancellationToken.None)); + _ = Assert.Single(handler.Requests); + } + + [Theory] + [InlineData(HttpStatusCode.Unauthorized)] + [InlineData(HttpStatusCode.Forbidden)] + public async Task IsEnabled_Unauthorized_IsDisabled(HttpStatusCode status) + { + var (client, handler) = CreateClient(Config(apiKey: "wrong"), _ => Json(status, "{}")); + + Assert.False(await client.IsEnabledAsync(CancellationToken.None)); + _ = Assert.Single(handler.Requests); + } + + [Fact] + public async Task IsEnabled_ModelsNotSupported_FallsBackToCompletion() + { + var (client, handler) = CreateClient(Config(), r => + r.Method == HttpMethod.Get + ? Json(HttpStatusCode.NotFound, "{}") + : Json(HttpStatusCode.OK, ChatCompletionResponse)); + + Assert.True(await client.IsEnabledAsync(CancellationToken.None)); + Assert.Equal(2, handler.Requests.Count); + Assert.Equal("https://example.test/v1/chat/completions", handler.Requests[1].RequestUri!.ToString()); + } + + [Fact] + public async Task IsEnabled_ModelNotListed_FallsBackToCompletion() + { + var (client, handler) = CreateClient(Config(), r => + r.Method == HttpMethod.Get + ? Json(HttpStatusCode.OK, """{"data":[{"id":"other"}]}""") + : Json(HttpStatusCode.OK, ChatCompletionResponse)); + + Assert.True(await client.IsEnabledAsync(CancellationToken.None)); + Assert.Equal(2, handler.Requests.Count); + } + + [Fact] + public async Task IsEnabled_AzureOpenAI_UsesApiKeyHeaderAndSkipsModelMatch() + { + var (client, handler) = CreateClient(Config(LanguageModelClient.AzureOpenAI, "azure-secret"), _ => + Json(HttpStatusCode.OK, """{"data":[{"id":"gpt-4.1"}]}""")); + + Assert.True(await client.IsEnabledAsync(CancellationToken.None)); + + var request = Assert.Single(handler.Requests); + Assert.Null(request.Headers.Authorization); + Assert.Equal("azure-secret", Assert.Single(request.Headers.GetValues("api-key"))); + } + + [Fact] + public async Task IsEnabled_AzureOpenAIWithoutApiKey_IsDisabled() + { + var (client, handler) = CreateClient(Config(LanguageModelClient.AzureOpenAI), _ => Json(HttpStatusCode.OK, "{}")); + + Assert.False(await client.IsEnabledAsync(CancellationToken.None)); + Assert.Empty(handler.Requests); + } + + [Fact] + public async Task ApiKey_EnvReference_IsResolvedFromEnvironment() + { + var envVar = $"DEVPROXY_TEST_LM_KEY_{Guid.NewGuid():N}"; + Environment.SetEnvironmentVariable(envVar, "from-env"); + try + { + var (client, handler) = CreateClient(Config(apiKey: $"@env:{envVar}"), _ => + Json(HttpStatusCode.OK, """{"data":[{"id":"gpt-4.1-mini"}]}""")); + + Assert.True(await client.IsEnabledAsync(CancellationToken.None)); + Assert.Equal("from-env", handler.Requests[0].Headers.Authorization?.Parameter); + } + finally + { + Environment.SetEnvironmentVariable(envVar, null); + } + } + + [Fact] + public async Task ApiKey_MissingEnvVar_SendsNoAuthHeader() + { + var (client, handler) = CreateClient(Config(apiKey: "@env:DEVPROXY_TEST_LM_KEY_DOES_NOT_EXIST"), _ => + Json(HttpStatusCode.OK, """{"data":[{"id":"gpt-4.1-mini"}]}""")); + + Assert.True(await client.IsEnabledAsync(CancellationToken.None)); + Assert.Null(handler.Requests[0].Headers.Authorization); + } + + [Fact] + public async Task ChatCompletion_SendsAuthHeader() + { + var (client, handler) = CreateClient(Config(apiKey: "secret"), r => + r.Method == HttpMethod.Get + ? Json(HttpStatusCode.OK, """{"data":[{"id":"gpt-4.1-mini"}]}""") + : Json(HttpStatusCode.OK, ChatCompletionResponse)); + + var response = await client.GenerateChatCompletionAsync( + [new OpenAIChatCompletionMessage { Role = "user", Content = "hi" }], null, CancellationToken.None); + + Assert.Equal("yes", response?.Response); + var post = handler.Requests.Single(r => r.Method == HttpMethod.Post); + Assert.Equal("secret", post.Headers.Authorization?.Parameter); + } + + [Fact] + public void ToString_DoesNotExposeApiKey() + { + Assert.DoesNotContain("secret", Config(apiKey: "secret").ToString(), StringComparison.Ordinal); + } +} diff --git a/DevProxy.Abstractions/LanguageModel/BaseLanguageModelClient.cs b/DevProxy.Abstractions/LanguageModel/BaseLanguageModelClient.cs index 93a2574a5..b2e772e5c 100644 --- a/DevProxy.Abstractions/LanguageModel/BaseLanguageModelClient.cs +++ b/DevProxy.Abstractions/LanguageModel/BaseLanguageModelClient.cs @@ -15,6 +15,10 @@ public abstract class BaseLanguageModelClient(LanguageModelConfiguration configu protected ILogger Logger { get; } = logger; private bool? _lmAvailable; + private readonly Lazy _apiKey = new(() => ResolveApiKey(configuration, logger)); + + // Never log this value + protected string? ApiKey => _apiKey.Value; private readonly ConcurrentDictionary?, CompletionOptions?)> _promptCache = new(); @@ -130,6 +134,55 @@ public async Task IsEnabledAsync(CancellationToken cancellationToken) return (messages, options); } + protected void AddAuthenticationHeader(HttpRequestMessage request) + { + ArgumentNullException.ThrowIfNull(request); + + if (string.IsNullOrEmpty(ApiKey)) + { + return; + } + + if (Configuration.Client == LanguageModelClient.AzureOpenAI) + { + _ = request.Headers.TryAddWithoutValidation("api-key", ApiKey); + } + else + { + request.Headers.Authorization = new("Bearer", ApiKey); + } + } + + private static string? ResolveApiKey(LanguageModelConfiguration? configuration, ILogger logger) + { + var apiKey = configuration?.ApiKey; + if (string.IsNullOrWhiteSpace(apiKey)) + { + return null; + } + + if (!apiKey.StartsWith(LanguageModelConfiguration.EnvVarPrefix, StringComparison.OrdinalIgnoreCase)) + { + return apiKey; + } + + var envVarName = apiKey[LanguageModelConfiguration.EnvVarPrefix.Length..].Trim(); + if (string.IsNullOrEmpty(envVarName)) + { + logger.LogWarning("Language model API key is set to {Prefix} but no environment variable name was specified", LanguageModelConfiguration.EnvVarPrefix); + return null; + } + + var value = Environment.GetEnvironmentVariable(envVarName); + if (string.IsNullOrEmpty(value)) + { + logger.LogWarning("Environment variable {EnvVarName} for the language model API key is not set", envVarName); + return null; + } + + return value; + } + private static string GetPromptCacheKey(string promptFileName, Dictionary parameters) { var parametersString = string.Join(";", parameters.Select(kvp => $"{kvp.Key}={kvp.Value}")); diff --git a/DevProxy.Abstractions/LanguageModel/LanguageModelClientFactory.cs b/DevProxy.Abstractions/LanguageModel/LanguageModelClientFactory.cs index 8c955194b..c97c8c9ec 100644 --- a/DevProxy.Abstractions/LanguageModel/LanguageModelClientFactory.cs +++ b/DevProxy.Abstractions/LanguageModel/LanguageModelClientFactory.cs @@ -21,6 +21,7 @@ public static ILanguageModelClient Create(IServiceProvider serviceProvider, ICon { LanguageModelClient.Ollama => ActivatorUtilities.CreateInstance(serviceProvider, config), LanguageModelClient.OpenAI => ActivatorUtilities.CreateInstance(serviceProvider, config), + LanguageModelClient.AzureOpenAI => ActivatorUtilities.CreateInstance(serviceProvider, config), _ => ActivatorUtilities.CreateInstance(serviceProvider, config) }; } diff --git a/DevProxy.Abstractions/LanguageModel/LanguageModelConfiguration.cs b/DevProxy.Abstractions/LanguageModel/LanguageModelConfiguration.cs index 208c7ba7a..2e390c9f3 100644 --- a/DevProxy.Abstractions/LanguageModel/LanguageModelConfiguration.cs +++ b/DevProxy.Abstractions/LanguageModel/LanguageModelConfiguration.cs @@ -7,15 +7,24 @@ namespace DevProxy.Abstractions.LanguageModel; public enum LanguageModelClient { Ollama, - OpenAI + OpenAI, + AzureOpenAI } public class LanguageModelConfiguration { + internal const string EnvVarPrefix = "@env:"; + + // API key used to authenticate with hosted language models. + // Use @env:VAR_NAME to read the key from an environment variable. + public string? ApiKey { get; set; } public bool CacheResponses { get; set; } = true; public LanguageModelClient Client { get; set; } = LanguageModelClient.OpenAI; public bool Enabled { get; set; } public string Model { get; set; } = "llama3.2"; // default Ollama URL public string? Url { get; set; } = "http://localhost:11434/v1/"; -} \ No newline at end of file + + public override string ToString() => + $"{nameof(LanguageModelConfiguration)} {{ Client = {Client}, Enabled = {Enabled}, Model = {Model}, Url = {Url}, ApiKey = {(string.IsNullOrEmpty(ApiKey) ? "" : "***")} }}"; +} diff --git a/DevProxy.Abstractions/LanguageModel/OllamaLanguageModelClient.cs b/DevProxy.Abstractions/LanguageModel/OllamaLanguageModelClient.cs index e6b7377e4..3d595a705 100644 --- a/DevProxy.Abstractions/LanguageModel/OllamaLanguageModelClient.cs +++ b/DevProxy.Abstractions/LanguageModel/OllamaLanguageModelClient.cs @@ -139,16 +139,19 @@ protected override async Task IsEnabledCoreAsync(CancellationToken cancell var url = $"{Configuration.Url?.TrimEnd('/')}/api/generate"; Logger.LogDebug("Requesting completion. Prompt: {Prompt}", prompt); - var response = await _httpClient.PostAsJsonAsync(url, - new + using var request = new HttpRequestMessage(HttpMethod.Post, url) + { + Content = JsonContent.Create(new { prompt, model = Configuration.Model, stream = false, options - }, - cancellationToken - ); + }) + }; + AddAuthenticationHeader(request); + + using var response = await _httpClient.SendAsync(request, cancellationToken); Logger.LogDebug("Response status: {Response}", response.StatusCode); if (!response.IsSuccessStatusCode) @@ -189,15 +192,19 @@ protected override async Task IsEnabledCoreAsync(CancellationToken cancell Logger.LogDebug("Requesting chat completion. Message: {LastMessage}", messages.Last().Content); } - var response = await _httpClient.PostAsJsonAsync(url, - new + using var request = new HttpRequestMessage(HttpMethod.Post, url) + { + Content = JsonContent.Create(new { messages, model = Configuration.Model, stream = false, options - } - ); + }) + }; + AddAuthenticationHeader(request); + + using var response = await _httpClient.SendAsync(request); Logger.LogDebug("Response: {Response}", response.StatusCode); if (!response.IsSuccessStatusCode) diff --git a/DevProxy.Abstractions/LanguageModel/OpenAILanguageModelClient.cs b/DevProxy.Abstractions/LanguageModel/OpenAILanguageModelClient.cs index fb4eed8b4..2c0bd159d 100644 --- a/DevProxy.Abstractions/LanguageModel/OpenAILanguageModelClient.cs +++ b/DevProxy.Abstractions/LanguageModel/OpenAILanguageModelClient.cs @@ -6,7 +6,9 @@ using DevProxy.Abstractions.Utils; using Microsoft.Extensions.Logging; using System.Diagnostics; +using System.Net; using System.Net.Http.Json; +using System.Text.Json; namespace DevProxy.Abstractions.LanguageModel; @@ -120,10 +122,24 @@ protected override async Task IsEnabledCoreAsync(CancellationToken cancell return false; } + if (Configuration.Client == LanguageModelClient.AzureOpenAI && string.IsNullOrEmpty(ApiKey)) + { + _logger.LogError("API key is not set. Azure OpenAI requires an API key. Language model will be disabled"); + return false; + } + _logger.LogDebug("Checking LM availability at {Url}...", Configuration.Url); try { + var modelsCheck = await CheckModelsEndpointAsync(cancellationToken); + if (modelsCheck.HasValue) + { + return modelsCheck.Value; + } + + // Fall back to a completion request when /models isn't supported + // or doesn't list the configured model var testCompletion = await GenerateChatCompletionInternalAsync([new() { Content = "Are you there? Reply with a yes or no.", @@ -144,6 +160,87 @@ protected override async Task IsEnabledCoreAsync(CancellationToken cancell } } + // Returns true/false when the /models endpoint gives a definitive answer, + // null when availability couldn't be determined this way + private async Task CheckModelsEndpointAsync(CancellationToken cancellationToken) + { + var url = $"{Configuration.Url?.TrimEnd('/')}/models"; + _logger.LogDebug("Checking available models at {Url}...", url); + + using var request = new HttpRequestMessage(HttpMethod.Get, url); + AddAuthenticationHeader(request); + + HttpResponseMessage response; + try + { + response = await _httpClient.SendAsync(request, cancellationToken); + } + catch (HttpRequestException ex) + { + _logger.LogError(ex, "Couldn't reach language model at {Url}", Configuration.Url); + return false; + } + + using (response) + { + _logger.LogDebug("Response: {Response}", response.StatusCode); + + if (response.StatusCode is HttpStatusCode.Unauthorized or HttpStatusCode.Forbidden) + { + _logger.LogError("Language model at {Url} rejected the request ({StatusCode}). Check the configured API key", Configuration.Url, (int)response.StatusCode); + return false; + } + + if (!response.IsSuccessStatusCode) + { + _logger.LogDebug("Models endpoint returned {StatusCode}. Falling back to a completion request", (int)response.StatusCode); + return null; + } + + // Azure OpenAI lists base models rather than deployments, so the + // configured deployment name can't be matched against the list + if (Configuration.Client == LanguageModelClient.AzureOpenAI) + { + return true; + } + + try + { + using var json = await JsonDocument.ParseAsync( + await response.Content.ReadAsStreamAsync(cancellationToken), + cancellationToken: cancellationToken); + if (json.RootElement.TryGetProperty("data", out var data) && + data.ValueKind == JsonValueKind.Array && + data.EnumerateArray().Any(m => + m.TryGetProperty("id", out var id) && + IsConfiguredModel(id.GetString()))) + { + return true; + } + } + catch (JsonException ex) + { + _logger.LogDebug(ex, "Couldn't parse models response"); + return null; + } + + _logger.LogDebug("Model {Model} not found in the models list. Falling back to a completion request", Configuration.Model); + return null; + } + } + + private bool IsConfiguredModel(string? modelId) + { + if (string.IsNullOrEmpty(modelId)) + { + return false; + } + + return string.Equals(modelId, Configuration.Model, StringComparison.OrdinalIgnoreCase) || + // Ollama lists models with their tag, eg. llama3.2:latest + string.Equals(modelId, $"{Configuration.Model}:latest", StringComparison.OrdinalIgnoreCase); + } + private async Task GenerateChatCompletionInternalAsync(OpenAIChatCompletionMessage[] messages, CompletionOptions? options, CancellationToken cancellationToken = default) { Debug.Assert(Configuration != null, "Configuration is null"); @@ -164,7 +261,13 @@ protected override async Task IsEnabledCoreAsync(CancellationToken cancell Temperature = options?.Temperature }; - var response = await _httpClient.PostAsJsonAsync(url, payload, ProxyUtils.JsonSerializerOptions, cancellationToken); + using var request = new HttpRequestMessage(HttpMethod.Post, url) + { + Content = JsonContent.Create(payload, options: ProxyUtils.JsonSerializerOptions) + }; + AddAuthenticationHeader(request); + + using var response = await _httpClient.SendAsync(request, cancellationToken); _logger.LogDebug("Response: {Response}", response.StatusCode); if (!response.IsSuccessStatusCode) diff --git a/schemas/v4.0.0/rc.schema.json b/schemas/v4.0.0/rc.schema.json index 4db88db85..88d4a2331 100644 --- a/schemas/v4.0.0/rc.schema.json +++ b/schemas/v4.0.0/rc.schema.json @@ -81,8 +81,12 @@ }, "languageModel": { "type": "object", - "description": "Configuration for using a local language model with Dev Proxy.", + "description": "Configuration for using a language model with Dev Proxy. Supports local models (eg. Ollama) and hosted OpenAI-compatible or Azure OpenAI endpoints.", "properties": { + "apiKey": { + "type": "string", + "description": "API key for hosted language models. Sent as a Bearer token for OpenAI-compatible and Ollama clients and as the api-key header for Azure OpenAI. Use @env:VAR_NAME to read the key from an environment variable and keep it out of the config file." + }, "cacheResponses": { "type": "boolean", "description": "Whether to cache responses from the language model." @@ -91,9 +95,10 @@ "type": "string", "enum": [ "Ollama", - "OpenAI" + "OpenAI", + "AzureOpenAI" ], - "description": "The client to use for the local language model." + "description": "The client to use for the language model. Use OpenAI for any OpenAI-compatible endpoint (eg. Ollama, OpenAI, GitHub Models, OpenRouter). Use AzureOpenAI for Azure OpenAI / Microsoft Foundry v1 endpoints." }, "enabled": { "type": "boolean", @@ -101,11 +106,11 @@ }, "model": { "type": "string", - "description": "The name of the language model to use." + "description": "The name of the language model to use. For Azure OpenAI, the name of the model deployment." }, "url": { "type": "string", - "description": "URL of the local language model server." + "description": "Base URL of the language model API, eg. http://localhost:11434/v1/ for Ollama, https://api.openai.com/v1/ for OpenAI or https://.openai.azure.com/openai/v1/ for Azure OpenAI." } } }, diff --git a/skills/dev-proxy/references/configuration.md b/skills/dev-proxy/references/configuration.md index 7cbd22228..77c3ed9eb 100644 --- a/skills/dev-proxy/references/configuration.md +++ b/skills/dev-proxy/references/configuration.md @@ -173,7 +173,7 @@ For the beta image, use `devproxy-beta` instead of `devproxy`. Add `--pid ` ## Local Language Model -Enable a local LLM (e.g., Ollama) to improve AI-powered plugins: +Enable a local LLM (e.g., Ollama) or a hosted model to improve AI-powered plugins: ```json { @@ -187,6 +187,20 @@ Enable a local LLM (e.g., Ollama) to improve AI-powered plugins: } ``` +To use a hosted model, set `url` and `apiKey`. Use `@env:VAR_NAME` to keep the key out of the config file. `OpenAI` works with any OpenAI-compatible endpoint (OpenAI, GitHub Models, OpenRouter) and sends the key as a Bearer token. `AzureOpenAI` sends it in the `api-key` header and expects the v1 endpoint, with `model` set to the deployment name: + +```json +{ + "languageModel": { + "enabled": true, + "client": "AzureOpenAI", + "model": "gpt-4.1-mini", + "url": "https://.openai.azure.com/openai/v1/", + "apiKey": "@env:AZURE_OPENAI_API_KEY" + } +} +``` + Used by: `OpenAIMockResponsePlugin`, `OpenApiSpecGeneratorPlugin`, `TypeSpecGeneratorPlugin`. ## Detached Mode diff --git a/skills/dev-proxy/references/test-llm-apps.md b/skills/dev-proxy/references/test-llm-apps.md index 0fe504d1e..3fae254af 100644 --- a/skills/dev-proxy/references/test-llm-apps.md +++ b/skills/dev-proxy/references/test-llm-apps.md @@ -52,10 +52,11 @@ ollama serve | Option | Description | Default | |--------|-------------|---------| -| `enabled` | Enable local language model | `false` | -| `client` | Client type: `Ollama` or `OpenAI` | `OpenAI` | -| `model` | Model name | `llama3.2` | +| `enabled` | Enable language model | `false` | +| `client` | Client type: `Ollama`, `OpenAI` (any OpenAI-compatible endpoint), or `AzureOpenAI` | `OpenAI` | +| `model` | Model name (for Azure OpenAI: deployment name) | `llama3.2` | | `url` | Language model endpoint URL | `http://localhost:11434/v1/` | +| `apiKey` | API key for hosted models. Use `@env:VAR_NAME` to read it from an environment variable | (none) | | `cacheResponses` | Cache responses for identical prompts | `true` | ## Simulate LLM Failures From 7fc83e48d6ad67471eb8e251da11149ec63355fb Mon Sep 17 00:00:00 2001 From: waldekmastykarz Date: Sat, 31 Oct 2026 10:59:02 +0100 Subject: [PATCH 2/4] Address review feedback: redact API key in logs, harden /models probe - Redact the configured API key from logged provider errors - Fall back to a completion probe on /models 403 (restricted keys); 401 still disables - Disable the LM when the fallback completion fails - Guard /models JSON parsing against unexpected payload shapes Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../OpenAILanguageModelClientTests.cs | 79 +++++++++++++++++-- .../LanguageModel/BaseLanguageModelClient.cs | 10 +++ .../OllamaLanguageModelClient.cs | 10 +-- .../OpenAILanguageModelClient.cs | 22 ++++-- 4 files changed, 104 insertions(+), 17 deletions(-) diff --git a/DevProxy.Abstractions.Tests/LanguageModel/OpenAILanguageModelClientTests.cs b/DevProxy.Abstractions.Tests/LanguageModel/OpenAILanguageModelClientTests.cs index 3cf5f18f1..dd6e3f3d2 100644 --- a/DevProxy.Abstractions.Tests/LanguageModel/OpenAILanguageModelClientTests.cs +++ b/DevProxy.Abstractions.Tests/LanguageModel/OpenAILanguageModelClientTests.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information. using DevProxy.Abstractions.LanguageModel; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using System.Net; using System.Text; @@ -27,6 +28,18 @@ protected override Task SendAsync(HttpRequestMessage reques } } + private sealed class CapturingLogger : ILogger + { + public List Messages { get; } = []; + + public IDisposable? BeginScope(TState state) where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) => + Messages.Add(formatter(state, exception)); + } + private static HttpResponseMessage Json(HttpStatusCode status, string body) => new(status) { Content = new StringContent(body, Encoding.UTF8, "application/json") }; @@ -77,17 +90,73 @@ public async Task IsEnabled_OllamaTaggedModelListed_DoesNotSendCompletion() _ = Assert.Single(handler.Requests); } - [Theory] - [InlineData(HttpStatusCode.Unauthorized)] - [InlineData(HttpStatusCode.Forbidden)] - public async Task IsEnabled_Unauthorized_IsDisabled(HttpStatusCode status) + [Fact] + public async Task IsEnabled_Unauthorized_IsDisabled() { - var (client, handler) = CreateClient(Config(apiKey: "wrong"), _ => Json(status, "{}")); + var (client, handler) = CreateClient(Config(apiKey: "wrong"), _ => Json(HttpStatusCode.Unauthorized, "{}")); Assert.False(await client.IsEnabledAsync(CancellationToken.None)); _ = Assert.Single(handler.Requests); } + [Fact] + public async Task IsEnabled_ModelsForbidden_FallsBackToCompletion() + { + var (client, handler) = CreateClient(Config(apiKey: "restricted"), r => + r.Method == HttpMethod.Get + ? Json(HttpStatusCode.Forbidden, "{}") + : Json(HttpStatusCode.OK, ChatCompletionResponse)); + + Assert.True(await client.IsEnabledAsync(CancellationToken.None)); + Assert.Equal(2, handler.Requests.Count); + } + + [Fact] + public async Task IsEnabled_FallbackCompletionFails_IsDisabled() + { + var (client, handler) = CreateClient(Config(), r => + r.Method == HttpMethod.Get + ? Json(HttpStatusCode.NotFound, "{}") + : Json(HttpStatusCode.InternalServerError, "{}")); + + Assert.False(await client.IsEnabledAsync(CancellationToken.None)); + Assert.Equal(2, handler.Requests.Count); + } + + [Theory] + [InlineData("[]")] + [InlineData("""{"data":{}}""")] + [InlineData("""{"data":[1]}""")] + [InlineData("""{"data":[{"id":42}]}""")] + public async Task IsEnabled_UnexpectedModelsPayload_FallsBackToCompletion(string payload) + { + var (client, handler) = CreateClient(Config(), r => + r.Method == HttpMethod.Get + ? Json(HttpStatusCode.OK, payload) + : Json(HttpStatusCode.OK, ChatCompletionResponse)); + + Assert.True(await client.IsEnabledAsync(CancellationToken.None)); + Assert.Equal(2, handler.Requests.Count); + } + + [Fact] + public async Task ErrorResponse_EchoingApiKey_IsRedactedInLogs() + { + var logger = new CapturingLogger(); + var handler = new FakeHandler(r => + r.Method == HttpMethod.Get + ? Json(HttpStatusCode.OK, """{"data":[{"id":"gpt-4.1-mini"}]}""") + : Json(HttpStatusCode.BadRequest, """{"error":{"message":"bad key super-secret-key"}}""")); + using var httpClient = new HttpClient(handler); + var client = new OpenAILanguageModelClient(httpClient, Config(apiKey: "super-secret-key"), logger); + + _ = await client.GenerateChatCompletionAsync( + [new OpenAIChatCompletionMessage { Role = "user", Content = "hi" }], null, CancellationToken.None); + + Assert.Contains(logger.Messages, m => m.Contains("bad key ***", StringComparison.Ordinal)); + Assert.DoesNotContain(logger.Messages, m => m.Contains("super-secret-key", StringComparison.Ordinal)); + } + [Fact] public async Task IsEnabled_ModelsNotSupported_FallsBackToCompletion() { diff --git a/DevProxy.Abstractions/LanguageModel/BaseLanguageModelClient.cs b/DevProxy.Abstractions/LanguageModel/BaseLanguageModelClient.cs index b2e772e5c..0f9075216 100644 --- a/DevProxy.Abstractions/LanguageModel/BaseLanguageModelClient.cs +++ b/DevProxy.Abstractions/LanguageModel/BaseLanguageModelClient.cs @@ -134,6 +134,16 @@ public async Task IsEnabledAsync(CancellationToken cancellationToken) return (messages, options); } + protected string? RedactApiKey(string? value) + { + if (string.IsNullOrEmpty(value) || string.IsNullOrEmpty(ApiKey)) + { + return value; + } + + return value.Replace(ApiKey, "***", StringComparison.Ordinal); + } + protected void AddAuthenticationHeader(HttpRequestMessage request) { ArgumentNullException.ThrowIfNull(request); diff --git a/DevProxy.Abstractions/LanguageModel/OllamaLanguageModelClient.cs b/DevProxy.Abstractions/LanguageModel/OllamaLanguageModelClient.cs index 3d595a705..d28df223d 100644 --- a/DevProxy.Abstractions/LanguageModel/OllamaLanguageModelClient.cs +++ b/DevProxy.Abstractions/LanguageModel/OllamaLanguageModelClient.cs @@ -36,7 +36,7 @@ public sealed class OllamaLanguageModelClient( if (response.Error is not null) { - Logger.LogError("{Error}", response.Error); + Logger.LogError("{Error}", RedactApiKey(response.Error)); return null; } @@ -68,7 +68,7 @@ public sealed class OllamaLanguageModelClient( } if (response.Error is not null) { - Logger.LogError("{Error}", response.Error); + Logger.LogError("{Error}", RedactApiKey(response.Error)); return null; } else @@ -117,7 +117,7 @@ protected override async Task IsEnabledCoreAsync(CancellationToken cancell var testCompletion = await GenerateCompletionInternalAsync("Are you there? Reply with a yes or no.", null, cancellationToken); if (testCompletion?.Error is not null) { - Logger.LogError("Error: {Error}", testCompletion.Error); + Logger.LogError("Error: {Error}", RedactApiKey(testCompletion.Error)); return false; } @@ -157,7 +157,7 @@ protected override async Task IsEnabledCoreAsync(CancellationToken cancell if (!response.IsSuccessStatusCode) { var errorResponse = await response.Content.ReadAsStringAsync(cancellationToken); - Logger.LogDebug("LM error: {ErrorResponse}", errorResponse); + Logger.LogDebug("LM error: {ErrorResponse}", RedactApiKey(errorResponse)); return null; } @@ -210,7 +210,7 @@ protected override async Task IsEnabledCoreAsync(CancellationToken cancell if (!response.IsSuccessStatusCode) { var errorResponse = await response.Content.ReadAsStringAsync(); - Logger.LogDebug("LM error: {ErrorResponse}", errorResponse); + Logger.LogDebug("LM error: {ErrorResponse}", RedactApiKey(errorResponse)); return null; } diff --git a/DevProxy.Abstractions/LanguageModel/OpenAILanguageModelClient.cs b/DevProxy.Abstractions/LanguageModel/OpenAILanguageModelClient.cs index 2c0bd159d..d40dbb647 100644 --- a/DevProxy.Abstractions/LanguageModel/OpenAILanguageModelClient.cs +++ b/DevProxy.Abstractions/LanguageModel/OpenAILanguageModelClient.cs @@ -30,7 +30,7 @@ public sealed class OpenAILanguageModelClient( } if (response.ErrorMessage is not null) { - _logger.LogError("Error: {Error}", response.ErrorMessage); + _logger.LogError("Error: {Error}", RedactApiKey(response.ErrorMessage)); return null; } var openAIResponse = (OpenAIChatCompletionResponse)response; @@ -78,7 +78,7 @@ public sealed class OpenAILanguageModelClient( } if (response.Error is not null) { - _logger.LogError("Error: {Error}. Code: {Code}", response.Error.Message, response.Error.Code); + _logger.LogError("Error: {Error}. Code: {Code}", RedactApiKey(response.Error.Message), response.Error.Code); return null; } else @@ -145,9 +145,14 @@ protected override async Task IsEnabledCoreAsync(CancellationToken cancell Content = "Are you there? Reply with a yes or no.", Role = "user" }], null, cancellationToken); - if (testCompletion?.ErrorMessage is not null) + if (testCompletion is null) { - _logger.LogError("Error: {Error}", testCompletion.ErrorMessage); + _logger.LogError("Language model at {Url} didn't return a valid completion. Language model will be disabled", Configuration.Url); + return false; + } + if (testCompletion.ErrorMessage is not null) + { + _logger.LogError("Error: {Error}", RedactApiKey(testCompletion.ErrorMessage)); return false; } @@ -185,7 +190,7 @@ protected override async Task IsEnabledCoreAsync(CancellationToken cancell { _logger.LogDebug("Response: {Response}", response.StatusCode); - if (response.StatusCode is HttpStatusCode.Unauthorized or HttpStatusCode.Forbidden) + if (response.StatusCode is HttpStatusCode.Unauthorized) { _logger.LogError("Language model at {Url} rejected the request ({StatusCode}). Check the configured API key", Configuration.Url, (int)response.StatusCode); return false; @@ -209,10 +214,13 @@ protected override async Task IsEnabledCoreAsync(CancellationToken cancell using var json = await JsonDocument.ParseAsync( await response.Content.ReadAsStreamAsync(cancellationToken), cancellationToken: cancellationToken); - if (json.RootElement.TryGetProperty("data", out var data) && + if (json.RootElement.ValueKind == JsonValueKind.Object && + json.RootElement.TryGetProperty("data", out var data) && data.ValueKind == JsonValueKind.Array && data.EnumerateArray().Any(m => + m.ValueKind == JsonValueKind.Object && m.TryGetProperty("id", out var id) && + id.ValueKind == JsonValueKind.String && IsConfiguredModel(id.GetString()))) { return true; @@ -273,7 +281,7 @@ private bool IsConfiguredModel(string? modelId) if (!response.IsSuccessStatusCode) { var errorResponse = await response.Content.ReadAsStringAsync(cancellationToken); - _logger.LogDebug("LM error: {ErrorResponse}", errorResponse); + _logger.LogDebug("LM error: {ErrorResponse}", RedactApiKey(errorResponse)); return null; } From ff88f97a7b6e3a6c293634c66f4e90955783a9c4 Mon Sep 17 00:00:00 2001 From: waldekmastykarz Date: Sat, 31 Oct 2026 11:07:57 +0100 Subject: [PATCH 3/4] Disable redirects for language model HttpClient Prevents forwarding the api-key header to another origin on redirect. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../LanguageModelClientFactoryTests.cs | 98 +++++++++++++++++++ .../LanguageModelClientFactory.cs | 22 ++++- 2 files changed, 116 insertions(+), 4 deletions(-) create mode 100644 DevProxy.Abstractions.Tests/LanguageModel/LanguageModelClientFactoryTests.cs diff --git a/DevProxy.Abstractions.Tests/LanguageModel/LanguageModelClientFactoryTests.cs b/DevProxy.Abstractions.Tests/LanguageModel/LanguageModelClientFactoryTests.cs new file mode 100644 index 000000000..c76829489 --- /dev/null +++ b/DevProxy.Abstractions.Tests/LanguageModel/LanguageModelClientFactoryTests.cs @@ -0,0 +1,98 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using DevProxy.Abstractions.LanguageModel; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using System.Net; +using System.Net.Sockets; +using Xunit; + +namespace DevProxy.Abstractions.Tests.LanguageModel; + +public class LanguageModelClientFactoryTests +{ + private static int GetFreePort() + { + using var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + return ((IPEndPoint)listener.LocalEndpoint).Port; + } + + [Fact] + public async Task Create_DoesNotFollowRedirects_SoCredentialsStayOnOrigin() + { + var originPort = GetFreePort(); + var otherPort = GetFreePort(); + using var origin = new HttpListener(); + origin.Prefixes.Add($"http://127.0.0.1:{originPort}/"); + origin.Start(); + using var other = new HttpListener(); + other.Prefixes.Add($"http://127.0.0.1:{otherPort}/"); + other.Start(); + + var otherHit = false; + var otherTask = Task.Run(async () => + { + try + { + var ctx = await other.GetContextAsync(); + otherHit = true; + ctx.Response.StatusCode = 401; + ctx.Response.Close(); + } + catch (HttpListenerException) + { + } + catch (ObjectDisposedException) + { + } + }); + var originTask = Task.Run(async () => + { + try + { + while (true) + { + var ctx = await origin.GetContextAsync(); + ctx.Response.StatusCode = 307; + ctx.Response.RedirectLocation = $"http://127.0.0.1:{otherPort}{ctx.Request.Url!.AbsolutePath}"; + ctx.Response.Close(); + } + } + catch (HttpListenerException) + { + } + catch (ObjectDisposedException) + { + } + }); + + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["LanguageModel:Enabled"] = "true", + ["LanguageModel:Client"] = "AzureOpenAI", + ["LanguageModel:Model"] = "deployment", + ["LanguageModel:Url"] = $"http://127.0.0.1:{originPort}/openai/v1/", + ["LanguageModel:ApiKey"] = "azure-secret" + }) + .Build(); + using var serviceProvider = new ServiceCollection() + .AddSingleton(NullLoggerFactory.Instance) + .AddSingleton(typeof(ILogger<>), typeof(Logger<>)) + .BuildServiceProvider(); + + var client = LanguageModelClientFactory.Create(serviceProvider, configuration); + + Assert.False(await client.IsEnabledAsync(CancellationToken.None)); + Assert.False(otherHit); + + origin.Stop(); + other.Stop(); + await Task.WhenAll(originTask, otherTask); + } +} diff --git a/DevProxy.Abstractions/LanguageModel/LanguageModelClientFactory.cs b/DevProxy.Abstractions/LanguageModel/LanguageModelClientFactory.cs index c97c8c9ec..34c48ee05 100644 --- a/DevProxy.Abstractions/LanguageModel/LanguageModelClientFactory.cs +++ b/DevProxy.Abstractions/LanguageModel/LanguageModelClientFactory.cs @@ -17,12 +17,26 @@ public static ILanguageModelClient Create(IServiceProvider serviceProvider, ICon var lmSection = configuration.GetSection("LanguageModel"); var config = lmSection?.Get() ?? new(); + var httpClient = CreateHttpClient(); + return config.Client switch { - LanguageModelClient.Ollama => ActivatorUtilities.CreateInstance(serviceProvider, config), - LanguageModelClient.OpenAI => ActivatorUtilities.CreateInstance(serviceProvider, config), - LanguageModelClient.AzureOpenAI => ActivatorUtilities.CreateInstance(serviceProvider, config), - _ => ActivatorUtilities.CreateInstance(serviceProvider, config) + LanguageModelClient.Ollama => ActivatorUtilities.CreateInstance(serviceProvider, httpClient, config), + LanguageModelClient.OpenAI => ActivatorUtilities.CreateInstance(serviceProvider, httpClient, config), + LanguageModelClient.AzureOpenAI => ActivatorUtilities.CreateInstance(serviceProvider, httpClient, config), + _ => ActivatorUtilities.CreateInstance(serviceProvider, httpClient, config) }; } + + // Redirects are disabled so that credentials (eg. the api-key header, + // which HttpClient doesn't strip on redirect) aren't forwarded to + // another origin + internal static HttpClient CreateHttpClient() => +#pragma warning disable CA2000 // The handler is owned by the long-lived HttpClient + new(new SocketsHttpHandler + { + AllowAutoRedirect = false, + PooledConnectionLifetime = TimeSpan.FromMinutes(5) + }, disposeHandler: true); +#pragma warning restore CA2000 } \ No newline at end of file From 97cb666990483e1be5607d8632332f0876f85a16 Mon Sep 17 00:00:00 2001 From: waldekmastykarz Date: Sat, 31 Oct 2026 11:14:21 +0100 Subject: [PATCH 4/4] Address review feedback: DI-owned LM HttpClient, case-sensitive model match, stable test ports Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../LanguageModelClientFactoryTests.cs | 5 ++++- .../OpenAILanguageModelClientTests.cs | 14 ++++++++++++++ .../LanguageModel/LanguageModelClientFactory.cs | 8 ++++++-- .../LanguageModel/OpenAILanguageModelClient.cs | 4 ++-- .../Extensions/IServiceCollectionExtensions.cs | 1 + 5 files changed, 27 insertions(+), 5 deletions(-) diff --git a/DevProxy.Abstractions.Tests/LanguageModel/LanguageModelClientFactoryTests.cs b/DevProxy.Abstractions.Tests/LanguageModel/LanguageModelClientFactoryTests.cs index c76829489..4e91caef8 100644 --- a/DevProxy.Abstractions.Tests/LanguageModel/LanguageModelClientFactoryTests.cs +++ b/DevProxy.Abstractions.Tests/LanguageModel/LanguageModelClientFactoryTests.cs @@ -25,11 +25,13 @@ private static int GetFreePort() [Fact] public async Task Create_DoesNotFollowRedirects_SoCredentialsStayOnOrigin() { + // Start the origin listener before reserving the second port so the OS + // can't hand out the same port twice var originPort = GetFreePort(); - var otherPort = GetFreePort(); using var origin = new HttpListener(); origin.Prefixes.Add($"http://127.0.0.1:{originPort}/"); origin.Start(); + var otherPort = GetFreePort(); using var other = new HttpListener(); other.Prefixes.Add($"http://127.0.0.1:{otherPort}/"); other.Start(); @@ -84,6 +86,7 @@ public async Task Create_DoesNotFollowRedirects_SoCredentialsStayOnOrigin() using var serviceProvider = new ServiceCollection() .AddSingleton(NullLoggerFactory.Instance) .AddSingleton(typeof(ILogger<>), typeof(Logger<>)) + .AddKeyedSingleton(LanguageModelClientFactory.HttpClientServiceKey, (_, _) => LanguageModelClientFactory.CreateHttpClient()) .BuildServiceProvider(); var client = LanguageModelClientFactory.Create(serviceProvider, configuration); diff --git a/DevProxy.Abstractions.Tests/LanguageModel/OpenAILanguageModelClientTests.cs b/DevProxy.Abstractions.Tests/LanguageModel/OpenAILanguageModelClientTests.cs index dd6e3f3d2..17eb8d743 100644 --- a/DevProxy.Abstractions.Tests/LanguageModel/OpenAILanguageModelClientTests.cs +++ b/DevProxy.Abstractions.Tests/LanguageModel/OpenAILanguageModelClientTests.cs @@ -170,6 +170,20 @@ public async Task IsEnabled_ModelsNotSupported_FallsBackToCompletion() Assert.Equal("https://example.test/v1/chat/completions", handler.Requests[1].RequestUri!.ToString()); } + [Fact] + public async Task IsEnabled_ModelListedWithDifferentCase_FallsBackToCompletion() + { + var config = Config(); + config.Model = "GPT-4.1-mini"; + var (client, handler) = CreateClient(config, r => + r.Method == HttpMethod.Get + ? Json(HttpStatusCode.OK, """{"data":[{"id":"gpt-4.1-mini"}]}""") + : Json(HttpStatusCode.OK, ChatCompletionResponse)); + + Assert.True(await client.IsEnabledAsync(CancellationToken.None)); + Assert.Equal(2, handler.Requests.Count); + } + [Fact] public async Task IsEnabled_ModelNotListed_FallsBackToCompletion() { diff --git a/DevProxy.Abstractions/LanguageModel/LanguageModelClientFactory.cs b/DevProxy.Abstractions/LanguageModel/LanguageModelClientFactory.cs index 34c48ee05..324a3b934 100644 --- a/DevProxy.Abstractions/LanguageModel/LanguageModelClientFactory.cs +++ b/DevProxy.Abstractions/LanguageModel/LanguageModelClientFactory.cs @@ -9,6 +9,10 @@ namespace DevProxy.Abstractions.LanguageModel; public static class LanguageModelClientFactory { + // Register an HttpClient created with CreateHttpClient() under this key + // so that the container owns and disposes it + public const string HttpClientServiceKey = "DevProxy.LanguageModel"; + public static ILanguageModelClient Create(IServiceProvider serviceProvider, IConfiguration configuration) { ArgumentNullException.ThrowIfNull(serviceProvider); @@ -17,7 +21,7 @@ public static ILanguageModelClient Create(IServiceProvider serviceProvider, ICon var lmSection = configuration.GetSection("LanguageModel"); var config = lmSection?.Get() ?? new(); - var httpClient = CreateHttpClient(); + var httpClient = serviceProvider.GetKeyedService(HttpClientServiceKey) ?? CreateHttpClient(); return config.Client switch { @@ -31,7 +35,7 @@ public static ILanguageModelClient Create(IServiceProvider serviceProvider, ICon // Redirects are disabled so that credentials (eg. the api-key header, // which HttpClient doesn't strip on redirect) aren't forwarded to // another origin - internal static HttpClient CreateHttpClient() => + public static HttpClient CreateHttpClient() => #pragma warning disable CA2000 // The handler is owned by the long-lived HttpClient new(new SocketsHttpHandler { diff --git a/DevProxy.Abstractions/LanguageModel/OpenAILanguageModelClient.cs b/DevProxy.Abstractions/LanguageModel/OpenAILanguageModelClient.cs index d40dbb647..60a02894d 100644 --- a/DevProxy.Abstractions/LanguageModel/OpenAILanguageModelClient.cs +++ b/DevProxy.Abstractions/LanguageModel/OpenAILanguageModelClient.cs @@ -244,9 +244,9 @@ private bool IsConfiguredModel(string? modelId) return false; } - return string.Equals(modelId, Configuration.Model, StringComparison.OrdinalIgnoreCase) || + return string.Equals(modelId, Configuration.Model, StringComparison.Ordinal) || // Ollama lists models with their tag, eg. llama3.2:latest - string.Equals(modelId, $"{Configuration.Model}:latest", StringComparison.OrdinalIgnoreCase); + string.Equals(modelId, $"{Configuration.Model}:latest", StringComparison.Ordinal); } private async Task GenerateChatCompletionInternalAsync(OpenAIChatCompletionMessage[] messages, CompletionOptions? options, CancellationToken cancellationToken = default) diff --git a/DevProxy/Extensions/IServiceCollectionExtensions.cs b/DevProxy/Extensions/IServiceCollectionExtensions.cs index a64e8bf26..e959b0076 100644 --- a/DevProxy/Extensions/IServiceCollectionExtensions.cs +++ b/DevProxy/Extensions/IServiceCollectionExtensions.cs @@ -82,6 +82,7 @@ static IServiceCollection AddApplicationServices( // Share the hosted instance so the command can release its startup-output gate. .AddSingleton() .AddHostedService(sp => sp.GetRequiredService()) + .AddKeyedSingleton(LanguageModelClientFactory.HttpClientServiceKey, (_, _) => LanguageModelClientFactory.CreateHttpClient()) .AddSingleton(sp => LanguageModelClientFactory.Create(sp, configuration)) .AddSingleton() .AddSingleton()