From 1158b5fca589154580a6df4d6d1bc32a19da45d0 Mon Sep 17 00:00:00 2001 From: YigitTurali Date: Tue, 25 Nov 2025 16:00:21 -0800 Subject: [PATCH] Fixed LiteLLM multi-hosted_vllm request address mixture. --- .../online/litellm_online_request_processor.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/bespokelabs/curator/request_processor/online/litellm_online_request_processor.py b/src/bespokelabs/curator/request_processor/online/litellm_online_request_processor.py index ab6e901e..34742658 100644 --- a/src/bespokelabs/curator/request_processor/online/litellm_online_request_processor.py +++ b/src/bespokelabs/curator/request_processor/online/litellm_online_request_processor.py @@ -45,8 +45,8 @@ class LiteLLMOnlineRequestProcessor(BaseOnlineRequestProcessor): def __init__(self, config: OnlineRequestProcessorConfig): """Initialize the LiteLLMOnlineRequestProcessor.""" super().__init__(config) - if self.config.base_url is not None: - litellm.api_base = self.config.base_url + # NOTE: Removed global litellm.api_base assignment - now passed per-request + # to allow multiple LLM instances with different base_urls self.client = instructor.from_litellm(litellm.acompletion) self.header_based_max_requests_per_minute, self.header_based_max_tokens_per_minute = self.get_header_based_rate_limits() @@ -185,6 +185,7 @@ def test_call(self): completion = litellm.completion( model=self.config.model, messages=[{"role": "user", "content": "hi"}], # Some models (e.g. Claude) require an non-empty message to get rate limits. + api_base=self.config.base_url, # Pass api_base per-request **self.config.generation_params, ) # Try the method of caculating cost @@ -252,6 +253,10 @@ def create_api_specific_request_online(self, generic_request: GenericRequest) -> for key, value in generic_request.generation_params.items(): request[key] = value + # Pass api_base per-request instead of globally + if self.config.base_url is not None: + request["api_base"] = self.config.base_url + # Add safety settings for Gemini models if "gemini" in generic_request.model.lower(): request["safety_settings"] = [