From 3249073346c259ed579c5eab956ae6ce865563a4 Mon Sep 17 00:00:00 2001 From: littlegy <787321726@qq.com> Date: Thu, 20 Aug 2026 16:13:30 +0800 Subject: [PATCH 1/3] improve(autotest): replace APIClient with OpenAI SDK and strict param checks --- .../restful/test_restful_anthropic_v1.py | 6 +- .../test_restful_chat_completions_v1.py | 915 +++++------------- .../restful/test_restful_completions_v1.py | 264 ++--- .../restful/test_restful_generate.py | 16 +- .../test_tool_call_anthropic_sdk.py | 32 +- autotest/utils/anthropic_messages.py | 8 +- autotest/utils/restful_return_check.py | 63 +- autotest/utils/run_restful_chat.py | 98 +- autotest/utils/tool_reasoning_definitions.py | 11 +- 9 files changed, 504 insertions(+), 909 deletions(-) diff --git a/autotest/interface/restful/test_restful_anthropic_v1.py b/autotest/interface/restful/test_restful_anthropic_v1.py index 6d3302b786..92abfb98c1 100644 --- a/autotest/interface/restful/test_restful_anthropic_v1.py +++ b/autotest/interface/restful/test_restful_anthropic_v1.py @@ -18,8 +18,7 @@ ) from utils.config_utils import get_config from utils.constant import BACKEND_LIST, BASE_URL, RESTFUL_MODEL_LIST - -from lmdeploy.serve.openai.api_client import APIClient +from utils.restful_return_check import get_client_and_model ANTHROPIC_VERSION = '2023-06-01' @@ -41,7 +40,8 @@ def deployed_model_name() -> str: """Single model id exposed by the RESTFUL api_server.""" - return APIClient(BASE_URL).available_models[0] + _, model_name = get_client_and_model(BASE_URL) + return model_name @lru_cache(maxsize=1) diff --git a/autotest/interface/restful/test_restful_chat_completions_v1.py b/autotest/interface/restful/test_restful_chat_completions_v1.py index 4d0ebe2cbe..07bcf0388c 100644 --- a/autotest/interface/restful/test_restful_chat_completions_v1.py +++ b/autotest/interface/restful/test_restful_chat_completions_v1.py @@ -1,635 +1,27 @@ -from typing import Literal - import pytest -from openai import BadRequestError, OpenAI +import requests +from openai import BadRequestError from utils.constant import BACKEND_LIST, BASE_URL, DEFAULT_MAX_COMPLETION_TOKENS, RESTFUL_MODEL_LIST from utils.restful_return_check import ( + CONTEXT_LENGTH_ERROR, assert_chat_completions_batch_return, assert_chat_completions_stream_return, - assert_chat_message_error, + assert_openai_invalid_request_error, + encode_prompt, get_chat_delta_text, get_chat_message_text, + get_client_and_model, has_repeated_fragment, ) -from lmdeploy.serve.openai.api_client import APIClient - _OVERSIZE_CHAT_PROMPT = 'Hi, pls intro yourself' * 60000 +_CHAT_COMPLETIONS_URL = f'{BASE_URL}/v1/chat/completions' +_CHAT_MESSAGES = [{'role': 'user', 'content': 'Hi, pls intro yourself'}] -@pytest.mark.order(8) -@pytest.mark.flaky(reruns=2) -@pytest.mark.parametrize('backend', BACKEND_LIST) -@pytest.mark.parametrize('model_case', RESTFUL_MODEL_LIST) -class TestRestfulInterfaceChatCompletions: - - def test_return_info_with_prompt(self, backend, model_case): - api_client = APIClient(BASE_URL) - model_name = api_client.available_models[0] - for output in api_client.chat_completions_v1(model=model_name, - messages=[ - { - 'role': 'user', - 'content': 'Hi, pls intro yourself' - }, - ], - temperature=0.01): - continue - assert_chat_completions_batch_return(output, model_name) - - def test_return_info_with_messegae(self, backend, model_case): - api_client = APIClient(BASE_URL) - model_name = api_client.available_models[0] - for output in api_client.chat_completions_v1(model=model_name, - messages=[{ - 'role': 'user', - 'content': 'Hi, pls intro yourself' - }], - temperature=0.01): - continue - assert_chat_completions_batch_return(output, model_name) - - def test_return_info_with_prompt_streaming(self, backend, model_case): - api_client = APIClient(BASE_URL) - model_name = api_client.available_models[0] - outputList = [] - for output in api_client.chat_completions_v1(model=model_name, - messages=[ - { - 'role': 'user', - 'content': 'Hi, pls intro yourself' - }, - ], - stream=True, - temperature=0.01): - outputList.append(output) - - assert_chat_completions_stream_return(outputList[-1], model_name, True) - for index in range(0, len(outputList) - 1): - assert_chat_completions_stream_return(outputList[index], model_name) - - def test_return_info_with_messegae_streaming(self, backend, model_case): - api_client = APIClient(BASE_URL) - model_name = api_client.available_models[0] - outputList = [] - for output in api_client.chat_completions_v1(model=model_name, - messages=[{ - 'role': 'user', - 'content': 'Hi, pls intro yourself' - }], - stream=True, - temperature=0.01): - outputList.append(output) - - assert_chat_completions_stream_return(outputList[-1], model_name, True) - for index in range(0, len(outputList) - 1): - assert_chat_completions_stream_return(outputList[index], model_name) - - def test_single_stopword(self, backend, model_case): - api_client = APIClient(BASE_URL) - model_name = api_client.available_models[0] - for output in api_client.chat_completions_v1(model=model_name, - messages=[ - { - 'role': 'user', - 'content': 'Shanghai is' - }, - ], - stop=' is', - temperature=0.01): - continue - assert_chat_completions_batch_return(output, model_name) - assert ' is' not in get_chat_message_text(output.get('choices')[0]) - assert output.get('choices')[0].get('finish_reason') == 'stop' - - def test_single_stopword_streaming(self, backend, model_case): - api_client = APIClient(BASE_URL) - model_name = api_client.available_models[0] - outputList = [] - for output in api_client.chat_completions_v1(model=model_name, - messages=[ - { - 'role': 'user', - 'content': 'Shanghai is' - }, - ], - stop=' is', - stream=True, - temperature=0.01): - outputList.append(output) - - assert_chat_completions_stream_return(outputList[-1], model_name, True) - for index in range(0, len(outputList) - 1): - assert_chat_completions_stream_return(outputList[index], model_name) - assert ' to' not in get_chat_delta_text(outputList[index].get('choices')[0]) - assert outputList[-1].get('choices')[0].get('finish_reason') == 'stop' - - def test_array_stopwords(self, backend, model_case): - api_client = APIClient(BASE_URL) - model_name = api_client.available_models[0] - for output in api_client.chat_completions_v1(model=model_name, - messages=[ - { - 'role': 'user', - 'content': 'Shanghai is' - }, - ], - stop=[' is', '上海', ' to'], - temperature=0.01): - continue - assert_chat_completions_batch_return(output, model_name) - assert ' is' not in get_chat_message_text(output.get('choices')[0]) - assert ' 上海' not in get_chat_message_text(output.get('choices')[0]) - assert ' to ' not in get_chat_message_text(output.get('choices')[0]) - assert output.get('choices')[0].get('finish_reason') == 'stop' - - def test_array_stopwords_streaming(self, backend, model_case): - api_client = APIClient(BASE_URL) - model_name = api_client.available_models[0] - outputList = [] - for output in api_client.chat_completions_v1(model=model_name, - messages=[ - { - 'role': 'user', - 'content': 'Shanghai is' - }, - ], - stop=[' is', '上海', ' to'], - stream=True, - temperature=0.01): - outputList.append(output) - - assert_chat_completions_stream_return(outputList[-1], model_name, True) - for index in range(0, len(outputList) - 1): - assert_chat_completions_stream_return(outputList[index], model_name) - assert ' is' not in get_chat_delta_text(outputList[index].get('choices')[0]) - assert '上海' not in get_chat_delta_text(outputList[index].get('choices')[0]) - assert ' to ' not in get_chat_delta_text(outputList[index].get('choices')[0]) - assert outputList[-1].get('choices')[0].get('finish_reason') == 'stop' - - def test_minimum_repetition_penalty(self, backend, model_case): - api_client = APIClient(BASE_URL) - model_name = api_client.available_models[0] - for output in api_client.chat_completions_v1(model=model_name, - messages=[ - { - 'role': 'user', - 'content': 'Shanghai is' - }, - ], - repetition_penalty=0.0000001, - temperature=0.01, - max_tokens=200, - min_new_tokens=100): - continue - assert_chat_completions_batch_return(output, model_name) - result, msg = has_repeated_fragment(get_chat_message_text(output.get('choices')[0])) - assert result, msg - - def test_minimum_repetition_penalty_streaming(self, backend, model_case): - api_client = APIClient(BASE_URL) - model_name = api_client.available_models[0] - outputList = [] - for output in api_client.chat_completions_v1(model=model_name, - messages=[ - { - 'role': 'user', - 'content': 'Hi, pls intro yourself' - }, - ], - stream=True, - repetition_penalty=0.0000001, - temperature=0.01, - max_tokens=200, - min_new_tokens=100): - outputList.append(output) - assert_chat_completions_stream_return(outputList[-1], model_name, True) - response = '' - for index in range(0, len(outputList) - 1): - assert_chat_completions_stream_return(outputList[index], model_name) - response += get_chat_delta_text(outputList[index].get('choices')[0]) - result, msg = has_repeated_fragment(response) - assert result, msg - - def test_repetition_penalty_bigger_than_1(self, backend, model_case): - api_client = APIClient(BASE_URL) - model_name = api_client.available_models[0] - for output in api_client.chat_completions_v1(model=model_name, - messages=[ - { - 'role': 'user', - 'content': 'Shanghai is' - }, - ], - repetition_penalty=1.2, - temperature=0.01, - max_tokens=200): - continue - assert_chat_completions_batch_return(output, model_name) - - def test_repetition_penalty_bigger_than_1_streaming(self, backend, model_case): - api_client = APIClient(BASE_URL) - model_name = api_client.available_models[0] - outputList = [] - for output in api_client.chat_completions_v1(model=model_name, - messages=[ - { - 'role': 'user', - 'content': 'Hi, pls intro yourself' - }, - ], - stream=True, - repetition_penalty=1.2, - temperature=0.01, - max_tokens=200): - outputList.append(output) - assert_chat_completions_stream_return(outputList[-1], model_name, True) - for index in range(0, len(outputList) - 1): - assert_chat_completions_stream_return(outputList[index], model_name) - continue - - def test_minimum_topp(self, backend, model_case): - api_client = APIClient(BASE_URL) - model_name = api_client.available_models[0] - outputList = [] - for i in range(3): - for output in api_client.chat_completions_v1(model=model_name, - messages=[ - { - 'role': 'user', - 'content': 'Shanghai is' - }, - ], - top_p=0.0000000001, - max_tokens=10): - outputList.append(output) - assert_chat_completions_batch_return(output, model_name) - texts = [get_chat_message_text(output.get('choices')[0]) for output in outputList] - assert texts[0] == texts[1] - assert texts[1] == texts[2] - - def test_minimum_topp_streaming(self, backend, model_case): - api_client = APIClient(BASE_URL) - model_name = api_client.available_models[0] - responseList = [] - for i in range(3): - outputList = [] - response = '' - for output in api_client.chat_completions_v1(model=model_name, - messages=[ - { - 'role': 'user', - 'content': 'Hi, pls intro yourself' - }, - ], - stream=True, - top_p=0.0000000001, - max_tokens=10): - outputList.append(output) - assert_chat_completions_stream_return(outputList[-1], model_name, True) - response = '' - for index in range(0, len(outputList) - 1): - assert_chat_completions_stream_return(outputList[index], model_name) - response += get_chat_delta_text(outputList[index].get('choices')[0]) - responseList.append(response) - assert responseList[0] == responseList[1] or responseList[1] == responseList[2] - - def test_mistake_modelname_return(self, backend, model_case): - api_client = APIClient(BASE_URL) - for output in api_client.chat_completions_v1(model='error', - messages=[ - { - 'role': 'user', - 'content': 'Hi, pls intro yourself' - }, - ], - temperature=0.01): - continue - assert output.get('code') == 404 - assert output.get('message') == 'The model \'error\' does not exist.' - assert output.get('object') == 'error' - - def test_mistake_modelname_return_streaming(self, backend, model_case): - api_client = APIClient(BASE_URL) - outputList = [] - for output in api_client.chat_completions_v1(model='error', - messages=[ - { - 'role': 'user', - 'content': 'Hi, pls intro yourself' - }, - ], - stream=True, - max_tokens=5, - temperature=0.01): - outputList.append(output) - assert output.get('code') == 404 - assert output.get('message') == 'The model \'error\' does not exist.' - assert output.get('object') == 'error' - assert len(outputList) == 1 - - def test_mutilple_times_response_should_not_same(self, backend, model_case): - api_client = APIClient(BASE_URL) - model_name = api_client.available_models[0] - outputList = [] - for i in range(3): - for output in api_client.chat_completions_v1(model=model_name, - messages=[ - { - 'role': 'user', - 'content': 'Shanghai is', - }, - ], - max_tokens=100): - outputList.append(output) - assert_chat_completions_batch_return(output, model_name) - texts = [get_chat_message_text(output.get('choices')[0]) for output in outputList] - assert texts[0] != texts[1] or texts[1] != texts[2] - - def test_mutilple_times_response_should_not_same_streaming(self, backend, model_case): - api_client = APIClient(BASE_URL) - model_name = api_client.available_models[0] - responseList = [] - for i in range(3): - outputList = [] - for output in api_client.chat_completions_v1(model=model_name, - messages=[ - { - 'role': 'user', - 'content': 'Shanghai is', - }, - ], - stream=True, - max_tokens=100): - outputList.append(output) - assert_chat_completions_stream_return(outputList[-1], model_name, True) - response = '' - for index in range(0, len(outputList) - 1): - assert_chat_completions_stream_return(outputList[index], model_name) - response += get_chat_delta_text(outputList[index].get('choices')[0]) - responseList.append(response) - assert responseList[0] != responseList[1] or responseList[1] == responseList[2] - - def test_longtext_input(self, backend, model_case): - api_client = APIClient(BASE_URL) - model_name = api_client.available_models[0] - for output in api_client.chat_completions_v1(model=model_name, - messages=[ - { - 'role': 'user', - 'content': _OVERSIZE_CHAT_PROMPT, - }, - ], - temperature=0.01): - continue - assert_chat_message_error(output) - - def test_longtext_input_streaming(self, backend, model_case): - api_client = APIClient(BASE_URL) - model_name = api_client.available_models[0] - outputList = [] - for output in api_client.chat_completions_v1(model=model_name, - messages=[ - { - 'role': 'user', - 'content': _OVERSIZE_CHAT_PROMPT, - }, - ], - stream=True, - temperature=0.01): - outputList.append(output) - assert len(outputList) == 1 - assert_chat_message_error(outputList[0]) - - def test_ignore_eos(self, backend, model_case): - api_client = APIClient(BASE_URL) - model_name = api_client.available_models[0] - for output in api_client.chat_completions_v1(model=model_name, - messages=[ - { - 'role': 'user', - 'content': 'Hi, what is your name?' - }, - ], - ignore_eos=True, - max_tokens=100, - temperature=0.01): - continue - assert_chat_completions_batch_return(output, model_name) - assert output.get('usage').get('completion_tokens') == 101 or output.get('usage').get( - 'completion_tokens') == 100 - assert output.get('choices')[0].get('finish_reason') == 'length' - - def test_max_tokens_default_cap_no_overshoot_followup(self, backend, model_case): - """Hit DEFAULT max_tokens (8192) with ignore_eos; no overshoot; follow- - up must succeed. - - Catches regressions where length-capped generation returns a few extra tokens and breaks the next - /v1/chat/completions request. - """ - api_client = APIClient(BASE_URL) - model_name = api_client.available_models[0] - max_tokens = DEFAULT_MAX_COMPLETION_TOKENS - overshoot_slack = 1 - - for output in api_client.chat_completions_v1( - model=model_name, - messages=[ - { - 'role': 'user', - 'content': 'Continue writing forever without stopping.', - }, - ], - ignore_eos=True, - max_tokens=max_tokens, - temperature=0.01, - ): - continue - assert_chat_completions_batch_return(output, model_name) - assert output.get('choices')[0].get('finish_reason') == 'length' - completion_tokens = output.get('usage', {}).get('completion_tokens') - assert completion_tokens is not None, 'Missing usage.completion_tokens' - assert completion_tokens <= max_tokens + overshoot_slack, ( - f'Length cap overshoot: completion_tokens={completion_tokens} > ' - f'max_tokens={max_tokens}+{overshoot_slack}') - - for followup in api_client.chat_completions_v1( - model=model_name, - messages=[ - { - 'role': 'user', - 'content': 'Say hi in one word.', - }, - ], - max_tokens=8, - temperature=0.01, - ): - continue - assert_chat_completions_batch_return(followup, model_name) - - def test_ignore_eos_streaming(self, backend, model_case): - api_client = APIClient(BASE_URL) - model_name = api_client.available_models[0] - outputList = [] - for output in api_client.chat_completions_v1(model=model_name, - messages=[ - { - 'role': 'user', - 'content': 'Hi, what is your name?' - }, - ], - ignore_eos=True, - stream=True, - max_tokens=100, - temperature=0.01): - outputList.append(output) - assert_chat_completions_stream_return(outputList[-1], model_name, True) - response = '' - for index in range(0, len(outputList) - 1): - assert_chat_completions_stream_return(outputList[index], model_name) - response += get_chat_delta_text(outputList[index].get('choices')[0]) - length = api_client.encode(response, add_bos=False)[1] - assert outputList[-1].get('choices')[0].get('finish_reason') == 'length' - assert length >= 99 and length <= 101 - - def __test_max_tokens_or_max_completion_tokens( - self, - max_tokens_or_max_completion_tokens: Literal['max_tokens', 'max_completion_tokens'], - ): - api_client = APIClient(BASE_URL) - model_name = api_client.available_models[0] - if max_tokens_or_max_completion_tokens == 'max_tokens': - for output in api_client.chat_completions_v1( - model=model_name, - messages=[ - { - 'role': 'user', - 'content': 'Hi, pls intro yourself' - }, - ], - max_tokens=5, - temperature=0.01, - ): - continue - else: - for output in api_client.chat_completions_v1( - model=model_name, - messages=[ - { - 'role': 'user', - 'content': 'Hi, pls intro yourself' - }, - ], - max_completion_tokens=5, - temperature=0.01, - ): - continue - assert_chat_completions_batch_return(output, model_name) - assert output.get('choices')[0].get('finish_reason') == 'length' - assert output.get('usage').get('completion_tokens') == 6 or output.get('usage').get('completion_tokens') == 5 - - def test_max_tokens(self, backend, model_case): - self.__test_max_tokens_or_max_completion_tokens('max_tokens') - - def test_max_completion_tokens(self, backend, model_case): - self.__test_max_tokens_or_max_completion_tokens('max_completion_tokens') - - def __test_max_tokens_streaming_or_max_completion_tokens_streaming( - self, - max_tokens_or_max_completion_tokens: Literal['max_tokens', 'max_completion_tokens'], - ): - api_client = APIClient(BASE_URL) - model_name = api_client.available_models[0] - outputList = [] - if max_tokens_or_max_completion_tokens == 'max_tokens': - for output in api_client.chat_completions_v1( - model=model_name, - messages=[ - { - 'role': 'user', - 'content': 'Hi, pls intro yourself' - }, - ], - stream=True, - max_tokens=5, - temperature=0.01, - ): - outputList.append(output) - else: - for output in api_client.chat_completions_v1( - model=model_name, - messages=[ - { - 'role': 'user', - 'content': 'Hi, pls intro yourself' - }, - ], - stream=True, - max_completion_tokens=5, - temperature=0.01, - ): - outputList.append(output) - assert_chat_completions_stream_return(outputList[-1], model_name, True) - response = '' - for index in range(0, len(outputList) - 1): - assert_chat_completions_stream_return(outputList[index], model_name) - response += get_chat_delta_text(outputList[index].get('choices')[0]) - length = api_client.encode(response, add_bos=False)[1] - assert outputList[-1].get('choices')[0].get('finish_reason') == 'length' - assert length == 5 or length == 6 - - def test_max_tokens_streaming(self, backend, model_case): - self.__test_max_tokens_streaming_or_max_completion_tokens_streaming('max_tokens') - - def test_max_completion_tokens_streaming(self, backend, model_case): - self.__test_max_tokens_streaming_or_max_completion_tokens_streaming('max_completion_tokens') - - @pytest.mark.not_pytorch - def test_logprobs(self, backend, model_case): - api_client = APIClient(BASE_URL) - model_name = api_client.available_models[0] - for output in api_client.chat_completions_v1(model=model_name, - messages=[ - { - 'role': 'user', - 'content': 'Hi, pls intro yourself' - }, - ], - max_tokens=5, - temperature=0.01, - logprobs=True, - top_logprobs=10): - continue - assert_chat_completions_batch_return(output, model_name, check_logprobs=True, logprobs_num=10) - assert output.get('choices')[0].get('finish_reason') == 'length' - assert output.get('usage').get('completion_tokens') == 6 or output.get('usage').get('completion_tokens') == 5 - - @pytest.mark.not_pytorch - def test_logprobs_streaming(self, backend, model_case): - api_client = APIClient(BASE_URL) - model_name = api_client.available_models[0] - outputList = [] - for output in api_client.chat_completions_v1(model=model_name, - messages=[ - { - 'role': 'user', - 'content': 'Hi, pls intro yourself' - }, - ], - stream=True, - max_tokens=5, - temperature=0.01, - logprobs=True, - top_logprobs=10): - outputList.append(output) - assert_chat_completions_stream_return(outputList[-1], model_name, True, check_logprobs=True, logprobs_num=10) - response = '' - for index in range(0, len(outputList) - 1): - assert_chat_completions_stream_return(outputList[index], model_name, check_logprobs=True, logprobs_num=10) - response += get_chat_delta_text(outputList[index].get('choices')[0]) - length = api_client.encode(response, add_bos=False)[1] - assert outputList[-1].get('choices')[0].get('finish_reason') == 'length' - assert length == 5 or length == 6 +@pytest.fixture(scope='class') +def openai_client_and_model(): + return get_client_and_model(BASE_URL) @pytest.mark.order(8) @@ -639,9 +31,8 @@ def test_logprobs_streaming(self, backend, model_case): class TestRestfulOpenAI: @pytest.mark.pr_test - def test_return_info(self, backend, model_case): - client = OpenAI(api_key='YOUR_API_KEY', base_url=f'{BASE_URL}/v1') - model_name = client.models.list().data[0].id + def test_return_info(self, backend, model_case, openai_client_and_model): + client, model_name = openai_client_and_model outputs = client.chat.completions.create(model=model_name, messages=[ { @@ -655,9 +46,8 @@ def test_return_info(self, backend, model_case): assert_chat_completions_batch_return(output, model_name) @pytest.mark.pr_test - def test_return_info_streaming(self, backend, model_case): - client = OpenAI(api_key='YOUR_API_KEY', base_url=f'{BASE_URL}/v1') - model_name = client.models.list().data[0].id + def test_return_info_streaming(self, backend, model_case, openai_client_and_model): + client, model_name = openai_client_and_model outputs = client.chat.completions.create(model=model_name, messages=[ { @@ -676,9 +66,8 @@ def test_return_info_streaming(self, backend, model_case): for index in range(0, len(outputList) - 1): assert_chat_completions_stream_return(outputList[index], model_name) - def test_single_stopword(self, backend, model_case): - client = OpenAI(api_key='YOUR_API_KEY', base_url=f'{BASE_URL}/v1') - model_name = client.models.list().data[0].id + def test_single_stopword(self, backend, model_case, openai_client_and_model): + client, model_name = openai_client_and_model outputs = client.chat.completions.create(model=model_name, messages=[ { @@ -695,9 +84,8 @@ def test_single_stopword(self, backend, model_case): assert output.get('choices')[0].get('finish_reason') == 'stop' @pytest.mark.pr_test - def test_single_stopword_streaming(self, backend, model_case): - client = OpenAI(api_key='YOUR_API_KEY', base_url=f'{BASE_URL}/v1') - model_name = client.models.list().data[0].id + def test_single_stopword_streaming(self, backend, model_case, openai_client_and_model): + client, model_name = openai_client_and_model outputs = client.chat.completions.create(model=model_name, messages=[ { @@ -719,9 +107,8 @@ def test_single_stopword_streaming(self, backend, model_case): assert ' is ' not in get_chat_delta_text(outputList[index].get('choices')[0]) assert outputList[-1].get('choices')[0].get('finish_reason') == 'stop' - def test_array_stopwords(self, backend, model_case): - client = OpenAI(api_key='YOUR_API_KEY', base_url=f'{BASE_URL}/v1') - model_name = client.models.list().data[0].id + def test_array_stopwords(self, backend, model_case, openai_client_and_model): + client, model_name = openai_client_and_model outputs = client.chat.completions.create( model=model_name, messages=[ @@ -741,9 +128,8 @@ def test_array_stopwords(self, backend, model_case): assert ' to' not in get_chat_message_text(output.get('choices')[0]) assert output.get('choices')[0].get('finish_reason') == 'stop' - def test_array_stopwords_streaming(self, backend, model_case): - client = OpenAI(api_key='YOUR_API_KEY', base_url=f'{BASE_URL}/v1') - model_name = client.models.list().data[0].id + def test_array_stopwords_streaming(self, backend, model_case, openai_client_and_model): + client, model_name = openai_client_and_model outputs = client.chat.completions.create(model=model_name, messages=[ { @@ -768,9 +154,8 @@ def test_array_stopwords_streaming(self, backend, model_case): assert outputList[-1].get('choices')[0].get('finish_reason') == 'stop' @pytest.mark.pr_test - def test_minimum_topp(self, backend, model_case): - client = OpenAI(api_key='YOUR_API_KEY', base_url=f'{BASE_URL}/v1') - model_name = client.models.list().data[0].id + def test_minimum_topp(self, backend, model_case, openai_client_and_model): + client, model_name = openai_client_and_model outputList = [] for i in range(3): outputs = client.chat.completions.create(model=model_name, @@ -790,9 +175,8 @@ def test_minimum_topp(self, backend, model_case): assert texts[0] == texts[1] assert texts[1] == texts[2] - def test_minimum_topp_streaming(self, backend, model_case): - client = OpenAI(api_key='YOUR_API_KEY', base_url=f'{BASE_URL}/v1') - model_name = client.models.list().data[0].id + def test_minimum_topp_streaming(self, backend, model_case, openai_client_and_model): + client, model_name = openai_client_and_model responseList = [] for i in range(3): outputs = client.chat.completions.create(model=model_name, @@ -818,8 +202,8 @@ def test_minimum_topp_streaming(self, backend, model_case): assert responseList[0] == responseList[1] or responseList[1] == responseList[2] @pytest.mark.pr_test - def test_mistake_modelname_return(self, backend, model_case): - client = OpenAI(api_key='YOUR_API_KEY', base_url=f'{BASE_URL}/v1') + def test_mistake_modelname_return(self, backend, model_case, openai_client_and_model): + client, _ = openai_client_and_model with pytest.raises(Exception, match='The model \'error\' does not exist.'): client.chat.completions.create( model='error', @@ -833,8 +217,8 @@ def test_mistake_modelname_return(self, backend, model_case): stop=[' is', '上海', ' to'], ) - def test_mistake_modelname_return_streaming(self, backend, model_case): - client = OpenAI(api_key='YOUR_API_KEY', base_url=f'{BASE_URL}/v1') + def test_mistake_modelname_return_streaming(self, backend, model_case, openai_client_and_model): + client, _ = openai_client_and_model with pytest.raises(Exception, match='The model \'error\' does not exist.'): client.chat.completions.create(model='error', @@ -849,9 +233,8 @@ def test_mistake_modelname_return_streaming(self, backend, model_case): stream=True) @pytest.mark.pr_test - def test_mutilple_times_response_should_not_same(self, backend, model_case): - client = OpenAI(api_key='YOUR_API_KEY', base_url=f'{BASE_URL}/v1') - model_name = client.models.list().data[0].id + def test_mutilple_times_response_should_not_same(self, backend, model_case, openai_client_and_model): + client, model_name = openai_client_and_model outputList = [] for i in range(3): outputs = client.chat.completions.create(model=model_name, @@ -868,9 +251,8 @@ def test_mutilple_times_response_should_not_same(self, backend, model_case): texts = [get_chat_message_text(output.get('choices')[0]) for output in outputList] assert texts[0] != texts[1] or texts[1] != texts[2] - def test_mutilple_times_response_should_not_same_streaming(self, backend, model_case): - client = OpenAI(api_key='YOUR_API_KEY', base_url=f'{BASE_URL}/v1') - model_name = client.models.list().data[0].id + def test_mutilple_times_response_should_not_same_streaming(self, backend, model_case, openai_client_and_model): + client, model_name = openai_client_and_model responseList = [] for i in range(3): outputs = client.chat.completions.create(model=model_name, @@ -894,9 +276,8 @@ def test_mutilple_times_response_should_not_same_streaming(self, backend, model_ responseList.append(response) assert responseList[0] != responseList[1] or responseList[1] == responseList[2] - def test_longtext_input(self, backend, model_case): - client = OpenAI(api_key='YOUR_API_KEY', base_url=f'{BASE_URL}/v1') - model_name = client.models.list().data[0].id + def test_longtext_input(self, backend, model_case, openai_client_and_model): + client, model_name = openai_client_and_model with pytest.raises(BadRequestError) as ei: client.chat.completions.create(model=model_name, messages=[ @@ -907,12 +288,11 @@ def test_longtext_input(self, backend, model_case): ], max_tokens=100) assert ei.value.status_code == 400 - assert_chat_message_error(ei.value.body) + assert_openai_invalid_request_error(ei.value.body, message_substr=CONTEXT_LENGTH_ERROR) @pytest.mark.pr_test - def test_longtext_input_streaming(self, backend, model_case): - client = OpenAI(api_key='YOUR_API_KEY', base_url=f'{BASE_URL}/v1') - model_name = client.models.list().data[0].id + def test_longtext_input_streaming(self, backend, model_case, openai_client_and_model): + client, model_name = openai_client_and_model with pytest.raises(BadRequestError) as ei: client.chat.completions.create(model=model_name, messages=[ @@ -924,12 +304,11 @@ def test_longtext_input_streaming(self, backend, model_case): max_tokens=100, stream=True) assert ei.value.status_code == 400 - assert_chat_message_error(ei.value.body) + assert_openai_invalid_request_error(ei.value.body, message_substr=CONTEXT_LENGTH_ERROR) @pytest.mark.pr_test - def test_max_tokens(self, backend, model_case): - client = OpenAI(api_key='YOUR_API_KEY', base_url=f'{BASE_URL}/v1') - model_name = client.models.list().data[0].id + def test_max_tokens(self, backend, model_case, openai_client_and_model): + client, model_name = openai_client_and_model outputs = client.chat.completions.create(model=model_name, messages=[ { @@ -944,9 +323,8 @@ def test_max_tokens(self, backend, model_case): assert output.get('choices')[0].get('finish_reason') == 'length' assert output.get('usage').get('completion_tokens') == 6 or output.get('usage').get('completion_tokens') == 5 - def test_max_tokens_streaming(self, backend, model_case): - client = OpenAI(api_key='YOUR_API_KEY', base_url=f'{BASE_URL}/v1') - model_name = client.models.list().data[0].id + def test_max_tokens_streaming(self, backend, model_case, openai_client_and_model): + client, model_name = openai_client_and_model outputs = client.chat.completions.create(model=model_name, messages=[ @@ -968,16 +346,14 @@ def test_max_tokens_streaming(self, backend, model_case): for index in range(0, len(outputList) - 1): assert_chat_completions_stream_return(outputList[index], model_name) response += get_chat_delta_text(outputList[index].get('choices')[0]) - api_client = APIClient(BASE_URL) - length = api_client.encode(response, add_bos=False)[1] + _, length = encode_prompt(BASE_URL, response, add_bos=False) assert outputList[-1].get('choices')[0].get('finish_reason') == 'length' assert length == 5 or length == 6 @pytest.mark.not_pytorch @pytest.mark.pr_test - def test_logprobs(self, backend, model_case): - client = OpenAI(api_key='YOUR_API_KEY', base_url=f'{BASE_URL}/v1') - model_name = client.models.list().data[0].id + def test_logprobs(self, backend, model_case, openai_client_and_model): + client, model_name = openai_client_and_model outputs = client.chat.completions.create(model=model_name, messages=[ { @@ -996,9 +372,8 @@ def test_logprobs(self, backend, model_case): @pytest.mark.not_pytorch @pytest.mark.pr_test - def test_logprobs_streaming(self, backend, model_case): - client = OpenAI(api_key='YOUR_API_KEY', base_url=f'{BASE_URL}/v1') - model_name = client.models.list().data[0].id + def test_logprobs_streaming(self, backend, model_case, openai_client_and_model): + client, model_name = openai_client_and_model outputs = client.chat.completions.create(model=model_name, messages=[ @@ -1022,14 +397,189 @@ def test_logprobs_streaming(self, backend, model_case): for index in range(0, len(outputList) - 1): assert_chat_completions_stream_return(outputList[index], model_name, check_logprobs=True, logprobs_num=10) response += get_chat_delta_text(outputList[index].get('choices')[0]) - api_client = APIClient(BASE_URL) - length = api_client.encode(response, add_bos=False)[1] + _, length = encode_prompt(BASE_URL, response, add_bos=False) assert outputList[-1].get('choices')[0].get('finish_reason') == 'length' assert length == 5 or length == 6 - def test_input_validation(self, backend, model_case): - client = OpenAI(api_key='YOUR_API_KEY', base_url=f'{BASE_URL}/v1') - model_name = client.models.list().data[0].id + def test_minimum_repetition_penalty(self, backend, model_case, openai_client_and_model): + client, model_name = openai_client_and_model + outputs = client.chat.completions.create( + model=model_name, + messages=[{'role': 'user', 'content': 'Shanghai is'}], + extra_body={'repetition_penalty': 0.0000001, 'min_new_tokens': 100}, + temperature=0.01, + max_tokens=200, + ) + output = outputs.model_dump() + assert_chat_completions_batch_return(output, model_name) + result, msg = has_repeated_fragment(get_chat_message_text(output.get('choices')[0])) + assert result, msg + + def test_minimum_repetition_penalty_streaming(self, backend, model_case, openai_client_and_model): + client, model_name = openai_client_and_model + outputs = client.chat.completions.create( + model=model_name, + messages=[{'role': 'user', 'content': 'Hi, pls intro yourself'}], + extra_body={'repetition_penalty': 0.0000001, 'min_new_tokens': 100}, + temperature=0.01, + max_tokens=200, + stream=True, + ) + outputList = [chunk.model_dump() for chunk in outputs] + assert_chat_completions_stream_return(outputList[-1], model_name, True) + response = '' + for index in range(0, len(outputList) - 1): + assert_chat_completions_stream_return(outputList[index], model_name) + response += get_chat_delta_text(outputList[index].get('choices')[0]) + result, msg = has_repeated_fragment(response) + assert result, msg + + def test_repetition_penalty_bigger_than_1(self, backend, model_case, openai_client_and_model): + client, model_name = openai_client_and_model + outputs = client.chat.completions.create( + model=model_name, + messages=[{'role': 'user', 'content': 'Shanghai is'}], + extra_body={'repetition_penalty': 1.2}, + temperature=0.01, + max_tokens=200, + ) + output = outputs.model_dump() + assert_chat_completions_batch_return(output, model_name) + + def test_repetition_penalty_bigger_than_1_streaming(self, backend, model_case, openai_client_and_model): + client, model_name = openai_client_and_model + outputs = client.chat.completions.create( + model=model_name, + messages=[{'role': 'user', 'content': 'Hi, pls intro yourself'}], + extra_body={'repetition_penalty': 1.2}, + temperature=0.01, + max_tokens=200, + stream=True, + ) + outputList = [chunk.model_dump() for chunk in outputs] + assert_chat_completions_stream_return(outputList[-1], model_name, True) + for index in range(0, len(outputList) - 1): + assert_chat_completions_stream_return(outputList[index], model_name) + + def test_ignore_eos(self, backend, model_case, openai_client_and_model): + client, model_name = openai_client_and_model + outputs = client.chat.completions.create( + model=model_name, + messages=[{'role': 'user', 'content': 'Hi, what is your name?'}], + extra_body={'ignore_eos': True}, + max_tokens=100, + temperature=0.01, + ) + output = outputs.model_dump() + assert_chat_completions_batch_return(output, model_name) + completion_tokens = output.get('usage', {}).get('completion_tokens') + assert completion_tokens == 101 or completion_tokens == 100 + assert output.get('choices')[0].get('finish_reason') == 'length' + + def test_ignore_eos_streaming(self, backend, model_case, openai_client_and_model): + client, model_name = openai_client_and_model + outputs = client.chat.completions.create( + model=model_name, + messages=[{'role': 'user', 'content': 'Hi, what is your name?'}], + extra_body={'ignore_eos': True}, + max_tokens=100, + temperature=0.01, + stream=True, + ) + outputList = [chunk.model_dump() for chunk in outputs] + assert_chat_completions_stream_return(outputList[-1], model_name, True) + response = '' + for index in range(0, len(outputList) - 1): + assert_chat_completions_stream_return(outputList[index], model_name) + response += get_chat_delta_text(outputList[index].get('choices')[0]) + _, length = encode_prompt(BASE_URL, response, add_bos=False) + assert outputList[-1].get('choices')[0].get('finish_reason') == 'length' + assert length >= 99 and length <= 101 + + def test_max_tokens_default_cap_no_overshoot_followup(self, backend, model_case, openai_client_and_model): + client, model_name = openai_client_and_model + max_tokens = DEFAULT_MAX_COMPLETION_TOKENS + overshoot_slack = 1 + outputs = client.chat.completions.create( + model=model_name, + messages=[{'role': 'user', 'content': 'Continue writing forever without stopping.'}], + extra_body={'ignore_eos': True}, + max_tokens=max_tokens, + temperature=0.01, + ) + output = outputs.model_dump() + assert_chat_completions_batch_return(output, model_name) + assert output.get('choices')[0].get('finish_reason') == 'length' + completion_tokens = output.get('usage', {}).get('completion_tokens') + assert completion_tokens is not None, 'Missing usage.completion_tokens' + assert completion_tokens <= max_tokens + overshoot_slack, ( + f'Length cap overshoot: completion_tokens={completion_tokens} > ' + f'max_tokens={max_tokens}+{overshoot_slack}') + followup = client.chat.completions.create( + model=model_name, + messages=[{'role': 'user', 'content': 'Say hi in one word.'}], + max_tokens=8, + temperature=0.01, + ) + assert_chat_completions_batch_return(followup.model_dump(), model_name) + + def test_max_completion_tokens(self, backend, model_case, openai_client_and_model): + client, model_name = openai_client_and_model + outputs = client.chat.completions.create( + model=model_name, + messages=[{'role': 'user', 'content': 'Hi, pls intro yourself'}], + max_completion_tokens=5, + temperature=0.01, + ) + output = outputs.model_dump() + assert_chat_completions_batch_return(output, model_name) + assert output.get('choices')[0].get('finish_reason') == 'length' + assert output.get('usage').get('completion_tokens') in (5, 6) + + def test_max_completion_tokens_streaming(self, backend, model_case, openai_client_and_model): + client, model_name = openai_client_and_model + outputs = client.chat.completions.create( + model=model_name, + messages=[{'role': 'user', 'content': 'Hi, pls intro yourself'}], + max_completion_tokens=5, + temperature=0.01, + stream=True, + ) + outputList = [chunk.model_dump() for chunk in outputs] + assert_chat_completions_stream_return(outputList[-1], model_name, True) + response = '' + for index in range(0, len(outputList) - 1): + assert_chat_completions_stream_return(outputList[index], model_name) + response += get_chat_delta_text(outputList[index].get('choices')[0]) + _, length = encode_prompt(BASE_URL, response, add_bos=False) + assert outputList[-1].get('choices')[0].get('finish_reason') == 'length' + assert length in (5, 6) + + @pytest.mark.parametrize( + 'invalid_payload', + [ + pytest.param({'max_tokens': 0}, id='max_tokens_zero'), + pytest.param({'max_tokens': -1}, id='max_tokens_negative'), + pytest.param({'temperature': True}, id='temperature_bool'), + ], + ) + def test_rejects_invalid_request_parameters( + self, backend, model_case, openai_client_and_model, invalid_payload): + """Invalid types/ranges must return HTTP 400 (raw JSON, not SDK).""" + _, model_name = openai_client_and_model + resp = requests.post( + _CHAT_COMPLETIONS_URL, + json={ + 'model': model_name, + 'messages': _CHAT_MESSAGES, + **invalid_payload, + }, + timeout=30, + ) + assert_openai_invalid_request_error(resp) + + def test_input_validation(self, backend, model_case, openai_client_and_model): + client, model_name = openai_client_and_model messages = [ { 'role': 'user', @@ -1060,9 +610,8 @@ def test_input_validation(self, backend, model_case): with pytest.raises(Exception): client.chat.completions.create(model=model_name, messages=messages, temperature='test') - def test_input_validation_streaming(self, backend, model_case): - client = OpenAI(api_key='YOUR_API_KEY', base_url=f'{BASE_URL}/v1') - model_name = client.models.list().data[0].id + def test_input_validation_streaming(self, backend, model_case, openai_client_and_model): + client, model_name = openai_client_and_model messages = [ { 'role': 'user', diff --git a/autotest/interface/restful/test_restful_completions_v1.py b/autotest/interface/restful/test_restful_completions_v1.py index 8e46c34d1a..0c95bbb351 100644 --- a/autotest/interface/restful/test_restful_completions_v1.py +++ b/autotest/interface/restful/test_restful_completions_v1.py @@ -1,108 +1,126 @@ import pytest +import requests from utils.constant import BACKEND_LIST, BASE_URL, RESTFUL_BASE_MODEL_LIST -from utils.restful_return_check import assert_completions_batch_return, assert_completions_stream_return +from utils.restful_return_check import ( + assert_completions_batch_return, + assert_completions_stream_return, + assert_openai_invalid_request_error, + get_client_and_model, +) -from lmdeploy.serve.openai.api_client import APIClient +_COMPLETIONS_URL = f'{BASE_URL}/v1/completions' + + +@pytest.fixture(scope='class') +def openai_client_and_model(): + return get_client_and_model(BASE_URL) @pytest.mark.parametrize('backend', BACKEND_LIST) @pytest.mark.parametrize('model_case', RESTFUL_BASE_MODEL_LIST) -class TestRestfulInterfaceBase: +class TestRestfulOpenAICompletions: - def test_return(self, backend, model_case): + def test_return(self, backend, model_case, openai_client_and_model): print(f'[test_return] backend={backend!r} model_case={model_case!r}') - api_client = APIClient(BASE_URL) - model_name = api_client.available_models[0] - for item in api_client.completions_v1( - model=model_name, - prompt='Hi, pls intro yourself', - max_tokens=16, - temperature=0.01, - ): - completion_tokens = item['usage']['completion_tokens'] - assert completion_tokens > 0 - assert completion_tokens <= 17 - assert completion_tokens >= 16 - assert item.get('choices')[0].get('finish_reason') in ['length'] + client, model_name = openai_client_and_model + response = client.completions.create( + model=model_name, + prompt='Hi, pls intro yourself', + max_tokens=16, + temperature=0.01, + ) + item = response.model_dump() + completion_tokens = item['usage']['completion_tokens'] + assert completion_tokens > 0 + assert completion_tokens <= 17 + assert completion_tokens >= 16 + assert item.get('choices')[0].get('finish_reason') in ['length'] print(f'[test_return] model_name={model_name!r} last_usage={item.get("usage")!r} ' f'finish_reason={item.get("choices")[0].get("finish_reason")!r}') assert_completions_batch_return(item, model_name) - def test_return_streaming(self, backend, model_case): + def test_return_streaming(self, backend, model_case, openai_client_and_model): print(f'[test_return_streaming] backend={backend!r} model_case={model_case!r}') - api_client = APIClient(BASE_URL) - model_name = api_client.available_models[0] - outputList = [] - for item in api_client.completions_v1(model=model_name, - prompt='Hi, pls intro yourself', - max_tokens=16, - stream=True, - temperature=0.01): - outputList.append(item) + client, model_name = openai_client_and_model + outputs = client.completions.create( + model=model_name, + prompt='Hi, pls intro yourself', + max_tokens=16, + stream=True, + temperature=0.01, + ) + outputList = [chunk.model_dump() for chunk in outputs] print(f'[test_return_streaming] model_name={model_name!r} stream_chunks={len(outputList)}') assert_completions_stream_return(outputList[-1], model_name, True) for index in range(0, len(outputList) - 1): assert_completions_stream_return(outputList[index], model_name) - def test_max_tokens(self, backend, model_case): + def test_max_tokens(self, backend, model_case, openai_client_and_model): print(f'[test_max_tokens] backend={backend!r} model_case={model_case!r}') - api_client = APIClient(BASE_URL) - model_name = api_client.available_models[0] - for item in api_client.completions_v1(model=model_name, - prompt='Hi, pls intro yourself', - max_tokens=16, - temperature=0.01): - completion_tokens = item['usage']['completion_tokens'] - assert completion_tokens > 0 - assert completion_tokens <= 17 - assert completion_tokens >= 16 - assert item.get('choices')[0].get('finish_reason') in ['length'] + client, model_name = openai_client_and_model + response = client.completions.create( + model=model_name, + prompt='Hi, pls intro yourself', + max_tokens=16, + temperature=0.01, + ) + item = response.model_dump() + completion_tokens = item['usage']['completion_tokens'] + assert completion_tokens > 0 + assert completion_tokens <= 17 + assert completion_tokens >= 16 + assert item.get('choices')[0].get('finish_reason') in ['length'] print(f'[test_max_tokens] completion_tokens={completion_tokens} ' f'finish_reason={item.get("choices")[0].get("finish_reason")!r}') - def test_single_stopword(self, backend, model_case): + def test_single_stopword(self, backend, model_case, openai_client_and_model): print(f'[test_single_stopword] backend={backend!r} model_case={model_case!r}') - api_client = APIClient(BASE_URL) - model_name = api_client.available_models[0] - for item in api_client.completions_v1(model=model_name, - prompt='Shanghai is', - max_tokens=200, - stop=' Shanghai', - temperature=0.01): - assert ' Shanghai' not in item.get('choices')[0].get('text') - assert item.get('choices')[0].get('finish_reason') in ['stop', 'length'] + client, model_name = openai_client_and_model + response = client.completions.create( + model=model_name, + prompt='Shanghai is', + max_tokens=200, + stop=' Shanghai', + temperature=0.01, + ) + item = response.model_dump() + assert ' Shanghai' not in item.get('choices')[0].get('text') + assert item.get('choices')[0].get('finish_reason') in ['stop', 'length'] print(f'[test_single_stopword] finish_reason={item.get("choices")[0].get("finish_reason")!r} ' f'text_preview={((item.get("choices")[0].get("text")) or "")[:120]!r}') - def test_array_stopwords(self, backend, model_case): + def test_array_stopwords(self, backend, model_case, openai_client_and_model): print(f'[test_array_stopwords] backend={backend!r} model_case={model_case!r}') - api_client = APIClient(BASE_URL) - model_name = api_client.available_models[0] - for item in api_client.completions_v1(model=model_name, - prompt='Shanghai is', - max_tokens=200, - stop=[' Shanghai', ' city', ' China'], - temperature=0.01): - assert ' Shanghai' not in item.get('choices')[0].get('text') - assert ' city' not in item.get('choices')[0].get('text') - assert ' China' not in item.get('choices')[0].get('text') - assert item.get('choices')[0].get('finish_reason') in ['stop', 'length'] + client, model_name = openai_client_and_model + response = client.completions.create( + model=model_name, + prompt='Shanghai is', + max_tokens=200, + stop=[' Shanghai', ' city', ' China'], + temperature=0.01, + ) + item = response.model_dump() + assert ' Shanghai' not in item.get('choices')[0].get('text') + assert ' city' not in item.get('choices')[0].get('text') + assert ' China' not in item.get('choices')[0].get('text') + assert item.get('choices')[0].get('finish_reason') in ['stop', 'length'] print(f'[test_array_stopwords] finish_reason={item.get("choices")[0].get("finish_reason")!r} ' f'text_preview={((item.get("choices")[0].get("text")) or "")[:120]!r}') - def test_completions_stream(self, backend, model_case): + def test_completions_stream(self, backend, model_case, openai_client_and_model): print(f'[test_completions_stream] backend={backend!r} model_case={model_case!r}') - api_client = APIClient(BASE_URL) - model_name = api_client.available_models[0] - outputList = [] - for output in api_client.completions_v1(model=model_name, prompt='Shanghai is', stream='true', - temperature=0.01): - outputList.append(output) - + client, model_name = openai_client_and_model + outputs = client.completions.create( + model=model_name, + prompt='Shanghai is', + stream=True, + temperature=0.01, + ) + outputList = [chunk.model_dump() for chunk in outputs] print(f'[test_completions_stream] model_name={model_name!r} stream_chunks={len(outputList)}') for index in range(1, len(outputList) - 1): output = outputList[index] - assert (output.get('model') == model_name) + assert output.get('model') == model_name for message in output.get('choices'): assert message.get('index') == 0 assert len(message.get('text')) > 0 @@ -111,23 +129,22 @@ def test_completions_stream(self, backend, model_case): assert output_last.get('choices')[0].get('finish_reason') in ['stop', 'length'] print(f'[test_completions_stream] last_finish_reason={output_last.get("choices")[0].get("finish_reason")!r}') - def test_completions_stream_stopword(self, backend, model_case): + def test_completions_stream_stopword(self, backend, model_case, openai_client_and_model): print(f'[test_completions_stream_stopword] backend={backend!r} model_case={model_case!r}') - api_client = APIClient(BASE_URL) - model_name = api_client.available_models[0] - outputList = [] - for output in api_client.completions_v1(model=model_name, - prompt='Beijing is', - stream='true', - stop=' is', - temperature=0.01): - outputList.append(output) - + client, model_name = openai_client_and_model + outputs = client.completions.create( + model=model_name, + prompt='Beijing is', + stream=True, + stop=' is', + temperature=0.01, + ) + outputList = [chunk.model_dump() for chunk in outputs] print(f'[test_completions_stream_stopword] model_name={model_name!r} stream_chunks={len(outputList)}') for index in range(1, len(outputList) - 2): output = outputList[index] - assert (output.get('model') == model_name) - assert (output.get('object') == 'text_completion') + assert output.get('model') == model_name + assert output.get('object') == 'text_completion' for message in output.get('choices'): assert ' is' not in message.get('text') assert message.get('index') == 0 @@ -139,23 +156,22 @@ def test_completions_stream_stopword(self, backend, model_case): print(f'[test_completions_stream_stopword] last_finish_reason=' f'{output_last.get("choices")[0].get("finish_reason")!r}') - def test_completions_stream_stopwords(self, backend, model_case): + def test_completions_stream_stopwords(self, backend, model_case, openai_client_and_model): print(f'[test_completions_stream_stopwords] backend={backend!r} model_case={model_case!r}') - api_client = APIClient(BASE_URL) - model_name = api_client.available_models[0] - outputList = [] - for output in api_client.completions_v1(model=model_name, - prompt='Beijing is', - stream='true', - stop=[' Beijing', ' city', ' China'], - temperature=0.01): - outputList.append(output) - + client, model_name = openai_client_and_model + outputs = client.completions.create( + model=model_name, + prompt='Beijing is', + stream=True, + stop=[' Beijing', ' city', ' China'], + temperature=0.01, + ) + outputList = [chunk.model_dump() for chunk in outputs] print(f'[test_completions_stream_stopwords] model_name={model_name!r} stream_chunks={len(outputList)}') for index in range(1, len(outputList) - 2): output = outputList[index] - assert (output.get('model') == model_name) - assert (output.get('object') == 'text_completion') + assert output.get('model') == model_name + assert output.get('object') == 'text_completion' for message in output.get('choices'): assert ' Beijing' not in message.get('text') assert ' city' not in message.get('text') @@ -169,20 +185,44 @@ def test_completions_stream_stopwords(self, backend, model_case): print(f'[test_completions_stream_stopwords] last_finish_reason=' f'{output_last.get("choices")[0].get("finish_reason")!r}') - def test_batch_prompt_order(self, backend, model_case): + def test_batch_prompt_order(self, backend, model_case, openai_client_and_model): print(f'[test_batch_prompt_order] backend={backend!r} model_case={model_case!r}') - api_client = APIClient(BASE_URL) - model_name = api_client.available_models[0] - for item in api_client.completions_v1(model=model_name, - prompt=['你好', '今天天气怎么样', '你是谁', - '帮我写一首以梅花为主题的五言律诗', '5+2等于多少'], - max_tokens=400, - min_tokens=50): - print(f'[test_batch_prompt_order] batch_response={item!r}') - assert '天' in item.get('choices')[1].get('text') or '雨' in item.get('choices')[1].get( - 'text') or '伞' in item.get('choices')[1].get('text'), item.get('choices')[1].get('text') - assert '梅' in item.get('choices')[3].get('text') or '对仗' in item.get('choices')[3].get( - 'text') or '仄' in item.get('choices')[3].get('text') or '诗' in item.get('choices')[3].get( - 'text'), item.get('choices')[3].get('text') - assert '7' in item.get('choices')[4].get('text') or '5+2' in item.get('choices')[4].get('text'), item.get( - 'choices')[4].get('text') + client, model_name = openai_client_and_model + response = client.completions.create( + model=model_name, + prompt=['你好', '今天天气怎么样', '你是谁', '帮我写一首以梅花为主题的五言律诗', '5+2等于多少'], + max_tokens=400, + extra_body={'min_new_tokens': 50}, + ) + item = response.model_dump() + print(f'[test_batch_prompt_order] batch_response={item!r}') + assert '天' in item.get('choices')[1].get('text') or '雨' in item.get('choices')[1].get( + 'text') or '伞' in item.get('choices')[1].get('text'), item.get('choices')[1].get('text') + assert '梅' in item.get('choices')[3].get('text') or '对仗' in item.get('choices')[3].get( + 'text') or '仄' in item.get('choices')[3].get('text') or '诗' in item.get('choices')[3].get( + 'text'), item.get('choices')[3].get('text') + assert '7' in item.get('choices')[4].get('text') or '5+2' in item.get('choices')[4].get('text'), item.get( + 'choices')[4].get('text') + + @pytest.mark.parametrize( + 'invalid_payload', + [ + pytest.param({'max_tokens': 0}, id='max_tokens_zero'), + pytest.param({'max_tokens': -1}, id='max_tokens_negative'), + pytest.param({'temperature': True}, id='temperature_bool'), + ], + ) + def test_rejects_invalid_request_parameters( + self, backend, model_case, openai_client_and_model, invalid_payload): + """Invalid types/ranges must return HTTP 400 (raw JSON, not SDK).""" + _, model_name = openai_client_and_model + resp = requests.post( + _COMPLETIONS_URL, + json={ + 'model': model_name, + 'prompt': 'Hi, pls intro yourself', + **invalid_payload, + }, + timeout=30, + ) + assert_openai_invalid_request_error(resp) diff --git a/autotest/interface/restful/test_restful_generate.py b/autotest/interface/restful/test_restful_generate.py index 495a3ddc43..935edcd07e 100644 --- a/autotest/interface/restful/test_restful_generate.py +++ b/autotest/interface/restful/test_restful_generate.py @@ -11,10 +11,9 @@ from transformers import AutoTokenizer from utils.config_utils import get_model_path_from_config from utils.constant import BACKEND_LIST, BASE_URL, DEFAULT_MAX_COMPLETION_TOKENS, RESTFUL_MODEL_LIST +from utils.restful_return_check import encode_prompt from utils.toolkit import encode_text, parse_sse_stream -from lmdeploy.serve.openai.api_client import APIClient - @pytest.mark.parametrize('backend', BACKEND_LIST) @pytest.mark.parametrize('model_name', RESTFUL_MODEL_LIST) @@ -733,8 +732,8 @@ def test_input_ids_rejected(self): self._post(payload) response = exc_info.value.response - assert response.status_code in [400, 422], (f"Bad Request for case '{test_desc}', " - f'but got {response.status_code}') + assert response.status_code == 400, (f"Bad Request for case '{test_desc}', " + f'but got {response.status_code}') def test_stress_concurrent_requests(self): print(f'\n[Model: {self.model_name}] Running stress concurrent requests test') @@ -1024,8 +1023,7 @@ def test_skip_special_tokens(self, config): def test_stop_token_ids(self): print(f'\n[Model: {self.model_name}] Running stop_token_ids test') - api_client = APIClient(BASE_URL) - input_ids1, length1 = api_client.encode('.', add_bos=False) + input_ids1, length1 = encode_prompt(BASE_URL, '.', add_bos=False) print(f'input_ids1={input_ids1}, length1={length1}') payload = { @@ -1105,7 +1103,7 @@ def test_invalid_temperature_values(self): with pytest.raises(requests.HTTPError) as exc_info: self._post({'prompt': 'Test', 'max_tokens': 3, 'temperature': -0.5, 'stream': False}) - assert exc_info.value.response.status_code in [400, 422] + assert exc_info.value.response.status_code == 400 print(' Invalid temperature values test passed') @@ -1113,7 +1111,7 @@ def test_invalid_top_p_values(self): print(f'\n[Model: {self.model_name}] Running invalid top_p values test') with pytest.raises(requests.HTTPError) as exc_info: self._post({'prompt': 'Test', 'max_tokens': 3, 'top_p': 1.5, 'stream': False}) - assert exc_info.value.response.status_code in [400, 422] + assert exc_info.value.response.status_code == 400 print(' Invalid top_p values test passed') @@ -1121,7 +1119,7 @@ def test_invalid_top_k_values(self): print(f'\n[Model: {self.model_name}] Running invalid top_k values test') with pytest.raises(requests.HTTPError) as exc_info: self._post({'prompt': 'Test', 'max_tokens': 3, 'top_k': -5, 'stream': False}) - assert exc_info.value.response.status_code in [400, 422] + assert exc_info.value.response.status_code == 400 print(' Invalid top_k values test passed') diff --git a/autotest/interface/restful/tool_parser/test_tool_call_anthropic_sdk.py b/autotest/interface/restful/tool_parser/test_tool_call_anthropic_sdk.py index ac5f6f5122..74bf48accc 100644 --- a/autotest/interface/restful/tool_parser/test_tool_call_anthropic_sdk.py +++ b/autotest/interface/restful/tool_parser/test_tool_call_anthropic_sdk.py @@ -31,13 +31,17 @@ ) from utils.config_utils import get_config from utils.constant import BASE_URL, DEFAULT_MAX_COMPLETION_TOKENS - -from lmdeploy.serve.openai.api_client import APIClient +from utils.restful_return_check import get_client_and_model from .conftest import _apply_marks, _ToolCallTestBase ANTHROPIC_VERSION = '2023-06-01' + +@lru_cache(maxsize=1) +def _deployed_model_name() -> str: + return get_client_and_model(BASE_URL)[1] + _EVAL_IMAGE_TIGER = 'tiger.jpeg' _TINY_PNG_BASE64 = ( 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==' @@ -312,7 +316,7 @@ class TestAnthropicHttpToolMessages(_ToolCallTestBase): """ def test_http_stream_tool_choice_force_named_tool(self, backend, model_case): - model_name = APIClient(BASE_URL).available_models[0] + model_name = _deployed_model_name() url = f'{BASE_URL}/v1/messages' req_json = { 'model': model_name, @@ -351,7 +355,7 @@ def test_http_stream_tool_choice_force_named_tool(self, backend, model_case): assert_weather_tool_city_state(inputs[0], ctx='test_http_stream_tool_choice_force_named_tool') def test_http_stream_single_location_weather_tool(self, backend, model_case): - model_name = APIClient(BASE_URL).available_models[0] + model_name = _deployed_model_name() url = f'{BASE_URL}/v1/messages' req_json = { 'model': model_name, @@ -387,7 +391,7 @@ def test_http_stream_single_location_weather_tool(self, backend, model_case): assert 'dallas' in loc, inputs def test_http_parallel_same_tool_stream(self, backend, model_case): - model_name = APIClient(BASE_URL).available_models[0] + model_name = _deployed_model_name() url = f'{BASE_URL}/v1/messages' req_json = { 'model': model_name, @@ -425,7 +429,7 @@ def test_http_parallel_same_tool_stream(self, backend, model_case): ) def test_http_full_roundtrip_single_tool_result(self, backend, model_case): - model_name = APIClient(BASE_URL).available_models[0] + model_name = _deployed_model_name() url = f'{BASE_URL}/v1/messages' turn1 = { 'model': model_name, @@ -483,7 +487,7 @@ def test_http_full_roundtrip_single_tool_result(self, backend, model_case): assert '98' in text or 'Dallas' in text or 'sunny' in text.lower(), text[:500] def test_http_history_tool_use_and_tool_result_blocks(self, backend, model_case): - model_name = APIClient(BASE_URL).available_models[0] + model_name = _deployed_model_name() url = f'{BASE_URL}/v1/messages' req_json = { 'model': model_name, @@ -515,7 +519,7 @@ def test_http_history_tool_use_and_tool_result_blocks(self, backend, model_case) ) def test_http_history_thinking_block_replay(self, backend, model_case): - model_name = APIClient(BASE_URL).available_models[0] + model_name = _deployed_model_name() url = f'{BASE_URL}/v1/messages' req_json = { 'model': model_name, @@ -549,7 +553,7 @@ def test_http_non_stream_tools_with_user_image_url(self, backend, model_case): """``tools`` + user ``content`` blocks with ``image`` (VLM matrix only; same tool contract as text-only).""" - model_name = APIClient(BASE_URL).available_models[0] + model_name = _deployed_model_name() if not _model_likely_supports_anthropic_vlm(model_name): pytest.skip(f'model {model_name!r} is not treated as vision-capable for this test') @@ -591,7 +595,7 @@ def test_http_stream_tools_with_user_image_url(self, backend, model_case): """Streaming ``tools`` + user image URL (VLM): SSE must still surface ``tool_use``.""" - model_name = APIClient(BASE_URL).available_models[0] + model_name = _deployed_model_name() if not _model_likely_supports_anthropic_vlm(model_name): pytest.skip(f'model {model_name!r} is not treated as vision-capable for this test') @@ -631,7 +635,7 @@ def test_http_stream_user_image_base64_solid_color_vlm(self, backend, model_case """Align with RESTful ``test_messages_user_image_base64_stream``: SSE text names the solid color.""" - model_name = APIClient(BASE_URL).available_models[0] + model_name = _deployed_model_name() if not _model_likely_supports_anthropic_vlm(model_name): pytest.skip(f'model {model_name!r} is not treated as vision-capable for this test') @@ -1143,7 +1147,7 @@ def test_tool_non_stream_tool_choice_any(self, backend, model_case): assert WEATHER_TOOL_ANTHROPIC['name'] in names, names def test_tool_non_stream_weather_with_user_image_url(self, backend, model_case): - model_name = APIClient(BASE_URL).available_models[0] + model_name = _deployed_model_name() if not _model_likely_supports_anthropic_vlm(model_name): pytest.skip(f'model {model_name!r} is not treated as vision-capable for this test') @@ -1155,7 +1159,7 @@ def test_tool_non_stream_weather_with_user_image_url(self, backend, model_case): assert_weather_tool_city_state(tool_blocks[0].input, ctx='test_tool_non_stream_weather_with_user_image_url') def test_tool_non_stream_weather_with_user_image_base64(self, backend, model_case): - model_name = APIClient(BASE_URL).available_models[0] + model_name = _deployed_model_name() if not _model_likely_supports_anthropic_vlm(model_name): pytest.skip(f'model {model_name!r} is not treated as vision-capable for this test') @@ -1172,7 +1176,7 @@ def test_sdk_stream_vlm_user_image_base64_solid_color(self, backend, model_case) """SDK streaming + 1×1 red PNG: final text (or raw event blob) should mention a red-ish color.""" - model_name = APIClient(BASE_URL).available_models[0] + model_name = _deployed_model_name() if not _model_likely_supports_anthropic_vlm(model_name): pytest.skip(f'model {model_name!r} is not treated as vision-capable for this test') diff --git a/autotest/utils/anthropic_messages.py b/autotest/utils/anthropic_messages.py index dcec9d2c82..209fa95bb4 100644 --- a/autotest/utils/anthropic_messages.py +++ b/autotest/utils/anthropic_messages.py @@ -368,14 +368,10 @@ def get_async_anthropic_client_and_model(base_url: str | None = None): routes.""" import anthropic - - from lmdeploy.serve.openai.api_client import get_model_list + from utils.restful_return_check import get_client_and_model url = base_url or BASE_URL - model_names = get_model_list(f'{url}/v1/models') - if not model_names: - raise RuntimeError(f'No models returned from {url}/v1/models') - model_name = model_names[0] + _, model_name = get_client_and_model(url) client = anthropic.AsyncAnthropic( api_key=os.getenv('ANTHROPIC_API_KEY', 'YOUR_API_KEY'), base_url=url, diff --git a/autotest/utils/restful_return_check.py b/autotest/utils/restful_return_check.py index da1d8cee89..cf3c37f5d8 100644 --- a/autotest/utils/restful_return_check.py +++ b/autotest/utils/restful_return_check.py @@ -1,9 +1,40 @@ import re +import requests +from openai import OpenAI +from utils.constant import BASE_URL + # Preprocess rejects oversize input with this OpenAI error substring. CONTEXT_LENGTH_ERROR = 'context length' +def assert_openai_invalid_request_error( + response: requests.Response | dict, + *, + message_substr: str | None = None, +) -> dict: + """Assert OpenAI invalid request (HTTP 400, ``invalid_request_error``). + + Accepts a ``requests.Response`` (raw HTTP) or an error body ``dict`` + (e.g. OpenAI SDK ``BadRequestError.body``). + """ + if isinstance(response, requests.Response): + assert response.status_code == 400, ( + f'expected 400, got {response.status_code}: {response.text[:500]}') + body = response.json() + else: + body = response + + assert body.get('object') == 'error' + assert body.get('type') == 'invalid_request_error' + assert body.get('code') == 400 + message = body.get('message') + assert message + if message_substr is not None: + assert message_substr.lower() in message.lower() + return body + + def get_chat_message_text(choice): msg = choice.get('message') or {} texts = [] @@ -24,14 +55,6 @@ def get_chat_delta_text(choice): return ''.join(texts) -def assert_chat_message_error(output, message_substr=CONTEXT_LENGTH_ERROR): - """Assert OpenAI preprocess/validation error envelope.""" - assert output.get('object') == 'error' - assert output.get('type') == 'invalid_request_error' - assert output.get('code') == 400 - assert message_substr.lower() in output.get('message').lower() - - def assert_chat_message_empty(choice): assert not get_chat_message_text(choice) @@ -169,3 +192,27 @@ def has_repeated_fragment(text, repeat_count=5): start_pos = match.start() return True, {'repeated_fragment': repeated_fragment, 'position': start_pos} return False, f'{text} does not contain repeated fragments' + + +def get_client_and_model(base_url: str | None = None) -> tuple[OpenAI, str]: + """Return ``(OpenAI client, deployed model id)`` for a running + api_server.""" + url = base_url or BASE_URL + client = OpenAI(api_key='YOUR_API_KEY', base_url=f'{url.rstrip("/")}/v1') + models = client.models.list().data + if not models: + raise RuntimeError(f'No model returned from GET {url}/v1/models') + return client, models[0].id + + +def encode_prompt(base_url: str, text: str, *, add_bos: bool = True) -> tuple[list, int]: + """Tokenize via ``POST /v1/encode``; returns ``(input_ids, length)``.""" + url = base_url.rstrip('/') + response = requests.post( + f'{url}/v1/encode', + json={'input': text, 'do_preprocess': False, 'add_bos': add_bos}, + timeout=30, + ) + response.raise_for_status() + output = response.json() + return output['input_ids'], output['length'] diff --git a/autotest/utils/run_restful_chat.py b/autotest/utils/run_restful_chat.py index 5d815d8d18..fef1929a29 100644 --- a/autotest/utils/run_restful_chat.py +++ b/autotest/utils/run_restful_chat.py @@ -7,7 +7,7 @@ import allure import psutil import requests -from openai import APIStatusError, BadRequestError, OpenAI +from openai import APIStatusError, BadRequestError from pytest_assume.plugin import assume from utils.ascend_multinode_utils import build_ascend_multinode_env, ensure_ascend_multinode_env from utils.config_utils import ( @@ -19,10 +19,9 @@ resolve_extra_params, ) from utils.constant import DEFAULT_PORT, DEFAULT_SERVER, MM_DEMO_TOMB_USER_PROMPT -from utils.restful_return_check import assert_chat_completions_batch_return +from utils.restful_return_check import assert_chat_completions_batch_return, get_client_and_model from utils.rule_condition_assert import assert_result -from lmdeploy.serve.openai.api_client import APIClient from lmdeploy.serve.parsers.response_parser import _parse_tool_call_arguments_dict BASE_HTTP_URL = f'http://{DEFAULT_SERVER}' @@ -178,8 +177,7 @@ def open_chat_test(log_path, case_name, case_info, url): result = True - client = OpenAI(api_key='YOUR_API_KEY', base_url=f'{url}/v1') - model_name = client.models.list().data[0].id + client, model_name = get_client_and_model(url) messages = [] msg = '' @@ -228,16 +226,13 @@ def open_chat_test(log_path, case_name, case_info, url): def health_check(url, model_name): try: - api_client = APIClient(url) - model_name_current = api_client.available_models[0] - messages = [] - messages.append({'role': 'user', 'content': '你好'}) - for output in api_client.chat_completions_v1(model=model_name, messages=messages, top_k=1): - if output.get('code') is not None and output.get('code') != 0: - return False - # Return True on first successful response - return model_name == model_name_current - return False # No output received + client, model_name_current = get_client_and_model(url) + response = client.chat.completions.create( + model=model_name, + messages=[{'role': 'user', 'content': '你好'}], + extra_body={'top_k': 1}, + ) + return model_name == model_name_current and response.choices is not None except Exception: return False @@ -245,28 +240,34 @@ def health_check(url, model_name): def get_model(url): print(url) try: - api_client = APIClient(url) - model_name = api_client.available_models[0] + _, model_name = get_client_and_model(url) return model_name.split('/')[-1] except Exception: return None +def _require_client_and_model(url): + """Return ``(client, model_name, short_model_name)``; fail if server is + down.""" + try: + client, model_name = get_client_and_model(url) + except Exception: + assert False, 'server not start correctly' + return client, model_name, model_name.split('/')[-1] + + def _run_logprobs_test(port: int = DEFAULT_PORT): http_url = ':'.join([BASE_HTTP_URL, str(port)]) - api_client = APIClient(http_url) - model_name = api_client.available_models[0] - output = None - for output in api_client.chat_completions_v1(model=model_name, - messages='Hi, pls intro yourself', - max_tokens=5, - temperature=0.01, - logprobs=True, - top_logprobs=10): - continue - if output is None: - assert False, 'No output received from logprobs test' - print(output) + client, model_name = get_client_and_model(http_url) + response = client.chat.completions.create( + model=model_name, + messages=[{'role': 'user', 'content': 'Hi, pls intro yourself'}], + max_tokens=5, + temperature=0.01, + logprobs=True, + top_logprobs=10, + ) + output = response.model_dump() assert_chat_completions_batch_return(output, model_name, check_logprobs=True, logprobs_num=10) assert output.get('choices')[0].get('finish_reason') == 'length' assert output.get('usage').get('completion_tokens') == 6 or output.get('usage').get('completion_tokens') == 5 @@ -503,16 +504,10 @@ def _is_video_mixed_whitelist_model(model_name: str) -> bool: def run_vl_testcase(log_path, resource_path, port: int = DEFAULT_PORT): http_url = ':'.join([BASE_HTTP_URL, str(port)]) - model = get_model(http_url) - if model is None: - assert False, 'server not start correctly' - - client = OpenAI(api_key='YOUR_API_KEY', base_url=http_url + '/v1') - model_name = client.models.list().data[0].id + client, model_name, simple_model_name = _require_client_and_model(http_url) timestamp = time.strftime('%Y%m%d_%H%M%S') - simple_model_name = model_name.split('/')[-1] restful_log = os.path.join(log_path, f'restful_vl_{simple_model_name}_{str(port)}_{timestamp}.log') # noqa file = open(restful_log, 'w') @@ -538,12 +533,6 @@ def run_vl_testcase(log_path, resource_path, port: int = DEFAULT_PORT): response = client.chat.completions.create(model=model_name, messages=prompt_messages, temperature=0.8, top_p=0.8) file.writelines(str(response).lower() + '\n') - api_client = APIClient(http_url) - model_name = api_client.available_models[0] - for item in api_client.chat_completions_v1(model=model_name, messages=prompt_messages): - continue - file.writelines(str(item) + '\n') - enable_video_mixed = _is_video_mixed_whitelist_model(model_name) if not enable_video_mixed: file.writelines( @@ -558,11 +547,6 @@ def run_vl_testcase(log_path, resource_path, port: int = DEFAULT_PORT): assert ( 'tiger' in resp_lower or '虎' in resp_lower or 'ski' in resp_lower or '滑雪' in resp_lower ), response - with assume: - item_lower = str(item).lower() - assert ( - 'tiger' in item_lower or '虎' in item_lower or 'ski' in item_lower or '滑雪' in item_lower - ), item return video_path = os.path.join(resource_path, VIDEO) @@ -831,26 +815,17 @@ def run_vl_testcase(log_path, resource_path, port: int = DEFAULT_PORT): with assume: assert 'tiger' in str(response).lower() or '虎' in str(response).lower() or 'ski' in str( response).lower() or '滑雪' in str(response).lower(), response - with assume: - assert 'tiger' in str(item).lower() or '虎' in str(item).lower() or 'ski' in str(item).lower() or '滑雪' in str( - item).lower(), item def _run_reasoning_case(log_path, port: int = DEFAULT_PORT): http_url = ':'.join([BASE_HTTP_URL, str(port)]) - model = get_model(http_url) - - if model is None: - assert False, 'server not start correctly' + client, model_name, model = _require_client_and_model(http_url) timestamp = time.strftime('%Y%m%d_%H%M%S') restful_log = os.path.join(log_path, f'restful_reasoning_{model}_{str(port)}_{timestamp}.log') file = open(restful_log, 'w') - client = OpenAI(api_key='YOUR_API_KEY', base_url=http_url + '/v1') - model_name = client.models.list().data[0].id - with allure.step('step1 - stream'): messages = [{'role': 'user', 'content': '9.11 and 9.8, which is greater?'}] response = client.chat.completions.create(model=model_name, messages=messages, temperature=0.01, stream=True) @@ -1132,15 +1107,10 @@ def get_function_by_name(name): def _run_tools_case(log_path, port: int = DEFAULT_PORT): http_url = ':'.join([BASE_HTTP_URL, str(port)]) - model = get_model(http_url) - - if model is None: - assert False, 'server not start correctly' + client, model_name, model = _require_client_and_model(http_url) timestamp = time.strftime('%Y%m%d_%H%M%S') restful_log = os.path.join(log_path, f'restful_toolcall_{model}_{str(port)}_{timestamp}.log') - client = OpenAI(api_key='YOUR_API_KEY', base_url=http_url + '/v1') - model_name = client.models.list().data[0].id with open(restful_log, 'a') as file: with allure.step('step1 - one_round_prompt'): diff --git a/autotest/utils/tool_reasoning_definitions.py b/autotest/utils/tool_reasoning_definitions.py index e09fc21b28..6d02f88be6 100644 --- a/autotest/utils/tool_reasoning_definitions.py +++ b/autotest/utils/tool_reasoning_definitions.py @@ -7,9 +7,9 @@ import aiohttp import requests -from openai import OpenAI from utils.config_utils import get_model_path_from_config from utils.constant import DEFAULT_MAX_COMPLETION_TOKENS, DEFAULT_PORT +from utils.restful_return_check import get_client_and_model from lmdeploy.serve.openai.protocol import ( ChatCompletionRequest, @@ -201,15 +201,6 @@ def get_reasoning_open_close_tags(reasoning_parser_name: str = 'default') -> tup } -def get_client_and_model(base_url=None): - url = base_url or BASE_URL - client = OpenAI(api_key='YOUR_API_KEY', base_url=f'{url}/v1') - models = client.models.list().data - if not models: - raise RuntimeError(f'No model returned from GET {url}/v1/models') - return client, models[0].id - - # -- Logging / client helpers ------------------------------------------------ From 3673f0192294c756cf826365c452350d350dd1b2 Mon Sep 17 00:00:00 2001 From: littlegy <787321726@qq.com> Date: Tue, 25 Aug 2026 16:13:31 +0800 Subject: [PATCH 2/3] update anthropic version --- .../test_restful_anthropic_sdk_messages.py | 11 ++++--- .../test_tool_call_anthropic_sdk.py | 31 ++++++++++--------- autotest/utils/anthropic_messages.py | 18 +++++++++++ requirements/test.txt | 2 +- 4 files changed, 42 insertions(+), 20 deletions(-) diff --git a/autotest/interface/restful/test_restful_anthropic_sdk_messages.py b/autotest/interface/restful/test_restful_anthropic_sdk_messages.py index 40cd845091..5f68967c6f 100644 --- a/autotest/interface/restful/test_restful_anthropic_sdk_messages.py +++ b/autotest/interface/restful/test_restful_anthropic_sdk_messages.py @@ -7,7 +7,10 @@ pytest.importorskip('anthropic') -from utils.anthropic_messages import get_async_anthropic_client_and_model +from utils.anthropic_messages import ( + anthropic_extra_body, + get_async_anthropic_client_and_model, +) from utils.constant import BACKEND_LIST, RESTFUL_MODEL_LIST @@ -33,7 +36,7 @@ async def _sdk_simple_non_stream() -> object: return await client.messages.create( model=model_name, max_tokens=1024, - temperature=0.01, + extra_body=anthropic_extra_body(temperature=0.01), messages=[{'role': 'user', 'content': 'how are you!'}], ) @@ -43,7 +46,7 @@ async def _sdk_system_non_stream() -> object: return await client.messages.create( model=model_name, max_tokens=1024, - temperature=0.01, + extra_body=anthropic_extra_body(temperature=0.01), system=[{'type': 'text', 'text': 'you are a helpful assistant'}], messages=[{'role': 'user', 'content': 'how are you!'}], ) @@ -54,7 +57,7 @@ async def _sdk_stream_events_and_final() -> tuple[list, object | None]: stream = await client.messages.create( model=model_name, max_tokens=1024, - temperature=0.01, + extra_body=anthropic_extra_body(temperature=0.01), messages=[{'role': 'user', 'content': 'how are you!'}], stream=True, ) diff --git a/autotest/interface/restful/tool_parser/test_tool_call_anthropic_sdk.py b/autotest/interface/restful/tool_parser/test_tool_call_anthropic_sdk.py index 74bf48accc..7fb469717a 100644 --- a/autotest/interface/restful/tool_parser/test_tool_call_anthropic_sdk.py +++ b/autotest/interface/restful/tool_parser/test_tool_call_anthropic_sdk.py @@ -21,6 +21,7 @@ USER_ASK_WEATHER_DALLAS_VLM, WEATHER_TOOL_ANTHROPIC, WEATHER_TOOL_SINGLE_LOCATION_ANTHROPIC, + anthropic_extra_body, assert_parallel_weather_tool_inputs, assert_tool_use_message, assert_warm_yes_answer, @@ -692,7 +693,7 @@ async def _async_weather_tool_single_location_non_stream(log_file: str): msg = await client.messages.create( model=model_name, max_tokens=_TOOL_MAX_TOKENS, - temperature=0, + extra_body=anthropic_extra_body(temperature=0), messages=[{'role': 'user', 'content': USER_ASK_WEATHER_DALLAS}], tools=[WEATHER_TOOL_SINGLE_LOCATION_ANTHROPIC], ) @@ -712,7 +713,7 @@ async def _async_tool_choice_force_named_tool(log_file: str): msg = await client.messages.create( model=model_name, max_tokens=_TOOL_MAX_TOKENS, - temperature=0, + extra_body=anthropic_extra_body(temperature=0), system=ANTHROPIC_SYSTEM_WEATHER, messages=ANTHROPIC_MESSAGES_ASKING_FOR_WEATHER, tools=[WEATHER_TOOL_ANTHROPIC, SEARCH_TOOL_ANTHROPIC], @@ -734,7 +735,7 @@ async def _async_tool_choice_any(log_file: str): msg = await client.messages.create( model=model_name, max_tokens=_TOOL_MAX_TOKENS, - temperature=0, + extra_body=anthropic_extra_body(temperature=0), system=ANTHROPIC_SYSTEM_WEATHER, messages=ANTHROPIC_MESSAGES_ASKING_FOR_WEATHER, tools=[WEATHER_TOOL_ANTHROPIC, SEARCH_TOOL_ANTHROPIC], @@ -756,7 +757,7 @@ async def _async_messages_tool_non_stream_with_user_image(log_file: str, image_u msg = await client.messages.create( model=model_name, max_tokens=_VLM_TOOL_MAX_TOKENS, - temperature=0, + extra_body=anthropic_extra_body(temperature=0), system=ANTHROPIC_SYSTEM_WEATHER, tools=[WEATHER_TOOL_ANTHROPIC, SEARCH_TOOL_ANTHROPIC], messages=[{ @@ -786,7 +787,7 @@ async def _async_messages_tool_non_stream_with_user_image_base64(log_file: str): msg = await client.messages.create( model=model_name, max_tokens=_VLM_TOOL_MAX_TOKENS, - temperature=0, + extra_body=anthropic_extra_body(temperature=0), system=ANTHROPIC_SYSTEM_WEATHER, tools=[WEATHER_TOOL_ANTHROPIC, SEARCH_TOOL_ANTHROPIC], messages=[{ @@ -820,7 +821,7 @@ async def _async_messages_tool_non_stream(log_file: str): msg = await client.messages.create( model=model_name, max_tokens=_TOOL_MAX_TOKENS, - temperature=0, + extra_body=anthropic_extra_body(temperature=0), system=ANTHROPIC_SYSTEM_WEATHER, messages=ANTHROPIC_MESSAGES_ASKING_FOR_WEATHER, tools=[WEATHER_TOOL_ANTHROPIC, SEARCH_TOOL_ANTHROPIC], @@ -841,7 +842,7 @@ async def _async_messages_tool_stream(log_file: str): stream = await client.messages.create( model=model_name, max_tokens=_TOOL_MAX_TOKENS, - temperature=0, + extra_body=anthropic_extra_body(temperature=0), system=ANTHROPIC_SYSTEM_WEATHER, messages=ANTHROPIC_MESSAGES_ASKING_FOR_WEATHER, tools=[WEATHER_TOOL_ANTHROPIC, SEARCH_TOOL_ANTHROPIC], @@ -881,7 +882,7 @@ async def _async_parallel_same_tool_non_stream(log_file: str): msg = await client.messages.create( model=model_name, max_tokens=_TOOL_MAX_TOKENS, - temperature=0, + extra_body=anthropic_extra_body(temperature=0), system=ANTHROPIC_SYSTEM_PARALLEL_WEATHER, messages=ANTHROPIC_MESSAGES_PARALLEL_WEATHER, tools=[WEATHER_TOOL_ANTHROPIC], @@ -895,7 +896,7 @@ async def _async_parallel_same_tool_stream(log_file: str): stream = await client.messages.create( model=model_name, max_tokens=_TOOL_MAX_TOKENS, - temperature=0, + extra_body=anthropic_extra_body(temperature=0), system=ANTHROPIC_SYSTEM_PARALLEL_WEATHER, messages=ANTHROPIC_MESSAGES_PARALLEL_WEATHER, tools=[WEATHER_TOOL_ANTHROPIC], @@ -919,7 +920,7 @@ async def _async_parallel_mixed_tools_non_stream(log_file: str): msg = await client.messages.create( model=model_name, max_tokens=_TOOL_MAX_TOKENS, - temperature=0, + extra_body=anthropic_extra_body(temperature=0), system=ANTHROPIC_SYSTEM_PARALLEL_MIXED, messages=ANTHROPIC_MESSAGES_PARALLEL_MIXED, tools=[WEATHER_TOOL_ANTHROPIC, CALCULATOR_TOOL_ANTHROPIC], @@ -934,7 +935,7 @@ async def _async_full_roundtrip_single_tool_result(log_file: str): msg1 = await client.messages.create( model=model_name, max_tokens=_TOOL_MAX_TOKENS, - temperature=0, + extra_body=anthropic_extra_body(temperature=0), system=ANTHROPIC_SYSTEM_WEATHER, messages=ANTHROPIC_MESSAGES_ASKING_FOR_WEATHER, tools=tools, @@ -955,7 +956,7 @@ async def _async_full_roundtrip_single_tool_result(log_file: str): msg2 = await client.messages.create( model=model_name, max_tokens=_TOOL_MAX_TOKENS, - temperature=0, + extra_body=anthropic_extra_body(temperature=0), system=ANTHROPIC_SYSTEM_WEATHER, messages=turn2_messages, tools=tools, @@ -969,7 +970,7 @@ async def _async_full_roundtrip_parallel_tool_results(log_file: str): msg1 = await client.messages.create( model=model_name, max_tokens=_TOOL_MAX_TOKENS, - temperature=0, + extra_body=anthropic_extra_body(temperature=0), system=ANTHROPIC_SYSTEM_PARALLEL_WEATHER, messages=ANTHROPIC_MESSAGES_PARALLEL_WEATHER, tools=[WEATHER_TOOL_ANTHROPIC], @@ -1002,7 +1003,7 @@ async def _async_full_roundtrip_parallel_tool_results(log_file: str): msg2 = await client.messages.create( model=model_name, max_tokens=_TOOL_MAX_TOKENS, - temperature=0, + extra_body=anthropic_extra_body(temperature=0), system=ANTHROPIC_SYSTEM_PARALLEL_WEATHER, messages=turn2_messages, tools=[WEATHER_TOOL_ANTHROPIC], @@ -1040,7 +1041,7 @@ async def _async_vlm_base64_solid_color_stream(log_file: str) -> tuple[str, str] stream = await client.messages.create( model=model_name, max_tokens=16384, - temperature=0.01, + extra_body=anthropic_extra_body(temperature=0.01), stream=True, messages=[{ 'role': 'user', diff --git a/autotest/utils/anthropic_messages.py b/autotest/utils/anthropic_messages.py index 209fa95bb4..f39e6262da 100644 --- a/autotest/utils/anthropic_messages.py +++ b/autotest/utils/anthropic_messages.py @@ -4,6 +4,7 @@ import os import re +from collections.abc import Mapping from utils.constant import BASE_URL @@ -363,6 +364,23 @@ def _matches_site(city: str, state: str, site: tuple[tuple[str, ...], str]) -> b # -- Client / message helpers ----------------------------------------------- +def anthropic_extra_body( + *parts: Mapping[str, object] | None, + **fields: object, +) -> dict[str, object]: + """Build ``extra_body`` for Anthropic SDK >=1.0 (fields not on + ``messages.create()``).""" + + body: dict[str, object] = {} + for part in parts: + if part: + body.update(part) + for key, value in fields.items(): + if value is not None: + body[key] = value + return body + + def get_async_anthropic_client_and_model(base_url: str | None = None): """Return ``(AsyncAnthropic, model_name)`` for LMDeploy Anthropic routes.""" diff --git a/requirements/test.txt b/requirements/test.txt index 3580dfc179..97135d74d3 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,5 +1,5 @@ allure-pytest -anthropic>=0.39.0 +anthropic>=1.0.0 coverage jsonschema matplotlib From af07df62b00c0a4cc769e6c70aca153e47d76dc2 Mon Sep 17 00:00:00 2001 From: littlegy <787321726@qq.com> Date: Fri, 28 Aug 2026 09:59:20 +0800 Subject: [PATCH 3/3] add session-payload helpers to align test caps with engine limits --- .../restful/test_restful_anthropic_v1.py | 17 +-- .../test_restful_chat_completions_v1.py | 48 +++++--- .../restful/test_restful_completions_v1.py | 1 - .../restful/test_restful_generate.py | 30 +++-- .../tool_parser/test_tool_call_advanced.py | 8 +- autotest/utils/restful_return_check.py | 111 ++++++++++++++++-- autotest/utils/toolkit.py | 7 +- 7 files changed, 166 insertions(+), 56 deletions(-) diff --git a/autotest/interface/restful/test_restful_anthropic_v1.py b/autotest/interface/restful/test_restful_anthropic_v1.py index 7d0433795f..076104ef6f 100644 --- a/autotest/interface/restful/test_restful_anthropic_v1.py +++ b/autotest/interface/restful/test_restful_anthropic_v1.py @@ -18,7 +18,10 @@ ) from utils.config_utils import get_config from utils.constant import BACKEND_LIST, BASE_URL, RESTFUL_MODEL_LIST -from utils.restful_return_check import get_client_and_model +from utils.restful_return_check import ( + build_session_sized_user_content, + get_client_and_model, +) ANTHROPIC_VERSION = '2023-06-01' @@ -135,14 +138,9 @@ def _assert_count_tokens_json(data: dict) -> int: return n -_LARGE_PAYLOAD_PREFIX = 'Reply with one word: OK. Context:\n' _LARGE_PAYLOAD_MAX_TOKENS = 8 -def _large_payload_user_content() -> str: - return f'{_LARGE_PAYLOAD_PREFIX}{"x" * (128 * 1024)}' - - def _assert_anthropic_error_envelope(body: dict) -> dict: assert body['type'] == 'error', body err = body['error'] @@ -983,11 +981,14 @@ def test_count_tokens_empty_messages(self, backend, model_case, deployed_model_n ) _assert_anthropic_invalid_request_error(resp) - def test_messages_large_user_payload(self, backend, model_case, deployed_model_name: str): + def test_messages_large_user_payload(self, backend, model_case, deployed_model_name: str, config): """Regression guard for large JSON bodies (CI-sized payload, not stress-test scale).""" - user_content = _large_payload_user_content() + user_content = build_session_sized_user_content( + config=config, model_id=model_case, + max_completion_tokens=_LARGE_PAYLOAD_MAX_TOKENS, + ) resp = requests.post( _MESSAGES_URL, headers=_anthropic_headers(), diff --git a/autotest/interface/restful/test_restful_chat_completions_v1.py b/autotest/interface/restful/test_restful_chat_completions_v1.py index 07bcf0388c..2fe7259ebd 100644 --- a/autotest/interface/restful/test_restful_chat_completions_v1.py +++ b/autotest/interface/restful/test_restful_chat_completions_v1.py @@ -1,20 +1,22 @@ import pytest import requests from openai import BadRequestError +from utils.config_utils import get_model_path_from_config from utils.constant import BACKEND_LIST, BASE_URL, DEFAULT_MAX_COMPLETION_TOKENS, RESTFUL_MODEL_LIST from utils.restful_return_check import ( CONTEXT_LENGTH_ERROR, assert_chat_completions_batch_return, assert_chat_completions_stream_return, assert_openai_invalid_request_error, - encode_prompt, + build_session_sized_user_content, + cap_completion_tokens_for_session, get_chat_delta_text, get_chat_message_text, get_client_and_model, has_repeated_fragment, ) +from utils.toolkit import encode_text -_OVERSIZE_CHAT_PROMPT = 'Hi, pls intro yourself' * 60000 _CHAT_COMPLETIONS_URL = f'{BASE_URL}/v1/chat/completions' _CHAT_MESSAGES = [{'role': 'user', 'content': 'Hi, pls intro yourself'}] @@ -276,14 +278,16 @@ def test_mutilple_times_response_should_not_same_streaming(self, backend, model_ responseList.append(response) assert responseList[0] != responseList[1] or responseList[1] == responseList[2] - def test_longtext_input(self, backend, model_case, openai_client_and_model): + def test_longtext_input(self, backend, model_case, openai_client_and_model, config): client, model_name = openai_client_and_model + oversize_content = build_session_sized_user_content( + config=config, model_id=model_case, oversize=True) with pytest.raises(BadRequestError) as ei: client.chat.completions.create(model=model_name, messages=[ { 'role': 'user', - 'content': _OVERSIZE_CHAT_PROMPT, + 'content': oversize_content, }, ], max_tokens=100) @@ -291,14 +295,16 @@ def test_longtext_input(self, backend, model_case, openai_client_and_model): assert_openai_invalid_request_error(ei.value.body, message_substr=CONTEXT_LENGTH_ERROR) @pytest.mark.pr_test - def test_longtext_input_streaming(self, backend, model_case, openai_client_and_model): + def test_longtext_input_streaming(self, backend, model_case, openai_client_and_model, config): client, model_name = openai_client_and_model + oversize_content = build_session_sized_user_content( + config=config, model_id=model_case, oversize=True) with pytest.raises(BadRequestError) as ei: client.chat.completions.create(model=model_name, messages=[ { 'role': 'user', - 'content': _OVERSIZE_CHAT_PROMPT, + 'content': oversize_content, }, ], max_tokens=100, @@ -323,8 +329,9 @@ def test_max_tokens(self, backend, model_case, openai_client_and_model): assert output.get('choices')[0].get('finish_reason') == 'length' assert output.get('usage').get('completion_tokens') == 6 or output.get('usage').get('completion_tokens') == 5 - def test_max_tokens_streaming(self, backend, model_case, openai_client_and_model): + def test_max_tokens_streaming(self, backend, model_case, openai_client_and_model, config): client, model_name = openai_client_and_model + model_path = get_model_path_from_config(config, model_case) outputs = client.chat.completions.create(model=model_name, messages=[ @@ -346,7 +353,7 @@ def test_max_tokens_streaming(self, backend, model_case, openai_client_and_model for index in range(0, len(outputList) - 1): assert_chat_completions_stream_return(outputList[index], model_name) response += get_chat_delta_text(outputList[index].get('choices')[0]) - _, length = encode_prompt(BASE_URL, response, add_bos=False) + length = len(encode_text(model_path, response, add_special_tokens=False)) assert outputList[-1].get('choices')[0].get('finish_reason') == 'length' assert length == 5 or length == 6 @@ -372,8 +379,9 @@ def test_logprobs(self, backend, model_case, openai_client_and_model): @pytest.mark.not_pytorch @pytest.mark.pr_test - def test_logprobs_streaming(self, backend, model_case, openai_client_and_model): + def test_logprobs_streaming(self, backend, model_case, openai_client_and_model, config): client, model_name = openai_client_and_model + model_path = get_model_path_from_config(config, model_case) outputs = client.chat.completions.create(model=model_name, messages=[ @@ -397,7 +405,7 @@ def test_logprobs_streaming(self, backend, model_case, openai_client_and_model): for index in range(0, len(outputList) - 1): assert_chat_completions_stream_return(outputList[index], model_name, check_logprobs=True, logprobs_num=10) response += get_chat_delta_text(outputList[index].get('choices')[0]) - _, length = encode_prompt(BASE_URL, response, add_bos=False) + length = len(encode_text(model_path, response, add_special_tokens=False)) assert outputList[-1].get('choices')[0].get('finish_reason') == 'length' assert length == 5 or length == 6 @@ -476,8 +484,9 @@ def test_ignore_eos(self, backend, model_case, openai_client_and_model): assert completion_tokens == 101 or completion_tokens == 100 assert output.get('choices')[0].get('finish_reason') == 'length' - def test_ignore_eos_streaming(self, backend, model_case, openai_client_and_model): + def test_ignore_eos_streaming(self, backend, model_case, openai_client_and_model, config): client, model_name = openai_client_and_model + model_path = get_model_path_from_config(config, model_case) outputs = client.chat.completions.create( model=model_name, messages=[{'role': 'user', 'content': 'Hi, what is your name?'}], @@ -492,17 +501,20 @@ def test_ignore_eos_streaming(self, backend, model_case, openai_client_and_model for index in range(0, len(outputList) - 1): assert_chat_completions_stream_return(outputList[index], model_name) response += get_chat_delta_text(outputList[index].get('choices')[0]) - _, length = encode_prompt(BASE_URL, response, add_bos=False) + length = len(encode_text(model_path, response, add_special_tokens=False)) assert outputList[-1].get('choices')[0].get('finish_reason') == 'length' assert length >= 99 and length <= 101 - def test_max_tokens_default_cap_no_overshoot_followup(self, backend, model_case, openai_client_and_model): + def test_max_tokens_default_cap_no_overshoot_followup(self, backend, model_case, openai_client_and_model, config): client, model_name = openai_client_and_model - max_tokens = DEFAULT_MAX_COMPLETION_TOKENS + prompt = 'Continue writing forever without stopping.' + max_tokens = cap_completion_tokens_for_session( + prompt, DEFAULT_MAX_COMPLETION_TOKENS, + config=config, model_id=model_case) overshoot_slack = 1 outputs = client.chat.completions.create( model=model_name, - messages=[{'role': 'user', 'content': 'Continue writing forever without stopping.'}], + messages=[{'role': 'user', 'content': prompt}], extra_body={'ignore_eos': True}, max_tokens=max_tokens, temperature=0.01, @@ -536,8 +548,9 @@ def test_max_completion_tokens(self, backend, model_case, openai_client_and_mode assert output.get('choices')[0].get('finish_reason') == 'length' assert output.get('usage').get('completion_tokens') in (5, 6) - def test_max_completion_tokens_streaming(self, backend, model_case, openai_client_and_model): + def test_max_completion_tokens_streaming(self, backend, model_case, openai_client_and_model, config): client, model_name = openai_client_and_model + model_path = get_model_path_from_config(config, model_case) outputs = client.chat.completions.create( model=model_name, messages=[{'role': 'user', 'content': 'Hi, pls intro yourself'}], @@ -551,7 +564,7 @@ def test_max_completion_tokens_streaming(self, backend, model_case, openai_clien for index in range(0, len(outputList) - 1): assert_chat_completions_stream_return(outputList[index], model_name) response += get_chat_delta_text(outputList[index].get('choices')[0]) - _, length = encode_prompt(BASE_URL, response, add_bos=False) + length = len(encode_text(model_path, response, add_special_tokens=False)) assert outputList[-1].get('choices')[0].get('finish_reason') == 'length' assert length in (5, 6) @@ -560,7 +573,6 @@ def test_max_completion_tokens_streaming(self, backend, model_case, openai_clien [ pytest.param({'max_tokens': 0}, id='max_tokens_zero'), pytest.param({'max_tokens': -1}, id='max_tokens_negative'), - pytest.param({'temperature': True}, id='temperature_bool'), ], ) def test_rejects_invalid_request_parameters( diff --git a/autotest/interface/restful/test_restful_completions_v1.py b/autotest/interface/restful/test_restful_completions_v1.py index 0c95bbb351..23374e5d12 100644 --- a/autotest/interface/restful/test_restful_completions_v1.py +++ b/autotest/interface/restful/test_restful_completions_v1.py @@ -209,7 +209,6 @@ def test_batch_prompt_order(self, backend, model_case, openai_client_and_model): [ pytest.param({'max_tokens': 0}, id='max_tokens_zero'), pytest.param({'max_tokens': -1}, id='max_tokens_negative'), - pytest.param({'temperature': True}, id='temperature_bool'), ], ) def test_rejects_invalid_request_parameters( diff --git a/autotest/interface/restful/test_restful_generate.py b/autotest/interface/restful/test_restful_generate.py index a4ba09beeb..c86de2c2a7 100644 --- a/autotest/interface/restful/test_restful_generate.py +++ b/autotest/interface/restful/test_restful_generate.py @@ -15,7 +15,7 @@ model_enables_return_routed_experts, ) from utils.constant import BACKEND_LIST, BASE_URL, DEFAULT_MAX_COMPLETION_TOKENS, RESTFUL_MODEL_LIST -from utils.restful_return_check import encode_prompt +from utils.restful_return_check import cap_completion_tokens_for_session from utils.toolkit import encode_text, parse_sse_stream @@ -951,21 +951,24 @@ def test_ignore_eos_parameter(self): assert reason_ignore == 'length', \ f'ignore_eos=True must end due to length, actual: {reason_ignore}' - def test_max_tokens_default_cap_no_overshoot_followup(self): + def test_max_tokens_default_cap_no_overshoot_followup(self, config): """Hit DEFAULT max_tokens (8192) with ignore_eos; no overshoot; follow- up must succeed. Catches regressions where length-capped generation returns a few extra tokens and breaks the next request. """ - print(f'\n[Model: {self.model_name}] Running max_tokens={DEFAULT_MAX_COMPLETION_TOKENS} ' + prompt = 'Continue writing forever without stopping.' + max_tokens = cap_completion_tokens_for_session( + prompt, DEFAULT_MAX_COMPLETION_TOKENS, + config=config, model_id=self.model_name) + print(f'\n[Model: {self.model_name}] Running max_tokens={max_tokens} ' 'length-cap / follow-up test') - max_tokens = DEFAULT_MAX_COMPLETION_TOKENS # Align with existing generate/chat length checks (allow at most +1). overshoot_slack = 1 resp = self._post( { - 'prompt': 'Continue writing forever without stopping.', + 'prompt': prompt, 'max_tokens': max_tokens, 'ignore_eos': True, 'stream': False, @@ -1025,9 +1028,11 @@ def test_skip_special_tokens(self, config): assert not any(pattern in generated_text for pattern in special_patterns), \ 'Expected no special pattern in the generated text but found one.' - def test_stop_token_ids(self): + def test_stop_token_ids(self, config): print(f'\n[Model: {self.model_name}] Running stop_token_ids test') - input_ids1, length1 = encode_prompt(BASE_URL, '.', add_bos=False) + model_path = get_model_path_from_config(config, self.model_name) + input_ids1 = encode_text(model_path, '.', add_special_tokens=False) + length1 = len(input_ids1) print(f'input_ids1={input_ids1}, length1={length1}') payload = { @@ -1270,7 +1275,7 @@ def test_request_returns_experts(self, backend): @pytest.mark.experts @pytest.mark.not_turbomind - def test_request_returns_experts_max_tokens_cap_followup(self, backend): + def test_request_returns_experts_max_tokens_cap_followup(self, backend, config): """Hit DEFAULT max_tokens with return_routed_experts; length/experts OK; follow-up OK. @@ -1280,14 +1285,17 @@ def test_request_returns_experts_max_tokens_cap_followup(self, backend): if not model_enables_return_routed_experts( self.model_name, backend, required_suites=frozenset({'experts'})): pytest.skip(ROUTED_EXPERTS_UNSUPPORTED_SKIP) + prompt = 'Continue writing forever without stopping.' + max_tokens = cap_completion_tokens_for_session( + prompt, DEFAULT_MAX_COMPLETION_TOKENS, + config=config, model_id=self.model_name) print(f'\n[Model: {self.model_name}] Running experts max_tokens=' - f'{DEFAULT_MAX_COMPLETION_TOKENS} length-cap / follow-up test') - max_tokens = DEFAULT_MAX_COMPLETION_TOKENS + f'{max_tokens} length-cap / follow-up test') overshoot_slack = 1 resp = self._post( { - 'prompt': 'Continue writing forever without stopping.', + 'prompt': prompt, 'max_tokens': max_tokens, 'ignore_eos': True, 'stream': False, diff --git a/autotest/interface/restful/tool_parser/test_tool_call_advanced.py b/autotest/interface/restful/tool_parser/test_tool_call_advanced.py index e96ea30414..6507e88534 100644 --- a/autotest/interface/restful/tool_parser/test_tool_call_advanced.py +++ b/autotest/interface/restful/tool_parser/test_tool_call_advanced.py @@ -1,6 +1,7 @@ import pytest from utils.config_utils import ROUTED_EXPERTS_UNSUPPORTED_SKIP from utils.constant import DEFAULT_MAX_COMPLETION_TOKENS +from utils.restful_return_check import cap_completion_tokens_for_session from utils.tool_reasoning_definitions import ( ALL_OPTIONAL_TOOL, CALCULATOR_TOOL, @@ -737,12 +738,15 @@ def test_streaming_routed_experts_max_tokens_cap_followup(self, backend, model_c """ if not self._validate_experts(): pytest.skip(ROUTED_EXPERTS_UNSUPPORTED_SKIP) - max_tokens = DEFAULT_MAX_COMPLETION_TOKENS + prompt = 'Continue writing forever without stopping.' + max_tokens = cap_completion_tokens_for_session( + prompt, DEFAULT_MAX_COMPLETION_TOKENS, + config=self._config, model_id=self._model_case) overshoot_slack = 1 messages = [ { 'role': 'user', - 'content': 'Continue writing forever without stopping.', + 'content': prompt, }, ] r = self._stream_tool_call_with_tokens( diff --git a/autotest/utils/restful_return_check.py b/autotest/utils/restful_return_check.py index cf3c37f5d8..578c285f41 100644 --- a/autotest/utils/restful_return_check.py +++ b/autotest/utils/restful_return_check.py @@ -1,12 +1,25 @@ import re +from typing import Any import requests from openai import OpenAI +from utils.config_utils import ( + _entry_engine_config, + get_model_path_from_config, + iter_model_yaml_entries, +) from utils.constant import BASE_URL +from utils.toolkit import _load_tokenizer_cached, encode_text # Preprocess rejects oversize input with this OpenAI error substring. CONTEXT_LENGTH_ERROR = 'context length' +# Upper bound for "large but valid" payload tests (CI scale, not full context). +CI_LARGE_PAYLOAD_TOKEN_CAP = 32_000 + +# Legacy anthropic large-payload test body size (128 KiB repeated filler). +CI_LARGE_PAYLOAD_CHAR_BUDGET = 128 * 1024 + def assert_openai_invalid_request_error( response: requests.Response | dict, @@ -205,14 +218,90 @@ def get_client_and_model(base_url: str | None = None) -> tuple[OpenAI, str]: return client, models[0].id -def encode_prompt(base_url: str, text: str, *, add_bos: bool = True) -> tuple[list, int]: - """Tokenize via ``POST /v1/encode``; returns ``(input_ids, length)``.""" - url = base_url.rstrip('/') - response = requests.post( - f'{url}/v1/encode', - json={'input': text, 'do_preprocess': False, 'add_bos': add_bos}, - timeout=30, - ) - response.raise_for_status() - output = response.json() - return output['input_ids'], output['length'] +def resolve_effective_session_len(config: dict[str, Any], model_id: str) -> int: + """Context limit aligned with ``async_engine.session_len``. + + Uses yaml ``session-len`` when set, otherwise HF ``_get_and_verify_max_len``. + Does not apply ``tokenizer.model_max_length`` (server preprocess does not either). + """ + model_path = get_model_path_from_config(config, model_id) + session_len = None + for entry in iter_model_yaml_entries(model_id): + extra = _entry_engine_config(entry).get('extra') or {} + if extra.get('session-len') is not None: + session_len = int(extra['session-len']) + break + if session_len is None: + from transformers import AutoConfig + + from lmdeploy.utils import _get_and_verify_max_len + + hf_cfg = AutoConfig.from_pretrained(model_path, trust_remote_code=True) + session_len = _get_and_verify_max_len(hf_cfg, None) + return session_len + + +def build_session_sized_user_content( + *, + config: dict[str, Any], + model_id: str, + oversize: bool = False, + max_completion_tokens: int = 0, + reserve: int = 256, + slack: int = 32, + unit: str = 'x', + token_cap: int | None = CI_LARGE_PAYLOAD_TOKEN_CAP, +) -> str: + """Size user text relative to server ``session_len`` (local tokenizer). + + ``oversize=True``: raw user token count exceeds server ``session_len`` (400 tests). + ``oversize=False``: as large as possible while fitting; also caps by finite + ``tokenizer.model_max_length`` when tighter than HF context (e.g. InternVL3-38B). + """ + session_len = resolve_effective_session_len(config, model_id) + model_path = get_model_path_from_config(config, model_id) + if not oversize: + tok_mml = getattr(_load_tokenizer_cached(model_path), 'model_max_length', None) + if tok_mml is not None and tok_mml < 1_000_000: + session_len = min(session_len, int(tok_mml)) + text = '' + token_len = len(encode_text(model_path, text, add_special_tokens=False)) + + if oversize: + target = session_len + slack + 1 + while token_len < target: + deficit = target - token_len + text += unit * max(deficit, 1) + token_len = len(encode_text(model_path, text, add_special_tokens=False)) + return text + + input_limit = session_len - max_completion_tokens - reserve + if token_cap is not None: + input_limit = min(input_limit, token_cap) + while token_len < input_limit: + deficit = input_limit - token_len + text += unit * max(deficit, 1) + token_len = len(encode_text(model_path, text, add_special_tokens=False)) + + session_input_limit = session_len - max_completion_tokens + while token_len >= session_input_limit and text: + text = text[:-(max(1, len(text) // 20))] + token_len = len(encode_text(model_path, text, add_special_tokens=False)) + return text + + +def cap_completion_tokens_for_session( + prompt_text: str, + default_cap: int, + *, + config: dict[str, Any], + model_id: str, + reserve: int = 128, + min_cap: int = 64, +) -> int: + """Cap ``max_tokens`` so prompt + completion fits ``session_len``.""" + session_len = resolve_effective_session_len(config, model_id) + model_path = get_model_path_from_config(config, model_id) + prompt_tokens = len(encode_text(model_path, prompt_text, add_special_tokens=False)) + available = session_len - prompt_tokens - reserve + return min(default_cap, max(min_cap, available)) diff --git a/autotest/utils/toolkit.py b/autotest/utils/toolkit.py index 28078c1336..d6d54bb5a7 100644 --- a/autotest/utils/toolkit.py +++ b/autotest/utils/toolkit.py @@ -30,9 +30,6 @@ def _load_tokenizer_cached(model_path: str): raise RuntimeError(f"Failed to load tokenizer from '{model_path}': {e}") -def encode_text(model_path: str, text: str) -> list[int]: +def encode_text(model_path: str, text: str, *, add_special_tokens: bool = True) -> list[int]: tokenizer = _load_tokenizer_cached(model_path) - - encoded = tokenizer.encode(text) - - return encoded + return tokenizer.encode(text, add_special_tokens=add_special_tokens)