Skip to content

Commit 000e97f

Browse files
zxuexingzhijiezxuexingzhijie
authored
test: validate live Chat and Embedding contracts (#601)
* ci: report live model results on pull requests * test: switch live model validation to DeepSeek * test: apply live test formatting * test: add DashScope embedding live contract
1 parent f694a46 commit 000e97f

5 files changed

Lines changed: 221 additions & 71 deletions

File tree

‎.github/workflows/live-model-test.yml‎

Lines changed: 60 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
# See the License for the specific language governing permissions and
1414
# limitations under the License.
1515
#
16-
name: Alibaba Cloud Live Model Test
16+
name: Live Model Contract Test
1717

1818
on:
1919
workflow_dispatch:
@@ -23,9 +23,7 @@ on:
2323
required: true
2424
type: number
2525

26-
permissions:
27-
contents: read
28-
pull-requests: read
26+
permissions: {}
2927

3028
concurrency:
3129
group: live-model-pr-${{ inputs.pr_number }}
@@ -36,6 +34,10 @@ jobs:
3634
name: Resolve immutable PR head
3735
if: github.repository == 'spring-ai-alibaba/DataAgent'
3836
runs-on: ubuntu-22.04
37+
permissions:
38+
contents: read
39+
pull-requests: read
40+
statuses: write
3941
outputs:
4042
repository: ${{ steps.resolve.outputs.repository }}
4143
sha: ${{ steps.resolve.outputs.sha }}
@@ -48,6 +50,7 @@ jobs:
4850
GH_TOKEN: ${{ github.token }}
4951
REPOSITORY: ${{ github.repository }}
5052
PR_NUMBER: ${{ inputs.pr_number }}
53+
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
5154
run: |
5255
set -euo pipefail
5356
@@ -71,13 +74,21 @@ jobs:
7174
echo "url=${pr_url}"
7275
} >>"${GITHUB_OUTPUT}"
7376
77+
gh api --method POST "repos/${REPOSITORY}/statuses/${head_sha}" \
78+
-f state=pending \
79+
-f context='Live Model Contract Test' \
80+
-f description='DeepSeek Chat and DashScope Embedding validation is running' \
81+
-f target_url="${RUN_URL}"
82+
7483
printf 'Resolved %s to %s@%s\n' "${pr_url}" "${head_repository}" "${head_sha}"
7584
76-
live-model:
85+
live_model:
7786
name: "PR #${{ inputs.pr_number }} @ ${{ needs.resolve_pr.outputs.short_sha }}"
7887
needs: resolve_pr
7988
runs-on: ubuntu-22.04
8089
timeout-minutes: 15
90+
permissions:
91+
contents: read
8192
environment:
8293
name: live-model
8394
url: ${{ needs.resolve_pr.outputs.url }}
@@ -94,9 +105,52 @@ jobs:
94105
run: test "$(git rev-parse HEAD)" = "${EXPECTED_SHA}"
95106
- uses: ./CI/github-actions/setup-deps
96107
- run: make tools
97-
- name: Run Alibaba Cloud live model contract
108+
- name: Run live Chat and Embedding contracts
98109
env:
110+
DATAAGENT_LIVE_DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK }}
99111
DATAAGENT_LIVE_DASHSCOPE_API_KEY: ${{ secrets.DASHSCOPE_API_KEY }}
100112
run: |
113+
test -n "${DATAAGENT_LIVE_DEEPSEEK_API_KEY}"
101114
test -n "${DATAAGENT_LIVE_DASHSCOPE_API_KEY}"
102115
make live-model-test
116+
117+
report_status:
118+
name: Report PR live model status
119+
needs: [resolve_pr, live_model]
120+
if: ${{ always() && needs.resolve_pr.result == 'success' }}
121+
runs-on: ubuntu-22.04
122+
permissions:
123+
statuses: write
124+
steps:
125+
- name: Publish final PR status
126+
env:
127+
GH_TOKEN: ${{ github.token }}
128+
REPOSITORY: ${{ github.repository }}
129+
SHA: ${{ needs.resolve_pr.outputs.sha }}
130+
RESULT: ${{ needs.live_model.result }}
131+
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
132+
run: |
133+
case "${RESULT}" in
134+
success)
135+
state=success
136+
description='DeepSeek Chat and DashScope Embedding contracts passed'
137+
;;
138+
cancelled)
139+
state=error
140+
description='Live model validation was cancelled'
141+
;;
142+
skipped)
143+
state=error
144+
description='Live model validation was skipped'
145+
;;
146+
*)
147+
state=failure
148+
description='Live model workflow failed; inspect run details'
149+
;;
150+
esac
151+
152+
gh api --method POST "repos/${REPOSITORY}/statuses/${SHA}" \
153+
-f state="${state}" \
154+
-f context='Live Model Contract Test' \
155+
-f description="${description}" \
156+
-f target_url="${RUN_URL}"

data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/aimodelconfig/AlibabaCloudModelLiveIT.java renamed to data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/aimodelconfig/DashScopeEmbeddingLiveIT.java

Lines changed: 12 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -15,19 +15,14 @@
1515
*/
1616
package com.alibaba.cloud.ai.dataagent.service.aimodelconfig;
1717

18-
import java.time.Duration;
1918
import java.util.Arrays;
2019
import java.util.List;
21-
import java.util.Objects;
2220
import java.util.concurrent.TimeUnit;
2321

2422
import com.alibaba.cloud.ai.dataagent.dto.ModelConfigDTO;
2523
import org.junit.jupiter.api.Tag;
2624
import org.junit.jupiter.api.Test;
2725
import org.junit.jupiter.api.Timeout;
28-
import org.springframework.ai.chat.model.ChatModel;
29-
import org.springframework.ai.chat.model.ChatResponse;
30-
import org.springframework.ai.chat.prompt.Prompt;
3126
import org.springframework.ai.embedding.EmbeddingModel;
3227
import org.springframework.ai.embedding.EmbeddingResponse;
3328
import org.springframework.util.StringUtils;
@@ -36,59 +31,23 @@
3631

3732
@Tag("live")
3833
@Timeout(value = 2, unit = TimeUnit.MINUTES)
39-
class AlibabaCloudModelLiveIT {
34+
class DashScopeEmbeddingLiveIT {
4035

41-
private static final String EXPECTED_CHAT_RESPONSE = "DATAAGENT_LIVE_OK";
36+
private static final String DEFAULT_BASE_URL = "https://dashscope.aliyuncs.com/compatible-mode/v1";
4237

43-
private static final String DEFAULT_BASE_URL = "https://dashscope.aliyuncs.com/compatible-mode";
38+
private static final String DEFAULT_EMBEDDINGS_PATH = "/embeddings";
4439

45-
private final DynamicModelFactory modelFactory = new DynamicModelFactory();
46-
47-
@Test
48-
void streamingChat_callsAlibabaCloudAndReturnsContentWithUsage() {
49-
ChatModel chatModel = modelFactory.createChatModel(ModelConfigDTO.builder()
50-
.provider("qwen")
51-
.apiKey(requiredApiKey())
52-
.baseUrl(baseUrl())
53-
.modelName(environmentOrDefault("DATAAGENT_LIVE_DASHSCOPE_CHAT_MODEL", "qwen-plus"))
54-
.modelType("CHAT")
55-
.temperature(0.0)
56-
.maxTokens(32)
57-
.build());
40+
private static final int EXPECTED_DIMENSIONS = 1024;
5841

59-
Prompt prompt = new Prompt(
60-
"Return exactly DATAAGENT_LIVE_OK with no punctuation, explanation, or Markdown formatting.");
61-
List<ChatResponse> responses = chatModel.stream(prompt).collectList().block(Duration.ofSeconds(90));
62-
63-
assertThat(responses).isNotNull().isNotEmpty();
64-
String content = responses.stream()
65-
.map(ChatResponse::getResult)
66-
.filter(Objects::nonNull)
67-
.map(result -> result.getOutput().getText())
68-
.filter(Objects::nonNull)
69-
.reduce("", String::concat)
70-
.trim();
71-
assertThat(content).isEqualTo(EXPECTED_CHAT_RESPONSE);
72-
73-
int totalTokens = responses.stream()
74-
.map(ChatResponse::getMetadata)
75-
.filter(Objects::nonNull)
76-
.map(metadata -> metadata.getUsage())
77-
.filter(Objects::nonNull)
78-
.map(usage -> usage.getTotalTokens())
79-
.filter(Objects::nonNull)
80-
.mapToInt(Integer::intValue)
81-
.max()
82-
.orElse(0);
83-
assertThat(totalTokens).isPositive();
84-
}
42+
private final DynamicModelFactory modelFactory = new DynamicModelFactory();
8543

8644
@Test
87-
void embeddings_callAlibabaCloudAndReturnDistinctFiniteVectors() {
45+
void embeddings_callDashScopeAndReturnDistinctFiniteVectorsWithUsage() {
8846
EmbeddingModel embeddingModel = modelFactory.createEmbeddingModel(ModelConfigDTO.builder()
8947
.provider("qwen")
9048
.apiKey(requiredApiKey())
9149
.baseUrl(baseUrl())
50+
.embeddingsPath(environmentOrDefault("DATAAGENT_LIVE_DASHSCOPE_EMBEDDINGS_PATH", DEFAULT_EMBEDDINGS_PATH))
9251
.modelName(environmentOrDefault("DATAAGENT_LIVE_DASHSCOPE_EMBEDDING_MODEL", "text-embedding-v4"))
9352
.modelType("EMBEDDING")
9453
.build());
@@ -108,7 +67,7 @@ void embeddings_callAlibabaCloudAndReturnDistinctFiniteVectors() {
10867
}
10968

11069
private void assertVector(float[] vector) {
111-
assertThat(vector).hasSize(1024);
70+
assertThat(vector).hasSize(EXPECTED_DIMENSIONS);
11271
double squaredNorm = 0.0;
11372
for (float value : vector) {
11473
assertThat(Float.isFinite(value)).as("embedding value must be finite").isTrue();
@@ -123,13 +82,14 @@ private String requiredApiKey() {
12382
throw new IllegalStateException(
12483
"Set DATAAGENT_LIVE_DASHSCOPE_API_KEY or DASHSCOPE_API_KEY before running live-model tests");
12584
}
85+
if (!apiKey.equals(apiKey.strip())) {
86+
throw new IllegalStateException("DashScope API key must not contain leading or trailing whitespace");
87+
}
12688
return apiKey;
12789
}
12890

12991
private String baseUrl() {
130-
String value = environmentOrDefault("DATAAGENT_LIVE_DASHSCOPE_BASE_URL", DEFAULT_BASE_URL).replaceAll("/+$",
131-
"");
132-
return value.endsWith("/v1") ? value.substring(0, value.length() - 3) : value;
92+
return environmentOrDefault("DATAAGENT_LIVE_DASHSCOPE_BASE_URL", DEFAULT_BASE_URL).replaceAll("/+$", "");
13393
}
13494

13595
private String environmentOrDefault(String name, String defaultValue) {

‎data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/aimodelconfig/DeepSeekModelLiveIT.java‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
/*
2+
* Copyright 2026 the original author or authors.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package com.alibaba.cloud.ai.dataagent.service.aimodelconfig;
17+
18+
import java.time.Duration;
19+
import java.util.List;
20+
import java.util.Objects;
21+
import java.util.concurrent.TimeUnit;
22+
23+
import com.alibaba.cloud.ai.dataagent.dto.ModelConfigDTO;
24+
import org.junit.jupiter.api.Tag;
25+
import org.junit.jupiter.api.Test;
26+
import org.junit.jupiter.api.Timeout;
27+
import org.springframework.ai.chat.model.ChatModel;
28+
import org.springframework.ai.chat.model.ChatResponse;
29+
import org.springframework.ai.chat.prompt.Prompt;
30+
import org.springframework.util.StringUtils;
31+
32+
import static org.assertj.core.api.Assertions.assertThat;
33+
34+
@Tag("live")
35+
@Timeout(value = 2, unit = TimeUnit.MINUTES)
36+
class DeepSeekModelLiveIT {
37+
38+
private static final String EXPECTED_CHAT_RESPONSE = "DATAAGENT_LIVE_OK";
39+
40+
private static final String DEFAULT_BASE_URL = "https://api.deepseek.com";
41+
42+
private static final String DEFAULT_COMPLETIONS_PATH = "/chat/completions";
43+
44+
private final DynamicModelFactory modelFactory = new DynamicModelFactory();
45+
46+
@Test
47+
void streamingChat_callsDeepSeekAndReturnsContentWithUsage() {
48+
ChatModel chatModel = modelFactory.createChatModel(ModelConfigDTO.builder()
49+
.provider("deepseek")
50+
.apiKey(requiredApiKey())
51+
.baseUrl(baseUrl())
52+
.completionsPath(environmentOrDefault("DATAAGENT_LIVE_DEEPSEEK_COMPLETIONS_PATH", DEFAULT_COMPLETIONS_PATH))
53+
.modelName(environmentOrDefault("DATAAGENT_LIVE_DEEPSEEK_CHAT_MODEL", "deepseek-v4-flash"))
54+
.modelType("CHAT")
55+
.temperature(0.0)
56+
.maxTokens(256)
57+
.build());
58+
59+
Prompt prompt = new Prompt(
60+
"Return exactly DATAAGENT_LIVE_OK with no punctuation, explanation, or Markdown formatting.");
61+
List<ChatResponse> responses = chatModel.stream(prompt).collectList().block(Duration.ofSeconds(90));
62+
63+
assertThat(responses).isNotNull().isNotEmpty();
64+
String content = responses.stream()
65+
.map(ChatResponse::getResult)
66+
.filter(Objects::nonNull)
67+
.map(result -> result.getOutput().getText())
68+
.filter(Objects::nonNull)
69+
.reduce("", String::concat)
70+
.trim();
71+
assertThat(content).isEqualTo(EXPECTED_CHAT_RESPONSE);
72+
73+
int totalTokens = responses.stream()
74+
.map(ChatResponse::getMetadata)
75+
.filter(Objects::nonNull)
76+
.map(metadata -> metadata.getUsage())
77+
.filter(Objects::nonNull)
78+
.map(usage -> usage.getTotalTokens())
79+
.filter(Objects::nonNull)
80+
.mapToInt(Integer::intValue)
81+
.max()
82+
.orElse(0);
83+
assertThat(totalTokens).isPositive();
84+
}
85+
86+
private String requiredApiKey() {
87+
String apiKey = environmentOrDefault("DATAAGENT_LIVE_DEEPSEEK_API_KEY", System.getenv("DEEPSEEK_API_KEY"));
88+
if (!StringUtils.hasText(apiKey)) {
89+
throw new IllegalStateException(
90+
"Set DATAAGENT_LIVE_DEEPSEEK_API_KEY or DEEPSEEK_API_KEY before running live-model tests");
91+
}
92+
if (!apiKey.equals(apiKey.strip())) {
93+
throw new IllegalStateException("DeepSeek API key must not contain leading or trailing whitespace");
94+
}
95+
return apiKey;
96+
}
97+
98+
private String baseUrl() {
99+
return environmentOrDefault("DATAAGENT_LIVE_DEEPSEEK_BASE_URL", DEFAULT_BASE_URL).replaceAll("/+$", "");
100+
}
101+
102+
private String environmentOrDefault(String name, String defaultValue) {
103+
String value = System.getenv(name);
104+
return StringUtils.hasText(value) ? value : defaultValue;
105+
}
106+
107+
}

‎data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/aimodelconfig/DynamicModelFactoryTest.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -221,6 +221,21 @@ void createEmbeddingModel_withEmbeddingsPath_succeeds() {
221221
assertEquals("/custom/embeddings", singleProviderRequest().path());
222222
}
223223

224+
@Test
225+
void createEmbeddingModel_withBasePathAndEmbeddingsPath_preservesBothPaths() {
226+
ModelConfigDTO config = ModelConfigDTO.builder()
227+
.provider("custom")
228+
.apiKey("")
229+
.baseUrl(providerBaseUrl + "/compatible-mode/v1")
230+
.modelName("local-embed")
231+
.embeddingsPath("/embeddings")
232+
.build();
233+
234+
EmbeddingModel embeddingModel = dynamicModelFactory.createEmbeddingModel(config);
235+
assertArrayEquals(new float[] { 0.125f, 0.875f }, embeddingModel.embed("Hello"));
236+
assertEquals("/compatible-mode/v1/embeddings", singleProviderRequest().path());
237+
}
238+
224239
@Test
225240
void createChatModel_withProxy_routesRequestThroughProxy() throws IOException {
226241
proxyRequests = new CopyOnWriteArrayList<>();

0 commit comments

Comments
 (0)