From edca6af29729caee931377471595f97918127267 Mon Sep 17 00:00:00 2001 From: zxuexingzhijie <2762006003@qq.com> Date: Tue, 28 Jul 2026 01:39:49 +0800 Subject: [PATCH 1/3] feat: integrate SAA Python sandbox --- data-agent-management/pom.xml | 24 +- .../config/DataAgentConfiguration.java | 10 - .../properties/CodeExecutorProperties.java | 98 ++-- .../code/CodePoolExecutorServiceFactory.java | 54 --- ...ce.java => PythonCodeExecutorService.java} | 24 +- .../code/docker/DockerClientConnector.java | 28 -- .../code/docker/DockerClientFactory.java | 79 ---- .../code/docker/DockerExecutorFactory.java | 64 --- .../code/docker/DockerHostResolver.java | 84 ---- .../code/docker/DockerImageManager.java | 65 --- .../docker/ZerodepDockerClientConnector.java | 43 -- .../AbstractCodePoolExecutorService.java | 428 ------------------ .../AiSimulationCodeExecutorService.java | 64 --- .../impls/DockerCodePoolExecutorService.java | 285 ------------ .../impls/LocalCodePoolExecutorService.java | 224 --------- .../code/local/ExecutionTimeoutParser.java | 60 --- .../local/PathExecutableProgramLocator.java | 71 --- .../PythonDependencyMetadata.java} | 17 +- .../PythonDependencyMetadataParser.java | 114 +++++ .../code/sandbox/PythonDependencyPolicy.java | 56 +++ .../PythonSandboxBootstrapBuilder.java | 154 +++++++ .../SaaSandboxPythonCodeExecutorService.java | 103 +++++ .../code/sandbox/SaaSandboxRuntime.java | 111 +++++ .../code/sandbox/SaaSandboxTaskRunner.java | 82 ++++ .../code/sandbox/SandboxExecutionResult.java} | 13 +- .../sandbox/SandboxExecutionResultParser.java | 74 +++ .../workflow/node/PythonExecuteNode.java | 21 +- .../workflow/node/PythonGenerateNode.java | 2 +- .../src/main/resources/application-h2.yml | 17 +- .../src/main/resources/application.yml | 17 +- .../resources/prompts/python-generator.txt | 13 +- .../config/DataAgentConfigurationTest.java | 4 +- .../config/VectorStoreConfigurationTest.java | 7 +- .../enums/CodePoolExecutorEnumTest.java | 43 -- .../PythonWorkflowIntegrationTest.java | 21 +- .../CodePoolExecutorServiceFactoryTest.java | 83 ---- .../code/CodePoolExecutorServiceTest.java | 94 ---- .../code/PythonCodeExecutorServiceTest.java | 48 ++ .../code/docker/DockerClientFactoryTest.java | 109 ----- .../docker/DockerExecutorFactoryTest.java | 101 ----- .../code/docker/DockerHostResolverTest.java | 73 --- .../code/docker/DockerImageManagerTest.java | 129 ------ .../AiSimulationCodeExecutorServiceTest.java | 77 ---- .../DockerCodePoolExecutorServiceIT.java | 50 -- .../DockerCodePoolExecutorServiceTest.java | 86 ---- .../LocalCodePoolExecutorServiceTest.java | 76 ---- .../local/ExecutionTimeoutParserTest.java | 44 -- .../PathExecutableProgramLocatorTest.java | 60 --- .../PythonDependencyMetadataParserTest.java | 77 ++++ .../PythonSandboxBootstrapBuilderTest.java | 54 +++ .../sandbox/PythonSandboxBootstrapTest.java | 81 ++++ ...aSandboxPythonCodeExecutorServiceTest.java | 68 +++ .../code/sandbox/SaaSandboxRuntimeTest.java | 38 ++ .../code/sandbox/SaaSandboxTaskRunnerIT.java | 57 +++ .../SandboxExecutionResultParserTest.java | 51 +++ .../node/python/PythonExecuteNodeTest.java | 63 ++- .../node/python/PythonGenerateNodeTest.java | 3 +- docs/ADVANCED_FEATURES-en.md | 58 +-- docs/ADVANCED_FEATURES.md | 57 +-- docs/ARCHITECTURE-en.md | 37 +- docs/ARCHITECTURE.md | 37 +- docs/DEVELOPER_GUIDE-en.md | 19 +- docs/DEVELOPER_GUIDE.md | 19 +- docs/superpowers/mock-strategy-guide.md | 20 +- docs/superpowers/nodes-dependency-analysis.md | 11 +- ...8-saa-python-sandbox-integration-design.md | 302 ++++++++++++ pom.xml | 19 +- 67 files changed, 1782 insertions(+), 2893 deletions(-) delete mode 100644 data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/CodePoolExecutorServiceFactory.java rename data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/{CodePoolExecutorService.java => PythonCodeExecutorService.java} (79%) delete mode 100644 data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerClientConnector.java delete mode 100644 data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerClientFactory.java delete mode 100644 data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerExecutorFactory.java delete mode 100644 data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerHostResolver.java delete mode 100644 data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerImageManager.java delete mode 100644 data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/docker/ZerodepDockerClientConnector.java delete mode 100644 data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/impls/AbstractCodePoolExecutorService.java delete mode 100644 data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/impls/AiSimulationCodeExecutorService.java delete mode 100644 data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/impls/DockerCodePoolExecutorService.java delete mode 100644 data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/impls/LocalCodePoolExecutorService.java delete mode 100644 data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/local/ExecutionTimeoutParser.java delete mode 100644 data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/local/PathExecutableProgramLocator.java rename data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/{local/ExecutableProgramLocator.java => sandbox/PythonDependencyMetadata.java} (63%) create mode 100644 data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/PythonDependencyMetadataParser.java create mode 100644 data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/PythonDependencyPolicy.java create mode 100644 data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/PythonSandboxBootstrapBuilder.java create mode 100644 data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SaaSandboxPythonCodeExecutorService.java create mode 100644 data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SaaSandboxRuntime.java create mode 100644 data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SaaSandboxTaskRunner.java rename data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/{enums/CodePoolExecutorEnum.java => service/code/sandbox/SandboxExecutionResult.java} (73%) create mode 100644 data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SandboxExecutionResultParser.java delete mode 100644 data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/enums/CodePoolExecutorEnumTest.java delete mode 100644 data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/CodePoolExecutorServiceFactoryTest.java delete mode 100644 data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/CodePoolExecutorServiceTest.java create mode 100644 data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/PythonCodeExecutorServiceTest.java delete mode 100644 data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerClientFactoryTest.java delete mode 100644 data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerExecutorFactoryTest.java delete mode 100644 data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerHostResolverTest.java delete mode 100644 data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerImageManagerTest.java delete mode 100644 data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/impls/AiSimulationCodeExecutorServiceTest.java delete mode 100644 data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/impls/DockerCodePoolExecutorServiceIT.java delete mode 100644 data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/impls/DockerCodePoolExecutorServiceTest.java delete mode 100644 data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/impls/LocalCodePoolExecutorServiceTest.java delete mode 100644 data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/local/ExecutionTimeoutParserTest.java delete mode 100644 data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/local/PathExecutableProgramLocatorTest.java create mode 100644 data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/PythonDependencyMetadataParserTest.java create mode 100644 data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/PythonSandboxBootstrapBuilderTest.java create mode 100644 data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/PythonSandboxBootstrapTest.java create mode 100644 data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SaaSandboxPythonCodeExecutorServiceTest.java create mode 100644 data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SaaSandboxRuntimeTest.java create mode 100644 data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SaaSandboxTaskRunnerIT.java create mode 100644 data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SandboxExecutionResultParserTest.java create mode 100644 docs/superpowers/specs/2026-07-28-saa-python-sandbox-integration-design.md diff --git a/data-agent-management/pom.xml b/data-agent-management/pom.xml index 6dd6a4b09..2528511bd 100644 --- a/data-agent-management/pom.xml +++ b/data-agent-management/pom.xml @@ -83,6 +83,10 @@ org.apache.commons commons-lang3 + + com.google.guava + guava + com.mysql mysql-connector-j @@ -269,6 +273,16 @@ spring-ai-alibaba-graph-core ${spring-ai-alibaba.version} + + com.alibaba.cloud.ai + spring-ai-alibaba-sandbox + ${spring-ai-alibaba.version} + + + org.tomlj + tomlj + ${tomlj.version} + org.springframework.ai spring-ai-starter-model-chat-memory-repository-jdbc @@ -371,16 +385,6 @@ easyexcel 3.3.4 - - - com.github.docker-java - docker-java-core - - - com.github.docker-java - docker-java-transport-zerodep - - diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/config/DataAgentConfiguration.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/config/DataAgentConfiguration.java index fa4277321..92780fece 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/config/DataAgentConfiguration.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/config/DataAgentConfiguration.java @@ -19,9 +19,6 @@ import com.alibaba.cloud.ai.dataagent.properties.DataAgentProperties; import com.alibaba.cloud.ai.dataagent.properties.FileStorageProperties; import com.alibaba.cloud.ai.dataagent.properties.OssStorageProperties; -import com.alibaba.cloud.ai.dataagent.service.code.CodePoolExecutorService; -import com.alibaba.cloud.ai.dataagent.service.code.CodePoolExecutorServiceFactory; -import com.alibaba.cloud.ai.dataagent.service.code.docker.DockerExecutorFactory; import com.alibaba.cloud.ai.dataagent.service.file.FileStorageService; import com.alibaba.cloud.ai.dataagent.service.file.FileStorageServiceFactory; import com.alibaba.cloud.ai.dataagent.service.llm.LlmService; @@ -127,13 +124,6 @@ public FileStorageService fileStorageService(FileStorageProperties properties, return new FileStorageServiceFactory(properties, ossStorageProperties).getObject(); } - @Bean - @ConditionalOnMissingBean(CodePoolExecutorService.class) - public CodePoolExecutorService codePoolExecutorService(CodeExecutorProperties properties, LlmService llmService, - DockerExecutorFactory dockerExecutorFactory) { - return new CodePoolExecutorServiceFactory(properties, llmService, dockerExecutorFactory).getObject(); - } - @Bean @ConditionalOnMissingBean(RestClientCustomizer.class) public RestClientCustomizer restClientCustomizer(@Value("${rest.connect.timeout:600}") long connectTimeout, diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/properties/CodeExecutorProperties.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/properties/CodeExecutorProperties.java index e368be92b..355f5be4b 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/properties/CodeExecutorProperties.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/properties/CodeExecutorProperties.java @@ -15,11 +15,12 @@ */ package com.alibaba.cloud.ai.dataagent.properties; -import com.alibaba.cloud.ai.dataagent.enums.CodePoolExecutorEnum; import lombok.Getter; import lombok.Setter; import org.springframework.boot.context.properties.ConfigurationProperties; +import java.time.Duration; + import static com.alibaba.cloud.ai.dataagent.constant.Constant.PROJECT_PROPERTIES_PREFIX; /** @@ -34,94 +35,59 @@ public class CodeExecutorProperties { public static final String CONFIG_PREFIX = PROJECT_PROPERTIES_PREFIX + ".code-executor"; /** - * Specify implementation class of code container pool runtime service + * Maximum container memory, in MB */ - CodePoolExecutorEnum codePoolExecutor = CodePoolExecutorEnum.DOCKER; + Long limitMemory = 500L; /** - * Service host, use default address if null + * Number of container CPU cores */ - String host = null; + Long cpuCore = 1L; /** - * Image name, can customize image with common third-party dependencies to replace - * this configuration + * Python code execution time limit */ - String imageName = "continuumio/anaconda3:latest"; + Duration codeTimeout = Duration.ofSeconds(60); /** - * Container name prefix + * Python执行的最大重试次数 */ - String containerNamePrefix = "nl2sql-python-exec-"; + Integer pythonMaxTriesCount = 5; - /** - * Task blocking queue size - */ - Integer taskQueueSize = 5; + private Sandbox sandbox = new Sandbox(); - /** - * Maximum number of core containers - */ - Integer coreContainerNum = 2; + @Getter + @Setter + public static class Sandbox { - /** - * Maximum number of temporary containers - */ - Integer tempContainerNum = 2; + String dockerHost = "unix:///var/run/docker.sock"; - /** - * Core thread count of thread pool - */ - Integer coreThreadSize = 5; + String imageName = "agentscope-registry.ap-southeast-1.cr.aliyuncs.com/agentscope/runtime-sandbox-base:latest"; - /** - * Maximum thread count of thread pool - */ - Integer maxThreadSize = 5; + String containerPrefix = "dataagent-sandbox-"; - /** - * Survival time of temporary containers, in minutes - */ - Integer tempContainerAliveTime = 5; + Integer maxConcurrency = 4; - /** - * Task survival time of thread pool, in seconds - */ - Long keepThreadAliveTime = 60L; + Integer queueCapacity = 10; - /** - * Task blocking queue size of thread pool - */ - Integer threadQueueSize = 10; + Integer maxCodeBytes = 256 * 1024; - /** - * Maximum container memory, in MB - */ - Long limitMemory = 500L; + Integer maxInputBytes = 10 * 1024 * 1024; - /** - * Number of container CPU cores - */ - Long cpuCore = 1L; + Integer maxOutputBytes = 1024 * 1024; - /** - * Python code execution time limit - */ - String codeTimeout = "60s"; + Integer maxErrorBytes = 256 * 1024; - /** - * Maximum container runtime - */ - Long containerTimeout = 3000L; + Integer maxMetadataBytes = 8 * 1024; - /** - * Container network mode - */ - String networkMode = "none"; + Integer maxDependencies = 20; - /** - * Python执行的最大重试次数 - */ - Integer pythonMaxTriesCount = 5; + String packageIndexUrl = "https://pypi.org/simple"; + + Duration dependencyInstallTimeout = Duration.ofMinutes(3); + + Integer maxConnections = 4096; + + } } diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/CodePoolExecutorServiceFactory.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/CodePoolExecutorServiceFactory.java deleted file mode 100644 index 3a5e35f41..000000000 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/CodePoolExecutorServiceFactory.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright 2024-2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code; - -import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; -import com.alibaba.cloud.ai.dataagent.service.code.docker.DockerExecutorFactory; -import com.alibaba.cloud.ai.dataagent.service.code.impls.AiSimulationCodeExecutorService; -import com.alibaba.cloud.ai.dataagent.service.code.impls.LocalCodePoolExecutorService; -import com.alibaba.cloud.ai.dataagent.service.llm.LlmService; -import lombok.AllArgsConstructor; - -/** - * 运行 Python 任务的容器池实现选择器 - * - * @author vlsmb - * @since 2025/7/28 - */ -@AllArgsConstructor -public class CodePoolExecutorServiceFactory { - - private final CodeExecutorProperties properties; - - private final LlmService llmService; - - private final DockerExecutorFactory dockerExecutorFactory; - - public CodePoolExecutorService getObject() { - return switch (properties.getCodePoolExecutor()) { - case DOCKER -> dockerExecutorFactory.create(properties); - case LOCAL -> new LocalCodePoolExecutorService(properties); - case AI_SIMULATION -> new AiSimulationCodeExecutorService(llmService); - default -> - throw new IllegalStateException("This option does not have a corresponding implementation class yet."); - }; - } - - public Class getObjectType() { - return CodePoolExecutorService.class; - } - -} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/CodePoolExecutorService.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/PythonCodeExecutorService.java similarity index 79% rename from data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/CodePoolExecutorService.java rename to data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/PythonCodeExecutorService.java index ae6b54dc8..9e59a76c7 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/CodePoolExecutorService.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/PythonCodeExecutorService.java @@ -15,34 +15,34 @@ */ package com.alibaba.cloud.ai.dataagent.service.code; +import java.util.List; + /** - * 运行Python任务的容器池接口 - * - * @author vlsmb - * @since 2025/7/12 + * Executes generated Python code in an isolated sandbox. */ -public interface CodePoolExecutorService { +public interface PythonCodeExecutorService { TaskResponse runTask(TaskRequest request); - record TaskRequest(String code, String input, String requirement) { + record TaskRequest(String code, String input, List dependencies) { + + public TaskRequest { + dependencies = dependencies == null ? List.of() : List.copyOf(dependencies); + } } record TaskResponse(boolean isSuccess, boolean executionSuccessButResultFailed, String stdOut, String stdErr, String exceptionMsg) { - // 执行运行代码任务时发生异常 public static TaskResponse exception(String msg) { return new TaskResponse(false, false, null, null, "An exception occurred while executing the task: " + msg); } - // 执行运行代码任务成功,并且代码正常返回 public static TaskResponse success(String stdOut) { return new TaskResponse(true, false, stdOut, null, null); } - // 执行运行代码任务成功,但是代码异常返回 public static TaskResponse failure(String stdOut, String stdErr) { return new TaskResponse(false, true, stdOut, stdErr, "StdErr: " + stdErr); } @@ -54,10 +54,4 @@ public String toString() { } } - enum State { - - READY, RUNNING, REMOVING - - } - } diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerClientConnector.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerClientConnector.java deleted file mode 100644 index 016419a98..000000000 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerClientConnector.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2024-2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code.docker; - -import com.github.dockerjava.api.DockerClient; - -/** - * Creates a Docker client for one endpoint without probing the daemon. - */ -@FunctionalInterface -public interface DockerClientConnector { - - DockerClient connect(String host); - -} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerClientFactory.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerClientFactory.java deleted file mode 100644 index 06e1a14b9..000000000 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerClientFactory.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright 2024-2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code.docker; - -import com.github.dockerjava.api.DockerClient; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Component; - -import java.io.IOException; -import java.util.List; - -/** - * Connects to the first reachable Docker endpoint. - */ -@Component -@RequiredArgsConstructor -@Slf4j -public class DockerClientFactory { - - private final DockerClientConnector connector; - - public DockerClient create(List candidateHosts) { - return connect(candidateHosts).client(); - } - - public DockerConnection connect(List candidateHosts) { - if (candidateHosts == null || candidateHosts.isEmpty()) { - throw new IllegalArgumentException("At least one Docker host candidate is required"); - } - - RuntimeException lastFailure = null; - for (String host : candidateHosts) { - DockerClient client = null; - try { - client = connector.connect(host); - client.pingCmd().exec(); - log.info("Connected to Docker using {}", host); - return new DockerConnection(client, host); - } - catch (RuntimeException exception) { - lastFailure = exception; - closeFailedClient(client, exception); - log.warn("Could not connect to Docker using {}: {}", host, exception.getMessage()); - } - } - - throw new IllegalStateException("Failed to connect to Docker. Attempted hosts: " + candidateHosts, lastFailure); - } - - public record DockerConnection(DockerClient client, String host) { - } - - private void closeFailedClient(DockerClient client, RuntimeException connectionFailure) { - if (client == null) { - return; - } - try { - client.close(); - } - catch (IOException closeFailure) { - connectionFailure.addSuppressed(closeFailure); - } - } - -} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerExecutorFactory.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerExecutorFactory.java deleted file mode 100644 index f60ed93e2..000000000 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerExecutorFactory.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright 2024-2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code.docker; - -import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; -import com.alibaba.cloud.ai.dataagent.service.code.docker.DockerClientFactory.DockerConnection; -import com.alibaba.cloud.ai.dataagent.service.code.impls.DockerCodePoolExecutorService; -import com.github.dockerjava.api.DockerClient; -import lombok.RequiredArgsConstructor; -import org.springframework.stereotype.Component; - -import java.io.IOException; -import java.util.List; - -/** - * Composition root for the Docker-backed code executor. - */ -@Component -@RequiredArgsConstructor -public class DockerExecutorFactory { - - private final DockerHostResolver hostResolver; - - private final DockerClientFactory clientFactory; - - private final DockerImageManager imageManager; - - public DockerCodePoolExecutorService create(CodeExecutorProperties properties) { - List candidateHosts = hostResolver.candidates(properties.getHost(), System.getProperty("os.name")); - DockerConnection connection = clientFactory.connect(candidateHosts); - DockerClient client = connection.client(); - try { - imageManager.ensureAvailable(client, properties.getImageName()); - return new DockerCodePoolExecutorService(properties, client, hostResolver.isRemote(connection.host())); - } - catch (RuntimeException constructionFailure) { - closeAfterFailedConstruction(client, constructionFailure); - throw constructionFailure; - } - } - - private void closeAfterFailedConstruction(DockerClient client, RuntimeException constructionFailure) { - try { - client.close(); - } - catch (IOException closeFailure) { - constructionFailure.addSuppressed(closeFailure); - } - } - -} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerHostResolver.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerHostResolver.java deleted file mode 100644 index 2f3a64989..000000000 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerHostResolver.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright 2024-2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code.docker; - -import org.springframework.stereotype.Component; -import org.springframework.util.StringUtils; - -import java.net.URI; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Locale; -import java.util.Objects; - -/** - * Resolves Docker connection candidates and classifies their network location. - */ -@Component -public class DockerHostResolver { - - private static final String UNIX_SOCKET = "unix:///var/run/docker.sock"; - - private static final String WINDOWS_NAMED_PIPE = "npipe://./pipe/docker_engine"; - - private static final String LOCAL_TCP = "tcp://localhost:2375"; - - public List candidates(String configuredHost, String osName) { - String normalizedOs = Objects.requireNonNullElse(osName, "").toLowerCase(Locale.ROOT); - if (normalizedOs.contains("win")) { - LinkedHashSet hosts = new LinkedHashSet<>(); - if (StringUtils.hasText(configuredHost)) { - hosts.add(configuredHost); - } - hosts.add(WINDOWS_NAMED_PIPE); - hosts.add(LOCAL_TCP); - return List.copyOf(hosts); - } - if (StringUtils.hasText(configuredHost)) { - return List.of(configuredHost); - } - return List.of(UNIX_SOCKET); - } - - public boolean isRemote(String host) { - if (!StringUtils.hasText(host)) { - return false; - } - - URI uri; - try { - uri = URI.create(host); - } - catch (IllegalArgumentException exception) { - throw new IllegalArgumentException("Invalid Docker host URI: " + host, exception); - } - - String scheme = uri.getScheme(); - if ("unix".equalsIgnoreCase(scheme) || "npipe".equalsIgnoreCase(scheme)) { - return false; - } - if (!"tcp".equalsIgnoreCase(scheme) || !StringUtils.hasText(uri.getHost())) { - throw new IllegalArgumentException("Unsupported Docker host URI: " + host); - } - - String hostname = uri.getHost(); - if (hostname.startsWith("[") && hostname.endsWith("]")) { - hostname = hostname.substring(1, hostname.length() - 1); - } - return !("localhost".equalsIgnoreCase(hostname) || "127.0.0.1".equals(hostname) || "::1".equals(hostname)); - } - -} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerImageManager.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerImageManager.java deleted file mode 100644 index 3ce1d1c4a..000000000 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerImageManager.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright 2024-2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code.docker; - -import com.github.dockerjava.api.DockerClient; -import com.github.dockerjava.api.model.Image; -import com.github.dockerjava.core.command.PullImageResultCallback; -import org.springframework.stereotype.Component; - -import java.util.Arrays; -import java.util.List; -import java.util.Objects; -import java.util.function.Supplier; - -/** - * Ensures the configured Docker image is locally available. - */ -@Component -public class DockerImageManager { - - private final Supplier callbackFactory; - - public DockerImageManager() { - this(PullImageResultCallback::new); - } - - DockerImageManager(Supplier callbackFactory) { - this.callbackFactory = Objects.requireNonNull(callbackFactory, "callbackFactory"); - } - - public void ensureAvailable(DockerClient client, String imageName) { - List images = client.listImagesCmd().withImageNameFilter(imageName).exec(); - boolean imageExists = images != null && images.stream() - .map(Image::getRepoTags) - .filter(Objects::nonNull) - .flatMap(Arrays::stream) - .anyMatch(imageName::equals); - if (imageExists) { - return; - } - - PullImageResultCallback callback = callbackFactory.get(); - try { - client.pullImageCmd(imageName).exec(callback).awaitCompletion(); - } - catch (InterruptedException exception) { - Thread.currentThread().interrupt(); - throw new IllegalStateException("Interrupted while pulling Docker image " + imageName, exception); - } - } - -} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/docker/ZerodepDockerClientConnector.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/docker/ZerodepDockerClientConnector.java deleted file mode 100644 index 3834758b2..000000000 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/docker/ZerodepDockerClientConnector.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2024-2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code.docker; - -import com.github.dockerjava.api.DockerClient; -import com.github.dockerjava.core.DefaultDockerClientConfig; -import com.github.dockerjava.core.DockerClientConfig; -import com.github.dockerjava.core.DockerClientImpl; -import com.github.dockerjava.zerodep.ZerodepDockerHttpClient; -import org.springframework.stereotype.Component; - -/** - * docker-java connector backed by its dependency-free HTTP transport. - */ -@Component -public class ZerodepDockerClientConnector implements DockerClientConnector { - - @Override - public DockerClient connect(String host) { - DockerClientConfig config = DefaultDockerClientConfig.createDefaultConfigBuilder() - .withDockerHost(host) - .withDockerTlsVerify(false) - .build(); - ZerodepDockerHttpClient httpClient = new ZerodepDockerHttpClient.Builder().dockerHost(config.getDockerHost()) - .sslConfig(config.getSSLConfig()) - .build(); - return DockerClientImpl.getInstance(config, httpClient); - } - -} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/impls/AbstractCodePoolExecutorService.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/impls/AbstractCodePoolExecutorService.java deleted file mode 100644 index f08ab77a4..000000000 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/impls/AbstractCodePoolExecutorService.java +++ /dev/null @@ -1,428 +0,0 @@ -/* - * Copyright 2024-2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code.impls; - -import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; -import com.alibaba.cloud.ai.dataagent.service.code.CodePoolExecutorService; -import lombok.extern.slf4j.Slf4j; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; - -import java.io.IOException; -import java.nio.file.FileVisitResult; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.attribute.BasicFileAttributes; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; -import java.util.concurrent.FutureTask; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * 运行Python任务的容器池 - * - * @author vlsmb - * @since 2025/7/12 - */ -@Slf4j -public abstract class AbstractCodePoolExecutorService implements CodePoolExecutorService, AutoCloseable { - - private final AtomicBoolean closed = new AtomicBoolean(); - - // Record core container status - protected final ConcurrentHashMap coreContainerState; - - // Record temporary container status - protected final ConcurrentHashMap tempContainerState; - - // Record Future for temporary container destruction - protected final ConcurrentHashMap> tempContainerRemoveFuture; - - // Task queue (temporarily store tasks when containers are full) - protected final ArrayBlockingQueue> taskQueue; - - // Ready core containers - protected final ArrayBlockingQueue readyCoreContainer; - - // Ready temporary containers - protected final ArrayBlockingQueue readyTempContainer; - - // Current number of core containers - protected final AtomicInteger currentCoreContainerSize; - - // Current number of temporary containers - protected final AtomicInteger currentTempContainerSize; - - // Thread pool, running temporarily stored tasks - protected final ExecutorService consumerThreadPool; - - // Configuration properties - protected final CodeExecutorProperties properties; - - public AbstractCodePoolExecutorService(CodeExecutorProperties properties) { - this.properties = properties; - this.coreContainerState = new ConcurrentHashMap<>(); - this.tempContainerState = new ConcurrentHashMap<>(); - this.tempContainerRemoveFuture = new ConcurrentHashMap<>(); - this.taskQueue = new ArrayBlockingQueue<>(properties.getTaskQueueSize()); - this.readyCoreContainer = new ArrayBlockingQueue<>(properties.getCoreContainerNum()); - this.readyTempContainer = new ArrayBlockingQueue<>(properties.getTempContainerNum()); - this.consumerThreadPool = new ThreadPoolExecutor(properties.getCoreThreadSize(), properties.getMaxThreadSize(), - properties.getKeepThreadAliveTime(), TimeUnit.SECONDS, - new ArrayBlockingQueue<>(properties.getThreadQueueSize())); - this.currentCoreContainerSize = new AtomicInteger(0); - this.currentTempContainerSize = new AtomicInteger(0); - } - - @Override - public final void close() { - if (!closed.compareAndSet(false, true)) { - return; - } - try { - this.shutdownPool(); - } - catch (Exception exception) { - throw new IllegalStateException("Failed to close code pool executor", exception); - } - } - - /** - * 创建新的容器 - * @return 容器ID - */ - protected abstract String createNewContainer() throws Exception; - - /** - * 在指定容器ID的容器运行任务 - * @param request 任务请求对象 - * @param containerId 容器ID - * @return 运行结果对象 - */ - protected abstract TaskResponse execTaskInContainer(TaskRequest request, String containerId); - - /** - * 停止指定容器 - * @param containerId 容器ID - */ - protected abstract void stopContainer(String containerId) throws Exception; - - /** - * 删除指定容器 - * @param containerId 容器ID - */ - protected abstract void removeContainer(String containerId) throws Exception; - - protected void shutdownPool() throws Exception { - // Shutdown thread pool - this.consumerThreadPool.shutdownNow(); - // Stop and delete all containers - this.tempContainerState.keySet().forEach(id -> this.removeContainerAndState(id, false, true)); - this.coreContainerState.keySet().forEach(id -> this.removeContainerAndState(id, true, true)); - this.tempContainerState.clear(); - this.coreContainerState.clear(); - this.tempContainerRemoveFuture.clear(); - this.readyCoreContainer.clear(); - this.readyTempContainer.clear(); - this.taskQueue.clear(); - } - - private void removeContainerAndState(String containerId, boolean isCore, boolean isForce) { - try { - if (isCore) { - // Remove core container - State state = this.coreContainerState.replace(containerId, State.REMOVING); - if (state == State.RUNNING) { - if (isForce) { - this.stopContainer(containerId); - } - else { - throw new RuntimeException("Container is still Running!"); - } - } - this.removeContainer(containerId); - this.coreContainerState.remove(containerId); - this.currentCoreContainerSize.decrementAndGet(); - log.info("Core Container {} has been removed successfully", containerId); - } - else { - // Remove temporary container - State state = this.tempContainerState.replace(containerId, State.REMOVING); - if (state == State.RUNNING) { - if (isForce) { - this.stopContainer(containerId); - } - else { - throw new RuntimeException("Container is still Running!"); - } - } - this.removeContainer(containerId); - this.tempContainerState.remove(containerId); - this.tempContainerRemoveFuture.remove(containerId); - this.currentTempContainerSize.decrementAndGet(); - log.info("Temp Container {} has been removed successfully", containerId); - } - } - catch (Exception e) { - log.error("Error when trying to remove a container, containerId: {}, info: {}", containerId, e.getMessage(), - e); - } - } - - // Create thread to delete temporary containers - private Future registerRemoveTempContainer(String containerId) { - return consumerThreadPool.submit(() -> { - try { - Thread.sleep(this.properties.getKeepThreadAliveTime() * 1000L * 60L); - if (this.tempContainerState.get(containerId) == State.RUNNING) { - throw new InterruptedException("Container " + containerId + " is already running"); - } - } - catch (InterruptedException e) { - // Cancel temporary container destruction thread due to container running - log.debug("Interrupted while waiting for temp container to be removed, info: {}", e.getMessage()); - return; - } - this.removeContainerAndState(containerId, false, false); - }); - } - - // Use core container - private TaskResponse useCoreContainer(String containerId, TaskRequest request) { - try { - // Execute task - this.coreContainerState.replace(containerId, State.RUNNING); - TaskResponse resp = this.execTaskInContainer(request, containerId); - // 如果运行代码任务时出现了异常,认为容器损坏,执行容器清除,并将当前任务放进队列里重新执行 - if (!resp.isSuccess() && !resp.executionSuccessButResultFailed()) { - log.error("use core container failed, {}", resp.exceptionMsg()); - this.coreContainerState.replace(containerId, State.REMOVING); - this.removeContainerAndState(containerId, true, true); - return this.pushTaskQueue(request); - } - this.coreContainerState.replace(containerId, State.READY); - // Put back into blocking queue - this.readyCoreContainer.add(containerId); - // Run tasks in task queue if any - this.popTaskQueue(); - return resp; - } - catch (Exception e) { - log.error("use core container failed, {}", e.getMessage(), e); - return TaskResponse.exception(e.getMessage()); - } - } - - // Use temporary container - private TaskResponse useTempContainer(String containerId, TaskRequest request) { - try { - Future future = this.tempContainerRemoveFuture.remove(containerId); - // Cancel temporary container destruction thread - if (future != null) { - if (future.isDone()) { - // Container is destroyed, reselect usage strategy - log.debug("reselect strategy: {} ...", request.toString()); - return this.runTask(request); - } - future.cancel(true); - } - // Execute task - this.tempContainerState.replace(containerId, State.RUNNING); - TaskResponse resp = this.execTaskInContainer(request, containerId); - // 如果运行代码任务时出现了异常,认为容器损坏,执行容器清除,并将当前任务放进队列里重新执行 - if (!resp.isSuccess() && !resp.executionSuccessButResultFailed()) { - log.error("use temp container failed, {}", resp.exceptionMsg()); - this.tempContainerState.replace(containerId, State.REMOVING); - this.removeContainerAndState(containerId, false, true); - return this.pushTaskQueue(request); - } - this.tempContainerState.replace(containerId, State.READY); - // Put back into blocking queue - this.readyTempContainer.add(containerId); - // Recreate temporary container destruction thread - this.tempContainerRemoveFuture.put(containerId, this.registerRemoveTempContainer(containerId)); - // Run tasks in task queue if any - this.popTaskQueue(); - return resp; - } - catch (Exception e) { - log.error("use temp container failed, {}", e.getMessage(), e); - return TaskResponse.exception(e.getMessage()); - } - } - - // Create and use core container - private TaskResponse createAndUseCoreContainer(TaskRequest request) { - String containerId; - try { - containerId = this.createNewContainer(); - } - catch (Exception e) { - log.error("create new container failed, {}", e.getMessage(), e); - return TaskResponse.exception(e.getMessage()); - } - // Record newly added container - this.coreContainerState.put(containerId, State.READY); - // Use container - return this.useCoreContainer(containerId, request); - } - - // Create and use temporary container - private TaskResponse createAndUseTempContainer(TaskRequest request) { - String containerId; - try { - containerId = this.createNewContainer(); - } - catch (Exception e) { - log.error("create new container failed, {}", e.getMessage(), e); - return TaskResponse.exception(e.getMessage()); - } - // Record newly added container - this.tempContainerState.put(containerId, State.READY); - // Use container - return this.useTempContainer(containerId, request); - } - - private TaskResponse pushTaskQueue(TaskRequest request) throws ExecutionException, InterruptedException { - FutureTask ft = new FutureTask<>(() -> { - log.info("Execute tasks in the BlockingQueue {} ...", request.toString()); - return this.runTask(request); - }); - this.taskQueue.put(ft); - - if (!this.readyCoreContainer.isEmpty() || !this.readyTempContainer.isEmpty()) { - this.popTaskQueue(); - } - - return ft.get(); - } - - // Run tasks in task queue if any - private void popTaskQueue() { - FutureTask future = this.taskQueue.poll(); - if (future == null) { - return; - } - log.info("Get task from the BlockingQueue ..."); - this.consumerThreadPool.submit(future); - } - - @Override - public TaskResponse runTask(TaskRequest request) { - // Use available core container - String freeCoreId = this.readyCoreContainer.poll(); - if (freeCoreId != null) { - log.debug("Use free core container to run task {} ...", request.toString()); - return this.useCoreContainer(freeCoreId, request); - } - - // Use available temporary container - String freeTempId = this.readyTempContainer.poll(); - if (freeTempId != null) { - log.debug("Use free temp container to run task {} ...", request.toString()); - return this.useTempContainer(freeTempId, request); - } - - // Create new core container - int currentCore; - boolean useCoreContainer = true; - do { - currentCore = this.currentCoreContainerSize.get(); - if (currentCore >= properties.getCoreContainerNum()) { - useCoreContainer = false; - break; - } - } - while (!this.currentCoreContainerSize.compareAndSet(currentCore, currentCore + 1)); - if (useCoreContainer) { - log.debug("Create new core container to run task {} ...", request.toString()); - return this.createAndUseCoreContainer(request); - } - - // Create new temporary container - int currentTemp; - boolean useTempContainer = true; - do { - currentTemp = this.currentTempContainerSize.get(); - if (currentTemp >= properties.getTempContainerNum()) { - useTempContainer = false; - break; - } - } - while (!this.currentTempContainerSize.compareAndSet(currentTemp, currentTemp + 1)); - if (useTempContainer) { - log.debug("Create new temp container to run task {} ...", request.toString()); - return this.createAndUseTempContainer(request); - } - - // Put into task queue to wait - try { - log.debug("push task into BlockingQueue: {} ...", request.toString()); - return this.pushTaskQueue(request); - } - catch (Exception e) { - log.error("An exception occurred while executing the task: {}", e.getMessage(), e); - return TaskResponse.exception(e.getMessage()); - } - } - - /** - * Delete temporary directory - */ - protected void clearTempDir(Path tempDir) { - try { - Files.walkFileTree(tempDir, new SimpleFileVisitor<>() { - @NotNull - @Override - public FileVisitResult visitFile(@NotNull Path file, @NotNull BasicFileAttributes attrs) - throws IOException { - Files.delete(file); - return super.visitFile(file, attrs); - } - - @NotNull - @Override - public FileVisitResult postVisitDirectory(@NotNull Path dir, @Nullable IOException exc) - throws IOException { - if (exc != null) - throw exc; - Files.delete(dir); - return super.postVisitDirectory(dir, exc); - } - }); - log.info("Temp directory has been deleted."); - } - catch (Exception e) { - log.warn("Exception in clean temp directory: {}", e.getMessage()); - } - } - - /** - * Generate unique container name - */ - protected String generateContainerName() { - return this.properties.getContainerNamePrefix() + "_" + System.currentTimeMillis() + "_" - + Thread.currentThread().getName(); - } - -} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/impls/AiSimulationCodeExecutorService.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/impls/AiSimulationCodeExecutorService.java deleted file mode 100644 index 8a073193d..000000000 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/impls/AiSimulationCodeExecutorService.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright 2024-2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code.impls; - -import com.alibaba.cloud.ai.dataagent.service.code.CodePoolExecutorService; -import com.alibaba.cloud.ai.dataagent.service.llm.LlmService; -import lombok.extern.slf4j.Slf4j; - -/** - * 使用AI模拟运行Python代码(便于在无Docker环境测试) - * - * @author vlsmb - * @since 2025/7/30 - */ -@Slf4j -public class AiSimulationCodeExecutorService implements CodePoolExecutorService { - - private static final String SYSTEM_PROMPT = """ - 你将模拟Python的执行,根据我提供的代码和输入数据,并给出最终的数据结果。 - 在模拟运行时,请按照以下要求操作: - 1. 仔细理解代码和输入数据的内容。 - 2. 输出模拟运行结果。 - **要求**:仅输出模拟运行结果,禁止包含任何额外说明或自然语言。 - """; - - private final LlmService llmService; - - public AiSimulationCodeExecutorService(LlmService llmService) { - this.llmService = llmService; - } - - @Override - public TaskResponse runTask(TaskRequest request) { - String userPrompt = String.format(""" - 【代码】 - ```python - %s - ``` - 【标准输入】 - ```json - %s - ``` - """, request.code(), request.input()); - String output = llmService.toStringFlux(llmService.call(SYSTEM_PROMPT, userPrompt)) - .collect(StringBuilder::new, StringBuilder::append) - .map(StringBuilder::toString) - .block(); - return TaskResponse.success(output); - } - -} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/impls/DockerCodePoolExecutorService.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/impls/DockerCodePoolExecutorService.java deleted file mode 100644 index 64e5f8d8d..000000000 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/impls/DockerCodePoolExecutorService.java +++ /dev/null @@ -1,285 +0,0 @@ -/* - * Copyright 2024-2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code.impls; - -import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; -import com.alibaba.cloud.ai.dataagent.service.code.CodePoolExecutorService; -import com.github.dockerjava.api.DockerClient; -import com.github.dockerjava.api.command.CreateContainerResponse; -import com.github.dockerjava.api.command.InspectContainerResponse; -import com.github.dockerjava.api.async.ResultCallback; -import com.github.dockerjava.api.model.AccessMode; -import com.github.dockerjava.api.model.Bind; -import com.github.dockerjava.api.model.Capability; -import com.github.dockerjava.api.model.Frame; -import com.github.dockerjava.api.model.HostConfig; -import com.github.dockerjava.api.model.StreamType; -import com.github.dockerjava.api.model.Volume; -import lombok.extern.slf4j.Slf4j; -import org.springframework.util.StringUtils; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; - -import static com.github.dockerjava.api.model.HostConfig.newHostConfig; - -/** - * 运行Python任务的容器池(Docker实现类) - * - * @author vlsmb - * @since 2025/7/12 - */ -@Slf4j -public class DockerCodePoolExecutorService extends AbstractCodePoolExecutorService implements CodePoolExecutorService { - - private final DockerClient dockerClient; - - private final boolean isRemote; - - private final ConcurrentHashMap containerTempPath; - - public DockerCodePoolExecutorService(CodeExecutorProperties properties, DockerClient dockerClient, - boolean isRemote) { - super(properties); - this.dockerClient = Objects.requireNonNull(dockerClient, "dockerClient"); - this.isRemote = isRemote; - this.containerTempPath = new ConcurrentHashMap<>(); - log.info("Docker Code Pool initialized. Mode: {}", - this.isRemote ? "Remote (Copy Files)" : "Local (Bind Mounts)"); - } - - /** - * Create container's HostConfig - */ - private HostConfig createHostConfig(Path tempDir) { - HostConfig config = newHostConfig().withMemory(this.properties.getLimitMemory() * 1024L * 1024L) - .withCpuCount(this.properties.getCpuCore()) - .withCapDrop(Capability.ALL) - .withAutoRemove(false) - .withTmpFs(Map.of("/tmp", "")) - .withNetworkMode(this.properties.getNetworkMode()); - - if (!this.isRemote) { - List binds = new ArrayList<>(); - binds.add(new Bind(tempDir.resolve("script.py").toAbsolutePath().toString(), new Volume("/app/script.py"), - AccessMode.ro)); - binds.add(new Bind(tempDir.resolve("requirements.txt").toAbsolutePath().toString(), - new Volume("/app/requirements.txt"), AccessMode.ro)); - binds.add(new Bind(tempDir.resolve("input_data.txt").toAbsolutePath().toString(), - new Volume("/app/input_data.txt"), AccessMode.ro)); - config.withBinds(binds.toArray(new Bind[0])); - } - return config; - } - - /** - * Clean up existing container with same name - */ - private void cleanupExistingResources(String containName) { - try { - // Try to delete container with same name - dockerClient.removeContainerCmd(containName).withForce(true).exec(); - log.info("Removed existing container: {}", containName); - } - catch (Exception e) { - log.warn("Failed to remove container {}: {}", containName, e.getMessage()); - } - } - - @Override - protected String createNewContainer() throws Exception { - String containerName = this.generateContainerName(); - // First clean up possibly existing container with same name - this.cleanupExistingResources(containerName); - - // Generate temporary directory and files - Path tempDir = Files.createTempDirectory(containerName); - Files.createFile(tempDir.resolve("requirements.txt")); - Files.createFile(tempDir.resolve("script.py")); - Files.createFile(tempDir.resolve("input_data.txt")); - - // Create container - HostConfig hostConfig = this.createHostConfig(tempDir); - String cmd = this.buildExecutionCommand(tempDir); - - CreateContainerResponse container = dockerClient.createContainerCmd(properties.getImageName()) - .withName(containerName) - .withWorkingDir("/app") - .withHostConfig(hostConfig) - .withCmd("sh", "-c", cmd) - .exec(); - String containerId = container.getId(); - // Save temporary directory object - this.containerTempPath.put(containerId, tempDir); - return containerId; - } - - @Override - protected TaskResponse execTaskInContainer(TaskRequest request, String containerId) { - // Get temporary directory object - Path tempDir = this.containerTempPath.get(containerId); - if (tempDir == null) { - log.error("Container '{}' does not exist work dir", containerId); - return TaskResponse.exception("Container '" + containerId + "' does not exist work dir"); - } - - try { - // 1. Prepare files - this.writeContextFiles(tempDir, request); - this.uploadFilesIfRemote(containerId, tempDir); - - // 2. Start container and wait - dockerClient.startContainerCmd(containerId).exec(); - dockerClient.waitContainerCmd(containerId) - .start() - .awaitCompletion(this.properties.getContainerTimeout(), TimeUnit.SECONDS); - - // 3. Fetch logs - LogResult logs = this.fetchExecutionLogs(containerId, tempDir); - String stdout = logs.stdout; - String stderr = logs.stderr; - - // 4. Check exit code - InspectContainerResponse inspectResponse = dockerClient.inspectContainerCmd(containerId).exec(); - int exitCode = Objects.requireNonNull(inspectResponse.getState().getExitCodeLong()).intValue(); - if (exitCode != 0) { - String errorMessage = "Docker exit code " + exitCode + ". Stderr: " + stderr + ". Stdout: " + stdout; - log.error("Error executing Docker container {}: {}", containerId, errorMessage); - return TaskResponse.failure(stdout, stderr); - } - return TaskResponse.success(stdout); - } - catch (Exception e) { - log.error("Error executing task in container: {}", e.getMessage()); - return TaskResponse.exception(e.getMessage()); - } - } - - // --- Helper Methods --- - - private String buildExecutionCommand(Path tempDir) { - return String.format( - "if [ -s requirements.txt ]; then pip3 install --no-cache-dir -r requirements.txt > /dev/null; fi && timeout -s SIGKILL %s python3 -u script.py < input_data.txt", - properties.getCodeTimeout()); - } - - private void writeContextFiles(Path tempDir, TaskRequest request) throws IOException { - Files.write(tempDir.resolve("script.py"), - StringUtils.hasText(request.code()) ? request.code().getBytes() : "".getBytes()); - Files.write(tempDir.resolve("requirements.txt"), - StringUtils.hasText(request.requirement()) ? request.requirement().getBytes() : "".getBytes()); - Files.write(tempDir.resolve("input_data.txt"), - StringUtils.hasText(request.input()) ? request.input().getBytes() : "".getBytes()); - } - - private void uploadFilesIfRemote(String containerId, Path tempDir) { - if (!this.isRemote) { - return; - } - String[] files = { "script.py", "requirements.txt", "input_data.txt" }; - for (String file : files) { - dockerClient.copyArchiveToContainerCmd(containerId) - .withHostResource(tempDir.resolve(file).toString()) - .withRemotePath("/app/") - .exec(); - } - } - - private record LogResult(String stdout, String stderr) { - } - - private LogResult fetchExecutionLogs(String containerId, Path tempDir) throws InterruptedException { - StringBuilder stdoutBuilder = new StringBuilder(); - StringBuilder stderrBuilder = new StringBuilder(); - - final int MAX_LOG_SIZE = 5 * 1024 * 1024; // 5MB limit - dockerClient.logContainerCmd(containerId) - .withStdOut(true) - .withStdErr(true) - .exec(new ResultCallback.Adapter() { - @Override - public void onNext(Frame item) { - String payload = new String(item.getPayload(), StandardCharsets.UTF_8); - if (item.getStreamType() == StreamType.STDOUT) { - appendWithLimit(stdoutBuilder, payload, MAX_LOG_SIZE); - } - else if (item.getStreamType() == StreamType.STDERR) { - appendWithLimit(stderrBuilder, payload, MAX_LOG_SIZE); - } - } - }) - .awaitCompletion(); - - return new LogResult(stdoutBuilder.toString(), stderrBuilder.toString()); - } - - private void appendWithLimit(StringBuilder builder, String payload, int limit) { - if (builder.length() < limit) { - builder.append(payload); - } - else if (builder.length() == limit) { - builder.append("\n...[Output truncated due to size limit]..."); - builder.append(" "); // Prevent re-entry - } - } - - @Override - protected void stopContainer(String containerId) throws Exception { - try { - this.dockerClient.stopContainerCmd(containerId).exec(); - log.info("Successfully stopped container: {}", containerId); - } - catch (Exception e) { - log.warn("Failed to stop container: {}, message: {}", containerId, e.getMessage()); - } - } - - @Override - protected void removeContainer(String containerId) throws Exception { - try { - this.dockerClient.removeContainerCmd(containerId).withForce(true).exec(); - Path tempDir = containerTempPath.get(containerId); - if (tempDir != null) { - this.clearTempDir(tempDir); - } - containerTempPath.remove(containerId); - log.info("Successfully removed container: {}", containerId); - } - catch (Exception e) { - log.warn("Failed to remove container: {}, message: {}", containerId, e.getMessage()); - } - } - - @Override - protected void shutdownPool() throws Exception { - try { - super.shutdownPool(); - this.dockerClient.close(); - } - catch (IOException ignored) { - - } - } - -} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/impls/LocalCodePoolExecutorService.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/impls/LocalCodePoolExecutorService.java deleted file mode 100644 index a0d5cca57..000000000 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/impls/LocalCodePoolExecutorService.java +++ /dev/null @@ -1,224 +0,0 @@ -/* - * Copyright 2024-2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code.impls; - -import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; -import com.alibaba.cloud.ai.dataagent.service.code.CodePoolExecutorService; -import com.alibaba.cloud.ai.dataagent.service.code.local.ExecutableProgramLocator; -import com.alibaba.cloud.ai.dataagent.service.code.local.ExecutionTimeoutParser; -import com.alibaba.cloud.ai.dataagent.service.code.local.PathExecutableProgramLocator; -import lombok.extern.slf4j.Slf4j; -import org.springframework.util.StringUtils; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.StringWriter; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; - -/** - * 使用本地Python3环境运行代码的实现类,要求本地的Python3需要有pandas等数据分析库。 - * - * @author vlsmb - * @since 2025/8/23 - */ -@Slf4j -public class LocalCodePoolExecutorService extends AbstractCodePoolExecutorService implements CodePoolExecutorService { - - private final ConcurrentHashMap containers; - - private static final List PYTHON_NAMES = List.of("python3", "pypy3", "py3", "python", "pypy", "py"); - - private static final List PIP_NAMES = List.of("pip3", "pip"); - - private final String pythonExecutable; - - private final String pipExecutable; - - private final ExecutionTimeoutParser timeoutParser; - - // 对于本地运行这个实现类,“容器”为临时文件夹 - public LocalCodePoolExecutorService(CodeExecutorProperties properties) { - this(properties, new PathExecutableProgramLocator(), new ExecutionTimeoutParser()); - } - - LocalCodePoolExecutorService(CodeExecutorProperties properties, ExecutableProgramLocator programLocator, - ExecutionTimeoutParser timeoutParser) { - this(properties, timeoutParser, locatePrograms(programLocator)); - } - - private LocalCodePoolExecutorService(CodeExecutorProperties properties, ExecutionTimeoutParser timeoutParser, - LocalPrograms programs) { - super(properties); - this.containers = new ConcurrentHashMap<>(); - this.pythonExecutable = programs.pythonExecutable(); - this.pipExecutable = programs.pipExecutable(); - this.timeoutParser = timeoutParser; - } - - @Override - protected String createNewContainer() throws Exception { - Path container = Files.createTempDirectory(this.properties.getContainerNamePrefix()); - String containerId = container.toString(); - this.containers.put(containerId, container); - return containerId; - } - - @Override - protected TaskResponse execTaskInContainer(TaskRequest request, String containerId) { - Path container = this.containers.get(containerId); - - // 写入Py代码和标准输入 - Path scriptFile = container.resolve("script.py"); - Path stdinFile = container.resolve("stdin.txt"); - Path requirementFile = container.resolve("requirements.txt"); - try { - Files.write(scriptFile, Optional.ofNullable(request.code()).orElse("").getBytes()); - Files.write(stdinFile, Optional.ofNullable(request.input()).orElse("").getBytes()); - Files.write(requirementFile, Optional.ofNullable(request.requirement()).orElse("").getBytes()); - } - catch (Exception e) { - log.error("Create temp file failed: {}", e.getMessage(), e); - return TaskResponse.exception(e.getMessage()); - } - - // 如果有requirements,则先安装依赖 - if (this.pipExecutable != null && StringUtils.hasText(request.requirement())) { - ProcessBuilder pip = new ProcessBuilder(this.pipExecutable, "install", "--no-cache-dir", "-r", - requirementFile.toAbsolutePath().toString()); - pip.redirectOutput(ProcessBuilder.Redirect.DISCARD); - pip.redirectError(ProcessBuilder.Redirect.DISCARD); - Process process = null; - - try { - process = pip.start(); - boolean completed = process.waitFor(this.properties.getContainerTimeout(), TimeUnit.MINUTES); - if (!completed) { - process.destroy(); - if (process.isAlive()) { - process.destroyForcibly(); - } - throw new RuntimeException("Pip command timed out."); - } - } - catch (Exception e) { - // 即使PIP安装失败,仍然尝试运行Python代码 - log.warn("Pip install failed: {}", e.getMessage(), e); - } - finally { - if (process != null && process.isAlive()) { - process.destroyForcibly(); - } - } - } - - // 运行Python代码 - Process process = null; - try { - ProcessBuilder pb = new ProcessBuilder(this.pythonExecutable, scriptFile.toAbsolutePath().toString()); - pb.directory(container.toFile()); - pb.redirectInput(stdinFile.toFile()); - process = pb.start(); - - // 读取stdout和stderr - StringWriter stdoutWriter = new StringWriter(); - StringWriter stderrWriter = new StringWriter(); - try (BufferedReader stdoutReader = new BufferedReader(new InputStreamReader(process.getInputStream())); - BufferedReader stderrReader = new BufferedReader(new InputStreamReader(process.getErrorStream()))) { - CompletableFuture stdoutFuture = CompletableFuture.runAsync(() -> { - try { - stdoutReader.transferTo(stdoutWriter); - } - catch (IOException e) { - stderrWriter.write("Error reading stdout: " + e.getMessage()); - } - }); - CompletableFuture stderrFuture = CompletableFuture.runAsync(() -> { - try { - stderrReader.transferTo(stderrWriter); - } - catch (IOException e) { - stderrWriter.write("Error reading stderr: " + e.getMessage()); - } - }); - - // 等待进程完成,带超时限制 - boolean completed = process.waitFor(this.timeoutParser.parse(this.properties.getCodeTimeout()), - TimeUnit.MILLISECONDS); - if (!completed) { - process.destroy(); - if (process.isAlive()) { - process.destroyForcibly(); - } - return TaskResponse.failure("", "python code timeout, Killed."); - } - - // 等待输出读取完成,给输出读取额外2秒时间 - CompletableFuture.allOf(stdoutFuture, stderrFuture).get(2, TimeUnit.SECONDS); - } - - // 返回结果 - int exitCode = process.exitValue(); - String stdout = stdoutWriter.toString(); - String stderr = stderrWriter.toString(); - if (exitCode != 0) { - return TaskResponse.failure(stdout, stderr); - } - else { - return TaskResponse.success(stdout); - } - - } - catch (Exception e) { - log.error("Python execution failed: {}", e.getMessage(), e); - return TaskResponse.exception(e.getMessage()); - } - finally { - if (process != null && process.isAlive()) { - process.destroyForcibly(); - } - } - } - - @Override - protected void stopContainer(String containerId) throws Exception { - // 临时文件夹没有停止方法 - } - - @Override - protected void removeContainer(String containerId) throws Exception { - Path container = this.containers.remove(containerId); - this.clearTempDir(container); - } - - private static LocalPrograms locatePrograms(ExecutableProgramLocator locator) { - String python = locator.findFirst(PYTHON_NAMES) - .orElseThrow(() -> new IllegalStateException( - "No valid Python interpreter was found in PATH. Please install Python 3.")); - String pip = locator.findFirst(PIP_NAMES).orElse(null); - return new LocalPrograms(python, pip); - } - - private record LocalPrograms(String pythonExecutable, String pipExecutable) { - } - -} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/local/ExecutionTimeoutParser.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/local/ExecutionTimeoutParser.java deleted file mode 100644 index f7d294941..000000000 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/local/ExecutionTimeoutParser.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright 2024-2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code.local; - -import java.util.Locale; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * Parses the compact timeout format used by the code executor. - */ -public final class ExecutionTimeoutParser { - - public static final long DEFAULT_TIMEOUT_MILLIS = 60_000L; - - private static final Pattern TIMEOUT_PATTERN = Pattern.compile("(\\d+)(ms|[smhd])"); - - public long parse(String value) { - if (value == null) { - return DEFAULT_TIMEOUT_MILLIS; - } - Matcher matcher = TIMEOUT_PATTERN.matcher(value.trim().toLowerCase(Locale.ROOT)); - if (!matcher.matches()) { - return DEFAULT_TIMEOUT_MILLIS; - } - - try { - long amount = Long.parseLong(matcher.group(1)); - return Math.multiplyExact(amount, multiplier(matcher.group(2))); - } - catch (ArithmeticException | NumberFormatException exception) { - return DEFAULT_TIMEOUT_MILLIS; - } - } - - private long multiplier(String unit) { - return switch (unit) { - case "ms" -> 1L; - case "s" -> 1_000L; - case "m" -> 60_000L; - case "h" -> 3_600_000L; - case "d" -> 86_400_000L; - default -> throw new IllegalArgumentException("Unsupported timeout unit: " + unit); - }; - } - -} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/local/PathExecutableProgramLocator.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/local/PathExecutableProgramLocator.java deleted file mode 100644 index d59d1761a..000000000 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/local/PathExecutableProgramLocator.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright 2024-2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code.local; - -import java.io.File; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.List; -import java.util.Locale; -import java.util.Optional; -import java.util.regex.Pattern; - -/** - * Locates executables from the process {@code PATH}. - */ -public final class PathExecutableProgramLocator implements ExecutableProgramLocator { - - private final String pathEnvironment; - - private final boolean windows; - - public PathExecutableProgramLocator() { - this(System.getenv("PATH"), System.getProperty("os.name", "").toLowerCase(Locale.ROOT).contains("win")); - } - - PathExecutableProgramLocator(String pathEnvironment, boolean windows) { - this.pathEnvironment = pathEnvironment; - this.windows = windows; - } - - @Override - public Optional findFirst(List programNames) { - if (pathEnvironment == null || pathEnvironment.isBlank() || programNames == null || programNames.isEmpty()) { - return Optional.empty(); - } - - for (String programName : programNames) { - if (programName == null || programName.isBlank()) { - continue; - } - for (String directory : pathEnvironment.split(Pattern.quote(File.pathSeparator))) { - if (directory.isBlank()) { - continue; - } - if (isExecutable(Path.of(directory, programName)) - || windows && isExecutable(Path.of(directory, programName + ".exe"))) { - return Optional.of(programName); - } - } - } - return Optional.empty(); - } - - private boolean isExecutable(Path path) { - return Files.isRegularFile(path) && Files.isExecutable(path); - } - -} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/local/ExecutableProgramLocator.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/PythonDependencyMetadata.java similarity index 63% rename from data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/local/ExecutableProgramLocator.java rename to data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/PythonDependencyMetadata.java index 3fa7b0607..51a5ee454 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/local/ExecutableProgramLocator.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/PythonDependencyMetadata.java @@ -13,17 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.alibaba.cloud.ai.dataagent.service.code.local; +package com.alibaba.cloud.ai.dataagent.service.code.sandbox; import java.util.List; -import java.util.Optional; -/** - * Finds the first available executable from an ordered candidate list. - */ -@FunctionalInterface -public interface ExecutableProgramLocator { +public record PythonDependencyMetadata(List dependencies, String requiresPython) { + + public static PythonDependencyMetadata empty() { + return new PythonDependencyMetadata(List.of(), null); + } - Optional findFirst(List programNames); + public PythonDependencyMetadata { + dependencies = dependencies == null ? List.of() : List.copyOf(dependencies); + } } diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/PythonDependencyMetadataParser.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/PythonDependencyMetadataParser.java new file mode 100644 index 000000000..2c1a28e06 --- /dev/null +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/PythonDependencyMetadataParser.java @@ -0,0 +1,114 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.cloud.ai.dataagent.service.code.sandbox; + +import org.springframework.stereotype.Component; +import org.tomlj.Toml; +import org.tomlj.TomlArray; +import org.tomlj.TomlParseResult; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +@Component +public class PythonDependencyMetadataParser { + + private static final String BLOCK_START = "# /// script"; + + private static final String BLOCK_END = "# ///"; + + private static final Set SUPPORTED_KEYS = Set.of("dependencies", "requires-python"); + + private final PythonDependencyPolicy dependencyPolicy; + + public PythonDependencyMetadataParser(PythonDependencyPolicy dependencyPolicy) { + this.dependencyPolicy = dependencyPolicy; + } + + public PythonDependencyMetadata parse(String code) { + String metadata = extractMetadata(code); + if (metadata == null) { + return PythonDependencyMetadata.empty(); + } + dependencyPolicy.validateMetadataSize(metadata); + + TomlParseResult result = Toml.parse(metadata); + if (result.hasErrors()) { + throw new IllegalArgumentException("Invalid PEP 723 dependency metadata: " + result.errors().get(0)); + } + if (!SUPPORTED_KEYS.containsAll(result.keySet())) { + throw new IllegalArgumentException("PEP 723 metadata contains unsupported fields"); + } + + List dependencies = new ArrayList<>(); + TomlArray dependencyArray = result.getArray("dependencies"); + if (dependencyArray != null) { + for (int index = 0; index < dependencyArray.size(); index++) { + Object dependency = dependencyArray.get(index); + if (!(dependency instanceof String dependencyString)) { + throw new IllegalArgumentException("PEP 723 dependencies must be strings"); + } + dependencies.add(dependencyString); + } + } + String requiresPython = result.getString("requires-python"); + return new PythonDependencyMetadata(dependencyPolicy.validateDependencies(dependencies), requiresPython); + } + + private String extractMetadata(String code) { + String[] lines = code.split("\\R", -1); + StringBuilder metadata = null; + boolean insideBlock = false; + boolean foundBlock = false; + for (int lineIndex = 0; lineIndex < lines.length; lineIndex++) { + String line = lines[lineIndex]; + if (BLOCK_START.equals(line.strip())) { + if (insideBlock || foundBlock) { + throw new IllegalArgumentException("Python code must contain at most one PEP 723 script block"); + } + metadata = new StringBuilder(); + insideBlock = true; + foundBlock = true; + continue; + } + if (!insideBlock) { + continue; + } + if (BLOCK_END.equals(line.strip())) { + insideBlock = false; + continue; + } + String stripped = line.stripLeading(); + if (stripped.isEmpty() && lineIndex == lines.length - 1) { + continue; + } + if (!stripped.startsWith("#")) { + throw new IllegalArgumentException("Each PEP 723 metadata line must be a Python comment"); + } + String value = stripped.substring(1); + if (value.startsWith(" ")) { + value = value.substring(1); + } + metadata.append(value).append('\n'); + } + if (insideBlock) { + throw new IllegalArgumentException("PEP 723 script block is not closed"); + } + return foundBlock ? metadata.toString() : null; + } + +} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/PythonDependencyPolicy.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/PythonDependencyPolicy.java new file mode 100644 index 000000000..619de8706 --- /dev/null +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/PythonDependencyPolicy.java @@ -0,0 +1,56 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.cloud.ai.dataagent.service.code.sandbox; + +import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; +import org.springframework.stereotype.Component; + +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.regex.Pattern; + +@Component +public class PythonDependencyPolicy { + + private static final Pattern PACKAGE_SPECIFIER = Pattern.compile( + "^[A-Za-z0-9][A-Za-z0-9._-]*(?:\\[[A-Za-z0-9._,-]+])?(?:\\s*[!<>=~]{1,2}\\s*[A-Za-z0-9.*+!_-]+(?:\\s*,\\s*[!<>=~]{1,2}\\s*[A-Za-z0-9.*+!_-]+)*)?$"); + + private final CodeExecutorProperties properties; + + public PythonDependencyPolicy(CodeExecutorProperties properties) { + this.properties = properties; + } + + public void validateMetadataSize(String metadata) { + int bytes = metadata.getBytes(StandardCharsets.UTF_8).length; + if (bytes > properties.getSandbox().getMaxMetadataBytes()) { + throw new IllegalArgumentException("Python dependency metadata exceeds the configured size limit"); + } + } + + public List validateDependencies(List dependencies) { + if (dependencies.size() > properties.getSandbox().getMaxDependencies()) { + throw new IllegalArgumentException("Too many Python dependencies"); + } + for (String dependency : dependencies) { + if (dependency == null || !PACKAGE_SPECIFIER.matcher(dependency).matches()) { + throw new IllegalArgumentException("Unsupported Python dependency specifier: " + dependency); + } + } + return List.copyOf(dependencies); + } + +} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/PythonSandboxBootstrapBuilder.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/PythonSandboxBootstrapBuilder.java new file mode 100644 index 000000000..484d12482 --- /dev/null +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/PythonSandboxBootstrapBuilder.java @@ -0,0 +1,154 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.cloud.ai.dataagent.service.code.sandbox; + +import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; +import com.alibaba.cloud.ai.dataagent.util.JsonUtil; +import com.fasterxml.jackson.core.JsonProcessingException; +import org.springframework.stereotype.Component; + +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.List; + +@Component +public class PythonSandboxBootstrapBuilder { + + static final String RESULT_MARKER = "__DATAAGENT_RESULT_V1__:"; + + private final CodeExecutorProperties properties; + + public PythonSandboxBootstrapBuilder(CodeExecutorProperties properties) { + this.properties = properties; + } + + public String buildDependencyInstaller(List dependencies) { + String dependenciesJson; + try { + dependenciesJson = JsonUtil.getObjectMapper().writeValueAsString(dependencies); + } + catch (JsonProcessingException ex) { + throw new IllegalArgumentException("Failed to serialize Python dependencies", ex); + } + String encodedDependencies = encode(dependenciesJson); + String encodedIndex = encode(properties.getSandbox().getPackageIndexUrl()); + long timeoutSeconds = properties.getSandbox().getDependencyInstallTimeout().toSeconds(); + int maxOutputCharacters = properties.getSandbox().getMaxOutputBytes(); + int maxErrorCharacters = properties.getSandbox().getMaxErrorBytes(); + return """ + import base64 + import json + import subprocess + import sys + + _dependencies = json.loads(base64.b64decode("%s").decode("utf-8")) + _index_url = base64.b64decode("%s").decode("utf-8") + _envelope = {"success": True, "stdout": "", "stderr": "", "errorType": None, "errorMessage": None} + try: + _command = [ + sys.executable, "-m", "pip", "install", + "--disable-pip-version-check", "--no-input", "--no-cache-dir", + "--target", "/tmp/dataagent-deps", "--index-url", _index_url, + *_dependencies + ] + _completed = subprocess.run( + _command, capture_output=True, text=True, timeout=%d, check=False + ) + _envelope["stdout"] = _completed.stdout[-%d:] + _envelope["stderr"] = _completed.stderr[-%d:] + if _completed.returncode != 0: + _envelope["success"] = False + _envelope["errorType"] = "DependencyInstallError" + _envelope["errorMessage"] = "pip exited with status " + str(_completed.returncode) + except subprocess.TimeoutExpired as _error: + _envelope["success"] = False + _envelope["errorType"] = "DependencyInstallTimeout" + _envelope["errorMessage"] = str(_error) + except BaseException as _error: + _envelope["success"] = False + _envelope["errorType"] = type(_error).__name__ + _envelope["errorMessage"] = str(_error) + _payload = base64.b64encode( + json.dumps(_envelope, ensure_ascii=False).encode("utf-8") + ).decode("ascii") + print("%s" + _payload) + """.formatted(encodedDependencies, encodedIndex, timeoutSeconds, maxOutputCharacters, + maxErrorCharacters, RESULT_MARKER); + } + + public String buildCodeExecution(String code, String input) { + String encodedCode = encode(code); + String encodedInput = encode(input); + long timeoutSeconds = properties.getCodeTimeout().toSeconds(); + int maxOutputCharacters = properties.getSandbox().getMaxOutputBytes(); + int maxErrorCharacters = properties.getSandbox().getMaxErrorBytes(); + return """ + import base64 + import json + import os + import subprocess + import sys + + _code = base64.b64decode("%s").decode("utf-8") + _input = base64.b64decode("%s").decode("utf-8") + _envelope = {"success": True, "stdout": "", "stderr": "", "errorType": None, "errorMessage": None} + + def _dataagent_as_text(_value): + if isinstance(_value, bytes): + return _value.decode("utf-8", errors="replace") + return _value or "" + + try: + _environment = dict(os.environ) + _environment["PYTHONPATH"] = "/tmp/dataagent-deps" + _completed = subprocess.run( + [sys.executable, "-c", _code], + input=_input, + capture_output=True, + text=True, + timeout=%d, + check=False, + env=_environment + ) + _envelope["stdout"] = _completed.stdout[-%d:] + _envelope["stderr"] = _completed.stderr[-%d:] + if _completed.returncode != 0: + _envelope["success"] = False + _envelope["errorType"] = "PythonProcessError" + _envelope["errorMessage"] = "Python exited with status " + str(_completed.returncode) + except subprocess.TimeoutExpired as _error: + _envelope["success"] = False + _envelope["errorType"] = "PythonTimeout" + _envelope["errorMessage"] = "Python code execution exceeded %d seconds" + _envelope["stdout"] = _dataagent_as_text(_error.stdout)[-%d:] + _envelope["stderr"] = _dataagent_as_text(_error.stderr)[-%d:] + except BaseException as _error: + _envelope["success"] = False + _envelope["errorType"] = type(_error).__name__ + _envelope["errorMessage"] = str(_error) + _payload = base64.b64encode( + json.dumps(_envelope, ensure_ascii=False).encode("utf-8") + ).decode("ascii") + print("%s" + _payload) + """.formatted(encodedCode, encodedInput, timeoutSeconds, maxOutputCharacters, maxErrorCharacters, + timeoutSeconds, maxOutputCharacters, maxErrorCharacters, RESULT_MARKER); + } + + private String encode(String value) { + return Base64.getEncoder().encodeToString(value.getBytes(StandardCharsets.UTF_8)); + } + +} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SaaSandboxPythonCodeExecutorService.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SaaSandboxPythonCodeExecutorService.java new file mode 100644 index 000000000..83f77aecd --- /dev/null +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SaaSandboxPythonCodeExecutorService.java @@ -0,0 +1,103 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.cloud.ai.dataagent.service.code.sandbox; + +import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; +import com.alibaba.cloud.ai.dataagent.service.code.PythonCodeExecutorService; +import jakarta.annotation.PreDestroy; +import org.springframework.stereotype.Service; + +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +@Service +public class SaaSandboxPythonCodeExecutorService implements PythonCodeExecutorService { + + private final CodeExecutorProperties properties; + + private final SaaSandboxTaskRunner taskRunner; + + private final ThreadPoolExecutor executor; + + public SaaSandboxPythonCodeExecutorService(CodeExecutorProperties properties, SaaSandboxTaskRunner taskRunner) { + this.properties = properties; + this.taskRunner = taskRunner; + int concurrency = properties.getSandbox().getMaxConcurrency(); + this.executor = new ThreadPoolExecutor(concurrency, concurrency, 0L, TimeUnit.MILLISECONDS, + new ArrayBlockingQueue<>(properties.getSandbox().getQueueCapacity()), + new ThreadPoolExecutor.AbortPolicy()); + } + + @Override + public TaskResponse runTask(TaskRequest request) { + try { + validateSize("Python code", request.code(), properties.getSandbox().getMaxCodeBytes()); + validateSize("Python input", request.input(), properties.getSandbox().getMaxInputBytes()); + } + catch (IllegalArgumentException ex) { + return TaskResponse.exception(ex.getMessage()); + } + + Future future; + try { + future = executor.submit(() -> taskRunner.run(request)); + } + catch (RuntimeException ex) { + return TaskResponse.exception("Python sandbox capacity is exhausted"); + } + + Duration taskTimeout = properties.getSandbox() + .getDependencyInstallTimeout() + .plus(properties.getCodeTimeout()) + .plusSeconds(30); + try { + return future.get(taskTimeout.toMillis(), TimeUnit.MILLISECONDS); + } + catch (TimeoutException ex) { + future.cancel(true); + return TaskResponse.exception("Python sandbox task timed out"); + } + catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + return TaskResponse.exception("Python sandbox task was interrupted"); + } + catch (ExecutionException ex) { + Throwable cause = ex.getCause(); + return TaskResponse.exception(cause == null ? ex.getMessage() : cause.getMessage()); + } + } + + private void validateSize(String label, String value, int maximumBytes) { + if (value == null) { + throw new IllegalArgumentException(label + " must not be null"); + } + if (value.getBytes(StandardCharsets.UTF_8).length > maximumBytes) { + throw new IllegalArgumentException(label + " exceeds the configured size limit"); + } + } + + @PreDestroy + public void close() { + executor.shutdownNow(); + } + +} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SaaSandboxRuntime.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SaaSandboxRuntime.java new file mode 100644 index 000000000..21d08c9d1 --- /dev/null +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SaaSandboxRuntime.java @@ -0,0 +1,111 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.cloud.ai.dataagent.service.code.sandbox; + +import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; +import io.agentscope.runtime.sandbox.box.BaseSandbox; +import io.agentscope.runtime.sandbox.manager.ManagerConfig; +import io.agentscope.runtime.sandbox.manager.SandboxService; +import io.agentscope.runtime.sandbox.manager.client.container.docker.DockerClientStarter; +import io.agentscope.runtime.sandbox.manager.model.sandbox.SandboxConfig; +import io.agentscope.runtime.sandbox.manager.registry.SandboxRegistryService; +import jakarta.annotation.PreDestroy; +import org.springframework.stereotype.Component; + +import java.util.Map; + +@Component +public class SaaSandboxRuntime { + + private final CodeExecutorProperties properties; + + private volatile SandboxService sandboxService; + + public SaaSandboxRuntime(CodeExecutorProperties properties) { + this.properties = properties; + } + + public SandboxService getOrStart() { + SandboxService current = sandboxService; + if (current != null) { + return current; + } + synchronized (this) { + if (sandboxService == null) { + sandboxService = startSandboxService(); + } + return sandboxService; + } + } + + private SandboxService startSandboxService() { + CodeExecutorProperties.Sandbox sandboxProperties = properties.getSandbox(); + Map runtimeConfig = Map.of("mem_limit", properties.getLimitMemory() + "m", "nano_cpus", + properties.getCpuCore() * 1_000_000_000L, "max_connections", sandboxProperties.getMaxConnections(), + "privileged", false); + SandboxConfig sandboxConfig = new SandboxConfig.Builder().imageName(sandboxProperties.getImageName()) + .sandboxType("base") + .securityLevel("high") + .timeout(sandboxTimeoutSeconds()) + .description("DataAgent generated Python sandbox") + .runtimeConfig(runtimeConfig) + .build(); + SandboxRegistryService.register(BaseSandbox.class, sandboxConfig); + + DockerClientStarter dockerClientStarter = createDockerClientStarter(sandboxProperties.getDockerHost()); + ManagerConfig managerConfig = ManagerConfig.builder() + .containerPrefixKey(sandboxProperties.getContainerPrefix()) + .clientStarter(dockerClientStarter) + .build(); + SandboxService service = new SandboxService(managerConfig); + service.start(); + return service; + } + + private DockerClientStarter createDockerClientStarter(String dockerHost) { + if (dockerHost != null && dockerHost.startsWith("tcp://")) { + String address = dockerHost.substring("tcp://".length()); + int separator = address.lastIndexOf(':'); + if (separator > 0) { + return DockerClientStarter.builder() + .host(address.substring(0, separator)) + .port(Integer.parseInt(address.substring(separator + 1))) + .build(); + } + } + // AgentScope 1.0.2 falls back to docker-java's standard local socket + // discovery when its default TCP endpoint is unavailable. + return DockerClientStarter.builder().build(); + } + + int sandboxTimeoutSeconds() { + return Math.toIntExact(properties.getSandbox() + .getDependencyInstallTimeout() + .plus(properties.getCodeTimeout()) + .plusSeconds(30) + .toSeconds()); + } + + @PreDestroy + public synchronized void close() { + if (sandboxService != null) { + sandboxService.cleanupAllSandboxes(); + sandboxService.close(); + sandboxService = null; + } + } + +} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SaaSandboxTaskRunner.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SaaSandboxTaskRunner.java new file mode 100644 index 000000000..b5f4ca5a4 --- /dev/null +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SaaSandboxTaskRunner.java @@ -0,0 +1,82 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.cloud.ai.dataagent.service.code.sandbox; + +import com.alibaba.cloud.ai.dataagent.service.code.PythonCodeExecutorService; +import com.alibaba.cloud.ai.sandbox.tools.base.SaaBasePythonRunner; +import io.agentscope.runtime.sandbox.box.BaseSandbox; +import org.springframework.stereotype.Component; + +import java.util.UUID; + +@Component +public class SaaSandboxTaskRunner { + + private final SaaSandboxRuntime sandboxRuntime; + + private final PythonSandboxBootstrapBuilder bootstrapBuilder; + + private final SandboxExecutionResultParser resultParser; + + public SaaSandboxTaskRunner(SaaSandboxRuntime sandboxRuntime, PythonSandboxBootstrapBuilder bootstrapBuilder, + SandboxExecutionResultParser resultParser) { + this.sandboxRuntime = sandboxRuntime; + this.bootstrapBuilder = bootstrapBuilder; + this.resultParser = resultParser; + } + + public PythonCodeExecutorService.TaskResponse run(PythonCodeExecutorService.TaskRequest request) { + try (BaseSandbox sandbox = new BaseSandbox(sandboxRuntime.getOrStart(), "dataagent", + UUID.randomUUID().toString())) { + SaaBasePythonRunner pythonRunner = new SaaBasePythonRunner(); + pythonRunner.setSandbox(sandbox); + + if (!request.dependencies().isEmpty()) { + SandboxExecutionResult installResult = run(pythonRunner, + bootstrapBuilder.buildDependencyInstaller(request.dependencies())); + if (!installResult.success()) { + return PythonCodeExecutorService.TaskResponse.failure(installResult.stdout(), + formatError(installResult)); + } + } + + SandboxExecutionResult executionResult = run(pythonRunner, + bootstrapBuilder.buildCodeExecution(request.code(), request.input())); + if (executionResult.success()) { + return PythonCodeExecutorService.TaskResponse.success(executionResult.stdout()); + } + return PythonCodeExecutorService.TaskResponse.failure(executionResult.stdout(), + formatError(executionResult)); + } + catch (Exception ex) { + return PythonCodeExecutorService.TaskResponse.exception(ex.getMessage()); + } + } + + private SandboxExecutionResult run(SaaBasePythonRunner pythonRunner, String code) { + SaaBasePythonRunner.RunPythonToolResponse response = pythonRunner + .apply(new SaaBasePythonRunner.RunPythonToolRequest(code), null); + return resultParser.parse(response.output().result()); + } + + private String formatError(SandboxExecutionResult result) { + String errorType = result.errorType() == null ? "PythonError" : result.errorType(); + String errorMessage = result.errorMessage() == null ? "" : ": " + result.errorMessage(); + String stderr = result.stderr() == null ? "" : "\n" + result.stderr(); + return errorType + errorMessage + stderr; + } + +} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/enums/CodePoolExecutorEnum.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SandboxExecutionResult.java similarity index 73% rename from data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/enums/CodePoolExecutorEnum.java rename to data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SandboxExecutionResult.java index c324ba244..c970a2eab 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/enums/CodePoolExecutorEnum.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SandboxExecutionResult.java @@ -13,16 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.alibaba.cloud.ai.dataagent.enums; +package com.alibaba.cloud.ai.dataagent.service.code.sandbox; -/** - * 运行Python任务的容器池(实现枚举) - * - * @author vlsmb - * @since 2025/7/28 - */ -public enum CodePoolExecutorEnum { - - DOCKER, CONTAINERD, KATA, AI_SIMULATION, LOCAL; +public record SandboxExecutionResult(boolean success, String stdout, String stderr, String errorType, + String errorMessage) { } diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SandboxExecutionResultParser.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SandboxExecutionResultParser.java new file mode 100644 index 000000000..17b617378 --- /dev/null +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SandboxExecutionResultParser.java @@ -0,0 +1,74 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.cloud.ai.dataagent.service.code.sandbox; + +import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; +import com.alibaba.cloud.ai.dataagent.util.JsonUtil; +import com.fasterxml.jackson.core.JsonProcessingException; +import org.springframework.stereotype.Component; + +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +@Component +public class SandboxExecutionResultParser { + + private static final Pattern ENCODED_RESULT = Pattern + .compile(Pattern.quote(PythonSandboxBootstrapBuilder.RESULT_MARKER) + "([A-Za-z0-9+/=]+)"); + + private final CodeExecutorProperties properties; + + public SandboxExecutionResultParser(CodeExecutorProperties properties) { + this.properties = properties; + } + + public SandboxExecutionResult parse(String rawResult) { + if (rawResult == null) { + throw new IllegalArgumentException("Sandbox returned an empty response"); + } + Matcher matcher = ENCODED_RESULT.matcher(rawResult); + String encodedResult = null; + while (matcher.find()) { + encodedResult = matcher.group(1); + } + if (encodedResult == null) { + throw new IllegalArgumentException("Sandbox response does not contain the DataAgent result marker"); + } + + byte[] decoded; + try { + decoded = Base64.getDecoder().decode(encodedResult); + } + catch (IllegalArgumentException ex) { + throw new IllegalArgumentException("Sandbox returned an invalid encoded result", ex); + } + int maximumBytes = properties.getSandbox().getMaxOutputBytes() + properties.getSandbox().getMaxErrorBytes(); + if (decoded.length > maximumBytes + 4096) { + throw new IllegalArgumentException("Sandbox result exceeds the configured size limit"); + } + + try { + return JsonUtil.getObjectMapper() + .readValue(new String(decoded, StandardCharsets.UTF_8), SandboxExecutionResult.class); + } + catch (JsonProcessingException ex) { + throw new IllegalArgumentException("Sandbox returned an invalid result envelope", ex); + } + } + +} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/workflow/node/PythonExecuteNode.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/workflow/node/PythonExecuteNode.java index 5cc1531d5..94184358e 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/workflow/node/PythonExecuteNode.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/workflow/node/PythonExecuteNode.java @@ -16,13 +16,15 @@ package com.alibaba.cloud.ai.dataagent.workflow.node; import com.alibaba.cloud.ai.dataagent.enums.TextType; +import com.alibaba.cloud.ai.dataagent.service.code.PythonCodeExecutorService; +import com.alibaba.cloud.ai.dataagent.service.code.sandbox.PythonDependencyMetadata; +import com.alibaba.cloud.ai.dataagent.service.code.sandbox.PythonDependencyMetadataParser; import com.alibaba.cloud.ai.dataagent.util.JsonParseUtil; import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; import com.alibaba.cloud.ai.graph.GraphResponse; import com.alibaba.cloud.ai.graph.OverAllState; import com.alibaba.cloud.ai.graph.action.NodeAction; import com.alibaba.cloud.ai.graph.streaming.StreamingOutput; -import com.alibaba.cloud.ai.dataagent.service.code.CodePoolExecutorService; import com.alibaba.cloud.ai.dataagent.util.ChatResponseUtil; import com.alibaba.cloud.ai.dataagent.util.FluxUtil; import com.alibaba.cloud.ai.dataagent.util.JsonUtil; @@ -49,7 +51,9 @@ @Component public class PythonExecuteNode implements NodeAction { - private final CodePoolExecutorService codePoolExecutor; + private final PythonCodeExecutorService pythonCodeExecutor; + + private final PythonDependencyMetadataParser dependencyMetadataParser; private final ObjectMapper objectMapper; @@ -57,9 +61,11 @@ public class PythonExecuteNode implements NodeAction { private final CodeExecutorProperties codeExecutorProperties; - public PythonExecuteNode(CodePoolExecutorService codePoolExecutor, JsonParseUtil jsonParseUtil, + public PythonExecuteNode(PythonCodeExecutorService pythonCodeExecutor, + PythonDependencyMetadataParser dependencyMetadataParser, JsonParseUtil jsonParseUtil, CodeExecutorProperties codeExecutorProperties) { - this.codePoolExecutor = codePoolExecutor; + this.pythonCodeExecutor = pythonCodeExecutor; + this.dependencyMetadataParser = dependencyMetadataParser; this.objectMapper = JsonUtil.getObjectMapper(); this.jsonParseUtil = jsonParseUtil; this.codeExecutorProperties = codeExecutorProperties; @@ -77,11 +83,12 @@ public Map apply(OverAllState state) throws Exception { // 检查重试次数 int triesCount = StateUtil.getObjectValue(state, PYTHON_TRIES_COUNT, Integer.class, 0); - CodePoolExecutorService.TaskRequest taskRequest = new CodePoolExecutorService.TaskRequest(pythonCode, - objectMapper.writeValueAsString(sqlResults), null); + PythonDependencyMetadata metadata = dependencyMetadataParser.parse(pythonCode); + PythonCodeExecutorService.TaskRequest taskRequest = new PythonCodeExecutorService.TaskRequest(pythonCode, + objectMapper.writeValueAsString(sqlResults), metadata.dependencies()); // Run Python code - CodePoolExecutorService.TaskResponse taskResponse = this.codePoolExecutor.runTask(taskRequest); + PythonCodeExecutorService.TaskResponse taskResponse = this.pythonCodeExecutor.runTask(taskRequest); if (!taskResponse.isSuccess()) { String errorMsg = "Python Execute Failed!\nStdOut: " + taskResponse.stdOut() + "\nStdErr: " + taskResponse.stdErr() + "\nExceptionMsg: " + taskResponse.exceptionMsg(); diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/workflow/node/PythonGenerateNode.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/workflow/node/PythonGenerateNode.java index 70f0ecb4c..d120ba763 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/workflow/node/PythonGenerateNode.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/workflow/node/PythonGenerateNode.java @@ -103,7 +103,7 @@ public Map apply(OverAllState state) throws Exception { // Load Python code generation template String systemPrompt = PromptConstant.getPythonGeneratorPromptTemplate() .render(Map.of("python_memory", codeExecutorProperties.getLimitMemory().toString(), "python_timeout", - codeExecutorProperties.getCodeTimeout(), "database_schema", + codeExecutorProperties.getCodeTimeout().toSeconds() + "s", "database_schema", objectMapper.writeValueAsString(schemaDTO), "sample_input", objectMapper.writeValueAsString(sqlResults.stream().limit(SAMPLE_DATA_NUMBER).toList()), "plan_description", objectMapper.writeValueAsString(toolParameters))); diff --git a/data-agent-management/src/main/resources/application-h2.yml b/data-agent-management/src/main/resources/application-h2.yml index 3bf8b2c5e..f76aab97c 100644 --- a/data-agent-management/src/main/resources/application-h2.yml +++ b/data-agent-management/src/main/resources/application-h2.yml @@ -3,6 +3,9 @@ server: # 存储 DataAgent 业务数据的数据库,并非Agent分析数据的来源 spring: + codec: + # Python workflow timelines can contain generated code, result data and the final report. + max-in-memory-size: 10MB datasource: url: jdbc:h2:mem:nl2sql_database;DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=true;MODE=MySQL;DB_CLOSE_ON_EXIT=FALSE username: root @@ -38,10 +41,18 @@ spring: default-topk-limit: 8 default-similarity-threshold: 0.4 code-executor: - # 运行Python代码的环境(生产环境建议使用docker,不建议使用local) - code-pool-executor: local - # Python执行的最大重试次数 python-max-tries-count: 5 + code-timeout: 60s + limit-memory: 500 + cpu-core: 1 + sandbox: + docker-host: ${DATAAGENT_SANDBOX_DOCKER_HOST:unix:///var/run/docker.sock} + image-name: ${DATAAGENT_SANDBOX_IMAGE:agentscope-registry.ap-southeast-1.cr.aliyuncs.com/agentscope/runtime-sandbox-base:latest} + max-concurrency: 4 + queue-capacity: 10 + max-connections: 4096 + package-index-url: ${DATAAGENT_PYPI_INDEX_URL:https://pypi.org/simple} + dependency-install-timeout: 3m file: type: local path-prefix: data-agent diff --git a/data-agent-management/src/main/resources/application.yml b/data-agent-management/src/main/resources/application.yml index a8beeefea..e1be57528 100644 --- a/data-agent-management/src/main/resources/application.yml +++ b/data-agent-management/src/main/resources/application.yml @@ -9,6 +9,9 @@ springdoc: # 存储 DataAgent 业务数据的数据库,并非Agent分析数据的来源 spring: + codec: + # Python workflow timelines can contain generated code, result data and the final report. + max-in-memory-size: 10MB datasource: url: ${DATA_AGENT_DATASOURCE_URL:jdbc:mysql://127.0.0.1:3306/saa_data_agent?useUnicode=true&characterEncoding=utf-8&zeroDateTimeBehavior=convertToNull&transformedBitIsBoolean=true&allowMultiQueries=true&allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=Asia/Shanghai} username: ${DATA_AGENT_DATASOURCE_USERNAME:root} @@ -38,10 +41,18 @@ spring: default-topk-limit: 8 default-similarity-threshold: 0.4 code-executor: - # 运行Python代码的环境(生产环境建议使用docker,不建议使用local) - code-pool-executor: local - # Python执行的最大重试次数 python-max-tries-count: 5 + code-timeout: 60s + limit-memory: 500 + cpu-core: 1 + sandbox: + docker-host: ${DATAAGENT_SANDBOX_DOCKER_HOST:unix:///var/run/docker.sock} + image-name: ${DATAAGENT_SANDBOX_IMAGE:agentscope-registry.ap-southeast-1.cr.aliyuncs.com/agentscope/runtime-sandbox-base:latest} + max-concurrency: 4 + queue-capacity: 10 + max-connections: 4096 + package-index-url: ${DATAAGENT_PYPI_INDEX_URL:https://pypi.org/simple} + dependency-install-timeout: 3m file: type: local path-prefix: data-agent diff --git a/data-agent-management/src/main/resources/prompts/python-generator.txt b/data-agent-management/src/main/resources/prompts/python-generator.txt index c000c8a42..ee54b11e1 100644 --- a/data-agent-management/src/main/resources/prompts/python-generator.txt +++ b/data-agent-management/src/main/resources/prompts/python-generator.txt @@ -16,7 +16,10 @@ traceback.print_exc(file=sys.stderr) sys.exit(1) ``` -5. **依赖限制**:所有使用的库必须是`continuumio/anaconda3`默认安装的库,如`pandas`, `numpy`, `json`, `sys`等。 +5. **动态依赖**: + - 所有非标准库依赖必须在脚本顶部使用 [PEP 723] 的 `# /// script` 元数据声明,执行器会在隔离沙盒中动态安装。 + - `dependencies` 只能填写 PyPI 包名及版本约束,例如 `"pandas>=2,<3"`;禁止 URL、VCS、文件路径、环境标记和 pip 参数。 + - 每一个非标准库 `import` 都必须有对应的依赖声明;只使用标准库时可以省略元数据块。 6. **动态处理**:禁止硬编码列名/值,所有逻辑基于输入数据动态构建。 7. **安全限制**:禁止以下操作: - 任何文件/网络操作(open/requests等)。 @@ -32,6 +35,10 @@ 以下是生成代码的模板,请根据具体需求填充逻辑: ```python +# /// script +# dependencies = ["pandas>=2,<3"] +# /// + import sys import json import traceback @@ -94,6 +101,10 @@ except Exception: 假设用户需求是“统计每个渠道的线索数量和转化率”,生成的代码可能如下: ```python +# /// script +# dependencies = ["pandas>=2,<3"] +# /// + import sys import json import traceback diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/config/DataAgentConfigurationTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/config/DataAgentConfigurationTest.java index da649bc2a..205ca61a1 100644 --- a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/config/DataAgentConfigurationTest.java +++ b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/config/DataAgentConfigurationTest.java @@ -15,7 +15,6 @@ */ package com.alibaba.cloud.ai.dataagent.config; -import com.alibaba.cloud.ai.dataagent.service.code.CodePoolExecutorServiceFactory; import com.alibaba.cloud.ai.dataagent.service.file.FileStorageServiceFactory; import com.alibaba.cloud.ai.graph.CompileConfig; import com.alibaba.cloud.ai.graph.RunnableConfig; @@ -34,9 +33,8 @@ class DataAgentConfigurationTest { @Test - void runtimeServiceFactories_arePlainSelectorsInsteadOfSpringFactoryBeans() { + void fileStorageServiceFactory_isPlainSelectorInsteadOfSpringFactoryBean() { assertThat(FactoryBean.class.isAssignableFrom(FileStorageServiceFactory.class)).isFalse(); - assertThat(FactoryBean.class.isAssignableFrom(CodePoolExecutorServiceFactory.class)).isFalse(); } @Test diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/config/VectorStoreConfigurationTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/config/VectorStoreConfigurationTest.java index 79c1ab129..5c14652b3 100644 --- a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/config/VectorStoreConfigurationTest.java +++ b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/config/VectorStoreConfigurationTest.java @@ -15,12 +15,9 @@ */ package com.alibaba.cloud.ai.dataagent.config; -import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; import com.alibaba.cloud.ai.dataagent.properties.FileStorageProperties; import com.alibaba.cloud.ai.dataagent.properties.OssStorageProperties; import com.alibaba.cloud.ai.dataagent.service.aimodelconfig.AiModelRegistry; -import com.alibaba.cloud.ai.dataagent.service.code.docker.DockerExecutorFactory; -import com.alibaba.cloud.ai.dataagent.service.llm.LlmService; import org.junit.jupiter.api.Test; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.context.properties.bind.Binder; @@ -41,7 +38,7 @@ class VectorStoreConfigurationTest { @Test void vectorStoreProfiles_useExplicitSpringAiStarterTypes() throws Exception { - assertThat(property("application.yml", "spring.ai.vectorstore.type")).isEqualTo("simple"); + assertThat(property("application.yml", "spring.ai.vectorstore.type")).isEqualTo("milvus"); assertThat(property("application-milvus.yml", "spring.ai.vectorstore.type")).isEqualTo("milvus"); assertThat(property("application-elasticsearch.yml", "spring.ai.vectorstore.type")).isEqualTo("elasticsearch"); } @@ -67,8 +64,6 @@ void milvusProfile_bindsConnectionPropertiesUsedBySpringAi() throws Exception { void replaceableRuntimeServices_areConditionalOnMissingBean() throws Exception { assertConditional("llmService", AiModelRegistry.class); assertConditional("fileStorageService", FileStorageProperties.class, OssStorageProperties.class); - assertConditional("codePoolExecutorService", CodeExecutorProperties.class, LlmService.class, - DockerExecutorFactory.class); } private Object property(String resource, String name) throws Exception { diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/enums/CodePoolExecutorEnumTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/enums/CodePoolExecutorEnumTest.java deleted file mode 100644 index 06c949b31..000000000 --- a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/enums/CodePoolExecutorEnumTest.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.enums; - -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.*; - -class CodePoolExecutorEnumTest { - - @Test - void testValues() { - assertEquals(5, CodePoolExecutorEnum.values().length); - } - - @Test - void testValueOf() { - assertEquals(CodePoolExecutorEnum.DOCKER, CodePoolExecutorEnum.valueOf("DOCKER")); - assertEquals(CodePoolExecutorEnum.CONTAINERD, CodePoolExecutorEnum.valueOf("CONTAINERD")); - assertEquals(CodePoolExecutorEnum.KATA, CodePoolExecutorEnum.valueOf("KATA")); - assertEquals(CodePoolExecutorEnum.AI_SIMULATION, CodePoolExecutorEnum.valueOf("AI_SIMULATION")); - assertEquals(CodePoolExecutorEnum.LOCAL, CodePoolExecutorEnum.valueOf("LOCAL")); - } - - @Test - void testValueOf_invalid() { - assertThrows(IllegalArgumentException.class, () -> CodePoolExecutorEnum.valueOf("UNKNOWN")); - } - -} diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/integration/PythonWorkflowIntegrationTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/integration/PythonWorkflowIntegrationTest.java index 0d7408373..27cc00c74 100644 --- a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/integration/PythonWorkflowIntegrationTest.java +++ b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/integration/PythonWorkflowIntegrationTest.java @@ -24,6 +24,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.time.Duration; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -35,7 +36,9 @@ import com.alibaba.cloud.ai.dataagent.common.TestFixtures; import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; -import com.alibaba.cloud.ai.dataagent.service.code.CodePoolExecutorService; +import com.alibaba.cloud.ai.dataagent.service.code.PythonCodeExecutorService; +import com.alibaba.cloud.ai.dataagent.service.code.sandbox.PythonDependencyMetadata; +import com.alibaba.cloud.ai.dataagent.service.code.sandbox.PythonDependencyMetadataParser; import com.alibaba.cloud.ai.dataagent.service.llm.LlmService; import com.alibaba.cloud.ai.dataagent.util.ChatResponseUtil; import com.alibaba.cloud.ai.dataagent.util.JsonParseUtil; @@ -55,7 +58,10 @@ class PythonWorkflowIntegrationTest { private LlmService llmService; @Mock - private CodePoolExecutorService codePoolExecutor; + private PythonCodeExecutorService pythonCodeExecutor; + + @Mock + private PythonDependencyMetadataParser dependencyMetadataParser; @Mock private JsonParseUtil jsonParseUtil; @@ -72,11 +78,13 @@ class PythonWorkflowIntegrationTest { @BeforeEach void setUp() { when(codeExecutorProperties.getLimitMemory()).thenReturn(500L); - when(codeExecutorProperties.getCodeTimeout()).thenReturn("60s"); + when(codeExecutorProperties.getCodeTimeout()).thenReturn(Duration.ofSeconds(60)); when(codeExecutorProperties.getPythonMaxTriesCount()).thenReturn(5); + when(dependencyMetadataParser.parse(anyString())).thenReturn(PythonDependencyMetadata.empty()); pythonGenerateNode = new PythonGenerateNode(codeExecutorProperties, llmService); - pythonExecuteNode = new PythonExecuteNode(codePoolExecutor, jsonParseUtil, codeExecutorProperties); + pythonExecuteNode = new PythonExecuteNode(pythonCodeExecutor, dependencyMetadataParser, jsonParseUtil, + codeExecutorProperties); pythonAnalyzeNode = new PythonAnalyzeNode(llmService); } @@ -125,7 +133,8 @@ void endToEnd_pythonAnalysis_generatesExecutesAnalyzes_returnsResults() throws E state.updateState(Map.of(PYTHON_GENERATE_NODE_OUTPUT, generatedCode, PYTHON_TRIES_COUNT, 1)); String executionOutput = "{\"count\": 3, \"mean\": 200.0}"; - when(codePoolExecutor.runTask(any())).thenReturn(CodePoolExecutorService.TaskResponse.success(executionOutput)); + when(pythonCodeExecutor.runTask(any())) + .thenReturn(PythonCodeExecutorService.TaskResponse.success(executionOutput)); when(jsonParseUtil.tryConvertToObject(anyString(), any(Class.class))).thenReturn(null); Map executeResult = pythonExecuteNode.apply(state); @@ -143,7 +152,7 @@ void endToEnd_pythonAnalysis_generatesExecutesAnalyzes_returnsResults() throws E assertTrue(analyzeResult.containsKey(PYTHON_ANALYSIS_NODE_OUTPUT)); verify(llmService).call(anyString(), anyString()); - verify(codePoolExecutor).runTask(any()); + verify(pythonCodeExecutor).runTask(any()); verify(llmService).callSystem(anyString()); } diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/CodePoolExecutorServiceFactoryTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/CodePoolExecutorServiceFactoryTest.java deleted file mode 100644 index 604c4f4a4..000000000 --- a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/CodePoolExecutorServiceFactoryTest.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright 2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code; - -import com.alibaba.cloud.ai.dataagent.enums.CodePoolExecutorEnum; -import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; -import com.alibaba.cloud.ai.dataagent.service.code.docker.DockerExecutorFactory; -import com.alibaba.cloud.ai.dataagent.service.code.impls.AiSimulationCodeExecutorService; -import com.alibaba.cloud.ai.dataagent.service.code.impls.DockerCodePoolExecutorService; -import com.alibaba.cloud.ai.dataagent.service.llm.LlmService; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.Mockito.when; - -@ExtendWith(MockitoExtension.class) -class CodePoolExecutorServiceFactoryTest { - - @Mock - private CodeExecutorProperties properties; - - @Mock - private LlmService llmService; - - @Mock - private DockerExecutorFactory dockerExecutorFactory; - - @Mock - private DockerCodePoolExecutorService dockerExecutor; - - private CodePoolExecutorServiceFactory factory; - - @BeforeEach - void setUp() { - factory = new CodePoolExecutorServiceFactory(properties, llmService, dockerExecutorFactory); - } - - @Test - void testGetObject_docker() { - when(properties.getCodePoolExecutor()).thenReturn(CodePoolExecutorEnum.DOCKER); - when(dockerExecutorFactory.create(properties)).thenReturn(dockerExecutor); - - assertSame(dockerExecutor, factory.getObject()); - } - - @Test - void testGetObject_aiSimulation() { - when(properties.getCodePoolExecutor()).thenReturn(CodePoolExecutorEnum.AI_SIMULATION); - - CodePoolExecutorService result = factory.getObject(); - assertInstanceOf(AiSimulationCodeExecutorService.class, result); - } - - @Test - void testGetObject_unsupported() { - when(properties.getCodePoolExecutor()).thenReturn(CodePoolExecutorEnum.CONTAINERD); - - assertThrows(IllegalStateException.class, () -> factory.getObject()); - } - - @Test - void testGetObjectType() { - assertEquals(CodePoolExecutorService.class, factory.getObjectType()); - } - -} diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/CodePoolExecutorServiceTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/CodePoolExecutorServiceTest.java deleted file mode 100644 index 891256962..000000000 --- a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/CodePoolExecutorServiceTest.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright 2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code; - -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.*; - -class CodePoolExecutorServiceTest { - - @Test - void taskResponse_success_hasCorrectFields() { - CodePoolExecutorService.TaskResponse response = CodePoolExecutorService.TaskResponse.success("output"); - - assertTrue(response.isSuccess()); - assertFalse(response.executionSuccessButResultFailed()); - assertEquals("output", response.stdOut()); - assertNull(response.stdErr()); - assertNull(response.exceptionMsg()); - } - - @Test - void taskResponse_failure_hasCorrectFields() { - CodePoolExecutorService.TaskResponse response = CodePoolExecutorService.TaskResponse.failure("stdout", - "stderr msg"); - - assertFalse(response.isSuccess()); - assertTrue(response.executionSuccessButResultFailed()); - assertEquals("stdout", response.stdOut()); - assertEquals("stderr msg", response.stdErr()); - assertTrue(response.exceptionMsg().contains("stderr msg")); - } - - @Test - void taskResponse_exception_hasCorrectFields() { - CodePoolExecutorService.TaskResponse response = CodePoolExecutorService.TaskResponse - .exception("connection timeout"); - - assertFalse(response.isSuccess()); - assertFalse(response.executionSuccessButResultFailed()); - assertNull(response.stdOut()); - assertNull(response.stdErr()); - assertTrue(response.exceptionMsg().contains("connection timeout")); - } - - @Test - void taskResponse_toString_containsFields() { - CodePoolExecutorService.TaskResponse response = CodePoolExecutorService.TaskResponse.success("hello"); - - String str = response.toString(); - assertTrue(str.contains("isSuccess=true")); - assertTrue(str.contains("hello")); - } - - @Test - void taskRequest_fieldsAccessible() { - CodePoolExecutorService.TaskRequest request = new CodePoolExecutorService.TaskRequest("print('hi')", - "{\"key\": 1}", "analyze data"); - - assertEquals("print('hi')", request.code()); - assertEquals("{\"key\": 1}", request.input()); - assertEquals("analyze data", request.requirement()); - } - - @Test - void taskResponse_success_nullStdErr() { - CodePoolExecutorService.TaskResponse response = CodePoolExecutorService.TaskResponse.success(null); - - assertTrue(response.isSuccess()); - assertNull(response.stdOut()); - } - - @Test - void stateEnum_values() { - assertEquals(3, CodePoolExecutorService.State.values().length); - assertNotNull(CodePoolExecutorService.State.READY); - assertNotNull(CodePoolExecutorService.State.RUNNING); - assertNotNull(CodePoolExecutorService.State.REMOVING); - } - -} diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/PythonCodeExecutorServiceTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/PythonCodeExecutorServiceTest.java new file mode 100644 index 000000000..39c14a676 --- /dev/null +++ b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/PythonCodeExecutorServiceTest.java @@ -0,0 +1,48 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.cloud.ai.dataagent.service.code; + +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +class PythonCodeExecutorServiceTest { + + @Test + void taskRequest_copiesDependenciesAndUsesEmptyListForNull() { + PythonCodeExecutorService.TaskRequest empty = new PythonCodeExecutorService.TaskRequest("print(1)", "[]", null); + PythonCodeExecutorService.TaskRequest populated = new PythonCodeExecutorService.TaskRequest("print(1)", "[]", + List.of("pandas>=2,<3")); + + assertThat(empty.dependencies()).isEmpty(); + assertThat(populated.dependencies()).containsExactly("pandas>=2,<3"); + } + + @Test + void taskResponse_distinguishesPythonFailureFromInfrastructureFailure() { + PythonCodeExecutorService.TaskResponse pythonFailure = PythonCodeExecutorService.TaskResponse.failure("partial", + "ValueError"); + PythonCodeExecutorService.TaskResponse infrastructureFailure = PythonCodeExecutorService.TaskResponse + .exception("Docker unavailable"); + + assertThat(pythonFailure.executionSuccessButResultFailed()).isTrue(); + assertThat(infrastructureFailure.executionSuccessButResultFailed()).isFalse(); + assertThat(infrastructureFailure.exceptionMsg()).contains("Docker unavailable"); + } + +} diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerClientFactoryTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerClientFactoryTest.java deleted file mode 100644 index a59cc651b..000000000 --- a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerClientFactoryTest.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright 2024-2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code.docker; - -import com.github.dockerjava.api.DockerClient; -import com.github.dockerjava.api.command.PingCmd; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.InOrder; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -import java.io.IOException; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertSame; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.inOrder; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -@ExtendWith(MockitoExtension.class) -class DockerClientFactoryTest { - - private static final String PRIMARY = "npipe://./pipe/docker_engine"; - - private static final String FALLBACK = "tcp://localhost:2375"; - - @Mock - private DockerClientConnector connector; - - @Mock - private DockerClient primaryClient; - - @Mock - private DockerClient fallbackClient; - - @Mock - private PingCmd primaryPing; - - @Mock - private PingCmd fallbackPing; - - @Test - void create_primaryCandidateWorks_returnsItWithoutTryingFallback() { - when(connector.connect(PRIMARY)).thenReturn(primaryClient); - when(primaryClient.pingCmd()).thenReturn(primaryPing); - - DockerClient result = new DockerClientFactory(connector).create(List.of(PRIMARY, FALLBACK)); - - assertSame(primaryClient, result); - verify(primaryPing).exec(); - verify(connector, never()).connect(FALLBACK); - } - - @Test - void create_primaryPingFails_closesItThenUsesFallback() throws IOException { - when(connector.connect(PRIMARY)).thenReturn(primaryClient); - when(primaryClient.pingCmd()).thenReturn(primaryPing); - when(primaryPing.exec()).thenThrow(new IllegalStateException("pipe unavailable")); - when(connector.connect(FALLBACK)).thenReturn(fallbackClient); - when(fallbackClient.pingCmd()).thenReturn(fallbackPing); - - DockerClient result = new DockerClientFactory(connector).create(List.of(PRIMARY, FALLBACK)); - - assertSame(fallbackClient, result); - InOrder order = inOrder(primaryClient, connector); - order.verify(primaryClient).close(); - order.verify(connector).connect(FALLBACK); - verify(fallbackPing).exec(); - } - - @Test - void create_allCandidatesFail_reportsEveryHostAndLastCause() { - when(connector.connect(PRIMARY)).thenThrow(new IllegalStateException("pipe unavailable")); - when(connector.connect(FALLBACK)).thenThrow(new IllegalStateException("tcp unavailable")); - - IllegalStateException error = assertThrows(IllegalStateException.class, - () -> new DockerClientFactory(connector).create(List.of(PRIMARY, FALLBACK))); - - assertTrue(error.getMessage().contains(PRIMARY)); - assertTrue(error.getMessage().contains(FALLBACK)); - assertTrue(error.getCause().getMessage().contains("tcp unavailable")); - } - - @Test - void create_withoutCandidates_rejectsInvalidConfiguration() { - IllegalArgumentException error = assertThrows(IllegalArgumentException.class, - () -> new DockerClientFactory(connector).create(List.of())); - - assertTrue(error.getMessage().contains("candidate")); - } - -} diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerExecutorFactoryTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerExecutorFactoryTest.java deleted file mode 100644 index a8e0f480b..000000000 --- a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerExecutorFactoryTest.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright 2024-2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code.docker; - -import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; -import com.alibaba.cloud.ai.dataagent.service.code.docker.DockerClientFactory.DockerConnection; -import com.alibaba.cloud.ai.dataagent.service.code.impls.DockerCodePoolExecutorService; -import com.github.dockerjava.api.DockerClient; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.InOrder; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -import java.io.IOException; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertSame; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.Mockito.inOrder; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -@ExtendWith(MockitoExtension.class) -class DockerExecutorFactoryTest { - - private static final String REMOTE = "tcp://docker.example:2375"; - - private static final String LOCAL = "tcp://localhost:2375"; - - @Mock - private DockerHostResolver hostResolver; - - @Mock - private DockerClientFactory clientFactory; - - @Mock - private DockerImageManager imageManager; - - @Mock - private DockerClient client; - - @Test - void create_remotePrimaryFallsBackToLocal_usesConnectedHostClassification() { - CodeExecutorProperties properties = properties(); - List candidates = List.of(REMOTE, LOCAL); - when(hostResolver.candidates(REMOTE, System.getProperty("os.name"))).thenReturn(candidates); - when(clientFactory.connect(candidates)).thenReturn(new DockerConnection(client, LOCAL)); - when(hostResolver.isRemote(LOCAL)).thenReturn(false); - - DockerCodePoolExecutorService executor = new DockerExecutorFactory(hostResolver, clientFactory, imageManager) - .create(properties); - - verify(hostResolver).isRemote(LOCAL); - executor.close(); - } - - @Test - void create_imagePreparationFails_closesClientAndRethrowsOriginalFailure() throws IOException { - CodeExecutorProperties properties = properties(); - List candidates = List.of(REMOTE); - IllegalStateException imageFailure = new IllegalStateException("pull failed"); - when(hostResolver.candidates(REMOTE, System.getProperty("os.name"))).thenReturn(candidates); - when(clientFactory.connect(candidates)).thenReturn(new DockerConnection(client, REMOTE)); - org.mockito.Mockito.doThrow(imageFailure).when(imageManager).ensureAvailable(client, properties.getImageName()); - - IllegalStateException actual = assertThrows(IllegalStateException.class, - () -> new DockerExecutorFactory(hostResolver, clientFactory, imageManager).create(properties)); - - assertSame(imageFailure, actual); - InOrder order = inOrder(imageManager, client); - order.verify(imageManager).ensureAvailable(client, properties.getImageName()); - order.verify(client).close(); - } - - private CodeExecutorProperties properties() { - CodeExecutorProperties properties = new CodeExecutorProperties(); - properties.setHost(REMOTE); - properties.setTaskQueueSize(2); - properties.setCoreContainerNum(1); - properties.setTempContainerNum(1); - properties.setCoreThreadSize(1); - properties.setMaxThreadSize(1); - properties.setThreadQueueSize(2); - return properties; - } - -} diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerHostResolverTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerHostResolverTest.java deleted file mode 100644 index a119ad22f..000000000 --- a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerHostResolverTest.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright 2024-2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code.docker; - -import org.junit.jupiter.api.Test; - -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -class DockerHostResolverTest { - - private final DockerHostResolver resolver = new DockerHostResolver(); - - @Test - void candidates_explicitUnixHost_keepsOnlyConfiguredHost() { - assertEquals(List.of("tcp://docker.example:2375"), resolver.candidates("tcp://docker.example:2375", "Linux")); - } - - @Test - void candidates_explicitWindowsHost_keepsFallbacksWithoutDuplicates() { - assertEquals(List.of("tcp://docker.example:2375", "npipe://./pipe/docker_engine", "tcp://localhost:2375"), - resolver.candidates("tcp://docker.example:2375", "Windows 11")); - } - - @Test - void candidates_windowsWithoutHost_returnsNamedPipeThenLocalTcp() { - assertEquals(List.of("npipe://./pipe/docker_engine", "tcp://localhost:2375"), - resolver.candidates(null, "Windows 11")); - } - - @Test - void candidates_unixWithoutHost_returnsUnixSocket() { - assertEquals(List.of("unix:///var/run/docker.sock"), resolver.candidates(null, "Mac OS X")); - assertEquals(List.of("unix:///var/run/docker.sock"), resolver.candidates(null, "Linux")); - assertEquals(List.of("unix:///var/run/docker.sock"), resolver.candidates(null, null)); - } - - @Test - void isRemote_distinguishesLocalAndRemoteHosts() { - assertFalse(resolver.isRemote(null)); - assertFalse(resolver.isRemote("")); - assertFalse(resolver.isRemote("unix:///var/run/docker.sock")); - assertFalse(resolver.isRemote("npipe://./pipe/docker_engine")); - assertFalse(resolver.isRemote("tcp://localhost:2375")); - assertFalse(resolver.isRemote("tcp://127.0.0.1:2375")); - assertFalse(resolver.isRemote("tcp://[::1]:2375")); - assertTrue(resolver.isRemote("tcp://docker.example:2375")); - } - - @Test - void isRemote_malformedOrUnsupportedHost_isRejected() { - assertThrows(IllegalArgumentException.class, () -> resolver.isRemote("not a docker uri")); - assertThrows(IllegalArgumentException.class, () -> resolver.isRemote("http://docker.example:2375")); - } - -} diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerImageManagerTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerImageManagerTest.java deleted file mode 100644 index 780681433..000000000 --- a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerImageManagerTest.java +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Copyright 2024-2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code.docker; - -import com.github.dockerjava.api.DockerClient; -import com.github.dockerjava.api.command.ListImagesCmd; -import com.github.dockerjava.api.command.PullImageCmd; -import com.github.dockerjava.api.model.Image; -import com.github.dockerjava.core.command.PullImageResultCallback; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -import java.util.List; -import java.util.function.Supplier; - -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -@ExtendWith(MockitoExtension.class) -class DockerImageManagerTest { - - private static final String IMAGE_NAME = "python:test"; - - @Mock - private DockerClient client; - - @Mock - private ListImagesCmd listImagesCmd; - - @Mock - private PullImageCmd pullImageCmd; - - @Mock - private PullImageResultCallback pullCallback; - - @Mock - private Image image; - - @Mock - private Supplier callbackFactory; - - private DockerImageManager imageManager; - - @BeforeEach - void setUp() { - imageManager = new DockerImageManager(callbackFactory); - when(client.listImagesCmd()).thenReturn(listImagesCmd); - when(listImagesCmd.withImageNameFilter(IMAGE_NAME)).thenReturn(listImagesCmd); - } - - @AfterEach - void clearInterruptedFlag() { - Thread.interrupted(); - } - - @Test - void ensureAvailable_existingExactTag_doesNotPull() { - when(listImagesCmd.exec()).thenReturn(List.of(image)); - when(image.getRepoTags()).thenReturn(new String[] { "other:test", IMAGE_NAME }); - - imageManager.ensureAvailable(client, IMAGE_NAME); - - verify(client, never()).pullImageCmd(anyString()); - } - - @Test - void ensureAvailable_missingTag_pullsAndWaitsForCompletion() throws InterruptedException { - when(listImagesCmd.exec()).thenReturn(List.of(image)); - when(image.getRepoTags()).thenReturn(new String[] { "other:test" }); - stubPull(); - - imageManager.ensureAvailable(client, IMAGE_NAME); - - verify(client).pullImageCmd(IMAGE_NAME); - verify(pullCallback).awaitCompletion(); - } - - @Test - void ensureAvailable_nullTags_areTreatedAsMissing() throws InterruptedException { - when(listImagesCmd.exec()).thenReturn(List.of(image)); - when(image.getRepoTags()).thenReturn(null); - stubPull(); - - imageManager.ensureAvailable(client, IMAGE_NAME); - - verify(pullCallback).awaitCompletion(); - } - - @Test - void ensureAvailable_interruptedPull_restoresInterruptAndThrowsSpecificFailure() throws InterruptedException { - when(listImagesCmd.exec()).thenReturn(List.of()); - stubPull(); - when(pullCallback.awaitCompletion()).thenThrow(new InterruptedException("cancelled")); - - IllegalStateException error = assertThrows(IllegalStateException.class, - () -> imageManager.ensureAvailable(client, IMAGE_NAME)); - - assertTrue(error.getMessage().contains(IMAGE_NAME)); - assertTrue(Thread.currentThread().isInterrupted()); - } - - private void stubPull() { - when(callbackFactory.get()).thenReturn(pullCallback); - when(client.pullImageCmd(IMAGE_NAME)).thenReturn(pullImageCmd); - when(pullImageCmd.exec(pullCallback)).thenReturn(pullCallback); - } - -} diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/impls/AiSimulationCodeExecutorServiceTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/impls/AiSimulationCodeExecutorServiceTest.java deleted file mode 100644 index 44c417c6d..000000000 --- a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/impls/AiSimulationCodeExecutorServiceTest.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright 2024-2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code.impls; - -import com.alibaba.cloud.ai.dataagent.service.code.CodePoolExecutorService.TaskRequest; -import com.alibaba.cloud.ai.dataagent.service.code.CodePoolExecutorService.TaskResponse; -import com.alibaba.cloud.ai.dataagent.service.llm.LlmService; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.springframework.ai.chat.model.ChatResponse; -import reactor.core.publisher.Flux; - -import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.ArgumentMatchers.*; -import static org.mockito.Mockito.*; - -@ExtendWith(MockitoExtension.class) -class AiSimulationCodeExecutorServiceTest { - - private AiSimulationCodeExecutorService service; - - @Mock - private LlmService llmService; - - @BeforeEach - void setUp() { - service = new AiSimulationCodeExecutorService(llmService); - } - - @Test - void runTask_returnsSuccessWithLlmOutput() { - TaskRequest request = new TaskRequest("print('hello')", "{}", "run it"); - - Flux chatResponseFlux = Flux.empty(); - when(llmService.call(anyString(), anyString())).thenReturn(chatResponseFlux); - when(llmService.toStringFlux(chatResponseFlux)).thenReturn(Flux.just("hello")); - - TaskResponse response = service.runTask(request); - - assertTrue(response.isSuccess()); - assertEquals("hello", response.stdOut()); - assertNull(response.stdErr()); - assertNull(response.exceptionMsg()); - verify(llmService).call(anyString(), contains("print('hello')")); - } - - @Test - void runTask_concatenatesMultipleFluxElements() { - TaskRequest request = new TaskRequest("code", "input", "req"); - - Flux chatResponseFlux = Flux.empty(); - when(llmService.call(anyString(), anyString())).thenReturn(chatResponseFlux); - when(llmService.toStringFlux(chatResponseFlux)).thenReturn(Flux.just("part1", " part2", " part3")); - - TaskResponse response = service.runTask(request); - - assertTrue(response.isSuccess()); - assertEquals("part1 part2 part3", response.stdOut()); - } - -} diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/impls/DockerCodePoolExecutorServiceIT.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/impls/DockerCodePoolExecutorServiceIT.java deleted file mode 100644 index 1f383a66b..000000000 --- a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/impls/DockerCodePoolExecutorServiceIT.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright 2024-2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code.impls; - -import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; -import com.alibaba.cloud.ai.dataagent.service.code.docker.DockerClientFactory; -import com.alibaba.cloud.ai.dataagent.service.code.docker.DockerExecutorFactory; -import com.alibaba.cloud.ai.dataagent.service.code.docker.DockerHostResolver; -import com.alibaba.cloud.ai.dataagent.service.code.docker.DockerImageManager; -import com.alibaba.cloud.ai.dataagent.service.code.docker.ZerodepDockerClientConnector; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertNotNull; - -/** - * Opt-in smoke test for a real Docker daemon. Maven Surefire does not select the - * {@code *IT} suffix by default. Run explicitly with - * {@code ./mvnw -pl data-agent-management -Dtest=DockerCodePoolExecutorServiceIT test}. - */ -@Tag("docker-integration") -class DockerCodePoolExecutorServiceIT { - - @Test - void create_withAvailableDaemon_preparesAndClosesExecutor() { - CodeExecutorProperties properties = new CodeExecutorProperties(); - properties.setImageName("alpine:3.20"); - DockerHostResolver resolver = new DockerHostResolver(); - DockerExecutorFactory factory = new DockerExecutorFactory(resolver, - new DockerClientFactory(new ZerodepDockerClientConnector()), new DockerImageManager()); - - try (DockerCodePoolExecutorService service = factory.create(properties)) { - assertNotNull(service); - } - } - -} diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/impls/DockerCodePoolExecutorServiceTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/impls/DockerCodePoolExecutorServiceTest.java deleted file mode 100644 index 061bdd9c5..000000000 --- a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/impls/DockerCodePoolExecutorServiceTest.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright 2024-2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code.impls; - -import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; -import com.alibaba.cloud.ai.dataagent.service.code.CodePoolExecutorService.TaskRequest; -import com.alibaba.cloud.ai.dataagent.service.code.CodePoolExecutorService.TaskResponse; -import com.github.dockerjava.api.DockerClient; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -import java.io.IOException; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; - -@ExtendWith(MockitoExtension.class) -class DockerCodePoolExecutorServiceTest { - - @Mock - private DockerClient dockerClient; - - private CodeExecutorProperties properties; - - @BeforeEach - void setUp() { - properties = new CodeExecutorProperties(); - properties.setContainerNamePrefix("test-docker-"); - properties.setTaskQueueSize(2); - properties.setCoreContainerNum(1); - properties.setTempContainerNum(1); - properties.setCoreThreadSize(1); - properties.setMaxThreadSize(1); - properties.setThreadQueueSize(2); - } - - @Test - void constructor_withInjectedClient_hasNoDockerSideEffects() { - DockerCodePoolExecutorService service = new DockerCodePoolExecutorService(properties, dockerClient, false); - - verifyNoInteractions(dockerClient); - - service.close(); - } - - @Test - void execute_missingContainerWorkDirectory_returnsSpecificFailure() { - DockerCodePoolExecutorService service = new DockerCodePoolExecutorService(properties, dockerClient, false); - - TaskResponse response = service.execTaskInContainer(new TaskRequest("print(1)", "", ""), "missing"); - - assertFalse(response.isSuccess()); - assertEquals("An exception occurred while executing the task: Container 'missing' does not exist work dir", - response.exceptionMsg()); - service.close(); - } - - @Test - void close_calledTwice_closesDockerClientExactlyOnce() throws IOException { - DockerCodePoolExecutorService service = new DockerCodePoolExecutorService(properties, dockerClient, false); - - service.close(); - service.close(); - - verify(dockerClient).close(); - } - -} diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/impls/LocalCodePoolExecutorServiceTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/impls/LocalCodePoolExecutorServiceTest.java deleted file mode 100644 index 4ce92b563..000000000 --- a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/impls/LocalCodePoolExecutorServiceTest.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright 2024-2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code.impls; - -import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; -import com.alibaba.cloud.ai.dataagent.service.code.local.ExecutableProgramLocator; -import com.alibaba.cloud.ai.dataagent.service.code.local.ExecutionTimeoutParser; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -import java.util.Optional; - -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.anyList; -import static org.mockito.Mockito.when; - -@ExtendWith(MockitoExtension.class) -class LocalCodePoolExecutorServiceTest { - - @Mock - private ExecutableProgramLocator programLocator; - - private final ExecutionTimeoutParser timeoutParser = new ExecutionTimeoutParser(); - - @Test - void constructor_pythonAvailable_constructsWithoutInspectingMachinePath() { - when(programLocator.findFirst(anyList())).thenReturn(Optional.of("python-test"), Optional.empty()); - - LocalCodePoolExecutorService service = - new LocalCodePoolExecutorService(properties(), programLocator, timeoutParser); - - assertDoesNotThrow(service::close); - } - - @Test - void constructor_pythonUnavailable_throwsSpecificConfigurationFailure() { - when(programLocator.findFirst(anyList())).thenReturn(Optional.empty()); - - IllegalStateException error = assertThrows(IllegalStateException.class, - () -> new LocalCodePoolExecutorService(properties(), programLocator, timeoutParser)); - - assertTrue(error.getMessage().contains("Python")); - } - - private CodeExecutorProperties properties() { - CodeExecutorProperties properties = new CodeExecutorProperties(); - properties.setContainerNamePrefix("test-local-"); - properties.setCodeTimeout("60s"); - properties.setContainerTimeout(5L); - properties.setTaskQueueSize(2); - properties.setCoreContainerNum(1); - properties.setTempContainerNum(1); - properties.setCoreThreadSize(1); - properties.setMaxThreadSize(1); - properties.setThreadQueueSize(2); - return properties; - } - -} diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/local/ExecutionTimeoutParserTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/local/ExecutionTimeoutParserTest.java deleted file mode 100644 index 66fcf4190..000000000 --- a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/local/ExecutionTimeoutParserTest.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright 2024-2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code.local; - -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.CsvSource; -import org.junit.jupiter.params.provider.NullAndEmptySource; -import org.junit.jupiter.params.provider.ValueSource; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -class ExecutionTimeoutParserTest { - - private static final long DEFAULT_TIMEOUT_MILLIS = 60_000; - - private final ExecutionTimeoutParser parser = new ExecutionTimeoutParser(); - - @ParameterizedTest - @CsvSource({ "500ms,500", "0s,0", "30s,30000", "2m,120000", "1h,3600000", "1d,86400000", "100S,100000" }) - void parse_validValue_returnsMilliseconds(String value, long expected) { - assertEquals(expected, parser.parse(value)); - } - - @ParameterizedTest - @NullAndEmptySource - @ValueSource(strings = { " ", "invalid", "30s trailing", "-1s", "9223372036854775807d" }) - void parse_invalidOrOverflowingValue_returnsDocumentedDefault(String value) { - assertEquals(DEFAULT_TIMEOUT_MILLIS, parser.parse(value)); - } - -} diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/local/PathExecutableProgramLocatorTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/local/PathExecutableProgramLocatorTest.java deleted file mode 100644 index 123c1dad3..000000000 --- a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/local/PathExecutableProgramLocatorTest.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright 2024-2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code.local; - -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.List; -import java.util.Optional; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - -class PathExecutableProgramLocatorTest { - - @TempDir - private Path tempDirectory; - - @Test - void findFirst_secondCandidateExists_returnsSecondCandidate() throws IOException { - Path executable = Files.createFile(tempDirectory.resolve("python3")); - assertTrue(executable.toFile().setExecutable(true)); - PathExecutableProgramLocator locator = new PathExecutableProgramLocator(tempDirectory.toString(), false); - - assertEquals(Optional.of("python3"), locator.findFirst(List.of("missing", "python3"))); - } - - @Test - void findFirst_windowsExecutableSuffixExists_returnsProgramName() throws IOException { - Path executable = Files.createFile(tempDirectory.resolve("python.exe")); - assertTrue(executable.toFile().setExecutable(true)); - PathExecutableProgramLocator locator = new PathExecutableProgramLocator(tempDirectory.toString(), true); - - assertEquals(Optional.of("python"), locator.findFirst(List.of("python"))); - } - - @Test - void findFirst_missingPathOrCandidates_returnsEmpty() { - assertFalse(new PathExecutableProgramLocator(null, false).findFirst(List.of("python")).isPresent()); - assertFalse(new PathExecutableProgramLocator(tempDirectory.toString(), false).findFirst(List.of()).isPresent()); - } - -} diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/PythonDependencyMetadataParserTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/PythonDependencyMetadataParserTest.java new file mode 100644 index 000000000..cb0ac468e --- /dev/null +++ b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/PythonDependencyMetadataParserTest.java @@ -0,0 +1,77 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.cloud.ai.dataagent.service.code.sandbox; + +import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class PythonDependencyMetadataParserTest { + + private PythonDependencyMetadataParser parser; + + @BeforeEach + void setUp() { + CodeExecutorProperties properties = new CodeExecutorProperties(); + parser = new PythonDependencyMetadataParser(new PythonDependencyPolicy(properties)); + } + + @Test + void parse_readsCanonicalPep723ScriptBlock() { + PythonDependencyMetadata metadata = parser.parse(""" + # /// script + # requires-python = ">=3.10" + # dependencies = ["pandas>=2,<3", "numpy==2.2.1"] + # /// + import pandas + """); + + assertThat(metadata.requiresPython()).isEqualTo(">=3.10"); + assertThat(metadata.dependencies()).containsExactly("pandas>=2,<3", "numpy==2.2.1"); + } + + @Test + void parse_returnsEmptyMetadataWhenBlockIsAbsent() { + assertThat(parser.parse("import json").dependencies()).isEmpty(); + } + + @Test + void parse_rejectsUrlsPathsMarkersAndPipFlags() { + for (String dependency : new String[] { "pkg @ https://example.com/pkg.whl", "../local-package", + "pandas; python_version > '3.10'", "--extra-index-url=https://example.com" }) { + String code = "# /// script\n# dependencies = [\"" + dependency + "\"]\n# ///\n"; + assertThatThrownBy(() -> parser.parse(code)).isInstanceOf(IllegalArgumentException.class); + } + } + + @Test + void parse_rejectsMultipleOrUnclosedBlocks() { + assertThatThrownBy(() -> parser.parse(""" + # /// script + # dependencies = [] + # /// + # /// script + # dependencies = [] + # /// + """)).isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> parser.parse("# /// script\n# dependencies = []\n")) + .isInstanceOf(IllegalArgumentException.class); + } + +} diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/PythonSandboxBootstrapBuilderTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/PythonSandboxBootstrapBuilderTest.java new file mode 100644 index 000000000..4b33492a4 --- /dev/null +++ b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/PythonSandboxBootstrapBuilderTest.java @@ -0,0 +1,54 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.cloud.ai.dataagent.service.code.sandbox; + +import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +class PythonSandboxBootstrapBuilderTest { + + private final PythonSandboxBootstrapBuilder builder = new PythonSandboxBootstrapBuilder( + new CodeExecutorProperties()); + + @Test + void dependencyInstaller_usesFixedArgumentListAndDoesNotInterpolateModelText() { + String dependency = "safe-package==1.0"; + String installer = builder.buildDependencyInstaller(List.of(dependency)); + + assertThat(installer).contains("sys.executable, \"-m\", \"pip\", \"install\"") + .contains("\"--target\", \"/tmp/dataagent-deps\"") + .doesNotContain(dependency) + .contains(PythonSandboxBootstrapBuilder.RESULT_MARKER); + } + + @Test + void codeExecution_encodesGeneratedCodeAndRunsItInChildProcess() { + String code = "print('sensitive generated code')"; + String bootstrap = builder.buildCodeExecution(code, "[]"); + + assertThat(bootstrap).doesNotContain(code) + .contains("[sys.executable, \"-c\", _code]") + .contains("input=_input") + .contains("timeout=60") + .contains("_environment[\"PYTHONPATH\"] = \"/tmp/dataagent-deps\"") + .contains(PythonSandboxBootstrapBuilder.RESULT_MARKER); + } + +} diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/PythonSandboxBootstrapTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/PythonSandboxBootstrapTest.java new file mode 100644 index 000000000..2a53363fd --- /dev/null +++ b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/PythonSandboxBootstrapTest.java @@ -0,0 +1,81 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.cloud.ai.dataagent.service.code.sandbox; + +import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; +import com.alibaba.cloud.ai.dataagent.util.JsonUtil; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class PythonSandboxBootstrapTest { + + private PythonSandboxBootstrapBuilder builder; + + private SandboxExecutionResultParser parser; + + @BeforeEach + void setUp() { + CodeExecutorProperties properties = new CodeExecutorProperties(); + builder = new PythonSandboxBootstrapBuilder(properties); + parser = new SandboxExecutionResultParser(properties); + } + + @Test + void dependencyInstaller_usesFixedPipCommandAndDoesNotInterpolateDependency() { + String dependency = "pandas>=2,<3"; + String code = builder.buildDependencyInstaller(List.of(dependency)); + + assertThat(code).contains("sys.executable, \"-m\", \"pip\", \"install\""); + assertThat(code).contains("\"--target\", \"/tmp/dataagent-deps\""); + assertThat(code).doesNotContain(dependency); + } + + @Test + void codeExecution_encodesUntrustedCodeAndInput() { + String code = "print('secret marker')"; + String input = "[{\"value\":\"input marker\"}]"; + String wrapper = builder.buildCodeExecution(code, input); + + assertThat(wrapper).doesNotContain(code).doesNotContain("input marker"); + assertThat(wrapper).contains("[sys.executable, \"-c\", _code]"); + } + + @Test + void resultParser_extractsLastEnvelopeFromEscapedSandboxResponse() throws Exception { + SandboxExecutionResult expected = new SandboxExecutionResult(true, "{\"ok\":true}", "", null, null); + String json = JsonUtil.getObjectMapper().writeValueAsString(expected); + String encoded = Base64.getEncoder().encodeToString(json.getBytes(StandardCharsets.UTF_8)); + String raw = "{\"content\":[{\"text\":\"old " + PythonSandboxBootstrapBuilder.RESULT_MARKER + + "aW52YWxpZA==\\\\n" + PythonSandboxBootstrapBuilder.RESULT_MARKER + encoded + "\\\\n\"}]}"; + + SandboxExecutionResult actual = parser.parse(raw); + + assertThat(actual).isEqualTo(expected); + } + + @Test + void resultParser_rejectsUnstructuredSandboxOutput() { + assertThatThrownBy(() -> parser.parse("{\"isError\":true}")).hasMessageContaining("result marker"); + } + +} diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SaaSandboxPythonCodeExecutorServiceTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SaaSandboxPythonCodeExecutorServiceTest.java new file mode 100644 index 000000000..bc76cb0e7 --- /dev/null +++ b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SaaSandboxPythonCodeExecutorServiceTest.java @@ -0,0 +1,68 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.cloud.ai.dataagent.service.code.sandbox; + +import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; +import com.alibaba.cloud.ai.dataagent.service.code.PythonCodeExecutorService; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class SaaSandboxPythonCodeExecutorServiceTest { + + @Test + void runTask_delegatesAcceptedTaskToSandboxRunner() { + CodeExecutorProperties properties = new CodeExecutorProperties(); + SaaSandboxTaskRunner taskRunner = mock(SaaSandboxTaskRunner.class); + PythonCodeExecutorService.TaskRequest request = new PythonCodeExecutorService.TaskRequest("print(1)", "[]", + List.of()); + when(taskRunner.run(request)).thenReturn(PythonCodeExecutorService.TaskResponse.success("1")); + + try (TestExecutor service = new TestExecutor(properties, taskRunner)) { + assertThat(service.runTask(request).stdOut()).isEqualTo("1"); + } + + verify(taskRunner).run(request); + } + + @Test + void runTask_rejectsOversizedCodeBeforeCreatingSandbox() { + CodeExecutorProperties properties = new CodeExecutorProperties(); + properties.getSandbox().setMaxCodeBytes(3); + SaaSandboxTaskRunner taskRunner = mock(SaaSandboxTaskRunner.class); + + try (TestExecutor service = new TestExecutor(properties, taskRunner)) { + PythonCodeExecutorService.TaskResponse response = service + .runTask(new PythonCodeExecutorService.TaskRequest("print(1)", "[]", List.of())); + assertThat(response.isSuccess()).isFalse(); + assertThat(response.exceptionMsg()).contains("size limit"); + } + } + + private static final class TestExecutor extends SaaSandboxPythonCodeExecutorService implements AutoCloseable { + + private TestExecutor(CodeExecutorProperties properties, SaaSandboxTaskRunner taskRunner) { + super(properties, taskRunner); + } + + } + +} diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SaaSandboxRuntimeTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SaaSandboxRuntimeTest.java new file mode 100644 index 000000000..543f625a4 --- /dev/null +++ b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SaaSandboxRuntimeTest.java @@ -0,0 +1,38 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.cloud.ai.dataagent.service.code.sandbox; + +import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; +import org.junit.jupiter.api.Test; + +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; + +class SaaSandboxRuntimeTest { + + @Test + void shouldKeepSandboxAliveForDependencyInstallationAndCodeExecution() { + CodeExecutorProperties properties = new CodeExecutorProperties(); + properties.setCodeTimeout(Duration.ofSeconds(45)); + properties.getSandbox().setDependencyInstallTimeout(Duration.ofMinutes(2)); + + SaaSandboxRuntime runtime = new SaaSandboxRuntime(properties); + + assertThat(runtime.sandboxTimeoutSeconds()).isEqualTo(195); + } + +} diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SaaSandboxTaskRunnerIT.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SaaSandboxTaskRunnerIT.java new file mode 100644 index 000000000..8a18d2217 --- /dev/null +++ b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SaaSandboxTaskRunnerIT.java @@ -0,0 +1,57 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.cloud.ai.dataagent.service.code.sandbox; + +import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; +import com.alibaba.cloud.ai.dataagent.service.code.PythonCodeExecutorService; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Requires a running local Docker daemon and the configured AgentScope base image. + */ +class SaaSandboxTaskRunnerIT { + + @Test + void run_installsDeclaredDependencyAndExecutesPythonInSaaSandbox() { + CodeExecutorProperties properties = new CodeExecutorProperties(); + SaaSandboxRuntime runtime = new SaaSandboxRuntime(properties); + try { + PythonSandboxBootstrapBuilder builder = new PythonSandboxBootstrapBuilder(properties); + SandboxExecutionResultParser parser = new SandboxExecutionResultParser(properties); + SaaSandboxTaskRunner runner = new SaaSandboxTaskRunner(runtime, builder, parser); + PythonCodeExecutorService.TaskRequest request = new PythonCodeExecutorService.TaskRequest(""" + import json + import six + import sys + input_data = json.load(sys.stdin) + print(json.dumps({"six_version": six.__version__, "rows": len(input_data)})) + """, "[{\"id\":1}]", List.of("six==1.17.0")); + + PythonCodeExecutorService.TaskResponse response = runner.run(request); + + assertThat(response.isSuccess()).as(response.toString()).isTrue(); + assertThat(response.stdOut()).contains("\"six_version\": \"1.17.0\"").contains("\"rows\": 1"); + } + finally { + runtime.close(); + } + } + +} diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SandboxExecutionResultParserTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SandboxExecutionResultParserTest.java new file mode 100644 index 000000000..cc6b243a8 --- /dev/null +++ b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SandboxExecutionResultParserTest.java @@ -0,0 +1,51 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.cloud.ai.dataagent.service.code.sandbox; + +import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; +import com.alibaba.cloud.ai.dataagent.util.JsonUtil; +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.util.Base64; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class SandboxExecutionResultParserTest { + + private final SandboxExecutionResultParser parser = new SandboxExecutionResultParser(new CodeExecutorProperties()); + + @Test + void parse_extractsLastEnvelopeFromEscapedSandboxProtocol() throws Exception { + SandboxExecutionResult expected = new SandboxExecutionResult(true, "{\"ok\":true}", "", null, null); + String json = JsonUtil.getObjectMapper().writeValueAsString(expected); + String encoded = Base64.getEncoder().encodeToString(json.getBytes(StandardCharsets.UTF_8)); + String response = "{\"content\":[{\"text\":\"cell output\\\\n" + PythonSandboxBootstrapBuilder.RESULT_MARKER + + encoded + "\\\\n\"}]}"; + + SandboxExecutionResult actual = parser.parse(response); + + assertThat(actual).isEqualTo(expected); + } + + @Test + void parse_rejectsResponsesWithoutResultMarker() { + assertThatThrownBy(() -> parser.parse("{\"isError\":true}")).isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("result marker"); + } + +} diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/workflow/node/python/PythonExecuteNodeTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/workflow/node/python/PythonExecuteNodeTest.java index 258d571f9..206ef4dcf 100644 --- a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/workflow/node/python/PythonExecuteNodeTest.java +++ b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/workflow/node/python/PythonExecuteNodeTest.java @@ -29,12 +29,15 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; +import org.mockito.ArgumentCaptor; import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoSettings; import org.mockito.quality.Strictness; import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; -import com.alibaba.cloud.ai.dataagent.service.code.CodePoolExecutorService; +import com.alibaba.cloud.ai.dataagent.service.code.PythonCodeExecutorService; +import com.alibaba.cloud.ai.dataagent.service.code.sandbox.PythonDependencyMetadata; +import com.alibaba.cloud.ai.dataagent.service.code.sandbox.PythonDependencyMetadataParser; import com.alibaba.cloud.ai.dataagent.util.JsonParseUtil; import com.alibaba.cloud.ai.dataagent.workflow.node.PythonExecuteNode; import com.alibaba.cloud.ai.graph.OverAllState; @@ -45,7 +48,10 @@ class PythonExecuteNodeTest { @Mock - private CodePoolExecutorService codePoolExecutor; + private PythonCodeExecutorService pythonCodeExecutor; + + @Mock + private PythonDependencyMetadataParser dependencyMetadataParser; @Mock private JsonParseUtil jsonParseUtil; @@ -58,7 +64,9 @@ class PythonExecuteNodeTest { @BeforeEach void setUp() { when(codeExecutorProperties.getPythonMaxTriesCount()).thenReturn(5); - pythonExecuteNode = new PythonExecuteNode(codePoolExecutor, jsonParseUtil, codeExecutorProperties); + when(dependencyMetadataParser.parse(anyString())).thenReturn(PythonDependencyMetadata.empty()); + pythonExecuteNode = new PythonExecuteNode(pythonCodeExecutor, dependencyMetadataParser, jsonParseUtil, + codeExecutorProperties); } private OverAllState createTestState() { @@ -81,7 +89,8 @@ void apply_validCode_executesSuccessfully() throws Exception { OverAllState state = createTestState(); setupBasicState(state); - when(codePoolExecutor.runTask(any())).thenReturn(CodePoolExecutorService.TaskResponse.success("hello world")); + when(pythonCodeExecutor.runTask(any())) + .thenReturn(PythonCodeExecutorService.TaskResponse.success("hello world")); when(jsonParseUtil.tryConvertToObject(anyString(), any(Class.class))).thenReturn(null); Map result = pythonExecuteNode.apply(state); @@ -96,7 +105,7 @@ void apply_jsonOutput_parsesCorrectly() throws Exception { state.updateState(Map.of(PYTHON_GENERATE_NODE_OUTPUT, "import json\nprint(json.dumps({'key': 'value'}))")); String jsonOutput = "{\"key\": \"value\"}"; - when(codePoolExecutor.runTask(any())).thenReturn(CodePoolExecutorService.TaskResponse.success(jsonOutput)); + when(pythonCodeExecutor.runTask(any())).thenReturn(PythonCodeExecutorService.TaskResponse.success(jsonOutput)); Map parsed = Map.of("key", "value"); when(jsonParseUtil.tryConvertToObject(anyString(), any(Class.class))).thenReturn(parsed); @@ -113,8 +122,8 @@ void apply_executionError_setsFailureState() throws Exception { setupBasicState(state); state.updateState(Map.of(PYTHON_TRIES_COUNT, 1)); - when(codePoolExecutor.runTask(any())) - .thenReturn(CodePoolExecutorService.TaskResponse.failure("", "NameError: name 'x' is not defined")); + when(pythonCodeExecutor.runTask(any())) + .thenReturn(PythonCodeExecutorService.TaskResponse.failure("", "NameError: name 'x' is not defined")); Map result = pythonExecuteNode.apply(state); assertNotNull(result); @@ -128,8 +137,8 @@ void apply_maxRetryExceeded_setsFallbackMode() throws Exception { setupBasicState(state); state.updateState(Map.of(PYTHON_TRIES_COUNT, 6)); - when(codePoolExecutor.runTask(any())) - .thenReturn(CodePoolExecutorService.TaskResponse.failure("", "SyntaxError")); + when(pythonCodeExecutor.runTask(any())) + .thenReturn(PythonCodeExecutorService.TaskResponse.failure("", "SyntaxError")); Map result = pythonExecuteNode.apply(state); assertNotNull(result); @@ -143,7 +152,7 @@ void apply_jsonParseFailure_usesRawOutput() throws Exception { setupBasicState(state); String rawOutput = "not json content"; - when(codePoolExecutor.runTask(any())).thenReturn(CodePoolExecutorService.TaskResponse.success(rawOutput)); + when(pythonCodeExecutor.runTask(any())).thenReturn(PythonCodeExecutorService.TaskResponse.success(rawOutput)); when(jsonParseUtil.tryConvertToObject(anyString(), any(Class.class))).thenReturn(null); Map result = pythonExecuteNode.apply(state); @@ -158,7 +167,8 @@ void apply_unicodeInOutput_handlesCorrectly() throws Exception { state.updateState(Map.of(PYTHON_GENERATE_NODE_OUTPUT, "print('你好世界')")); String unicodeOutput = "{\"message\": \"\\u4f60\\u597d\\u4e16\\u754c\"}"; - when(codePoolExecutor.runTask(any())).thenReturn(CodePoolExecutorService.TaskResponse.success(unicodeOutput)); + when(pythonCodeExecutor.runTask(any())) + .thenReturn(PythonCodeExecutorService.TaskResponse.success(unicodeOutput)); Map parsed = Map.of("message", "你好世界"); when(jsonParseUtil.tryConvertToObject(anyString(), any(Class.class))).thenReturn(parsed); @@ -174,7 +184,7 @@ void apply_emptyOutput_setsEmptyResult() throws Exception { OverAllState state = createTestState(); setupBasicState(state); - when(codePoolExecutor.runTask(any())).thenReturn(CodePoolExecutorService.TaskResponse.success("")); + when(pythonCodeExecutor.runTask(any())).thenReturn(PythonCodeExecutorService.TaskResponse.success("")); when(jsonParseUtil.tryConvertToObject(anyString(), any(Class.class))).thenReturn(null); Map result = pythonExecuteNode.apply(state); @@ -192,7 +202,7 @@ void apply_withSqlResultsAsCSV_passesCorrectly() throws Exception { sqlResults.add(Map.of("name", "Bob", "amount", "200")); state.updateState(Map.of(SQL_RESULT_LIST_MEMORY, sqlResults)); - when(codePoolExecutor.runTask(any())).thenReturn(CodePoolExecutorService.TaskResponse.success("processed")); + when(pythonCodeExecutor.runTask(any())).thenReturn(PythonCodeExecutorService.TaskResponse.success("processed")); when(jsonParseUtil.tryConvertToObject(anyString(), any(Class.class))).thenReturn(null); Map result = pythonExecuteNode.apply(state); @@ -211,8 +221,8 @@ void apply_largeOutput_handlesMemoryPressure() throws Exception { largeOutput.append("line ").append(i).append(": data_value_").append(i).append("\n"); } - when(codePoolExecutor.runTask(any())) - .thenReturn(CodePoolExecutorService.TaskResponse.success(largeOutput.toString())); + when(pythonCodeExecutor.runTask(any())) + .thenReturn(PythonCodeExecutorService.TaskResponse.success(largeOutput.toString())); when(jsonParseUtil.tryConvertToObject(anyString(), any(Class.class))).thenReturn(null); Map result = pythonExecuteNode.apply(state); @@ -221,4 +231,27 @@ void apply_largeOutput_handlesMemoryPressure() throws Exception { assertNotNull(result.get(PYTHON_EXECUTE_NODE_OUTPUT)); } + @Test + void apply_pep723Dependencies_passesStructuredDependenciesToSandbox() throws Exception { + OverAllState state = createTestState(); + String code = """ + # /// script + # dependencies = ["pandas>=2,<3"] + # /// + print("{}") + """; + state.updateState(Map.of(PYTHON_GENERATE_NODE_OUTPUT, code)); + when(dependencyMetadataParser.parse(code)) + .thenReturn(new PythonDependencyMetadata(List.of("pandas>=2,<3"), null)); + when(pythonCodeExecutor.runTask(any())).thenReturn(PythonCodeExecutorService.TaskResponse.success("{}")); + when(jsonParseUtil.tryConvertToObject(anyString(), any(Class.class))).thenReturn(null); + + pythonExecuteNode.apply(state); + + ArgumentCaptor requestCaptor = ArgumentCaptor + .forClass(PythonCodeExecutorService.TaskRequest.class); + org.mockito.Mockito.verify(pythonCodeExecutor).runTask(requestCaptor.capture()); + assertEquals(List.of("pandas>=2,<3"), requestCaptor.getValue().dependencies()); + } + } diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/workflow/node/python/PythonGenerateNodeTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/workflow/node/python/PythonGenerateNodeTest.java index c3e649cdc..a4064b939 100644 --- a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/workflow/node/python/PythonGenerateNodeTest.java +++ b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/workflow/node/python/PythonGenerateNodeTest.java @@ -24,6 +24,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.time.Duration; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -98,7 +99,7 @@ class PythonGenerateNodeTest { @BeforeEach void setUp() { when(codeExecutorProperties.getLimitMemory()).thenReturn(500L); - when(codeExecutorProperties.getCodeTimeout()).thenReturn("60s"); + when(codeExecutorProperties.getCodeTimeout()).thenReturn(Duration.ofSeconds(60)); pythonGenerateNode = new PythonGenerateNode(codeExecutorProperties, llmService); } diff --git a/docs/ADVANCED_FEATURES-en.md b/docs/ADVANCED_FEATURES-en.md index 8980214de..161c3cf98 100644 --- a/docs/ADVANCED_FEATURES-en.md +++ b/docs/ADVANCED_FEATURES-en.md @@ -383,15 +383,10 @@ WHERE u.name = 'Zhang San' ## Python Execution Environment Configuration -### Executor Types +Python always runs through Spring AI Alibaba Sandbox in a task-scoped container. Host-local, +legacy Docker-pool, and AI Simulation executors are no longer available. -The system supports three Python executors: - -1. **Docker Executor** (Recommended) -2. **Local Executor** -3. **AI Simulation Executor** - -### Docker Executor Configuration +### SAA Sandbox Configuration ```yaml spring: @@ -399,41 +394,28 @@ spring: alibaba: data-agent: code-executor: - type: docker - docker: - image: continuumio/anaconda3:latest - timeout: 300000 # 5 minute timeout - memory-limit: 512m - cpu-limit: 1.0 + code-timeout: 60s + limit-memory: 500 + cpu-core: 1 + sandbox: + image-name: ${DATAAGENT_SANDBOX_IMAGE} + max-concurrency: 4 + queue-capacity: 10 + package-index-url: ${DATAAGENT_PYPI_INDEX_URL:https://pypi.org/simple} + dependency-install-timeout: 3m ``` -### Local Executor Configuration +### Dynamic Dependencies -```yaml -spring: - ai: - alibaba: - data-agent: - code-executor: - type: local - local: - python-path: /usr/bin/python3 - timeout: 300000 - work-dir: /tmp/dataagent +```python +# /// script +# dependencies = ["pandas>=2,<3"] +# /// ``` -### AI Simulation Executor - -Used for testing environments, doesn't actually execute Python code, but simulates execution results through AI: - -```yaml -spring: - ai: - alibaba: - data-agent: - code-executor: - type: ai-simulation -``` +Dependencies are installed inside the task sandbox by a fixed backend command. The model cannot +specify repositories, URLs, file paths, pip options, or shell commands. Production should use an +enterprise private PyPI proxy. ## Advanced Configuration Options diff --git a/docs/ADVANCED_FEATURES.md b/docs/ADVANCED_FEATURES.md index 8f729b518..2daab80de 100644 --- a/docs/ADVANCED_FEATURES.md +++ b/docs/ADVANCED_FEATURES.md @@ -382,15 +382,10 @@ WHERE u.name = '张三' ## 🐍 Python 执行环境配置 -### 执行器类型 +Python 统一通过 Spring AI Alibaba Sandbox 在任务级独立容器中执行,不再提供宿主机 +Local、旧 Docker 容器池或 AI Simulation 执行器。 -系统支持三种Python执行器: - -1. **Docker Executor** (推荐) -2. **Local Executor** -3. **AI Simulation Executor** - -### Docker 执行器配置 +### SAA 沙盒配置 ```yaml spring: @@ -398,41 +393,27 @@ spring: alibaba: data-agent: code-executor: - type: docker - docker: - image: continuumio/anaconda3:latest - timeout: 300000 # 5分钟超时 - memory-limit: 512m - cpu-limit: 1.0 + code-timeout: 60s + limit-memory: 500 + cpu-core: 1 + sandbox: + image-name: ${DATAAGENT_SANDBOX_IMAGE} + max-concurrency: 4 + queue-capacity: 10 + package-index-url: ${DATAAGENT_PYPI_INDEX_URL:https://pypi.org/simple} + dependency-install-timeout: 3m ``` -### Local 执行器配置 +### 动态依赖 -```yaml -spring: - ai: - alibaba: - data-agent: - code-executor: - type: local - local: - python-path: /usr/bin/python3 - timeout: 300000 - work-dir: /tmp/dataagent +```python +# /// script +# dependencies = ["pandas>=2,<3"] +# /// ``` -### AI 模拟执行器 - -用于测试环境,不实际执行Python代码,而是通过AI模拟执行结果: - -```yaml -spring: - ai: - alibaba: - data-agent: - code-executor: - type: ai-simulation -``` +依赖由固定的后端命令在当前任务沙盒内安装。模型不能指定仓库、URL、文件路径、pip 参数 +或 shell 命令。生产环境应将包索引配置为企业私有 PyPI 代理。 ## ⚙️ 高级配置选项 diff --git a/docs/ARCHITECTURE-en.md b/docs/ARCHITECTURE-en.md index 1a14d063b..cba9cce03 100644 --- a/docs/ARCHITECTURE-en.md +++ b/docs/ARCHITECTURE-en.md @@ -32,7 +32,7 @@ flowchart LR ModelRegistry[AiModelRegistry] VectorSvc[AgentVectorStoreService] Hybrid[HybridRetrievalStrategy] - CodePool[CodePoolExecutorService] + CodeExec[PythonCodeExecutorService] McpSvc[McpServerService] end @@ -657,8 +657,9 @@ sequenceDiagram #### Key Points - **Code Generation**: `PythonGenerateNode` generates Python based on plan and SQL results -- **Code Execution**: `PythonExecuteNode` uses `CodePoolExecutorService` (Docker/Local/AI simulation) -- **Execution Configuration**: `spring.ai.alibaba.data-agent.code-executor.*` (default Docker image `continuumio/anaconda3:latest`) +- **Code Execution**: `PythonExecuteNode` uses `PythonCodeExecutorService` and always runs in a task-scoped SAA sandbox +- **Dynamic Dependencies**: Generated code declares dependencies with PEP 723; a fixed bootstrap installs them inside the sandbox +- **Execution Configuration**: `spring.ai.alibaba.data-agent.code-executor.*` - **Result Return**: Execution results are written back to `PYTHON_EXECUTE_NODE_OUTPUT`, `PythonAnalyzeNode` summarizes and writes to `SQL_EXECUTE_NODE_OUTPUT` for final report #### Architecture Diagram @@ -666,14 +667,13 @@ sequenceDiagram ```mermaid flowchart LR PyGen[PythonGenerateNode] --> PyExec[PythonExecuteNode] - PyExec --> ExecSvc[CodePoolExecutorService] - ExecSvc --> Queue[Task Queue] - ExecSvc --> Pool[Container Pool] - Pool --> Docker[Docker Executor] - Pool --> Local[Local Executor] - Pool --> AISim[AI Simulation Executor] - Docker --> TempFiles[Temp Files] - TempFiles --> StdIO[Stdout Stderr] + PyExec --> Parser[PEP 723 Parser] + Parser --> ExecSvc[PythonCodeExecutorService] + ExecSvc --> Queue[Bounded Queue] + Queue --> SAA[SAA SandboxService] + SAA --> Install[Controlled Dependency Install] + Install --> Run[Python Subprocess] + Run --> StdIO[Stdout Stderr] StdIO --> JsonParse[JsonParseUtil] JsonParse --> PyAnalyze[PythonAnalyzeNode] PyAnalyze --> Report[ReportGeneratorNode] @@ -683,8 +683,8 @@ flowchart LR classDef data fill:#FEF3C7,stroke:#F59E0B,stroke-width:1px,color:#1F2937; class PyGen,PyExec,PyAnalyze,Report service - class ExecSvc,Pool,Docker,Local,AISim exec - class Queue,TempFiles,StdIO,JsonParse data + class ExecSvc,SAA,Install,Run exec + class Parser,Queue,StdIO,JsonParse data ``` #### Flow Diagram @@ -697,19 +697,20 @@ sequenceDiagram participant G as PythonGenerateNode participant L as LlmService participant E as PythonExecuteNode - participant CP as CodePoolExecutorService - participant D as Docker Executor + participant CP as PythonCodeExecutorService + participant S as SAA Sandbox participant J as JsonParseUtil participant A as PythonAnalyzeNode participant R as ReportGeneratorNode P->>G: Enter Python step with instructions G->>L: generate python code - L-->>G: python code + L-->>G: python code + PEP 723 G->>E: pass code and sql results E->>CP: runTask - CP->>D: execute in container - D-->>CP: stdout stderr + CP->>S: create sandbox and install dependencies + S->>S: execute with stdin + S-->>CP: stdout stderr CP-->>E: task response E->>J: parse stdout json J-->>E: normalized output diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 31926e860..83653792d 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -32,7 +32,7 @@ flowchart LR ModelRegistry[AiModelRegistry] VectorSvc[AgentVectorStoreService] Hybrid[HybridRetrievalStrategy] - CodePool[CodePoolExecutorService] + CodeExec[PythonCodeExecutorService] McpSvc[McpServerService] LangfuseSvc[LangfuseService] end @@ -666,8 +666,9 @@ sequenceDiagram #### 说明要点 - **代码生成**: `PythonGenerateNode` 根据计划与 SQL 结果生成 Python -- **代码执行**: `PythonExecuteNode` 使用 `CodePoolExecutorService`(Docker/Local/AI 模拟) -- **执行配置**: `spring.ai.alibaba.data-agent.code-executor.*`(默认 Docker 镜像 `continuumio/anaconda3:latest`) +- **代码执行**: `PythonExecuteNode` 使用 `PythonCodeExecutorService`,统一进入任务级 SAA 沙盒 +- **动态依赖**: 生成代码通过 PEP 723 声明依赖,由固定 bootstrap 在沙盒内受控安装 +- **执行配置**: `spring.ai.alibaba.data-agent.code-executor.*` - **结果回传**: 执行结果写回 `PYTHON_EXECUTE_NODE_OUTPUT`,`PythonAnalyzeNode` 汇总后写入 `SQL_EXECUTE_NODE_OUTPUT`,用于最终报告 #### 架构图 @@ -675,14 +676,13 @@ sequenceDiagram ```mermaid flowchart LR PyGen[PythonGenerateNode] --> PyExec[PythonExecuteNode] - PyExec --> ExecSvc[CodePoolExecutorService] - ExecSvc --> Queue[Task Queue] - ExecSvc --> Pool[Container Pool] - Pool --> Docker[Docker Executor] - Pool --> Local[Local Executor] - Pool --> AISim[AI Simulation Executor] - Docker --> TempFiles[Temp Files] - TempFiles --> StdIO[Stdout Stderr] + PyExec --> Parser[PEP 723 Parser] + Parser --> ExecSvc[PythonCodeExecutorService] + ExecSvc --> Queue[Bounded Queue] + Queue --> SAA[SAA SandboxService] + SAA --> Install[Controlled Dependency Install] + Install --> Run[Python Subprocess] + Run --> StdIO[Stdout Stderr] StdIO --> JsonParse[JsonParseUtil] JsonParse --> PyAnalyze[PythonAnalyzeNode] PyAnalyze --> Report[ReportGeneratorNode] @@ -692,8 +692,8 @@ flowchart LR classDef data fill:#FEF3C7,stroke:#F59E0B,stroke-width:1px,color:#1F2937; class PyGen,PyExec,PyAnalyze,Report service - class ExecSvc,Pool,Docker,Local,AISim exec - class Queue,TempFiles,StdIO,JsonParse data + class ExecSvc,SAA,Install,Run exec + class Parser,Queue,StdIO,JsonParse data ``` #### 流程图 @@ -706,19 +706,20 @@ sequenceDiagram participant G as PythonGenerateNode participant L as LlmService participant E as PythonExecuteNode - participant CP as CodePoolExecutorService - participant D as Docker Executor + participant CP as PythonCodeExecutorService + participant S as SAA Sandbox participant J as JsonParseUtil participant A as PythonAnalyzeNode participant R as ReportGeneratorNode P->>G: 进入Python步骤并传入指令 G->>L: generate python code - L-->>G: python code + L-->>G: python code + PEP 723 G->>E: pass code and sql results E->>CP: runTask - CP->>D: execute in container - D-->>CP: stdout stderr + CP->>S: create sandbox and install dependencies + S->>S: execute with stdin + S-->>CP: stdout stderr CP-->>E: task response E->>J: parse stdout json J-->>E: normalized output diff --git a/docs/DEVELOPER_GUIDE-en.md b/docs/DEVELOPER_GUIDE-en.md index 3cb13f4ff..46783523d 100644 --- a/docs/DEVELOPER_GUIDE-en.md +++ b/docs/DEVELOPER_GUIDE-en.md @@ -300,19 +300,18 @@ Configuration prefix: `spring.ai.alibaba.data-agent.code-executor` | Configuration Item | Description | Default Value | |-------------------|-------------|---------------| -| `code-pool-executor` | Executor type (DOCKER/LOCAL) | DOCKER (default is local in application.yml) | -| `image-name` | Docker image name | continuumio/anaconda3:latest | -| `container-name-prefix` | Container name prefix | nl2sql-python-exec- | -| `host` | Service host address | null | -| `task-queue-size` | Task blocking queue size | 5 | -| `core-container-num` | Maximum core container count | 2 | -| `temp-container-num` | Maximum temporary container count | 2 | -| `core-thread-size` | Thread pool core thread count | 5 | -| `max-thread-size` | Thread pool maximum thread count | 5 | | `code-timeout` | Python code execution timeout | 60s | -| `container-timeout` | Maximum container runtime | 3000 (ms) | | `limit-memory` | Container memory limit (MB) | 500 | | `cpu-core` | Container CPU cores | 1 | +| `python-max-tries-count` | Maximum Python execution retries | 5 | +| `sandbox.image-name` | SAA base image | AgentScope runtime sandbox base | +| `sandbox.max-concurrency` | Maximum concurrent sandboxes | 4 | +| `sandbox.queue-capacity` | Bounded wait queue size | 10 | +| `sandbox.package-index-url` | Dynamic package index | https://pypi.org/simple | +| `sandbox.dependency-install-timeout` | Dependency installation timeout | 3m | + +Third-party dependencies must be declared in the generated script's PEP 723 `dependencies`. +Host-local, legacy Docker pool, and AI Simulation executors are no longer available. ### 6. File Storage Configuration diff --git a/docs/DEVELOPER_GUIDE.md b/docs/DEVELOPER_GUIDE.md index 8369fd36d..15ed810e7 100644 --- a/docs/DEVELOPER_GUIDE.md +++ b/docs/DEVELOPER_GUIDE.md @@ -353,19 +353,18 @@ public class AgentVectorStoreService { | 配置项 | 说明 | 默认值 | |--------|------|--------| -| `code-pool-executor` | 执行器类型 (DOCKER/LOCAL) | DOCKER (application.yml中默认为local) | -| `image-name` | Docker镜像名称 | continuumio/anaconda3:latest | -| `container-name-prefix` | 容器名称前缀 | nl2sql-python-exec- | -| `host` | 服务主机地址 | null | -| `task-queue-size` | 任务阻塞队列大小 | 5 | -| `core-container-num` | 核心容器数量最大值 | 2 | -| `temp-container-num` | 临时容器数量最大值 | 2 | -| `core-thread-size` | 线程池核心线程数 | 5 | -| `max-thread-size` | 线程池最大线程数 | 5 | | `code-timeout` | Python代码执行超时时间 | 60s | -| `container-timeout` | 容器最大运行时长 | 3000 (ms) | | `limit-memory` | 容器内存限制 (MB) | 500 | | `cpu-core` | 容器CPU核数 | 1 | +| `python-max-tries-count` | Python执行最大重试次数 | 5 | +| `sandbox.image-name` | SAA基础镜像 | AgentScope runtime sandbox base | +| `sandbox.max-concurrency` | 最大并发沙盒数 | 4 | +| `sandbox.queue-capacity` | 有界等待队列大小 | 10 | +| `sandbox.package-index-url` | 动态依赖包索引 | https://pypi.org/simple | +| `sandbox.dependency-install-timeout` | 依赖安装超时 | 3m | + +第三方依赖必须在生成脚本的 PEP 723 `dependencies` 中声明。系统不再提供宿主机 Local、 +旧 Docker 容器池或 AI Simulation 执行器。 ### 6. 文件存储配置 (File Storage) diff --git a/docs/superpowers/mock-strategy-guide.md b/docs/superpowers/mock-strategy-guide.md index 229814387..b98f9c416 100644 --- a/docs/superpowers/mock-strategy-guide.md +++ b/docs/superpowers/mock-strategy-guide.md @@ -441,7 +441,8 @@ class PythonGenerateNodeTest { ### PythonExecuteNode **Dependencies:** -- `CodePoolExecutorService codePoolExecutor` - Code execution +- `PythonCodeExecutorService pythonCodeExecutor` - SAA Sandbox execution +- `PythonDependencyMetadataParser dependencyMetadataParser` - PEP 723 dependency parsing - `JsonParseUtil jsonParseUtil` - JSON parsing - `CodeExecutorProperties codeExecutorProperties` - Config @@ -453,7 +454,10 @@ class PythonGenerateNodeTest { class PythonExecuteNodeTest { @Mock - private CodePoolExecutorService codePoolExecutor; + private PythonCodeExecutorService pythonCodeExecutor; + + @Mock + private PythonDependencyMetadataParser dependencyMetadataParser; @Mock private JsonParseUtil jsonParseUtil; @@ -465,8 +469,10 @@ class PythonExecuteNodeTest { @BeforeEach void setUp() { + when(dependencyMetadataParser.parse(anyString())) + .thenReturn(PythonDependencyMetadata.empty()); pythonExecuteNode = new PythonExecuteNode( - codePoolExecutor, jsonParseUtil, codeExecutorProperties); + pythonCodeExecutor, dependencyMetadataParser, jsonParseUtil, codeExecutorProperties); } @Test @@ -481,11 +487,11 @@ class PythonExecuteNodeTest { PYTHON_TRIES_COUNT, 0 )); - // Mock code pool executor - successful execution + // Mock SAA Sandbox execution String jsonOutput = "{\"result\": \"hello\"}"; - CodePoolExecutorService.TaskResponse taskResponse = - new CodePoolExecutorService.TaskResponse(true, jsonOutput, "", ""); - when(codePoolExecutor.runTask(any())).thenReturn(taskResponse); + PythonCodeExecutorService.TaskResponse taskResponse = + PythonCodeExecutorService.TaskResponse.success(jsonOutput); + when(pythonCodeExecutor.runTask(any())).thenReturn(taskResponse); when(codeExecutorProperties.getPythonMaxTriesCount()).thenReturn(3); when(jsonParseUtil.tryConvertToObject(any(), any())).thenReturn(Map.of("result", "hello")); diff --git a/docs/superpowers/nodes-dependency-analysis.md b/docs/superpowers/nodes-dependency-analysis.md index 75869d27d..f13b73b9c 100644 --- a/docs/superpowers/nodes-dependency-analysis.md +++ b/docs/superpowers/nodes-dependency-analysis.md @@ -14,7 +14,7 @@ | PlannerNode | LlmService | TABLE_RELATION_OUTPUT, EVIDENCE, GENEGRATED_SEMANTIC_MODEL_PROMPT, PLAN_VALIDATION_ERROR, IS_ONLY_NL2SQL, QUERY_ENHANCE_NODE_OUTPUT | PLANNER_NODE_OUTPUT | Medium | | EvidenceRecallNode | LlmService, AgentVectorStoreService, JsonParseUtil, AgentKnowledgeMapper | INPUT_KEY, AGENT_ID, MULTI_TURN_CONTEXT | EVIDENCE | High | | PythonGenerateNode | CodeExecutorProperties, LlmService | TABLE_RELATION_OUTPUT, SQL_RESULT_LIST_MEMORY, PYTHON_IS_SUCCESS, PYTHON_TRIES_COUNT, QUERY_ENHANCE_NODE_OUTPUT | PYTHON_GENERATE_NODE_OUTPUT, PYTHON_TRIES_COUNT | Medium | -| PythonExecuteNode | CodePoolExecutorService, JsonParseUtil, CodeExecutorProperties | PYTHON_GENERATE_NODE_OUTPUT, SQL_RESULT_LIST_MEMORY, PYTHON_TRIES_COUNT | PYTHON_EXECUTE_NODE_OUTPUT, PYTHON_IS_SUCCESS, PYTHON_FALLBACK_MODE | Medium | +| PythonExecuteNode | PythonCodeExecutorService, PythonDependencyMetadataParser, JsonParseUtil, CodeExecutorProperties | PYTHON_GENERATE_NODE_OUTPUT, SQL_RESULT_LIST_MEMORY, PYTHON_TRIES_COUNT | PYTHON_EXECUTE_NODE_OUTPUT, PYTHON_IS_SUCCESS, PYTHON_FALLBACK_MODE | Medium | | PythonAnalyzeNode | LlmService | PYTHON_EXECUTE_NODE_OUTPUT, SQL_EXECUTE_NODE_OUTPUT, PLAN_CURRENT_STEP, PYTHON_FALLBACK_MODE, QUERY_ENHANCE_NODE_OUTPUT | SQL_EXECUTE_NODE_OUTPUT, PLAN_CURRENT_STEP, PYTHON_ANALYSIS_NODE_OUTPUT | Low | | SemanticConsistencyNode | Nl2SqlService | EVIDENCE, TABLE_RELATION_OUTPUT, DB_DIALECT_TYPE, SQL_GENERATE_OUTPUT, QUERY_ENHANCE_NODE_OUTPUT | SEMANTIC_CONSISTENCY_NODE_OUTPUT, SQL_REGENERATE_REASON | Low | | PlanExecutorNode | None | PLANNER_NODE_OUTPUT, PLAN_CURRENT_STEP, IS_ONLY_NL2SQL, PLAN_REPAIR_COUNT, HUMAN_REVIEW_ENABLED | PLAN_VALIDATION_STATUS, PLAN_VALIDATION_ERROR, PLAN_NEXT_NODE, PLAN_CURRENT_STEP, PLAN_REPAIR_COUNT | Very Low | @@ -95,16 +95,17 @@ - Timeout handling #### PythonExecuteNode -- **Purpose**: Execute Python code via code pool -- **Dependencies**: 3 - - `CodePoolExecutorService.runTask(TaskRequest)` - Returns `TaskResponse` +- **Purpose**: Parse PEP 723 dependencies and execute Python in SAA Sandbox +- **Dependencies**: 4 + - `PythonCodeExecutorService.runTask(TaskRequest)` - Returns `TaskResponse` + - `PythonDependencyMetadataParser.parse()` - Validates dynamic dependencies - `JsonParseUtil.tryConvertToObject()` - For parsing JSON output - `CodeExecutorProperties.getPythonMaxTriesCount()` - Max retries - **Logic Flow**: 1. Get code from `PYTHON_GENERATE_NODE_OUTPUT` 2. Get SQL results from `SQL_RESULT_LIST_MEMORY` 3. Check retry count - 4. Execute via code pool + 4. Parse dependencies and execute via SAA Sandbox 5. Parse JSON output (handle Unicode escape) 6. Return streaming generator with output markers - **Test Scenarios**: diff --git a/docs/superpowers/specs/2026-07-28-saa-python-sandbox-integration-design.md b/docs/superpowers/specs/2026-07-28-saa-python-sandbox-integration-design.md new file mode 100644 index 000000000..a496d7bc3 --- /dev/null +++ b/docs/superpowers/specs/2026-07-28-saa-python-sandbox-integration-design.md @@ -0,0 +1,302 @@ +# SAA 1.1.2.2 Python 沙盒接入方案 + +## 1. 结论与范围 + +本次只改造 Python 生成与执行链路: + +- Spring AI Alibaba 升级到 `1.1.2.2` +- Spring AI 对齐到 `1.1.2` +- Python 统一通过 `spring-ai-alibaba-sandbox` 执行 +- 支持模型用 PEP 723 声明依赖,并在任务沙盒内动态安装 +- 删除本地进程、自建 Docker 容器池和 AI Simulation 执行器 +- SQL 生成、校验和执行链路不做改动 + +不存在本地执行或旧 Docker 执行器的回退。SAA 启动、依赖安装或代码执行失败时,结果直接 +进入现有 Python 重试与降级分支。 + +## 2. 改造后的链路 + +```mermaid +flowchart LR + PG["PythonGenerateNode
生成代码和 PEP 723 元数据"] + PE["PythonExecuteNode
解析依赖并组装 stdin JSON"] + PARSER["PythonDependencyMetadataParser"] + POLICY["PythonDependencyPolicy"] + EXEC["PythonCodeExecutorService
并发、队列、大小和总超时"] + RUNTIME["SaaSandboxRuntime
SandboxService"] + SB["一次任务一个 BaseSandbox"] + INSTALL["固定 bootstrap
python -m pip install"] + RUN["固定 bootstrap
python -c generated.py"] + RESULT["结构化结果 envelope"] + EXISTING["现有重试 / 降级 / SSE / 报告"] + + PG --> PE + PE --> PARSER --> POLICY + POLICY --> EXEC + PE --> EXEC + EXEC --> RUNTIME --> SB + SB --> INSTALL --> RUN --> RESULT + RESULT --> PE --> EXISTING +``` + +保持不变的业务协议: + +1. 生成代码仍通过 `json.load(sys.stdin)` 读取 SQL 结果 JSON。 +2. 业务结果仍从 stdout 返回,并由 `PythonExecuteNode` 做现有 JSON 解析。 +3. Python 失败仍进入现有重试,超过次数仍走现有降级逻辑。 +4. StateGraph 节点关系和 SSE 事件协议不变。 + +## 3. 动态依赖协议 + +生成代码使用标准 PEP 723 `script` 元数据块: + +```python +# /// script +# dependencies = ["pandas>=2,<3", "numpy>=1.24,<3"] +# /// + +import json +import sys +import pandas as pd +``` + +处理规则: + +1. 脚本最多包含一个 `# /// script` 块。 +2. 后端使用 TOML Parser 读取 `dependencies` 和 `requires-python`。 +3. 最多允许 20 个直接依赖,元数据最大 8 KiB。 +4. 允许 PyPI 包名、extras 和版本约束。 +5. 拒绝 URL、VCS、文件路径、`@`、环境标记和任何 pip 参数。 +6. 模型不能提供 index、证书、命令、安装目录或 shell 片段。 + +依赖声明只是一组结构化包规格,不是命令字符串: + +```text +TaskRequest +├── code +├── input +└── dependencies: List +``` + +## 4. 受控动态安装 + +依赖和业务代码在同一个任务级 Sandbox 中分两阶段执行: + +```mermaid +sequenceDiagram + participant E as PythonCodeExecutorService + participant S as SAA BaseSandbox + participant P as Fixed Installer Bootstrap + participant C as Fixed Code Bootstrap + + E->>S: 创建独立 Sandbox + E->>P: Base64 依赖清单 + 固定 index + P->>P: python -m pip install --target /tmp/dataagent-deps + P-->>E: 安装结果 envelope + E->>C: Base64 代码 + Base64 stdin + C->>C: PYTHONPATH=/tmp/dataagent-deps + C->>C: python -c code,stdin=input + C-->>E: stdout/stderr/error envelope + E->>S: close,停止并删除容器 +``` + +固定安装命令由后端生成: + +```text +python -m pip install + --disable-pip-version-check + --no-input + --no-cache-dir + --target /tmp/dataagent-deps + --index-url <服务端配置> + <校验后的依赖清单> +``` + +关键边界: + +- 不使用 shell。 +- 代码、stdin、依赖和 index 均先 Base64 编码,避免字符串拼接注入。 +- 安装目录只存在于本次容器,任务结束即删除。 +- 安装超时和代码超时分开控制。 +- 安装失败不会继续执行业务代码。 +- 每次调用创建新 Sandbox,不跨 Agent、会话或重试复用可写状态。 + +## 5. SAA 框架接入 + +直接使用 1.1.2.2 提供的对象: + +- `SandboxService` +- `ManagerConfig` +- `DockerClientStarter` +- `BaseSandbox` +- `SaaBasePythonRunner` +- `RunPythonToolRequest` / `RunPythonToolResponse` + +`SaaSandboxRuntime` 懒启动单例 `SandboxService`,避免普通 Spring 单元测试要求 Docker +在线;应用关闭时统一调用 `cleanupAllSandboxes()` 和 `close()`。 + +每次任务创建新的 `BaseSandbox` 和新的 `SaaBasePythonRunner`。Runner 持有可变 Sandbox +引用,不能作为单例跨线程复用。 + +当前 SAA 1.1.2.2 的 Docker runtime config 可落地: + +| 参数 | 默认值 | +|---|---:| +| 内存 | 500 MiB | +| CPU | 1 核 | +| nofile | 4096 | +| privileged | `false` | +| 代码超时 | 60 秒 | +| 依赖安装超时 | 3 分钟 | +| 最大并发 | 4 | +| 等待队列 | 10 | + +`nofile` 不能沿用原方案的 256。官方基础镜像中的 supervisord 要求至少 1024;本方案使用 +4096,并通过真实容器启动测试验证。 + +SAA 的 `SandboxConfig.timeout` 同时约束依赖安装和代码执行时的 HTTP 调用,不能只设置为 +`code-timeout`。本方案按 `dependency-install-timeout + code-timeout + 30 秒通信余量` +计算 Sandbox 请求超时,避免 pandas 等较大依赖仍在安装时连接被提前关闭。 + +## 6. 输入输出协议 + +SAA Runner 只有 `code` 字段,没有独立 stdin 字段,因此使用固定 bootstrap 适配: + +1. Java 校验代码和输入大小。 +2. Java 对代码和 stdin 做 UTF-8 Base64。 +3. Sandbox 内用固定 `subprocess.run([python, "-c", code])` 执行。 +4. stdin 通过 subprocess 的 `input` 参数传入。 +5. stdout、stderr、异常类型和错误信息封装为 JSON,再以固定 marker + Base64 返回。 +6. Java 从 SAA 的原始响应中定位 marker 并反序列化。 + +结果协议: + +```json +{ + "success": true, + "stdout": "{\"result\":42}\n", + "stderr": "", + "errorType": null, + "errorMessage": null +} +``` + +映射规则: + +| Sandbox 结果 | 工作流结果 | +|---|---| +| `success=true` | `TaskResponse.success(stdout)` | +| Python 非零退出 | `TaskResponse.failure(stdout, stderr)` | +| 依赖安装失败 | `TaskResponse.failure(install stdout, install stderr)` | +| 响应协议错误、Docker 或通信异常 | `TaskResponse.exception(...)` | +| 外层任务超时 | 取消任务并返回 timeout exception | + +## 7. 大小与并发限制 + +| 内容 | 默认上限 | +|---|---:| +| Python 源码 | 256 KiB | +| stdin JSON | 10 MiB | +| stdout | 1 MiB | +| stderr | 256 KiB | +| PEP 723 元数据 | 8 KiB | +| 直接依赖 | 20 | +| 并发 Sandbox | 4 | +| 等待队列 | 10 | + +代码和 stdin 超限时不创建 Sandbox。stdout/stderr 在返回 envelope 前截断,Java 再校验 +envelope 总大小。队列满时快速失败,不无限堆积请求。 + +完整工作流时间线会包含生成代码、安装日志和执行结果。WebFlux 请求缓冲上限设置为 +`spring.codec.max-in-memory-size: 10MB`,保证浏览器保存这一时间线时不会因默认缓冲上限 +触发 HTTP 413;Python stdout/stderr 仍受上表的独立沙盒限制。 + +## 8. 配置 + +```yaml +spring: + ai: + alibaba: + data-agent: + code-executor: + python-max-tries-count: 5 + code-timeout: 60s + limit-memory: 500 + cpu-core: 1 + sandbox: + docker-host: ${DATAAGENT_SANDBOX_DOCKER_HOST:unix:///var/run/docker.sock} + image-name: ${DATAAGENT_SANDBOX_IMAGE:agentscope-registry.ap-southeast-1.cr.aliyuncs.com/agentscope/runtime-sandbox-base:latest} + max-concurrency: 4 + queue-capacity: 10 + package-index-url: ${DATAAGENT_PYPI_INDEX_URL:https://pypi.org/simple} + dependency-install-timeout: 3m +``` + +开发环境可以使用公网 PyPI 完成端到端验证。生产环境必须通过环境变量切到企业私有 PyPI +代理和固定镜像 digest。 + +## 9. 删除项 + +以下实现不再保留: + +- `CodePoolExecutorServiceFactory` +- `CodePoolExecutorEnum` +- `AbstractCodePoolExecutorService` +- `LocalCodePoolExecutorService` +- `DockerCodePoolExecutorService` +- `AiSimulationCodeExecutorService` +- 自建 docker-java client、image、host 和 pool 辅助类 +- 对应单元测试和旧 Docker 集成测试 + +新的稳定边界为 `PythonCodeExecutorService`,当前只有 +`SaaSandboxPythonCodeExecutorService` 一个实现。 + +## 10. 安全边界与生产加固 + +当前版本已经具备: + +- 独立任务容器 +- 非 privileged +- CPU、内存、nofile、超时、并发和输入输出限制 +- 依赖声明策略 +- 固定 pip 命令 +- 任务完成后删除容器 +- 不回退到宿主机执行 + +需要明确:动态安装要求 Sandbox 在安装阶段能访问包仓库,而 SAA 1.1.2.2 没有公开 +“安装后立即断开容器网络”的 API。因此当前实现适用于本地和受控测试环境;生产上线还需 +补齐基础设施级网络控制: + +1. 仅允许访问私有 PyPI 代理,禁止访问其他公网和业务内网。 +2. 固定基础镜像 digest,镜像内使用非 root 用户。 +3. 不向 Sandbox 注入数据库、模型、OSS 或 Docker Socket 等业务凭据。 +4. 私有代理增加恶意包、CVE 和许可证扫描。 +5. 后续可增加 wheel-only、hash lock 和只读 wheel 缓存;这些是生产加固项,不作为 + 当前代码已实现能力描述。 + +## 11. 验收 + +| 场景 | 验收条件 | +|---|---| +| 动态依赖 | PEP 723 声明的包在任务 Sandbox 内安装成功 | +| stdin | 生成代码能通过 `json.load(sys.stdin)` 读取 SQL 结果 | +| stdout | JSON 结果保持现有工作流协议 | +| Python 异常 | 进入现有重试或降级分支 | +| 超时/超限 | 返回明确失败并释放 Sandbox | +| 隔离 | 两次执行不复用可写运行态 | +| 清理 | 成功和失败后均无残留 `dataagent-sandbox-*` 容器 | +| 端到端 | 浏览器发起真实分析后出现最终报告且 SSE 收到 `event:complete` | + +验证分层: + +1. Parser、依赖策略、bootstrap、响应映射和执行器单元测试。 +2. 真实 Docker 集成测试:SAA 创建容器、动态安装 `six==1.17.0`、读取 stdin、输出 + JSON、停止并删除容器。 +3. 完整后端测试与格式检查。 +4. 本地启动 Milvus、后端和前端,使用浏览器完成真实 StateGraph 请求。 + +## 12. 参考 + +- [Spring AI Alibaba v1.1.2.2 releases](https://github.com/alibaba/spring-ai-alibaba/releases) +- [Spring AI Alibaba Sandbox example](https://github.com/spring-ai-alibaba/examples/tree/main/spring-ai-alibaba-sandbox-example/sandbox-simple-tool) +- [PEP 723 – Inline script metadata](https://peps.python.org/pep-0723/) diff --git a/pom.xml b/pom.xml index 81140106b..426d0ea2a 100644 --- a/pom.xml +++ b/pom.xml @@ -21,7 +21,7 @@ - 1.1.0.0 + 1.1.2.2 1.0.0-SNAPSHOT @@ -35,7 +35,7 @@ ${java.version} - 1.1.0 + 1.1.2 3.4.8 2.8.8 0.0.39 @@ -49,7 +49,8 @@ 3.11.0 3.1.0 - 3.5.3 + 1.1.1 + 33.4.8-jre 3.0.0 1.2.22 42.4.1 @@ -87,18 +88,16 @@ pom import - - com.github.docker-java - docker-java-bom - ${docker-java.version} - pom - import - com.aliyun gpdb20160503 ${gpdb.version} + + com.google.guava + guava + ${guava.version} + com.alibaba druid From 5608f23696ac24949863049b332ec03a0a4dbdde Mon Sep 17 00:00:00 2001 From: zxuexingzhijie <2762006003@qq.com> Date: Tue, 28 Jul 2026 15:53:58 +0800 Subject: [PATCH 2/3] refactor: organize Python sandbox packages --- .../SaaSandboxPythonCodeExecutorService.java | 1 + .../PythonDependencyMetadata.java | 2 +- .../PythonDependencyMetadataParser.java | 2 +- .../PythonDependencyPolicy.java | 2 +- .../PythonSandboxBootstrapBuilder.java | 2 +- .../SandboxExecutionResult.java | 2 +- .../SandboxExecutionResultParser.java | 2 +- .../{ => runtime}/SaaSandboxRuntime.java | 2 +- .../{ => runtime}/SaaSandboxTaskRunner.java | 5 +- .../workflow/node/PythonExecuteNode.java | 4 +- .../config/VectorStoreConfigurationTest.java | 4 +- .../PythonWorkflowIntegrationTest.java | 4 +- .../sandbox/PythonSandboxBootstrapTest.java | 81 ------------------- ...aSandboxPythonCodeExecutorServiceTest.java | 1 + .../PythonDependencyMetadataParserTest.java | 2 +- .../PythonSandboxBootstrapBuilderTest.java | 2 +- .../SandboxExecutionResultParserTest.java | 2 +- .../{ => runtime}/SaaSandboxRuntimeTest.java | 2 +- .../{ => runtime}/SaaSandboxTaskRunnerIT.java | 4 +- .../node/python/PythonExecuteNodeTest.java | 4 +- ...8-saa-python-sandbox-integration-design.md | 25 ++++++ 21 files changed, 53 insertions(+), 102 deletions(-) rename data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/{ => dependency}/PythonDependencyMetadata.java (93%) rename data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/{ => dependency}/PythonDependencyMetadataParser.java (98%) rename data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/{ => dependency}/PythonDependencyPolicy.java (96%) rename data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/{ => execution}/PythonSandboxBootstrapBuilder.java (98%) rename data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/{ => execution}/SandboxExecutionResult.java (91%) rename data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/{ => execution}/SandboxExecutionResultParser.java (97%) rename data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/{ => runtime}/SaaSandboxRuntime.java (98%) rename data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/{ => runtime}/SaaSandboxTaskRunner.java (89%) delete mode 100644 data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/PythonSandboxBootstrapTest.java rename data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/{ => dependency}/PythonDependencyMetadataParserTest.java (97%) rename data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/{ => execution}/PythonSandboxBootstrapBuilderTest.java (96%) rename data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/{ => execution}/SandboxExecutionResultParserTest.java (96%) rename data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/{ => runtime}/SaaSandboxRuntimeTest.java (94%) rename data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/{ => runtime}/SaaSandboxTaskRunnerIT.java (88%) diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SaaSandboxPythonCodeExecutorService.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SaaSandboxPythonCodeExecutorService.java index 83f77aecd..b1fafb809 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SaaSandboxPythonCodeExecutorService.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SaaSandboxPythonCodeExecutorService.java @@ -17,6 +17,7 @@ import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; import com.alibaba.cloud.ai.dataagent.service.code.PythonCodeExecutorService; +import com.alibaba.cloud.ai.dataagent.service.code.sandbox.runtime.SaaSandboxTaskRunner; import jakarta.annotation.PreDestroy; import org.springframework.stereotype.Service; diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/PythonDependencyMetadata.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/dependency/PythonDependencyMetadata.java similarity index 93% rename from data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/PythonDependencyMetadata.java rename to data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/dependency/PythonDependencyMetadata.java index 51a5ee454..cf3de5882 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/PythonDependencyMetadata.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/dependency/PythonDependencyMetadata.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.alibaba.cloud.ai.dataagent.service.code.sandbox; +package com.alibaba.cloud.ai.dataagent.service.code.sandbox.dependency; import java.util.List; diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/PythonDependencyMetadataParser.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/dependency/PythonDependencyMetadataParser.java similarity index 98% rename from data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/PythonDependencyMetadataParser.java rename to data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/dependency/PythonDependencyMetadataParser.java index 2c1a28e06..b2c2b7401 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/PythonDependencyMetadataParser.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/dependency/PythonDependencyMetadataParser.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.alibaba.cloud.ai.dataagent.service.code.sandbox; +package com.alibaba.cloud.ai.dataagent.service.code.sandbox.dependency; import org.springframework.stereotype.Component; import org.tomlj.Toml; diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/PythonDependencyPolicy.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/dependency/PythonDependencyPolicy.java similarity index 96% rename from data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/PythonDependencyPolicy.java rename to data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/dependency/PythonDependencyPolicy.java index 619de8706..87770ad92 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/PythonDependencyPolicy.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/dependency/PythonDependencyPolicy.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.alibaba.cloud.ai.dataagent.service.code.sandbox; +package com.alibaba.cloud.ai.dataagent.service.code.sandbox.dependency; import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; import org.springframework.stereotype.Component; diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/PythonSandboxBootstrapBuilder.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/execution/PythonSandboxBootstrapBuilder.java similarity index 98% rename from data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/PythonSandboxBootstrapBuilder.java rename to data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/execution/PythonSandboxBootstrapBuilder.java index 484d12482..f5438ccec 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/PythonSandboxBootstrapBuilder.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/execution/PythonSandboxBootstrapBuilder.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.alibaba.cloud.ai.dataagent.service.code.sandbox; +package com.alibaba.cloud.ai.dataagent.service.code.sandbox.execution; import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; import com.alibaba.cloud.ai.dataagent.util.JsonUtil; diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SandboxExecutionResult.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/execution/SandboxExecutionResult.java similarity index 91% rename from data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SandboxExecutionResult.java rename to data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/execution/SandboxExecutionResult.java index c970a2eab..fbfc93db0 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SandboxExecutionResult.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/execution/SandboxExecutionResult.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.alibaba.cloud.ai.dataagent.service.code.sandbox; +package com.alibaba.cloud.ai.dataagent.service.code.sandbox.execution; public record SandboxExecutionResult(boolean success, String stdout, String stderr, String errorType, String errorMessage) { diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SandboxExecutionResultParser.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/execution/SandboxExecutionResultParser.java similarity index 97% rename from data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SandboxExecutionResultParser.java rename to data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/execution/SandboxExecutionResultParser.java index 17b617378..a373d2a4b 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SandboxExecutionResultParser.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/execution/SandboxExecutionResultParser.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.alibaba.cloud.ai.dataagent.service.code.sandbox; +package com.alibaba.cloud.ai.dataagent.service.code.sandbox.execution; import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; import com.alibaba.cloud.ai.dataagent.util.JsonUtil; diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SaaSandboxRuntime.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/runtime/SaaSandboxRuntime.java similarity index 98% rename from data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SaaSandboxRuntime.java rename to data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/runtime/SaaSandboxRuntime.java index 21d08c9d1..d3eee9217 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SaaSandboxRuntime.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/runtime/SaaSandboxRuntime.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.alibaba.cloud.ai.dataagent.service.code.sandbox; +package com.alibaba.cloud.ai.dataagent.service.code.sandbox.runtime; import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; import io.agentscope.runtime.sandbox.box.BaseSandbox; diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SaaSandboxTaskRunner.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/runtime/SaaSandboxTaskRunner.java similarity index 89% rename from data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SaaSandboxTaskRunner.java rename to data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/runtime/SaaSandboxTaskRunner.java index b5f4ca5a4..6333887b3 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SaaSandboxTaskRunner.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/runtime/SaaSandboxTaskRunner.java @@ -13,8 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.alibaba.cloud.ai.dataagent.service.code.sandbox; +package com.alibaba.cloud.ai.dataagent.service.code.sandbox.runtime; +import com.alibaba.cloud.ai.dataagent.service.code.sandbox.execution.PythonSandboxBootstrapBuilder; +import com.alibaba.cloud.ai.dataagent.service.code.sandbox.execution.SandboxExecutionResult; +import com.alibaba.cloud.ai.dataagent.service.code.sandbox.execution.SandboxExecutionResultParser; import com.alibaba.cloud.ai.dataagent.service.code.PythonCodeExecutorService; import com.alibaba.cloud.ai.sandbox.tools.base.SaaBasePythonRunner; import io.agentscope.runtime.sandbox.box.BaseSandbox; diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/workflow/node/PythonExecuteNode.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/workflow/node/PythonExecuteNode.java index 94184358e..f1a7e77dc 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/workflow/node/PythonExecuteNode.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/workflow/node/PythonExecuteNode.java @@ -17,8 +17,8 @@ import com.alibaba.cloud.ai.dataagent.enums.TextType; import com.alibaba.cloud.ai.dataagent.service.code.PythonCodeExecutorService; -import com.alibaba.cloud.ai.dataagent.service.code.sandbox.PythonDependencyMetadata; -import com.alibaba.cloud.ai.dataagent.service.code.sandbox.PythonDependencyMetadataParser; +import com.alibaba.cloud.ai.dataagent.service.code.sandbox.dependency.PythonDependencyMetadata; +import com.alibaba.cloud.ai.dataagent.service.code.sandbox.dependency.PythonDependencyMetadataParser; import com.alibaba.cloud.ai.dataagent.util.JsonParseUtil; import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; import com.alibaba.cloud.ai.graph.GraphResponse; diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/config/VectorStoreConfigurationTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/config/VectorStoreConfigurationTest.java index 5c14652b3..942c85d21 100644 --- a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/config/VectorStoreConfigurationTest.java +++ b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/config/VectorStoreConfigurationTest.java @@ -37,8 +37,8 @@ class VectorStoreConfigurationTest { private final YamlPropertySourceLoader loader = new YamlPropertySourceLoader(); @Test - void vectorStoreProfiles_useExplicitSpringAiStarterTypes() throws Exception { - assertThat(property("application.yml", "spring.ai.vectorstore.type")).isEqualTo("milvus"); + void vectorStoreProfiles_declareExpectedTypes() throws Exception { + assertThat(property("application.yml", "spring.ai.vectorstore.type")).isEqualTo("simple"); assertThat(property("application-milvus.yml", "spring.ai.vectorstore.type")).isEqualTo("milvus"); assertThat(property("application-elasticsearch.yml", "spring.ai.vectorstore.type")).isEqualTo("elasticsearch"); } diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/integration/PythonWorkflowIntegrationTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/integration/PythonWorkflowIntegrationTest.java index 27cc00c74..a0470cc92 100644 --- a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/integration/PythonWorkflowIntegrationTest.java +++ b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/integration/PythonWorkflowIntegrationTest.java @@ -37,8 +37,8 @@ import com.alibaba.cloud.ai.dataagent.common.TestFixtures; import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; import com.alibaba.cloud.ai.dataagent.service.code.PythonCodeExecutorService; -import com.alibaba.cloud.ai.dataagent.service.code.sandbox.PythonDependencyMetadata; -import com.alibaba.cloud.ai.dataagent.service.code.sandbox.PythonDependencyMetadataParser; +import com.alibaba.cloud.ai.dataagent.service.code.sandbox.dependency.PythonDependencyMetadata; +import com.alibaba.cloud.ai.dataagent.service.code.sandbox.dependency.PythonDependencyMetadataParser; import com.alibaba.cloud.ai.dataagent.service.llm.LlmService; import com.alibaba.cloud.ai.dataagent.util.ChatResponseUtil; import com.alibaba.cloud.ai.dataagent.util.JsonParseUtil; diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/PythonSandboxBootstrapTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/PythonSandboxBootstrapTest.java deleted file mode 100644 index 2a53363fd..000000000 --- a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/PythonSandboxBootstrapTest.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright 2024-2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code.sandbox; - -import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; -import com.alibaba.cloud.ai.dataagent.util.JsonUtil; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.nio.charset.StandardCharsets; -import java.util.Base64; -import java.util.List; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -class PythonSandboxBootstrapTest { - - private PythonSandboxBootstrapBuilder builder; - - private SandboxExecutionResultParser parser; - - @BeforeEach - void setUp() { - CodeExecutorProperties properties = new CodeExecutorProperties(); - builder = new PythonSandboxBootstrapBuilder(properties); - parser = new SandboxExecutionResultParser(properties); - } - - @Test - void dependencyInstaller_usesFixedPipCommandAndDoesNotInterpolateDependency() { - String dependency = "pandas>=2,<3"; - String code = builder.buildDependencyInstaller(List.of(dependency)); - - assertThat(code).contains("sys.executable, \"-m\", \"pip\", \"install\""); - assertThat(code).contains("\"--target\", \"/tmp/dataagent-deps\""); - assertThat(code).doesNotContain(dependency); - } - - @Test - void codeExecution_encodesUntrustedCodeAndInput() { - String code = "print('secret marker')"; - String input = "[{\"value\":\"input marker\"}]"; - String wrapper = builder.buildCodeExecution(code, input); - - assertThat(wrapper).doesNotContain(code).doesNotContain("input marker"); - assertThat(wrapper).contains("[sys.executable, \"-c\", _code]"); - } - - @Test - void resultParser_extractsLastEnvelopeFromEscapedSandboxResponse() throws Exception { - SandboxExecutionResult expected = new SandboxExecutionResult(true, "{\"ok\":true}", "", null, null); - String json = JsonUtil.getObjectMapper().writeValueAsString(expected); - String encoded = Base64.getEncoder().encodeToString(json.getBytes(StandardCharsets.UTF_8)); - String raw = "{\"content\":[{\"text\":\"old " + PythonSandboxBootstrapBuilder.RESULT_MARKER - + "aW52YWxpZA==\\\\n" + PythonSandboxBootstrapBuilder.RESULT_MARKER + encoded + "\\\\n\"}]}"; - - SandboxExecutionResult actual = parser.parse(raw); - - assertThat(actual).isEqualTo(expected); - } - - @Test - void resultParser_rejectsUnstructuredSandboxOutput() { - assertThatThrownBy(() -> parser.parse("{\"isError\":true}")).hasMessageContaining("result marker"); - } - -} diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SaaSandboxPythonCodeExecutorServiceTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SaaSandboxPythonCodeExecutorServiceTest.java index bc76cb0e7..e97f96533 100644 --- a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SaaSandboxPythonCodeExecutorServiceTest.java +++ b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SaaSandboxPythonCodeExecutorServiceTest.java @@ -17,6 +17,7 @@ import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; import com.alibaba.cloud.ai.dataagent.service.code.PythonCodeExecutorService; +import com.alibaba.cloud.ai.dataagent.service.code.sandbox.runtime.SaaSandboxTaskRunner; import org.junit.jupiter.api.Test; import java.util.List; diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/PythonDependencyMetadataParserTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/dependency/PythonDependencyMetadataParserTest.java similarity index 97% rename from data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/PythonDependencyMetadataParserTest.java rename to data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/dependency/PythonDependencyMetadataParserTest.java index cb0ac468e..3ce42e7f4 100644 --- a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/PythonDependencyMetadataParserTest.java +++ b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/dependency/PythonDependencyMetadataParserTest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.alibaba.cloud.ai.dataagent.service.code.sandbox; +package com.alibaba.cloud.ai.dataagent.service.code.sandbox.dependency; import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; import org.junit.jupiter.api.BeforeEach; diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/PythonSandboxBootstrapBuilderTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/execution/PythonSandboxBootstrapBuilderTest.java similarity index 96% rename from data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/PythonSandboxBootstrapBuilderTest.java rename to data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/execution/PythonSandboxBootstrapBuilderTest.java index 4b33492a4..da4cd6a0d 100644 --- a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/PythonSandboxBootstrapBuilderTest.java +++ b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/execution/PythonSandboxBootstrapBuilderTest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.alibaba.cloud.ai.dataagent.service.code.sandbox; +package com.alibaba.cloud.ai.dataagent.service.code.sandbox.execution; import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; import org.junit.jupiter.api.Test; diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SandboxExecutionResultParserTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/execution/SandboxExecutionResultParserTest.java similarity index 96% rename from data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SandboxExecutionResultParserTest.java rename to data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/execution/SandboxExecutionResultParserTest.java index cc6b243a8..862f3d499 100644 --- a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SandboxExecutionResultParserTest.java +++ b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/execution/SandboxExecutionResultParserTest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.alibaba.cloud.ai.dataagent.service.code.sandbox; +package com.alibaba.cloud.ai.dataagent.service.code.sandbox.execution; import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; import com.alibaba.cloud.ai.dataagent.util.JsonUtil; diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SaaSandboxRuntimeTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/runtime/SaaSandboxRuntimeTest.java similarity index 94% rename from data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SaaSandboxRuntimeTest.java rename to data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/runtime/SaaSandboxRuntimeTest.java index 543f625a4..840340cb6 100644 --- a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SaaSandboxRuntimeTest.java +++ b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/runtime/SaaSandboxRuntimeTest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.alibaba.cloud.ai.dataagent.service.code.sandbox; +package com.alibaba.cloud.ai.dataagent.service.code.sandbox.runtime; import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; import org.junit.jupiter.api.Test; diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SaaSandboxTaskRunnerIT.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/runtime/SaaSandboxTaskRunnerIT.java similarity index 88% rename from data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SaaSandboxTaskRunnerIT.java rename to data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/runtime/SaaSandboxTaskRunnerIT.java index 8a18d2217..d5275c6e9 100644 --- a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SaaSandboxTaskRunnerIT.java +++ b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/runtime/SaaSandboxTaskRunnerIT.java @@ -13,10 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.alibaba.cloud.ai.dataagent.service.code.sandbox; +package com.alibaba.cloud.ai.dataagent.service.code.sandbox.runtime; import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; import com.alibaba.cloud.ai.dataagent.service.code.PythonCodeExecutorService; +import com.alibaba.cloud.ai.dataagent.service.code.sandbox.execution.PythonSandboxBootstrapBuilder; +import com.alibaba.cloud.ai.dataagent.service.code.sandbox.execution.SandboxExecutionResultParser; import org.junit.jupiter.api.Test; import java.util.List; diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/workflow/node/python/PythonExecuteNodeTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/workflow/node/python/PythonExecuteNodeTest.java index 206ef4dcf..9519332a1 100644 --- a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/workflow/node/python/PythonExecuteNodeTest.java +++ b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/workflow/node/python/PythonExecuteNodeTest.java @@ -36,8 +36,8 @@ import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; import com.alibaba.cloud.ai.dataagent.service.code.PythonCodeExecutorService; -import com.alibaba.cloud.ai.dataagent.service.code.sandbox.PythonDependencyMetadata; -import com.alibaba.cloud.ai.dataagent.service.code.sandbox.PythonDependencyMetadataParser; +import com.alibaba.cloud.ai.dataagent.service.code.sandbox.dependency.PythonDependencyMetadata; +import com.alibaba.cloud.ai.dataagent.service.code.sandbox.dependency.PythonDependencyMetadataParser; import com.alibaba.cloud.ai.dataagent.util.JsonParseUtil; import com.alibaba.cloud.ai.dataagent.workflow.node.PythonExecuteNode; import com.alibaba.cloud.ai.graph.OverAllState; diff --git a/docs/superpowers/specs/2026-07-28-saa-python-sandbox-integration-design.md b/docs/superpowers/specs/2026-07-28-saa-python-sandbox-integration-design.md index a496d7bc3..c2910fb34 100644 --- a/docs/superpowers/specs/2026-07-28-saa-python-sandbox-integration-design.md +++ b/docs/superpowers/specs/2026-07-28-saa-python-sandbox-integration-design.md @@ -46,6 +46,31 @@ flowchart LR 3. Python 失败仍进入现有重试,超过次数仍走现有降级逻辑。 4. StateGraph 节点关系和 SSE 事件协议不变。 +### 2.1 包边界 + +`sandbox` 顶层只保留对外执行服务,内部按变化原因拆分: + +```text +service/code/sandbox +├── SaaSandboxPythonCodeExecutorService.java +├── dependency +│ ├── PythonDependencyMetadata.java +│ ├── PythonDependencyMetadataParser.java +│ └── PythonDependencyPolicy.java +├── execution +│ ├── PythonSandboxBootstrapBuilder.java +│ ├── SandboxExecutionResult.java +│ └── SandboxExecutionResultParser.java +└── runtime + ├── SaaSandboxRuntime.java + └── SaaSandboxTaskRunner.java +``` + +- `dependency` 只负责 PEP 723 和依赖策略,不感知 SAA。 +- `execution` 只负责固定 Python bootstrap 与结果协议,不管理容器。 +- `runtime` 只负责 SAA 服务、任务 Sandbox 和 Runner 生命周期。 +- 顶层服务只负责输入限制、并发队列和任务总超时。 + ## 3. 动态依赖协议 生成代码使用标准 PEP 723 `script` 元数据块: From 43affbf3aaf28f05d64cfe13bdfcf8aef450cb8c Mon Sep 17 00:00:00 2001 From: zxuexingzhijie <2762006003@qq.com> Date: Tue, 28 Jul 2026 18:03:41 +0800 Subject: [PATCH 3/3] =?UTF-8?q?fix:=20=E6=9B=B4=E6=96=B0=E4=B8=AD=E6=96=87?= =?UTF-8?q?=E6=96=87=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README-en.md | 29 +++--- README.md | 19 ++-- docs/ADVANCED_FEATURES-en.md | 99 +++++++++++++++++-- docs/ADVANCED_FEATURES.md | 90 +++++++++++++++-- docs/ARCHITECTURE-en.md | 64 ++++++++---- docs/ARCHITECTURE.md | 64 ++++++++---- docs/DEVELOPER_GUIDE-en.md | 86 +++++++++++++--- docs/DEVELOPER_GUIDE.md | 82 ++++++++++++--- docs/QUICK_START-en.md | 95 +++++++++++++----- docs/QUICK_START.md | 90 ++++++++++++----- ...8-saa-python-sandbox-integration-design.md | 83 +++++++++++++--- 11 files changed, 638 insertions(+), 163 deletions(-) diff --git a/README-en.md b/README-en.md index 686804058..ac361c3cf 100644 --- a/README-en.md +++ b/README-en.md @@ -9,7 +9,7 @@

- Spring AI Alibaba + Spring AI Alibaba Spring Boot Java License @@ -46,7 +46,7 @@ Additionally, this project natively supports **MCP (Model Context Protocol)**, e | Feature | Description | | :--- | :--- | | **Intelligent Data Analysis** | StateGraph-based Text-to-SQL conversion, supporting complex multi-table queries and multi-turn conversation intent understanding. | -| **Python Deep Analysis** | Built-in Docker/Local Python executor, automatically generating and executing Python code for statistical analysis and machine learning predictions. | +| **Python Deep Analysis** | Executes generated code in task-scoped Spring AI Alibaba Sandbox containers with PEP 723 dynamic dependencies, resource limits, retries, and automatic cleanup. | | **Intelligent Report Generation** | Analysis results are automatically summarized into HTML/Markdown reports with ECharts visualizations, WYSIWYG. | | **Human Feedback Mechanism** | Innovative Human-in-the-loop mechanism, supporting user intervention and adjustments during the plan generation phase. | | **RAG Retrieval Enhancement** | Integrated vector database, supporting semantic retrieval of business metadata and terminology libraries to improve SQL generation accuracy. | @@ -61,12 +61,14 @@ Additionally, this project natively supports **MCP (Model Context Protocol)**, e ## Quick Start -> For detailed installation and configuration guide, please refer to [Quick Start Guide](docs/QUICK_START.md). +> For detailed installation and configuration guidance, see the [Quick Start Guide](docs/QUICK_START-en.md). ### 1. Prerequisites - JDK 17+ - MySQL 5.7+ -- Node.js 16+ +- Node.js 22+ +- pnpm 11+ +- Docker (required when a workflow executes Python steps) ### 2. Start Services @@ -75,12 +77,11 @@ Additionally, this project natively supports **MCP (Model Context Protocol)**, e mysql -u root -p < data-agent-management/src/main/resources/sql/schema.sql # 2. Start backend -cd data-agent-management -./mvnw spring-boot:run +./mvnw -pl data-agent-management spring-boot:run -# 3. Start frontend -cd data-agent-frontend -npm install && npm run dev +# 3. Start the frontend in another terminal +cd data-agent-frontend-nuxt +pnpm install && pnpm dev ``` ### 3. Access the System @@ -90,11 +91,11 @@ Open your browser and visit `http://localhost:3000` to start creating your first | Document | Contents | | :--- | :--- | -| [Quick Start](docs/QUICK_START.md) | Environment requirements, database import, basic configuration, getting started | -| [Architecture Design](docs/ARCHITECTURE.md) | System layered architecture, StateGraph and workflow design, core module sequence diagrams | -| [Developer Guide](docs/DEVELOPER_GUIDE.md) | Development environment setup, detailed configuration manual, coding standards, extension development (vector DB/models) | -| [Advanced Features](docs/ADVANCED_FEATURES.md) | API Key invocation, MCP server configuration, custom hybrid retrieval strategies, Python executor configuration | -| [Knowledge Configuration Best Practices](docs/KNOWLEDGE_USAGE.md) | Explanation and usage of semantic models, business knowledge, and agent knowledge | +| [Quick Start](docs/QUICK_START-en.md) | Environment requirements, database import, basic configuration, getting started | +| [Architecture Design](docs/ARCHITECTURE-en.md) | System layered architecture, StateGraph and workflow design, core module sequence diagrams | +| [Developer Guide](docs/DEVELOPER_GUIDE-en.md) | Development environment setup, detailed configuration manual, coding standards, extension development (vector DB/models) | +| [Advanced Features](docs/ADVANCED_FEATURES-en.md) | API Key invocation, MCP server configuration, custom hybrid retrieval strategies, SAA Python sandbox, and dynamic dependencies | +| [Knowledge Configuration Best Practices](docs/KNOWLEDGE_USAGE-en.md) | Explanation and usage of semantic models, business knowledge, and agent knowledge | ## Community & Contribution diff --git a/README.md b/README.md index fbdab0591..99ba56734 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@

- Spring AI Alibaba + Spring AI Alibaba Spring Boot Java License @@ -46,7 +46,7 @@ | 特性 | 说明 | | :--- | :--- | | **智能数据分析** | 基于 StateGraph 的 Text-to-SQL 转换,支持复杂的多表查询和多轮对话意图理解。 | -| **Python 深度分析** | 内置 Docker/Local Python 执行器,自动生成并执行 Python 代码进行统计分析与机器学习预测。 | +| **Python 深度分析** | 基于 Spring AI Alibaba Sandbox 在任务级容器中执行生成代码,支持 PEP 723 动态依赖、资源限制、失败重试与自动清理。 | | **智能报告生成** | 分析结果自动汇总为包含 ECharts 图表的 HTML/Markdown 报告,所见即所得。 | | **人工反馈机制** | 独创的 Human-in-the-loop 机制,支持用户在计划生成阶段进行干预和调整。 | | **RAG 检索增强** | 集成向量数据库,支持对业务元数据、术语库的语义检索,提升 SQL生成准确率。 | @@ -66,7 +66,9 @@ ### 1. 准备环境 - JDK 17+ - MySQL 5.7+ -- Node.js 16+ +- Node.js 22+ +- pnpm 11+ +- Docker(工作流需要执行 Python 步骤时必需) ### 2. 启动服务 @@ -75,12 +77,11 @@ mysql -u root -p < data-agent-management/src/main/resources/sql/schema.sql # 2. 启动后端 -cd data-agent-management -./mvnw spring-boot:run +./mvnw -pl data-agent-management spring-boot:run -# 3. 启动前端 -cd data-agent-frontend -npm install && npm run dev +# 3. 在另一个终端启动前端 +cd data-agent-frontend-nuxt +pnpm install && pnpm dev ``` ### 3. 访问系统 @@ -93,7 +94,7 @@ npm install && npm run dev | [快速开始](docs/QUICK_START.md) | 环境要求、数据库导入、基础配置、系统初体验 | | [架构设计](docs/ARCHITECTURE.md) | 系统分层架构、StateGraph与工作流设计、核心模块时序图 | | [开发者指南](docs/DEVELOPER_GUIDE.md) | 开发环境搭建、详细配置手册、代码规范、扩展开发(向量库/模型) | -| [高级功能](docs/ADVANCED_FEATURES.md) | API Key 调用、MCP 服务器配置、自定义混合检索策略、Python执行器配置 | +| [高级功能](docs/ADVANCED_FEATURES.md) | API Key 调用、MCP 服务器配置、自定义混合检索策略、SAA Python 沙盒与动态依赖 | | [知识配置最佳实践](docs/KNOWLEDGE_USAGE.md) | 语义模型,业务知识,智能体知识的解释和使用 | ## 🤝 加入社区 & 贡献 diff --git a/docs/ADVANCED_FEATURES-en.md b/docs/ADVANCED_FEATURES-en.md index 161c3cf98..384da7f57 100644 --- a/docs/ADVANCED_FEATURES-en.md +++ b/docs/ADVANCED_FEATURES-en.md @@ -386,7 +386,17 @@ WHERE u.name = 'Zhang San' Python always runs through Spring AI Alibaba Sandbox in a task-scoped container. Host-local, legacy Docker-pool, and AI Simulation executors are no longer available. -### SAA Sandbox Configuration +### Prerequisites + +- Spring AI Alibaba `1.1.2.2` +- A reachable Docker daemon +- The sandbox image available to that Docker daemon +- Network access from the sandbox to the configured Python package index when installing dependencies + +SQL-only requests do not create a sandbox. Docker is required only when the execution plan contains +`PYTHON_GENERATE_NODE`. + +### Configure the SAA Sandbox ```yaml spring: @@ -397,25 +407,100 @@ spring: code-timeout: 60s limit-memory: 500 cpu-core: 1 + python-max-tries-count: 5 sandbox: - image-name: ${DATAAGENT_SANDBOX_IMAGE} + docker-host: ${DATAAGENT_SANDBOX_DOCKER_HOST:unix:///var/run/docker.sock} + image-name: ${DATAAGENT_SANDBOX_IMAGE:agentscope-registry.ap-southeast-1.cr.aliyuncs.com/agentscope/runtime-sandbox-base:latest} max-concurrency: 4 queue-capacity: 10 + max-connections: 4096 package-index-url: ${DATAAGENT_PYPI_INDEX_URL:https://pypi.org/simple} dependency-install-timeout: 3m ``` -### Dynamic Dependencies +When `docker-host` is `tcp://host:port`, DataAgent explicitly connects to that endpoint. Other +values use SAA/docker-java local Docker discovery. See +[Developer Guide - Code Executor Configuration](DEVELOPER_GUIDE-en.md#5-code-executor-configuration) +for all size, output, and metadata limits. + +### Declare Dynamic Dependencies ```python # /// script -# dependencies = ["pandas>=2,<3"] +# requires-python = ">=3.10" +# dependencies = ["pandas>=2,<3", "six==1.17.0"] # /// + +import pandas as pd +import six +``` + +A script may contain at most one PEP 723 `script` block. `dependencies` supports PyPI package +names, extras, and version constraints. URLs, VCS references, local paths, environment markers, +`@` references, and pip options are rejected before a container is created. The current version +parses and preserves `requires-python`, but does not use it to switch or reject the runtime Python +version. + +Every non-standard-library import should have a matching dependency. Scripts that use only the +standard library may omit the metadata block. + +### Execution Lifecycle + +```mermaid +sequenceDiagram + participant W as PythonExecuteNode + participant E as PythonCodeExecutorService + participant S as Task BaseSandbox + participant I as Fixed Installer Bootstrap + participant C as Fixed Code Bootstrap + + W->>E: code + stdin JSON + validated dependencies + E->>S: create isolated sandbox + opt dependencies are declared + E->>I: install into /tmp/dataagent-deps + I-->>E: bounded result envelope + end + E->>C: execute code with PYTHONPATH and stdin + C-->>E: bounded stdout/stderr envelope + E-->>W: TaskResponse + E->>S: close and remove container ``` -Dependencies are installed inside the task sandbox by a fixed backend command. The model cannot -specify repositories, URLs, file paths, pip options, or shell commands. Production should use an -enterprise private PyPI proxy. +The installer invokes `python -m pip install` with a fixed argument list and never uses a shell. +Code, stdin, the dependency list, and the package index are Base64 encoded before entering the +bootstrap. The dependency directory exists only in the current task container. + +### Verification + +Submit an analysis request that explicitly requires Python and a third-party dependency. Success +means: + +1. The timeline contains Python generation, execution, analysis, and a final report. +2. Python stdout contains the dependency version and business JSON. +3. Backend logs show sandbox creation, dependency installation, code execution, and container removal. +4. `docker ps --format '{{.Names}}' | grep '^dataagent-sandbox-'` prints nothing. + +### Troubleshooting + +| Symptom | Cause and action | +|---|---| +| Docker connection fails | Run `docker info`; use `tcp://host:port` for a remote daemon and default discovery for local Docker | +| `DependencyInstallError` | Check the package name, version, and index reachability; increase `dependency-install-timeout` or use a private caching proxy for large packages | +| `Unsupported Python dependency specifier` | Remove URLs, VCS references, paths, markers, and pip options; keep only a package name and version constraints | +| `Python sandbox capacity is exhausted` | Concurrency and the wait queue are full; reduce upstream concurrency or adjust `max-concurrency` and `queue-capacity` | +| `Python sandbox task timed out` | Total time exceeded “install timeout + code timeout + 30 seconds”; inspect package download and code duration | +| A container remains after a task | Check whether the application was force-killed; normal success and failure paths close the task sandbox, and application shutdown performs global cleanup | + +### Production Security Boundary + +Task containers are non-privileged and have CPU, memory, nofile, timeout, concurrency, input, and +output limits. SAA `1.1.2.2` does not expose an API to disconnect the container immediately after +dependency installation, so production must also: + +1. Use a pinned image digest and a non-root base image. +2. Allow sandbox access only to an enterprise PyPI proxy; block public internet and business networks. +3. Never inject database, model, OSS, or Docker credentials into the sandbox. +4. Scan packages for malware, CVEs, and license policy at the proxy. ## Advanced Configuration Options diff --git a/docs/ADVANCED_FEATURES.md b/docs/ADVANCED_FEATURES.md index 2daab80de..ba43285ee 100644 --- a/docs/ADVANCED_FEATURES.md +++ b/docs/ADVANCED_FEATURES.md @@ -385,7 +385,16 @@ WHERE u.name = '张三' Python 统一通过 Spring AI Alibaba Sandbox 在任务级独立容器中执行,不再提供宿主机 Local、旧 Docker 容器池或 AI Simulation 执行器。 -### SAA 沙盒配置 +### 前置条件 + +- Spring AI Alibaba `1.1.2.2` +- 可访问的 Docker daemon +- 沙盒镜像可从 Docker daemon 获取 +- 动态安装依赖时,沙盒网络可访问配置的 Python 包索引 + +仅执行 SQL 的请求不会创建沙盒;执行计划包含 `PYTHON_GENERATE_NODE` 时才需要 Docker。 + +### 配置 SAA 沙盒 ```yaml spring: @@ -396,24 +405,93 @@ spring: code-timeout: 60s limit-memory: 500 cpu-core: 1 + python-max-tries-count: 5 sandbox: - image-name: ${DATAAGENT_SANDBOX_IMAGE} + docker-host: ${DATAAGENT_SANDBOX_DOCKER_HOST:unix:///var/run/docker.sock} + image-name: ${DATAAGENT_SANDBOX_IMAGE:agentscope-registry.ap-southeast-1.cr.aliyuncs.com/agentscope/runtime-sandbox-base:latest} max-concurrency: 4 queue-capacity: 10 + max-connections: 4096 package-index-url: ${DATAAGENT_PYPI_INDEX_URL:https://pypi.org/simple} dependency-install-timeout: 3m ``` -### 动态依赖 +`docker-host` 使用 `tcp://host:port` 时会显式连接该端点;其他值走 SAA/docker-java 的 +本地 Docker 发现。完整大小、输出和元数据限制见 +[开发者指南 - 代码执行器配置](DEVELOPER_GUIDE.md#5-代码执行器配置-code-executor)。 + +### 声明动态依赖 ```python # /// script -# dependencies = ["pandas>=2,<3"] +# requires-python = ">=3.10" +# dependencies = ["pandas>=2,<3", "six==1.17.0"] # /// + +import pandas as pd +import six +``` + +脚本最多只能包含一个 PEP 723 `script` 块。`dependencies` 支持 PyPI 包名、extras 和 +版本约束;URL、VCS、本地路径、环境标记、`@` 引用和 pip 参数会在创建容器前被拒绝。 +当前版本会解析并保留 `requires-python`,但不会据此切换或拒绝运行时 Python 版本。 + +每一个非标准库 `import` 都应有对应依赖。只使用标准库时可以省略元数据块。 + +### 执行生命周期 + +```mermaid +sequenceDiagram + participant W as PythonExecuteNode + participant E as PythonCodeExecutorService + participant S as Task BaseSandbox + participant I as Fixed Installer Bootstrap + participant C as Fixed Code Bootstrap + + W->>E: code + stdin JSON + validated dependencies + E->>S: create isolated sandbox + opt dependencies are declared + E->>I: install into /tmp/dataagent-deps + I-->>E: bounded result envelope + end + E->>C: execute code with PYTHONPATH and stdin + C-->>E: bounded stdout/stderr envelope + E-->>W: TaskResponse + E->>S: close and remove container ``` -依赖由固定的后端命令在当前任务沙盒内安装。模型不能指定仓库、URL、文件路径、pip 参数 -或 shell 命令。生产环境应将包索引配置为企业私有 PyPI 代理。 +安装器固定执行 `python -m pip install` 参数列表,不经过 shell。代码、stdin、依赖清单 +和包索引在进入 bootstrap 前使用 Base64 编码。依赖安装目录只存在于本次任务容器。 + +### 验证 + +在数据问答页提交一个明确要求 Python 和第三方依赖的分析任务。成功标准: + +1. 时间线出现 Python 生成、执行、分析和最终报告。 +2. Python 标准输出包含依赖版本和业务 JSON。 +3. 后端日志包含沙盒创建、依赖安装、代码执行和容器删除。 +4. `docker ps --format '{{.Names}}' | grep '^dataagent-sandbox-'` 无输出。 + +### 故障排查 + +| 现象 | 原因与处理 | +|---|---| +| Docker 连接失败 | 先运行 `docker info`;远程 daemon 使用 `tcp://host:port`,本地 Docker 保持默认发现 | +| `DependencyInstallError` | 检查包名、版本和索引连通性;大包下载超时可增大 `dependency-install-timeout` 或使用私有缓存代理 | +| `Unsupported Python dependency specifier` | 移除 URL、VCS、路径、环境标记或 pip 参数,只保留包名和版本约束 | +| `Python sandbox capacity is exhausted` | 并发数和等待队列已满;降低上游并发或调整 `max-concurrency`、`queue-capacity` | +| `Python sandbox task timed out` | 总时间超过“依赖安装超时 + 代码超时 + 30 秒”;检查包下载或代码耗时 | +| 任务后仍有容器 | 检查应用是否被强制终止;正常成功和失败路径都会关闭任务沙盒,应用关闭还会执行全量清理 | + +### 生产安全边界 + +任务容器默认非 privileged,并限制 CPU、内存、nofile、超时、并发和输入输出大小。但 +SAA `1.1.2.2` 没有提供“安装完成后立即断网”的公开 API,因此生产环境还必须: + +1. 使用固定镜像 digest 和非 root 基础镜像。 +2. 仅允许沙盒访问企业私有 PyPI 代理,禁止访问公网和业务内网。 +3. 不向沙盒注入数据库、模型、OSS 或 Docker 凭据。 +4. 在代理侧执行恶意包、CVE 和许可证扫描。 ## ⚙️ 高级配置选项 diff --git a/docs/ARCHITECTURE-en.md b/docs/ARCHITECTURE-en.md index cba9cce03..0057429b4 100644 --- a/docs/ARCHITECTURE-en.md +++ b/docs/ARCHITECTURE-en.md @@ -10,7 +10,7 @@ This document provides a detailed introduction to DataAgent's system architectur %%{init: {"theme": "base", "flowchart": {"curve": "basis", "nodeSpacing": 35, "rankSpacing": 45}, "themeVariables": {"lineColor": "#475569", "primaryTextColor": "#1F2937"}}}%% flowchart LR subgraph Clients[Clients] - UserUI[data-agent-frontend UI] + UserUI[data-agent-frontend-nuxt UI] AdminUI[Admin Console] MCPClient[MCP Client] end @@ -48,10 +48,11 @@ flowchart LR EmbeddingLLM[Embedding Model] end - subgraph Exec[Python Runtime] - Docker[Docker Executor] - Local[Local Executor] - AISim[AI Simulation Executor] + subgraph Exec[Python Sandbox Runtime] + SandboxSvc[SAA SandboxService] + DockerEngine[Docker Engine] + TaskSandbox[Task-scoped BaseSandbox] + PackageIndex[(PyPI / Private Package Index)] end UserUI --> RestAPI @@ -74,10 +75,11 @@ flowchart LR GraphSvc --> ModelRegistry ModelRegistry --> ChatLLM ModelRegistry --> EmbeddingLLM - GraphSvc --> CodePool - CodePool --> Docker - CodePool --> Local - CodePool --> AISim + Graph --> CodeExec + CodeExec --> SandboxSvc + SandboxSvc --> DockerEngine + DockerEngine --> TaskSandbox + TaskSandbox --> PackageIndex AgentCtl --> MetaDB PromptCtl --> MetaDB ModelCtl --> MetaDB @@ -94,11 +96,11 @@ flowchart LR class UserUI,AdminUI,MCPClient client class RestAPI,SSE access class GraphCtl,AgentCtl,PromptCtl,ModelCtl api - class GraphSvc,Context,LlmSvc,ModelRegistry,VectorSvc,Hybrid,CodePool,McpSvc service + class GraphSvc,Context,LlmSvc,ModelRegistry,VectorSvc,Hybrid,CodeExec,McpSvc service class Graph workflow class BizDB,MetaDB,VectorDB,Files data class ChatLLM,EmbeddingLLM llm - class Docker,Local,AISim exec + class SandboxSvc,DockerEngine,TaskSandbox,PackageIndex exec style Clients fill:#FFF7ED,stroke:#D97706,stroke-width:1.5px style Access fill:#EFF6FF,stroke:#0284C7,stroke-width:1.5px @@ -660,8 +662,24 @@ sequenceDiagram - **Code Execution**: `PythonExecuteNode` uses `PythonCodeExecutorService` and always runs in a task-scoped SAA sandbox - **Dynamic Dependencies**: Generated code declares dependencies with PEP 723; a fixed bootstrap installs them inside the sandbox - **Execution Configuration**: `spring.ai.alibaba.data-agent.code-executor.*` +- **Isolation and Cleanup**: Every attempt creates a new `BaseSandbox`; the container is closed and removed after success or failure +- **Failure Handling**: Installation or execution failures are fed into the next code-generation attempt; the existing fallback or termination path applies after the retry limit - **Result Return**: Execution results are written back to `PYTHON_EXECUTE_NODE_OUTPUT`, `PythonAnalyzeNode` summarizes and writes to `SQL_EXECUTE_NODE_OUTPUT` for final report +#### Code Boundaries + +```text +service/code/sandbox +├── SaaSandboxPythonCodeExecutorService.java # input limits, bounded queue, total timeout +├── dependency/ # PEP 723 parsing and package policy +├── execution/ # fixed bootstraps and result envelope +└── runtime/ # SAA service and task sandbox lifecycle +``` + +`PythonCodeExecutorService` is the stable workflow interface. Its only implementation is +`SaaSandboxPythonCodeExecutorService`. DataAgent does not fall back to host-local execution, +local Python processes, the legacy Docker pool, or AI Simulation. + #### Architecture Diagram ```mermaid @@ -699,6 +717,7 @@ sequenceDiagram participant E as PythonExecuteNode participant CP as PythonCodeExecutorService participant S as SAA Sandbox + participant D as PythonExecutorDispatcher participant J as JsonParseUtil participant A as PythonAnalyzeNode participant R as ReportGeneratorNode @@ -708,15 +727,25 @@ sequenceDiagram L-->>G: python code + PEP 723 G->>E: pass code and sql results E->>CP: runTask - CP->>S: create sandbox and install dependencies + CP->>S: create task sandbox and install dependencies S->>S: execute with stdin S-->>CP: stdout stderr CP-->>E: task response - E->>J: parse stdout json - J-->>E: normalized output - E->>A: analyze result - A-->>P: update step results - P->>R: continue to report + alt execution succeeded + E->>J: parse stdout json + J-->>E: normalized output + E->>D: publish success state + D->>A: analyze result + A-->>P: update step results + P->>R: continue to report + else execution failed + E->>D: publish failure state + alt retry is available + D->>G: regenerate with previous code and error + else retry limit reached + D-->>P: fallback analysis or terminate + end + end ``` @@ -726,3 +755,4 @@ sequenceDiagram - [Quick Start](QUICK_START-en.md) - Installation and configuration guide - [Advanced Features](ADVANCED_FEATURES-en.md) - API calls and MCP server - [Developer Documentation](DEVELOPER_GUIDE-en.md) - Contribution guide +- [SAA 1.1.2.2 Python Sandbox Integration Design](superpowers/specs/2026-07-28-saa-python-sandbox-integration-design.md) - Detailed boundaries, security trade-offs, and acceptance evidence diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 83653792d..7ee4d9fd8 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -10,7 +10,7 @@ %%{init: {"theme": "base", "flowchart": {"curve": "basis", "nodeSpacing": 35, "rankSpacing": 45}, "themeVariables": {"lineColor": "#475569", "primaryTextColor": "#1F2937"}}}%% flowchart LR subgraph Clients[Clients] - UserUI[data-agent-frontend UI] + UserUI[data-agent-frontend-nuxt UI] AdminUI[Admin Console] MCPClient[MCP Client] end @@ -53,10 +53,11 @@ flowchart LR EmbeddingLLM[Embedding Model] end - subgraph Exec[Python Runtime] - Docker[Docker Executor] - Local[Local Executor] - AISim[AI Simulation Executor] + subgraph Exec[Python Sandbox Runtime] + SandboxSvc[SAA SandboxService] + DockerEngine[Docker Engine] + TaskSandbox[Task-scoped BaseSandbox] + PackageIndex[(PyPI / Private Package Index)] end UserUI --> RestAPI @@ -79,10 +80,11 @@ flowchart LR GraphSvc --> ModelRegistry ModelRegistry --> ChatLLM ModelRegistry --> EmbeddingLLM - GraphSvc --> CodePool - CodePool --> Docker - CodePool --> Local - CodePool --> AISim + Graph --> CodeExec + CodeExec --> SandboxSvc + SandboxSvc --> DockerEngine + DockerEngine --> TaskSandbox + TaskSandbox --> PackageIndex GraphSvc --> LangfuseSvc LangfuseSvc --> Langfuse AgentCtl --> MetaDB @@ -102,11 +104,11 @@ flowchart LR class UserUI,AdminUI,MCPClient client class RestAPI,SSE access class GraphCtl,AgentCtl,PromptCtl,ModelCtl api - class GraphSvc,Context,LlmSvc,ModelRegistry,VectorSvc,Hybrid,CodePool,McpSvc,LangfuseSvc service + class GraphSvc,Context,LlmSvc,ModelRegistry,VectorSvc,Hybrid,CodeExec,McpSvc,LangfuseSvc service class Graph workflow class BizDB,MetaDB,VectorDB,Files data class ChatLLM,EmbeddingLLM llm - class Docker,Local,AISim exec + class SandboxSvc,DockerEngine,TaskSandbox,PackageIndex exec class Langfuse observability style Clients fill:#FFF7ED,stroke:#D97706,stroke-width:1.5px @@ -669,8 +671,24 @@ sequenceDiagram - **代码执行**: `PythonExecuteNode` 使用 `PythonCodeExecutorService`,统一进入任务级 SAA 沙盒 - **动态依赖**: 生成代码通过 PEP 723 声明依赖,由固定 bootstrap 在沙盒内受控安装 - **执行配置**: `spring.ai.alibaba.data-agent.code-executor.*` +- **隔离与清理**: 每次尝试创建新的 `BaseSandbox`,成功或失败后均关闭并删除容器 +- **失败处理**: 安装或执行失败会把错误反馈给下一次代码生成;达到最大次数后走现有降级或终止分支 - **结果回传**: 执行结果写回 `PYTHON_EXECUTE_NODE_OUTPUT`,`PythonAnalyzeNode` 汇总后写入 `SQL_EXECUTE_NODE_OUTPUT`,用于最终报告 +#### 代码边界 + +```text +service/code/sandbox +├── SaaSandboxPythonCodeExecutorService.java # 输入限制、并发队列、任务总超时 +├── dependency/ # PEP 723 解析与包规格策略 +├── execution/ # 固定 bootstrap 与结果 envelope +└── runtime/ # SAA 服务和任务 Sandbox 生命周期 +``` + +`PythonCodeExecutorService` 是工作流依赖的稳定接口,当前只有 +`SaaSandboxPythonCodeExecutorService` 一个实现。系统不会回退到宿主机、本地 Python +进程、旧 Docker 容器池或 AI Simulation。 + #### 架构图 ```mermaid @@ -708,6 +726,7 @@ sequenceDiagram participant E as PythonExecuteNode participant CP as PythonCodeExecutorService participant S as SAA Sandbox + participant D as PythonExecutorDispatcher participant J as JsonParseUtil participant A as PythonAnalyzeNode participant R as ReportGeneratorNode @@ -717,15 +736,25 @@ sequenceDiagram L-->>G: python code + PEP 723 G->>E: pass code and sql results E->>CP: runTask - CP->>S: create sandbox and install dependencies + CP->>S: create task sandbox and install dependencies S->>S: execute with stdin S-->>CP: stdout stderr CP-->>E: task response - E->>J: parse stdout json - J-->>E: normalized output - E->>A: analyze result - A-->>P: update step results - P->>R: continue to report + alt execution succeeded + E->>J: parse stdout json + J-->>E: normalized output + E->>D: publish success state + D->>A: analyze result + A-->>P: update step results + P->>R: continue to report + else execution failed + E->>D: publish failure state + alt retry is available + D->>G: regenerate with previous code and error + else retry limit reached + D-->>P: fallback analysis or terminate + end + end ``` @@ -735,3 +764,4 @@ sequenceDiagram - [快速开始](QUICK_START.md) - 安装配置指南 - [高级功能](ADVANCED_FEATURES.md) - API调用和MCP服务器 - [开发者文档](DEVELOPER_GUIDE.md) - 贡献指南 +- [SAA 1.1.2.2 Python 沙盒接入方案](superpowers/specs/2026-07-28-saa-python-sandbox-integration-design.md) - 详细边界、安全取舍和验收证据 diff --git a/docs/DEVELOPER_GUIDE-en.md b/docs/DEVELOPER_GUIDE-en.md index 46783523d..6f204cba8 100644 --- a/docs/DEVELOPER_GUIDE-en.md +++ b/docs/DEVELOPER_GUIDE-en.md @@ -10,16 +10,18 @@ Welcome to participate in the development of the DataAgent project! This documen - **JDK**: 17 or higher - **Maven**: 3.6 or higher -- **Node.js**: 16 or higher +- **Node.js**: 22 +- **pnpm**: 11 - **MySQL**: 5.7 or higher +- **Docker**: Required only when running or verifying Python workflows - **Git**: Version control tool - **IDE**: IntelliJ IDEA or Eclipse (IntelliJ IDEA recommended) ### Clone Project ```bash -git clone https://github.com/your-org/spring-ai-alibaba-data-agent.git -cd spring-ai-alibaba-data-agent +git clone https://github.com/spring-ai-alibaba/DataAgent.git +cd DataAgent ``` ### Backend Development Environment @@ -34,21 +36,20 @@ cd spring-ai-alibaba-data-agent 3. **Start Backend Service** ```bash - cd data-agent-management - ./mvnw spring-boot:run + ./mvnw -pl data-agent-management spring-boot:run ``` ### Frontend Development Environment 1. **Install Dependencies** ```bash - cd data-agent-frontend - npm install + cd data-agent-frontend-nuxt + pnpm install ``` 2. **Start Development Server** ```bash - npm run dev + pnpm dev ``` 3. **Access Application** @@ -67,6 +68,8 @@ The workflow is based on Spring AI Alibaba's StateGraph implementation. Core nod - **PlannerNode**: Plan generation - **SqlGenerateNode**: SQL generation - **PythonGenerateNode**: Python code generation +- **PythonExecuteNode**: Parse PEP 723 metadata and dispatch SAA sandbox execution +- **PythonAnalyzeNode**: Analyze Python output and update step state - **ReportGeneratorNode**: Report generation ### 2. Multi-Model Scheduling @@ -304,15 +307,42 @@ Configuration prefix: `spring.ai.alibaba.data-agent.code-executor` | `limit-memory` | Container memory limit (MB) | 500 | | `cpu-core` | Container CPU cores | 1 | | `python-max-tries-count` | Maximum Python execution retries | 5 | -| `sandbox.image-name` | SAA base image | AgentScope runtime sandbox base | +| `sandbox.docker-host` | Docker Engine endpoint | `unix:///var/run/docker.sock` | +| `sandbox.image-name` | SAA base image | `agentscope-registry.ap-southeast-1.cr.aliyuncs.com/agentscope/runtime-sandbox-base:latest` | +| `sandbox.container-prefix` | Task container name prefix | `dataagent-sandbox-` | | `sandbox.max-concurrency` | Maximum concurrent sandboxes | 4 | | `sandbox.queue-capacity` | Bounded wait queue size | 10 | -| `sandbox.package-index-url` | Dynamic package index | https://pypi.org/simple | +| `sandbox.max-code-bytes` | Python source UTF-8 byte limit | 262144 (256 KiB) | +| `sandbox.max-input-bytes` | stdin JSON UTF-8 byte limit | 10485760 (10 MiB) | +| `sandbox.max-output-bytes` | stdout UTF-8 byte limit | 1048576 (1 MiB) | +| `sandbox.max-error-bytes` | stderr UTF-8 byte limit | 262144 (256 KiB) | +| `sandbox.max-metadata-bytes` | PEP 723 metadata UTF-8 byte limit | 8192 (8 KiB) | +| `sandbox.max-dependencies` | Maximum number of direct dependencies | 20 | +| `sandbox.package-index-url` | Dynamic package index | `https://pypi.org/simple` | | `sandbox.dependency-install-timeout` | Dependency installation timeout | 3m | +| `sandbox.max-connections` | Container `nofile` limit | 4096 | Third-party dependencies must be declared in the generated script's PEP 723 `dependencies`. Host-local, legacy Docker pool, and AI Simulation executors are no longer available. +Common environment variables: + +| Environment Variable | Configuration | Purpose | +|---|---|---| +| `DATAAGENT_SANDBOX_DOCKER_HOST` | `sandbox.docker-host` | Point to a local or remote Docker Engine | +| `DATAAGENT_SANDBOX_IMAGE` | `sandbox.image-name` | Pin the runtime image; use a digest in production | +| `DATAAGENT_PYPI_INDEX_URL` | `sandbox.package-index-url` | Point to an enterprise private PyPI proxy | + +Each Python task creates a separate `BaseSandbox`, installs dependencies and executes code in the same +container, then stops and removes that container. The service-side wait timeout is the dependency +installation timeout plus the code timeout plus a 30-second transport margin. `requires-python` is +currently parsed and retained, but it does not select or validate the sandbox Python version. + +See [Advanced Features - Python Execution Environment Configuration](ADVANCED_FEATURES-en.md#python-execution-environment-configuration) +for dependency syntax, security restrictions, runtime verification, and troubleshooting. See the +[SAA 1.1.2.2 Python Sandbox Integration Design](superpowers/specs/2026-07-28-saa-python-sandbox-integration-design.md) +for implementation boundaries. + ### 6. File Storage Configuration Configuration prefix: `spring.ai.alibaba.data-agent.file` @@ -344,7 +374,7 @@ Configuration prefix: `spring.sql.init` | Configuration Item | Description | Default Value | Notes | |-------------------|-------------|---------------|-------| -| `mode` | Initialization mode (always/never) | always | "always" executes schema.sql and data.sql on every startup. Recommended to set to "never" for production to avoid sample data overwriting business data | +| `mode` | Initialization mode (always/never) | never | Set to `always` only when initialization is explicitly required | | `schema-locations` | Table structure script path | classpath:sql/schema.sql | | | `data-locations` | Data script path | classpath:sql/data.sql | | @@ -378,13 +408,43 @@ Environment variables: `LANGFUSE_ENABLED`, `LANGFUSE_HOST`, `LANGFUSE_PUBLIC_KEY > For detailed usage, refer to [Advanced Features - Langfuse Observability](ADVANCED_FEATURES-en.md#langfuse-observability). +## Python Sandbox Verification + +Unit tests that do not require Docker: + +```bash +./mvnw -pl data-agent-management \ + -Dtest='PythonDependencyMetadataParserTest,PythonSandboxBootstrapBuilderTest,SandboxExecutionResultParserTest,SaaSandboxPythonCodeExecutorServiceTest,SaaSandboxRuntimeTest,PythonExecuteNodeTest,PythonWorkflowIntegrationTest' \ + test +``` + +Run the real SAA integration test while Docker is available: + +```bash +docker info +./mvnw -pl data-agent-management -Dtest=SaaSandboxTaskRunnerIT test +``` + +Run the CI-equivalent checks before submission: + +```bash +make format-check +make checkstyle-check +make test +``` + +A real end-to-end acceptance check must go beyond HTTP 200: the browser timeline must show dependency +installation and Python execution output, a final report, and SSE `event:complete`. The command +`docker ps -a --filter name=dataagent-sandbox-` must not show leftover task containers. + ## Learning Resources ### Official Documentation - [Spring AI Alibaba Documentation](https://springdoc.cn/spring-ai/) - [Spring Boot Documentation](https://spring.io/projects/spring-boot) -- [React Documentation](https://react.dev/) +- [Nuxt Documentation](https://nuxt.com/docs) +- [Vue Documentation](https://vuejs.org/guide/) - [TypeScript Documentation](https://www.typescriptlang.org/) ### Related Technologies @@ -396,7 +456,7 @@ Environment variables: `LANGFUSE_ENABLED`, `LANGFUSE_HOST`, `LANGFUSE_PUBLIC_KEY ## Contribution Guide -For detailed contribution guidelines, please refer to [CONTRIBUTING.md](../CONTRIBUTING.md). +For detailed contribution guidelines, see [CONTRIBUTING-en.md](../CONTRIBUTING-en.md). ### Contribution Types diff --git a/docs/DEVELOPER_GUIDE.md b/docs/DEVELOPER_GUIDE.md index 15ed810e7..8bc557c72 100644 --- a/docs/DEVELOPER_GUIDE.md +++ b/docs/DEVELOPER_GUIDE.md @@ -10,16 +10,18 @@ - **JDK**: 17 或更高版本 - **Maven**: 3.6 或更高版本 -- **Node.js**: 16 或更高版本 +- **Node.js**: 22 +- **pnpm**: 11 - **MySQL**: 5.7 或更高版本 +- **Docker**: 仅运行或验证 Python 工作流时需要 - **Git**: 版本控制工具 - **IDE**: IntelliJ IDEA 或 Eclipse (推荐 IntelliJ IDEA) ### 克隆项目 ```bash -git clone https://github.com/your-org/spring-ai-alibaba-data-agent.git -cd spring-ai-alibaba-data-agent +git clone https://github.com/spring-ai-alibaba/DataAgent.git +cd DataAgent ``` ### 后端开发环境 @@ -34,21 +36,20 @@ cd spring-ai-alibaba-data-agent 3. **启动后端服务** ```bash - cd data-agent-management - ./mvnw spring-boot:run + ./mvnw -pl data-agent-management spring-boot:run ``` ### 前端开发环境 1. **安装依赖** ```bash - cd data-agent-frontend - npm install + cd data-agent-frontend-nuxt + pnpm install ``` 2. **启动开发服务器** ```bash - npm run dev + pnpm dev ``` 3. **访问应用** @@ -67,6 +68,8 @@ cd spring-ai-alibaba-data-agent - **PlannerNode**: 计划生成 - **SqlGenerateNode**: SQL 生成 - **PythonGenerateNode**: Python 代码生成 +- **PythonExecuteNode**: 解析 PEP 723 元数据并调度 SAA 沙盒执行 +- **PythonAnalyzeNode**: 分析 Python 结果并更新步骤状态 - **ReportGeneratorNode**: 报告生成 ### 2. 多模型调度 @@ -357,15 +360,40 @@ public class AgentVectorStoreService { | `limit-memory` | 容器内存限制 (MB) | 500 | | `cpu-core` | 容器CPU核数 | 1 | | `python-max-tries-count` | Python执行最大重试次数 | 5 | -| `sandbox.image-name` | SAA基础镜像 | AgentScope runtime sandbox base | +| `sandbox.docker-host` | Docker Engine 地址 | `unix:///var/run/docker.sock` | +| `sandbox.image-name` | SAA 基础镜像 | `agentscope-registry.ap-southeast-1.cr.aliyuncs.com/agentscope/runtime-sandbox-base:latest` | +| `sandbox.container-prefix` | 任务容器名称前缀 | `dataagent-sandbox-` | | `sandbox.max-concurrency` | 最大并发沙盒数 | 4 | | `sandbox.queue-capacity` | 有界等待队列大小 | 10 | -| `sandbox.package-index-url` | 动态依赖包索引 | https://pypi.org/simple | +| `sandbox.max-code-bytes` | Python 源码 UTF-8 字节上限 | 262144(256 KiB) | +| `sandbox.max-input-bytes` | stdin JSON UTF-8 字节上限 | 10485760(10 MiB) | +| `sandbox.max-output-bytes` | stdout UTF-8 字节上限 | 1048576(1 MiB) | +| `sandbox.max-error-bytes` | stderr UTF-8 字节上限 | 262144(256 KiB) | +| `sandbox.max-metadata-bytes` | PEP 723 元数据 UTF-8 字节上限 | 8192(8 KiB) | +| `sandbox.max-dependencies` | 最大直接依赖数 | 20 | +| `sandbox.package-index-url` | 动态依赖包索引 | `https://pypi.org/simple` | | `sandbox.dependency-install-timeout` | 依赖安装超时 | 3m | +| `sandbox.max-connections` | 容器 `nofile` 上限 | 4096 | 第三方依赖必须在生成脚本的 PEP 723 `dependencies` 中声明。系统不再提供宿主机 Local、 旧 Docker 容器池或 AI Simulation 执行器。 +常用环境变量: + +| 环境变量 | 对应配置 | 用途 | +|---|---|---| +| `DATAAGENT_SANDBOX_DOCKER_HOST` | `sandbox.docker-host` | 指向本机或远程 Docker Engine | +| `DATAAGENT_SANDBOX_IMAGE` | `sandbox.image-name` | 固定运行时镜像;生产环境应使用 digest | +| `DATAAGENT_PYPI_INDEX_URL` | `sandbox.package-index-url` | 指向企业私有 PyPI 代理 | + +每次 Python 任务创建独立 `BaseSandbox`,在同一容器内先安装依赖、再执行代码,最后停止并 +删除容器。服务端总等待时间为“依赖安装超时 + 代码执行超时 + 30 秒通信余量”。 +`requires-python` 当前会被解析和保留,但不会切换或校验沙盒 Python 版本。 + +依赖格式、安全限制、运行验证和故障处理见 +[高级功能 - Python 执行环境配置](ADVANCED_FEATURES.md#-python-执行环境配置);实现边界见 +[SAA 1.1.2.2 Python 沙盒接入方案](superpowers/specs/2026-07-28-saa-python-sandbox-integration-design.md)。 + ### 6. 文件存储配置 (File Storage) 配置前缀: `spring.ai.alibaba.data-agent.file` @@ -397,7 +425,7 @@ public class AgentVectorStoreService { | 配置项 | 说明 | 默认值 | 备注 | |--------|------|--------|------| -| `mode` | 初始化模式 (always/never) | always | "always"会每次启动执行schema.sql和data.sql,建议生产环境设为"never" | +| `mode` | 初始化模式 (always/never) | never | 仅在明确需要初始化时设置为 `always` | | `schema-locations` | 表结构脚本路径 | classpath:sql/schema.sql | | | `data-locations` | 数据脚本路径 | classpath:sql/data.sql | | @@ -431,13 +459,43 @@ public class AgentVectorStoreService { > 详细使用说明请参考 [高级功能 - Langfuse 可观测性](ADVANCED_FEATURES.md#-langfuse-可观测性)。 +## ✅ Python 沙盒验证 + +不需要 Docker 的单元测试: + +```bash +./mvnw -pl data-agent-management \ + -Dtest='PythonDependencyMetadataParserTest,PythonSandboxBootstrapBuilderTest,SandboxExecutionResultParserTest,SaaSandboxPythonCodeExecutorServiceTest,SaaSandboxRuntimeTest,PythonExecuteNodeTest,PythonWorkflowIntegrationTest' \ + test +``` + +Docker 在线时运行真实 SAA 集成测试: + +```bash +docker info +./mvnw -pl data-agent-management -Dtest=SaaSandboxTaskRunnerIT test +``` + +提交前执行与 CI 对齐的检查: + +```bash +make format-check +make checkstyle-check +make test +``` + +真实端到端验收不能只看 HTTP 200:浏览器时间线应出现依赖安装和 Python 执行结果、最终报告, +SSE 应收到 `event:complete`,并且 `docker ps -a --filter name=dataagent-sandbox-` 不应留下 +任务容器。 + ## 📚 学习资源 ### 官方文档 - [Spring AI Alibaba 文档](https://springdoc.cn/spring-ai/) - [Spring Boot 文档](https://spring.io/projects/spring-boot) -- [React 文档](https://react.dev/) +- [Nuxt 文档](https://nuxt.com/docs) +- [Vue 文档](https://vuejs.org/guide/) - [TypeScript 文档](https://www.typescriptlang.org/) ### 相关技术 diff --git a/docs/QUICK_START-en.md b/docs/QUICK_START-en.md index bf6019a3a..b3708bb14 100644 --- a/docs/QUICK_START-en.md +++ b/docs/QUICK_START-en.md @@ -8,8 +8,9 @@ This document will guide you through the installation, configuration, and first - **JDK**: 17 or higher - **MySQL**: 5.7 or higher -- **Node.js**: 16 or higher -- **Docker**: (Optional) For Python code execution +- **Node.js**: 22 or higher +- **pnpm**: 11 or higher +- **Docker**: Required when a workflow executes Python steps; optional for SQL-only analysis - **Vector Database**: (Optional) Uses in-memory vector store by default ## 1. Business Database Preparation @@ -38,23 +39,25 @@ mysql -u root -p your_database < data-agent-management/src/main/resources/sql/pr Configure your MySQL database connection in `data-agent-management/src/main/resources/application.yml`. -> Initialization Behavior: Auto table creation and sample data insertion is enabled by default (`spring.sql.init.mode: always`). For production environments, it's recommended to disable this to avoid sample data overwriting your business data. +> Initialization behavior: the default is `spring.sql.init.mode: never`, so DataAgent does not +> create tables or insert sample data automatically. Run the SQL files above before the first +> start, or set `DATA_AGENT_DATASOURCE_SQL_INIT=always` only when sample initialization is +> explicitly required. ```yaml spring: datasource: url: jdbc:mysql://127.0.0.1:3306/saa_data_agent?useUnicode=true&characterEncoding=utf-8&zeroDateTimeBehavior=convertToNull&transformedBitIsBoolean=true&allowMultiQueries=true&allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=Asia/Shanghai - username: ${MYSQL_USERNAME:root} - password: ${MYSQL_PASSWORD:root} + username: ${DATA_AGENT_DATASOURCE_USERNAME:root} + password: ${DATA_AGENT_DATASOURCE_PASSWORD:root} driver-class-name: com.mysql.cj.jdbc.Driver - type: com.alibaba.druid.pool.DruidDataSource ``` ### 2.2 Data Initialization Configuration -Auto initialization is enabled by default (`spring.sql.init.mode: always`). +Auto initialization is disabled by default (`spring.sql.init.mode: never`). -> For information on how to disable auto initialization, please refer to [Developer Guide - Database Initialization Configuration](DEVELOPER_GUIDE-en.md#8-database-initialization). +> To change initialization behavior, see [Developer Guide - Database Initialization](DEVELOPER_GUIDE-en.md#8-database-initialization). ### 2.3 Configure Model @@ -77,7 +80,7 @@ Start the project, click on Model Configuration, add a new model and fill in you ### 2.4 Embedding Model Batch Processing Strategy Configuration -> For detailed configuration parameters, please refer to [Developer Guide - Development Configuration Manual](DEVELOPER_GUIDE-en.md#development-configuration-manual). +> For detailed configuration parameters, see [Developer Guide - Embedding Batch Configuration](DEVELOPER_GUIDE-en.md#2-embedding-batch-configuration). ### 2.5 Vector Store Configuration @@ -177,53 +180,93 @@ Below is the ES schema structure. For other vector stores like Milvus, PG, etc., #### 2.5.3 Vector Store Configuration Parameters -> For detailed configuration parameters, please refer to [Developer Guide - Development Configuration Manual](DEVELOPER_GUIDE-en.md#development-configuration-manual). +> For detailed configuration parameters, see [Developer Guide - Vector Store Configuration](DEVELOPER_GUIDE-en.md#3-vector-store-configuration). ### 2.6 Retrieval Fusion Strategy -> For detailed configuration parameters, please refer to [Developer Guide - Development Configuration Manual](DEVELOPER_GUIDE-en.md#development-configuration-manual). +> For detailed configuration parameters, see [Developer Guide - Vector Store Configuration](DEVELOPER_GUIDE-en.md#3-vector-store-configuration). ### 2.7 Replace Vector Store Implementation -> For information on how to replace the default in-memory vector store (e.g., using PGVector, Milvus, etc.), please refer to [Developer Guide - Dependency Extension Configuration](DEVELOPER_GUIDE-en.md#9-dependency-extension). +> To replace the default in-memory vector store (for example, with PGVector or Milvus), see [Developer Guide - Vector Store Dependency Extension](DEVELOPER_GUIDE-en.md#vector-store-dependency-extension). + +### 2.8 Configure the Python Sandbox + +DataAgent uses the Spring AI Alibaba `1.1.2.2` Sandbox to run generated Python code. Every +execution gets a task-scoped container; dynamic dependency installation, code execution, and +container cleanup all happen inside that task. + +Confirm that Docker is available: + +```bash +docker info +``` + +The default configuration uses local Docker discovery, public PyPI, and the AgentScope base +image. Development normally needs no override. To select a Docker endpoint, image, or private +package index, set: + +```bash +export DATAAGENT_SANDBOX_DOCKER_HOST=unix:///var/run/docker.sock +export DATAAGENT_SANDBOX_IMAGE=agentscope-registry.ap-southeast-1.cr.aliyuncs.com/agentscope/runtime-sandbox-base:latest +export DATAAGENT_PYPI_INDEX_URL=https://pypi.org/simple +``` + +Production should use a pinned image digest and an enterprise PyPI proxy, with infrastructure +network policy that allows the sandbox to reach only that proxy. See +[Advanced Features - Python Execution Environment Configuration](ADVANCED_FEATURES-en.md#python-execution-environment-configuration) +for the full configuration, dependency contract, and troubleshooting guidance. ## 3. Start Management Backend -In the `data-agent-management` directory, run the `DataAgentApplication.java` class. +Run the following command from the project root: ```bash -cd data-agent-management -./mvnw spring-boot:run +./mvnw -pl data-agent-management spring-boot:run ``` Or run `DataAgentApplication.java` directly in your IDE. ## 4. Start Web Frontend -Navigate to the `data-agent-frontend` directory +Navigate to the `data-agent-frontend-nuxt` directory. ### 4.1 Install Dependencies ```bash -# Using npm -npm install - -# Or using yarn -yarn install +pnpm install ``` ### 4.2 Start Service ```bash -# Using npm -npm run dev - -# Or using yarn -yarn dev +pnpm dev ``` After successful startup, access http://localhost:3000 +### 4.3 Verify Dynamic Python Dependencies + +After configuring the agent, models, and data source, submit a request that explicitly includes a +Python step: + +```text +Query the raw status and total_amount rows from orders, then aggregate them with Python. +Declare and import six==1.17.0 through PEP 723, and report the six version and status totals. +``` + +A successful timeline contains “Python Generation”, “Python Execution”, “Python Analysis”, and a +final report. The Python output should contain `six_version: 1.17.0`. No task container should +remain after completion: + +```bash +docker ps --format '{{.Names}}' | grep '^dataagent-sandbox-' +``` + +No output means the task sandbox was removed. If dependency installation fails, the workflow +feeds the error into the next Python generation attempt. After the configured retry limit, it +enters the existing fallback or termination path. + ## 5. System Experience ### 5.1 Creating and Configuring Data Agent diff --git a/docs/QUICK_START.md b/docs/QUICK_START.md index dfce88b54..9754c1193 100644 --- a/docs/QUICK_START.md +++ b/docs/QUICK_START.md @@ -8,8 +8,9 @@ - **JDK**: 17 或更高版本 - **MySQL**: 5.7 或更高版本 -- **Node.js**: 16 或更高版本 -- **Docker**: (可选) 用于Python代码执行 +- **Node.js**: 22 或更高版本 +- **pnpm**: 11 或更高版本 +- **Docker**: 工作流需要执行 Python 步骤时必需;仅使用 SQL 分析时可不启动 - **向量数据库**: (可选) 默认使用内存向量库 ## 🗄️ 1. 业务数据库准备 @@ -38,27 +39,28 @@ mysql -u root -p your_database < data-agent-management/src/main/resources/sql/pr 在`data-agent-management/src/main/resources/application.yml`中配置你的MySQL数据库连接信息。 -> 初始化行为说明:默认开启自动创建表并插入示例数据(`spring.sql.init.mode: always`)。生产环境建议关闭,避免示例数据回填覆盖你的业务数据。 +> 初始化行为说明:默认配置为 `spring.sql.init.mode: never`,不会自动创建表或插入示例数据。 +> 首次启动前请先执行上面的 SQL 文件,或在明确需要示例数据时通过环境变量 +> `DATA_AGENT_DATASOURCE_SQL_INIT=always` 开启初始化。 ```yaml spring: datasource: url: jdbc:mysql://127.0.0.1:3306/saa_data_agent?useUnicode=true&characterEncoding=utf-8&zeroDateTimeBehavior=convertToNull&transformedBitIsBoolean=true&allowMultiQueries=true&allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=Asia/Shanghai - username: ${MYSQL_USERNAME:root} - password: ${MYSQL_PASSWORD:root} + username: ${DATA_AGENT_DATASOURCE_USERNAME:root} + password: ${DATA_AGENT_DATASOURCE_PASSWORD:root} driver-class-name: com.mysql.cj.jdbc.Driver - type: com.alibaba.druid.pool.DruidDataSource ``` ### 2.2 数据初始化配置 -默认开启自动初始化 (`spring.sql.init.mode: always`)。 +默认关闭自动初始化(`spring.sql.init.mode: never`)。 -> 关于如何关闭自动初始化,请参考 [开发者指南 - 数据库初始化配置](DEVELOPER_GUIDE.md#8-数据库初始化配置-database-initialization)。 +> 关于如何调整初始化行为,请参考 [开发者指南 - 数据库初始化配置](DEVELOPER_GUIDE.md#8-数据库初始化配置-database-initialization)。 ### 2.3 配置模型 -> 如果涉及手动管理模型依赖(非默认 Starter),请参考 [开发者指南 - 扩展依赖配置](DEVELOPER_GUIDE.md#9-扩展依赖配置-dependency-extension)。 +> 如果涉及手动管理模型依赖(非默认 Starter),请参考 [开发者指南 - 模型依赖手动管理](DEVELOPER_GUIDE.md#9-模型依赖手动管理-manual-model-dependency)。 启动项目,点击模型配置,新增模型填写自己的apikey即可。 @@ -77,7 +79,7 @@ spring: ### 2.4 嵌入模型批处理策略配置 -> 详细配置参数请参考 [开发者指南 - 开发配置手册](DEVELOPER_GUIDE.md#⚙️-开发配置手册)。 +> 详细配置参数请参考 [开发者指南 - 嵌入模型批处理策略](DEVELOPER_GUIDE.md#2-嵌入模型批处理策略-embedding-batch)。 ### 2.5 向量库配置 @@ -177,53 +179,87 @@ spring: #### 2.5.3 向量库配置参数 -> 详细配置参数请参考 [开发者指南 - 开发配置手册](DEVELOPER_GUIDE.md#⚙️-开发配置手册)。 +> 详细配置参数请参考 [开发者指南 - 向量库配置](DEVELOPER_GUIDE.md#3-向量库配置-vector-store)。 ### 2.6 检索融合策略 -> 详细配置参数请参考 [开发者指南 - 开发配置手册](DEVELOPER_GUIDE.md#⚙️-开发配置手册)。 +> 详细配置参数请参考 [开发者指南 - 向量库配置](DEVELOPER_GUIDE.md#3-向量库配置-vector-store)。 ### 2.7 替换vector-store的实现类 -> 关于如何替换默认的内存向量库(如使用 PGVector, Milvus 等),请参考 [开发者指南 - 扩展依赖配置](DEVELOPER_GUIDE.md#9-扩展依赖配置-dependency-extension)。 +> 关于如何替换默认的内存向量库(如使用 PGVector、Milvus 等),请参考 [开发者指南 - 向量库依赖扩展](DEVELOPER_GUIDE.md#向量库依赖扩展)。 + +### 2.8 配置 Python 沙盒 + +DataAgent 使用 Spring AI Alibaba `1.1.2.2` 的 Sandbox 运行生成的 Python 代码。每次 +执行都会创建一个任务级容器,动态依赖安装、业务代码执行和容器清理都在该任务内完成。 + +先确认 Docker 可用: + +```bash +docker info +``` + +默认配置使用本地 Docker socket、公网 PyPI 和 AgentScope 基础镜像。开发环境通常无需 +修改;如需指定 Docker 地址、镜像或私有包索引,可设置: + +```bash +export DATAAGENT_SANDBOX_DOCKER_HOST=unix:///var/run/docker.sock +export DATAAGENT_SANDBOX_IMAGE=agentscope-registry.ap-southeast-1.cr.aliyuncs.com/agentscope/runtime-sandbox-base:latest +export DATAAGENT_PYPI_INDEX_URL=https://pypi.org/simple +``` + +生产环境应使用固定镜像 digest 和企业私有 PyPI 代理,并在基础设施层限制沙盒只能访问 +包代理。完整配置、依赖协议和故障处理见 +[高级功能 - Python 执行环境配置](ADVANCED_FEATURES.md#-python-执行环境配置)。 ## 🚀 3. 启动管理端 -在`data-agent-management`目录下,运行 `DataAgentApplication.java` 类。 +在项目根目录运行: ```bash -cd data-agent-management -./mvnw spring-boot:run +./mvnw -pl data-agent-management spring-boot:run ``` 或者在IDE中直接运行 `DataAgentApplication.java`。 ## 🌐 4. 启动WEB页面 -进入 `data-agent-frontend` 目录 +进入 `data-agent-frontend-nuxt` 目录 ### 4.1 安装依赖 ```bash -# 使用 npm -npm install - -# 或使用 yarn -yarn install +pnpm install ``` ### 4.2 启动服务 ```bash -# 使用 npm -npm run dev - -# 或使用 yarn -yarn dev +pnpm dev ``` 启动成功后,访问地址 http://localhost:3000 +### 4.3 验证 Python 动态依赖 + +完成智能体、模型和数据源配置后,在数据问答页提交一个明确包含 Python 步骤的请求,例如: + +```text +先查询 orders 表的 status 和 total_amount 原始数据,再使用 Python 聚合; +Python 通过 PEP 723 声明并导入 six==1.17.0,最终输出 six 版本和各状态汇总。 +``` + +成功时,时间线会依次出现“Python 生成”“Python 执行”“Python 分析”和最终报告,Python +输出中应包含 `six_version: 1.17.0`。任务结束后不应存在残留容器: + +```bash +docker ps --format '{{.Names}}' | grep '^dataagent-sandbox-' +``` + +命令无输出表示任务沙盒已经清理。若依赖安装失败,工作流会把错误反馈给下一次 Python +生成尝试;达到最大重试次数后进入现有降级或终止分支。 + ## 🎯 5. 系统体验 ### 5.1 数据智能体的创建与配置 diff --git a/docs/superpowers/specs/2026-07-28-saa-python-sandbox-integration-design.md b/docs/superpowers/specs/2026-07-28-saa-python-sandbox-integration-design.md index c2910fb34..a3463e4e9 100644 --- a/docs/superpowers/specs/2026-07-28-saa-python-sandbox-integration-design.md +++ b/docs/superpowers/specs/2026-07-28-saa-python-sandbox-integration-design.md @@ -1,8 +1,11 @@ # SAA 1.1.2.2 Python 沙盒接入方案 +> 状态:已通过 [PR #587](https://github.com/spring-ai-alibaba/DataAgent/pull/587) 合入上游。 +> 本文同时记录当前实现边界、生产约束和已完成的验证,不再作为待实施设计稿使用。 + ## 1. 结论与范围 -本次只改造 Python 生成与执行链路: +已完成的改造只覆盖 Python 生成与执行链路: - Spring AI Alibaba 升级到 `1.1.2.2` - Spring AI 对齐到 `1.1.2` @@ -11,8 +14,8 @@ - 删除本地进程、自建 Docker 容器池和 AI Simulation 执行器 - SQL 生成、校验和执行链路不做改动 -不存在本地执行或旧 Docker 执行器的回退。SAA 启动、依赖安装或代码执行失败时,结果直接 -进入现有 Python 重试与降级分支。 +不存在本地执行或旧 Docker 执行器的回退。SAA 启动、依赖安装或代码执行失败时,结果进入 +现有 Python 重试、降级或终止分支。 ## 2. 改造后的链路 @@ -43,7 +46,8 @@ flowchart LR 1. 生成代码仍通过 `json.load(sys.stdin)` 读取 SQL 结果 JSON。 2. 业务结果仍从 stdout 返回,并由 `PythonExecuteNode` 做现有 JSON 解析。 -3. Python 失败仍进入现有重试,超过次数仍走现有降级逻辑。 +3. Python 执行失败仍进入现有重试;达到上限后,普通执行失败进入降级分析,元数据解析或 + 协议异常等分支可能直接终止。 4. StateGraph 节点关系和 SSE 事件协议不变。 ### 2.1 包边界 @@ -94,6 +98,10 @@ import pandas as pd 5. 拒绝 URL、VCS、文件路径、`@`、环境标记和任何 pip 参数。 6. 模型不能提供 index、证书、命令、安装目录或 shell 片段。 +`requires-python` 会被 TOML Parser 解析并保留在元数据对象中,但当前版本不会据此切换 +基础镜像、选择解释器或校验 Sandbox 内的 Python 版本。需要固定 Python 版本时,应通过 +`sandbox.image-name` 选择并锁定镜像。 + 依赖声明只是一组结构化包规格,不是命令字符串: ```text @@ -251,8 +259,16 @@ spring: sandbox: docker-host: ${DATAAGENT_SANDBOX_DOCKER_HOST:unix:///var/run/docker.sock} image-name: ${DATAAGENT_SANDBOX_IMAGE:agentscope-registry.ap-southeast-1.cr.aliyuncs.com/agentscope/runtime-sandbox-base:latest} + container-prefix: dataagent-sandbox- max-concurrency: 4 queue-capacity: 10 + max-code-bytes: 262144 + max-input-bytes: 10485760 + max-output-bytes: 1048576 + max-error-bytes: 262144 + max-metadata-bytes: 8192 + max-dependencies: 20 + max-connections: 4096 package-index-url: ${DATAAGENT_PYPI_INDEX_URL:https://pypi.org/simple} dependency-install-timeout: 3m ``` @@ -299,18 +315,18 @@ spring: 5. 后续可增加 wheel-only、hash lock 和只读 wheel 缓存;这些是生产加固项,不作为 当前代码已实现能力描述。 -## 11. 验收 +## 11. 验收结果 -| 场景 | 验收条件 | -|---|---| -| 动态依赖 | PEP 723 声明的包在任务 Sandbox 内安装成功 | -| stdin | 生成代码能通过 `json.load(sys.stdin)` 读取 SQL 结果 | -| stdout | JSON 结果保持现有工作流协议 | -| Python 异常 | 进入现有重试或降级分支 | -| 超时/超限 | 返回明确失败并释放 Sandbox | -| 隔离 | 两次执行不复用可写运行态 | -| 清理 | 成功和失败后均无残留 `dataagent-sandbox-*` 容器 | -| 端到端 | 浏览器发起真实分析后出现最终报告且 SSE 收到 `event:complete` | +| 场景 | 验收条件 | 结果 | +|---|---|---| +| 动态依赖 | PEP 723 声明的包在任务 Sandbox 内安装成功 | 通过 | +| stdin | 生成代码能通过 `json.load(sys.stdin)` 读取 SQL 结果 | 通过 | +| stdout | JSON 结果保持现有工作流协议 | 通过 | +| Python 异常 | 进入现有重试、降级或终止分支 | 通过 | +| 超时/超限 | 返回明确失败并释放 Sandbox | 通过 | +| 隔离 | 两次执行不复用可写运行态 | 通过 | +| 清理 | 成功和失败后均无残留 `dataagent-sandbox-*` 容器 | 通过 | +| 端到端 | 浏览器发起真实分析后出现最终报告且 SSE 收到 `event:complete` | 通过 | 验证分层: @@ -320,8 +336,45 @@ spring: 3. 完整后端测试与格式检查。 4. 本地启动 Milvus、后端和前端,使用浏览器完成真实 StateGraph 请求。 +### 11.1 自动化验证 + +- 完整 Maven 测试:1628 个测试,0 失败、0 错误、1 跳过。 +- `spring-javaformat:validate` 和 Checkstyle 通过。 +- 上游 PR 的 Frontend Build、linter、license、secret、format、check-style、 + test-and-build 等 8 项检查全部通过。 + +### 11.2 浏览器端到端验证 + +端到端验证使用真实模型和真实运行时,而非 mock: + +1. 使用 Qwen Embedding 完成向量召回,使用 DeepSeek Chat 完成规划、代码生成和报告生成。 +2. 第一次生成的 `pandas + six` 依赖在下载超时后,工作流携带错误进入代码重生成。 +3. 重生成脚本仅声明 `six`,任务 Sandbox 动态安装 `six==1.17.0`,读取 SQL 结果 stdin + 并返回 JSON。 +4. 浏览器时间线出现依赖安装、Python 执行和最终报告,SSE 收到 `event:complete`,结果成功 + 持久化。 +5. 成功和失败路径结束后均未残留 `dataagent-sandbox-*` 容器;验证完成后前后端已关闭。 + +这组证据同时覆盖了“依赖安装失败后重试”和“动态依赖安装成功后完成报告”两条关键链路。 +单纯的后端 HTTP 200、单元测试或容器启动成功不能替代上述端到端验收。 + +### 11.3 CI 密钥边界 + +上游仓库可以把真实模型凭据放在 GitHub Actions Repository Secret 中,但从 fork 发起的 +`pull_request` 工作流不会获得上游 Secret。因此: + +- 常规 PR 的构建、格式、静态检查和无密钥测试必须保持可独立运行。 +- 真实模型端到端测试应放在上游分支手动触发的工作流,或受保护 Environment 中执行。 +- 不使用 `pull_request_target` 检出并执行 fork 代码来换取 Secret 访问,否则会把上游 + 凭据暴露给不受信任代码。 +- 日志、时间线和测试报告只记录模型/提供商名称,不输出 API Key。 + ## 12. 参考 +- [快速开始](../../QUICK_START.md) +- [高级功能:Python 执行环境](../../ADVANCED_FEATURES.md#-python-执行环境配置) +- [开发者指南:代码执行器配置](../../DEVELOPER_GUIDE.md#5-代码执行器配置-code-executor) +- [系统架构:Python 沙盒链路](../../ARCHITECTURE.md#8-python-执行与结果回传) - [Spring AI Alibaba v1.1.2.2 releases](https://github.com/alibaba/spring-ai-alibaba/releases) - [Spring AI Alibaba Sandbox example](https://github.com/spring-ai-alibaba/examples/tree/main/spring-ai-alibaba-sandbox-example/sandbox-simple-tool) - [PEP 723 – Inline script metadata](https://peps.python.org/pep-0723/)