diff --git a/data-agent-management/pom.xml b/data-agent-management/pom.xml index 6dd6a4b09..2528511bd 100644 --- a/data-agent-management/pom.xml +++ b/data-agent-management/pom.xml @@ -83,6 +83,10 @@ org.apache.commons commons-lang3 + + com.google.guava + guava + com.mysql mysql-connector-j @@ -269,6 +273,16 @@ spring-ai-alibaba-graph-core ${spring-ai-alibaba.version} + + com.alibaba.cloud.ai + spring-ai-alibaba-sandbox + ${spring-ai-alibaba.version} + + + org.tomlj + tomlj + ${tomlj.version} + org.springframework.ai spring-ai-starter-model-chat-memory-repository-jdbc @@ -371,16 +385,6 @@ easyexcel 3.3.4 - - - com.github.docker-java - docker-java-core - - - com.github.docker-java - docker-java-transport-zerodep - - diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/config/DataAgentConfiguration.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/config/DataAgentConfiguration.java index fa4277321..92780fece 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/config/DataAgentConfiguration.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/config/DataAgentConfiguration.java @@ -19,9 +19,6 @@ import com.alibaba.cloud.ai.dataagent.properties.DataAgentProperties; import com.alibaba.cloud.ai.dataagent.properties.FileStorageProperties; import com.alibaba.cloud.ai.dataagent.properties.OssStorageProperties; -import com.alibaba.cloud.ai.dataagent.service.code.CodePoolExecutorService; -import com.alibaba.cloud.ai.dataagent.service.code.CodePoolExecutorServiceFactory; -import com.alibaba.cloud.ai.dataagent.service.code.docker.DockerExecutorFactory; import com.alibaba.cloud.ai.dataagent.service.file.FileStorageService; import com.alibaba.cloud.ai.dataagent.service.file.FileStorageServiceFactory; import com.alibaba.cloud.ai.dataagent.service.llm.LlmService; @@ -127,13 +124,6 @@ public FileStorageService fileStorageService(FileStorageProperties properties, return new FileStorageServiceFactory(properties, ossStorageProperties).getObject(); } - @Bean - @ConditionalOnMissingBean(CodePoolExecutorService.class) - public CodePoolExecutorService codePoolExecutorService(CodeExecutorProperties properties, LlmService llmService, - DockerExecutorFactory dockerExecutorFactory) { - return new CodePoolExecutorServiceFactory(properties, llmService, dockerExecutorFactory).getObject(); - } - @Bean @ConditionalOnMissingBean(RestClientCustomizer.class) public RestClientCustomizer restClientCustomizer(@Value("${rest.connect.timeout:600}") long connectTimeout, diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/properties/CodeExecutorProperties.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/properties/CodeExecutorProperties.java index e368be92b..355f5be4b 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/properties/CodeExecutorProperties.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/properties/CodeExecutorProperties.java @@ -15,11 +15,12 @@ */ package com.alibaba.cloud.ai.dataagent.properties; -import com.alibaba.cloud.ai.dataagent.enums.CodePoolExecutorEnum; import lombok.Getter; import lombok.Setter; import org.springframework.boot.context.properties.ConfigurationProperties; +import java.time.Duration; + import static com.alibaba.cloud.ai.dataagent.constant.Constant.PROJECT_PROPERTIES_PREFIX; /** @@ -34,94 +35,59 @@ public class CodeExecutorProperties { public static final String CONFIG_PREFIX = PROJECT_PROPERTIES_PREFIX + ".code-executor"; /** - * Specify implementation class of code container pool runtime service + * Maximum container memory, in MB */ - CodePoolExecutorEnum codePoolExecutor = CodePoolExecutorEnum.DOCKER; + Long limitMemory = 500L; /** - * Service host, use default address if null + * Number of container CPU cores */ - String host = null; + Long cpuCore = 1L; /** - * Image name, can customize image with common third-party dependencies to replace - * this configuration + * Python code execution time limit */ - String imageName = "continuumio/anaconda3:latest"; + Duration codeTimeout = Duration.ofSeconds(60); /** - * Container name prefix + * Python执行的最大重试次数 */ - String containerNamePrefix = "nl2sql-python-exec-"; + Integer pythonMaxTriesCount = 5; - /** - * Task blocking queue size - */ - Integer taskQueueSize = 5; + private Sandbox sandbox = new Sandbox(); - /** - * Maximum number of core containers - */ - Integer coreContainerNum = 2; + @Getter + @Setter + public static class Sandbox { - /** - * Maximum number of temporary containers - */ - Integer tempContainerNum = 2; + String dockerHost = "unix:///var/run/docker.sock"; - /** - * Core thread count of thread pool - */ - Integer coreThreadSize = 5; + String imageName = "agentscope-registry.ap-southeast-1.cr.aliyuncs.com/agentscope/runtime-sandbox-base:latest"; - /** - * Maximum thread count of thread pool - */ - Integer maxThreadSize = 5; + String containerPrefix = "dataagent-sandbox-"; - /** - * Survival time of temporary containers, in minutes - */ - Integer tempContainerAliveTime = 5; + Integer maxConcurrency = 4; - /** - * Task survival time of thread pool, in seconds - */ - Long keepThreadAliveTime = 60L; + Integer queueCapacity = 10; - /** - * Task blocking queue size of thread pool - */ - Integer threadQueueSize = 10; + Integer maxCodeBytes = 256 * 1024; - /** - * Maximum container memory, in MB - */ - Long limitMemory = 500L; + Integer maxInputBytes = 10 * 1024 * 1024; - /** - * Number of container CPU cores - */ - Long cpuCore = 1L; + Integer maxOutputBytes = 1024 * 1024; - /** - * Python code execution time limit - */ - String codeTimeout = "60s"; + Integer maxErrorBytes = 256 * 1024; - /** - * Maximum container runtime - */ - Long containerTimeout = 3000L; + Integer maxMetadataBytes = 8 * 1024; - /** - * Container network mode - */ - String networkMode = "none"; + Integer maxDependencies = 20; - /** - * Python执行的最大重试次数 - */ - Integer pythonMaxTriesCount = 5; + String packageIndexUrl = "https://pypi.org/simple"; + + Duration dependencyInstallTimeout = Duration.ofMinutes(3); + + Integer maxConnections = 4096; + + } } diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/CodePoolExecutorServiceFactory.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/CodePoolExecutorServiceFactory.java deleted file mode 100644 index 3a5e35f41..000000000 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/CodePoolExecutorServiceFactory.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright 2024-2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code; - -import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; -import com.alibaba.cloud.ai.dataagent.service.code.docker.DockerExecutorFactory; -import com.alibaba.cloud.ai.dataagent.service.code.impls.AiSimulationCodeExecutorService; -import com.alibaba.cloud.ai.dataagent.service.code.impls.LocalCodePoolExecutorService; -import com.alibaba.cloud.ai.dataagent.service.llm.LlmService; -import lombok.AllArgsConstructor; - -/** - * 运行 Python 任务的容器池实现选择器 - * - * @author vlsmb - * @since 2025/7/28 - */ -@AllArgsConstructor -public class CodePoolExecutorServiceFactory { - - private final CodeExecutorProperties properties; - - private final LlmService llmService; - - private final DockerExecutorFactory dockerExecutorFactory; - - public CodePoolExecutorService getObject() { - return switch (properties.getCodePoolExecutor()) { - case DOCKER -> dockerExecutorFactory.create(properties); - case LOCAL -> new LocalCodePoolExecutorService(properties); - case AI_SIMULATION -> new AiSimulationCodeExecutorService(llmService); - default -> - throw new IllegalStateException("This option does not have a corresponding implementation class yet."); - }; - } - - public Class getObjectType() { - return CodePoolExecutorService.class; - } - -} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/CodePoolExecutorService.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/PythonCodeExecutorService.java similarity index 79% rename from data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/CodePoolExecutorService.java rename to data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/PythonCodeExecutorService.java index ae6b54dc8..9e59a76c7 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/CodePoolExecutorService.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/PythonCodeExecutorService.java @@ -15,34 +15,34 @@ */ package com.alibaba.cloud.ai.dataagent.service.code; +import java.util.List; + /** - * 运行Python任务的容器池接口 - * - * @author vlsmb - * @since 2025/7/12 + * Executes generated Python code in an isolated sandbox. */ -public interface CodePoolExecutorService { +public interface PythonCodeExecutorService { TaskResponse runTask(TaskRequest request); - record TaskRequest(String code, String input, String requirement) { + record TaskRequest(String code, String input, List dependencies) { + + public TaskRequest { + dependencies = dependencies == null ? List.of() : List.copyOf(dependencies); + } } record TaskResponse(boolean isSuccess, boolean executionSuccessButResultFailed, String stdOut, String stdErr, String exceptionMsg) { - // 执行运行代码任务时发生异常 public static TaskResponse exception(String msg) { return new TaskResponse(false, false, null, null, "An exception occurred while executing the task: " + msg); } - // 执行运行代码任务成功，并且代码正常返回 public static TaskResponse success(String stdOut) { return new TaskResponse(true, false, stdOut, null, null); } - // 执行运行代码任务成功，但是代码异常返回 public static TaskResponse failure(String stdOut, String stdErr) { return new TaskResponse(false, true, stdOut, stdErr, "StdErr: " + stdErr); } @@ -54,10 +54,4 @@ public String toString() { } } - enum State { - - READY, RUNNING, REMOVING - - } - } diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerClientConnector.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerClientConnector.java deleted file mode 100644 index 016419a98..000000000 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerClientConnector.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2024-2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code.docker; - -import com.github.dockerjava.api.DockerClient; - -/** - * Creates a Docker client for one endpoint without probing the daemon. - */ -@FunctionalInterface -public interface DockerClientConnector { - - DockerClient connect(String host); - -} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerClientFactory.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerClientFactory.java deleted file mode 100644 index 06e1a14b9..000000000 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerClientFactory.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright 2024-2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code.docker; - -import com.github.dockerjava.api.DockerClient; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Component; - -import java.io.IOException; -import java.util.List; - -/** - * Connects to the first reachable Docker endpoint. - */ -@Component -@RequiredArgsConstructor -@Slf4j -public class DockerClientFactory { - - private final DockerClientConnector connector; - - public DockerClient create(List candidateHosts) { - return connect(candidateHosts).client(); - } - - public DockerConnection connect(List candidateHosts) { - if (candidateHosts == null || candidateHosts.isEmpty()) { - throw new IllegalArgumentException("At least one Docker host candidate is required"); - } - - RuntimeException lastFailure = null; - for (String host : candidateHosts) { - DockerClient client = null; - try { - client = connector.connect(host); - client.pingCmd().exec(); - log.info("Connected to Docker using {}", host); - return new DockerConnection(client, host); - } - catch (RuntimeException exception) { - lastFailure = exception; - closeFailedClient(client, exception); - log.warn("Could not connect to Docker using {}: {}", host, exception.getMessage()); - } - } - - throw new IllegalStateException("Failed to connect to Docker. Attempted hosts: " + candidateHosts, lastFailure); - } - - public record DockerConnection(DockerClient client, String host) { - } - - private void closeFailedClient(DockerClient client, RuntimeException connectionFailure) { - if (client == null) { - return; - } - try { - client.close(); - } - catch (IOException closeFailure) { - connectionFailure.addSuppressed(closeFailure); - } - } - -} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerExecutorFactory.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerExecutorFactory.java deleted file mode 100644 index f60ed93e2..000000000 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerExecutorFactory.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright 2024-2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code.docker; - -import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; -import com.alibaba.cloud.ai.dataagent.service.code.docker.DockerClientFactory.DockerConnection; -import com.alibaba.cloud.ai.dataagent.service.code.impls.DockerCodePoolExecutorService; -import com.github.dockerjava.api.DockerClient; -import lombok.RequiredArgsConstructor; -import org.springframework.stereotype.Component; - -import java.io.IOException; -import java.util.List; - -/** - * Composition root for the Docker-backed code executor. - */ -@Component -@RequiredArgsConstructor -public class DockerExecutorFactory { - - private final DockerHostResolver hostResolver; - - private final DockerClientFactory clientFactory; - - private final DockerImageManager imageManager; - - public DockerCodePoolExecutorService create(CodeExecutorProperties properties) { - List candidateHosts = hostResolver.candidates(properties.getHost(), System.getProperty("os.name")); - DockerConnection connection = clientFactory.connect(candidateHosts); - DockerClient client = connection.client(); - try { - imageManager.ensureAvailable(client, properties.getImageName()); - return new DockerCodePoolExecutorService(properties, client, hostResolver.isRemote(connection.host())); - } - catch (RuntimeException constructionFailure) { - closeAfterFailedConstruction(client, constructionFailure); - throw constructionFailure; - } - } - - private void closeAfterFailedConstruction(DockerClient client, RuntimeException constructionFailure) { - try { - client.close(); - } - catch (IOException closeFailure) { - constructionFailure.addSuppressed(closeFailure); - } - } - -} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerHostResolver.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerHostResolver.java deleted file mode 100644 index 2f3a64989..000000000 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerHostResolver.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright 2024-2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code.docker; - -import org.springframework.stereotype.Component; -import org.springframework.util.StringUtils; - -import java.net.URI; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Locale; -import java.util.Objects; - -/** - * Resolves Docker connection candidates and classifies their network location. - */ -@Component -public class DockerHostResolver { - - private static final String UNIX_SOCKET = "unix:///var/run/docker.sock"; - - private static final String WINDOWS_NAMED_PIPE = "npipe://./pipe/docker_engine"; - - private static final String LOCAL_TCP = "tcp://localhost:2375"; - - public List candidates(String configuredHost, String osName) { - String normalizedOs = Objects.requireNonNullElse(osName, "").toLowerCase(Locale.ROOT); - if (normalizedOs.contains("win")) { - LinkedHashSet hosts = new LinkedHashSet<>(); - if (StringUtils.hasText(configuredHost)) { - hosts.add(configuredHost); - } - hosts.add(WINDOWS_NAMED_PIPE); - hosts.add(LOCAL_TCP); - return List.copyOf(hosts); - } - if (StringUtils.hasText(configuredHost)) { - return List.of(configuredHost); - } - return List.of(UNIX_SOCKET); - } - - public boolean isRemote(String host) { - if (!StringUtils.hasText(host)) { - return false; - } - - URI uri; - try { - uri = URI.create(host); - } - catch (IllegalArgumentException exception) { - throw new IllegalArgumentException("Invalid Docker host URI: " + host, exception); - } - - String scheme = uri.getScheme(); - if ("unix".equalsIgnoreCase(scheme) || "npipe".equalsIgnoreCase(scheme)) { - return false; - } - if (!"tcp".equalsIgnoreCase(scheme) || !StringUtils.hasText(uri.getHost())) { - throw new IllegalArgumentException("Unsupported Docker host URI: " + host); - } - - String hostname = uri.getHost(); - if (hostname.startsWith("[") && hostname.endsWith("]")) { - hostname = hostname.substring(1, hostname.length() - 1); - } - return !("localhost".equalsIgnoreCase(hostname) || "127.0.0.1".equals(hostname) || "::1".equals(hostname)); - } - -} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerImageManager.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerImageManager.java deleted file mode 100644 index 3ce1d1c4a..000000000 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerImageManager.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright 2024-2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code.docker; - -import com.github.dockerjava.api.DockerClient; -import com.github.dockerjava.api.model.Image; -import com.github.dockerjava.core.command.PullImageResultCallback; -import org.springframework.stereotype.Component; - -import java.util.Arrays; -import java.util.List; -import java.util.Objects; -import java.util.function.Supplier; - -/** - * Ensures the configured Docker image is locally available. - */ -@Component -public class DockerImageManager { - - private final Supplier callbackFactory; - - public DockerImageManager() { - this(PullImageResultCallback::new); - } - - DockerImageManager(Supplier callbackFactory) { - this.callbackFactory = Objects.requireNonNull(callbackFactory, "callbackFactory"); - } - - public void ensureAvailable(DockerClient client, String imageName) { - List images = client.listImagesCmd().withImageNameFilter(imageName).exec(); - boolean imageExists = images != null && images.stream() - .map(Image::getRepoTags) - .filter(Objects::nonNull) - .flatMap(Arrays::stream) - .anyMatch(imageName::equals); - if (imageExists) { - return; - } - - PullImageResultCallback callback = callbackFactory.get(); - try { - client.pullImageCmd(imageName).exec(callback).awaitCompletion(); - } - catch (InterruptedException exception) { - Thread.currentThread().interrupt(); - throw new IllegalStateException("Interrupted while pulling Docker image " + imageName, exception); - } - } - -} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/docker/ZerodepDockerClientConnector.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/docker/ZerodepDockerClientConnector.java deleted file mode 100644 index 3834758b2..000000000 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/docker/ZerodepDockerClientConnector.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2024-2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code.docker; - -import com.github.dockerjava.api.DockerClient; -import com.github.dockerjava.core.DefaultDockerClientConfig; -import com.github.dockerjava.core.DockerClientConfig; -import com.github.dockerjava.core.DockerClientImpl; -import com.github.dockerjava.zerodep.ZerodepDockerHttpClient; -import org.springframework.stereotype.Component; - -/** - * docker-java connector backed by its dependency-free HTTP transport. - */ -@Component -public class ZerodepDockerClientConnector implements DockerClientConnector { - - @Override - public DockerClient connect(String host) { - DockerClientConfig config = DefaultDockerClientConfig.createDefaultConfigBuilder() - .withDockerHost(host) - .withDockerTlsVerify(false) - .build(); - ZerodepDockerHttpClient httpClient = new ZerodepDockerHttpClient.Builder().dockerHost(config.getDockerHost()) - .sslConfig(config.getSSLConfig()) - .build(); - return DockerClientImpl.getInstance(config, httpClient); - } - -} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/impls/AbstractCodePoolExecutorService.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/impls/AbstractCodePoolExecutorService.java deleted file mode 100644 index f08ab77a4..000000000 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/impls/AbstractCodePoolExecutorService.java +++ /dev/null @@ -1,428 +0,0 @@ -/* - * Copyright 2024-2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code.impls; - -import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; -import com.alibaba.cloud.ai.dataagent.service.code.CodePoolExecutorService; -import lombok.extern.slf4j.Slf4j; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; - -import java.io.IOException; -import java.nio.file.FileVisitResult; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.attribute.BasicFileAttributes; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; -import java.util.concurrent.FutureTask; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * 运行Python任务的容器池 - * - * @author vlsmb - * @since 2025/7/12 - */ -@Slf4j -public abstract class AbstractCodePoolExecutorService implements CodePoolExecutorService, AutoCloseable { - - private final AtomicBoolean closed = new AtomicBoolean(); - - // Record core container status - protected final ConcurrentHashMap coreContainerState; - - // Record temporary container status - protected final ConcurrentHashMap tempContainerState; - - // Record Future for temporary container destruction - protected final ConcurrentHashMap> tempContainerRemoveFuture; - - // Task queue (temporarily store tasks when containers are full) - protected final ArrayBlockingQueue> taskQueue; - - // Ready core containers - protected final ArrayBlockingQueue readyCoreContainer; - - // Ready temporary containers - protected final ArrayBlockingQueue readyTempContainer; - - // Current number of core containers - protected final AtomicInteger currentCoreContainerSize; - - // Current number of temporary containers - protected final AtomicInteger currentTempContainerSize; - - // Thread pool, running temporarily stored tasks - protected final ExecutorService consumerThreadPool; - - // Configuration properties - protected final CodeExecutorProperties properties; - - public AbstractCodePoolExecutorService(CodeExecutorProperties properties) { - this.properties = properties; - this.coreContainerState = new ConcurrentHashMap<>(); - this.tempContainerState = new ConcurrentHashMap<>(); - this.tempContainerRemoveFuture = new ConcurrentHashMap<>(); - this.taskQueue = new ArrayBlockingQueue<>(properties.getTaskQueueSize()); - this.readyCoreContainer = new ArrayBlockingQueue<>(properties.getCoreContainerNum()); - this.readyTempContainer = new ArrayBlockingQueue<>(properties.getTempContainerNum()); - this.consumerThreadPool = new ThreadPoolExecutor(properties.getCoreThreadSize(), properties.getMaxThreadSize(), - properties.getKeepThreadAliveTime(), TimeUnit.SECONDS, - new ArrayBlockingQueue<>(properties.getThreadQueueSize())); - this.currentCoreContainerSize = new AtomicInteger(0); - this.currentTempContainerSize = new AtomicInteger(0); - } - - @Override - public final void close() { - if (!closed.compareAndSet(false, true)) { - return; - } - try { - this.shutdownPool(); - } - catch (Exception exception) { - throw new IllegalStateException("Failed to close code pool executor", exception); - } - } - - /** - * 创建新的容器 - * @return 容器ID - */ - protected abstract String createNewContainer() throws Exception; - - /** - * 在指定容器ID的容器运行任务 - * @param request 任务请求对象 - * @param containerId 容器ID - * @return 运行结果对象 - */ - protected abstract TaskResponse execTaskInContainer(TaskRequest request, String containerId); - - /** - * 停止指定容器 - * @param containerId 容器ID - */ - protected abstract void stopContainer(String containerId) throws Exception; - - /** - * 删除指定容器 - * @param containerId 容器ID - */ - protected abstract void removeContainer(String containerId) throws Exception; - - protected void shutdownPool() throws Exception { - // Shutdown thread pool - this.consumerThreadPool.shutdownNow(); - // Stop and delete all containers - this.tempContainerState.keySet().forEach(id -> this.removeContainerAndState(id, false, true)); - this.coreContainerState.keySet().forEach(id -> this.removeContainerAndState(id, true, true)); - this.tempContainerState.clear(); - this.coreContainerState.clear(); - this.tempContainerRemoveFuture.clear(); - this.readyCoreContainer.clear(); - this.readyTempContainer.clear(); - this.taskQueue.clear(); - } - - private void removeContainerAndState(String containerId, boolean isCore, boolean isForce) { - try { - if (isCore) { - // Remove core container - State state = this.coreContainerState.replace(containerId, State.REMOVING); - if (state == State.RUNNING) { - if (isForce) { - this.stopContainer(containerId); - } - else { - throw new RuntimeException("Container is still Running!"); - } - } - this.removeContainer(containerId); - this.coreContainerState.remove(containerId); - this.currentCoreContainerSize.decrementAndGet(); - log.info("Core Container {} has been removed successfully", containerId); - } - else { - // Remove temporary container - State state = this.tempContainerState.replace(containerId, State.REMOVING); - if (state == State.RUNNING) { - if (isForce) { - this.stopContainer(containerId); - } - else { - throw new RuntimeException("Container is still Running!"); - } - } - this.removeContainer(containerId); - this.tempContainerState.remove(containerId); - this.tempContainerRemoveFuture.remove(containerId); - this.currentTempContainerSize.decrementAndGet(); - log.info("Temp Container {} has been removed successfully", containerId); - } - } - catch (Exception e) { - log.error("Error when trying to remove a container, containerId: {}, info: {}", containerId, e.getMessage(), - e); - } - } - - // Create thread to delete temporary containers - private Future registerRemoveTempContainer(String containerId) { - return consumerThreadPool.submit(() -> { - try { - Thread.sleep(this.properties.getKeepThreadAliveTime() * 1000L * 60L); - if (this.tempContainerState.get(containerId) == State.RUNNING) { - throw new InterruptedException("Container " + containerId + " is already running"); - } - } - catch (InterruptedException e) { - // Cancel temporary container destruction thread due to container running - log.debug("Interrupted while waiting for temp container to be removed, info: {}", e.getMessage()); - return; - } - this.removeContainerAndState(containerId, false, false); - }); - } - - // Use core container - private TaskResponse useCoreContainer(String containerId, TaskRequest request) { - try { - // Execute task - this.coreContainerState.replace(containerId, State.RUNNING); - TaskResponse resp = this.execTaskInContainer(request, containerId); - // 如果运行代码任务时出现了异常，认为容器损坏，执行容器清除，并将当前任务放进队列里重新执行 - if (!resp.isSuccess() && !resp.executionSuccessButResultFailed()) { - log.error("use core container failed, {}", resp.exceptionMsg()); - this.coreContainerState.replace(containerId, State.REMOVING); - this.removeContainerAndState(containerId, true, true); - return this.pushTaskQueue(request); - } - this.coreContainerState.replace(containerId, State.READY); - // Put back into blocking queue - this.readyCoreContainer.add(containerId); - // Run tasks in task queue if any - this.popTaskQueue(); - return resp; - } - catch (Exception e) { - log.error("use core container failed, {}", e.getMessage(), e); - return TaskResponse.exception(e.getMessage()); - } - } - - // Use temporary container - private TaskResponse useTempContainer(String containerId, TaskRequest request) { - try { - Future future = this.tempContainerRemoveFuture.remove(containerId); - // Cancel temporary container destruction thread - if (future != null) { - if (future.isDone()) { - // Container is destroyed, reselect usage strategy - log.debug("reselect strategy: {} ...", request.toString()); - return this.runTask(request); - } - future.cancel(true); - } - // Execute task - this.tempContainerState.replace(containerId, State.RUNNING); - TaskResponse resp = this.execTaskInContainer(request, containerId); - // 如果运行代码任务时出现了异常，认为容器损坏，执行容器清除，并将当前任务放进队列里重新执行 - if (!resp.isSuccess() && !resp.executionSuccessButResultFailed()) { - log.error("use temp container failed, {}", resp.exceptionMsg()); - this.tempContainerState.replace(containerId, State.REMOVING); - this.removeContainerAndState(containerId, false, true); - return this.pushTaskQueue(request); - } - this.tempContainerState.replace(containerId, State.READY); - // Put back into blocking queue - this.readyTempContainer.add(containerId); - // Recreate temporary container destruction thread - this.tempContainerRemoveFuture.put(containerId, this.registerRemoveTempContainer(containerId)); - // Run tasks in task queue if any - this.popTaskQueue(); - return resp; - } - catch (Exception e) { - log.error("use temp container failed, {}", e.getMessage(), e); - return TaskResponse.exception(e.getMessage()); - } - } - - // Create and use core container - private TaskResponse createAndUseCoreContainer(TaskRequest request) { - String containerId; - try { - containerId = this.createNewContainer(); - } - catch (Exception e) { - log.error("create new container failed, {}", e.getMessage(), e); - return TaskResponse.exception(e.getMessage()); - } - // Record newly added container - this.coreContainerState.put(containerId, State.READY); - // Use container - return this.useCoreContainer(containerId, request); - } - - // Create and use temporary container - private TaskResponse createAndUseTempContainer(TaskRequest request) { - String containerId; - try { - containerId = this.createNewContainer(); - } - catch (Exception e) { - log.error("create new container failed, {}", e.getMessage(), e); - return TaskResponse.exception(e.getMessage()); - } - // Record newly added container - this.tempContainerState.put(containerId, State.READY); - // Use container - return this.useTempContainer(containerId, request); - } - - private TaskResponse pushTaskQueue(TaskRequest request) throws ExecutionException, InterruptedException { - FutureTask ft = new FutureTask<>(() -> { - log.info("Execute tasks in the BlockingQueue {} ...", request.toString()); - return this.runTask(request); - }); - this.taskQueue.put(ft); - - if (!this.readyCoreContainer.isEmpty() || !this.readyTempContainer.isEmpty()) { - this.popTaskQueue(); - } - - return ft.get(); - } - - // Run tasks in task queue if any - private void popTaskQueue() { - FutureTask future = this.taskQueue.poll(); - if (future == null) { - return; - } - log.info("Get task from the BlockingQueue ..."); - this.consumerThreadPool.submit(future); - } - - @Override - public TaskResponse runTask(TaskRequest request) { - // Use available core container - String freeCoreId = this.readyCoreContainer.poll(); - if (freeCoreId != null) { - log.debug("Use free core container to run task {} ...", request.toString()); - return this.useCoreContainer(freeCoreId, request); - } - - // Use available temporary container - String freeTempId = this.readyTempContainer.poll(); - if (freeTempId != null) { - log.debug("Use free temp container to run task {} ...", request.toString()); - return this.useTempContainer(freeTempId, request); - } - - // Create new core container - int currentCore; - boolean useCoreContainer = true; - do { - currentCore = this.currentCoreContainerSize.get(); - if (currentCore >= properties.getCoreContainerNum()) { - useCoreContainer = false; - break; - } - } - while (!this.currentCoreContainerSize.compareAndSet(currentCore, currentCore + 1)); - if (useCoreContainer) { - log.debug("Create new core container to run task {} ...", request.toString()); - return this.createAndUseCoreContainer(request); - } - - // Create new temporary container - int currentTemp; - boolean useTempContainer = true; - do { - currentTemp = this.currentTempContainerSize.get(); - if (currentTemp >= properties.getTempContainerNum()) { - useTempContainer = false; - break; - } - } - while (!this.currentTempContainerSize.compareAndSet(currentTemp, currentTemp + 1)); - if (useTempContainer) { - log.debug("Create new temp container to run task {} ...", request.toString()); - return this.createAndUseTempContainer(request); - } - - // Put into task queue to wait - try { - log.debug("push task into BlockingQueue: {} ...", request.toString()); - return this.pushTaskQueue(request); - } - catch (Exception e) { - log.error("An exception occurred while executing the task: {}", e.getMessage(), e); - return TaskResponse.exception(e.getMessage()); - } - } - - /** - * Delete temporary directory - */ - protected void clearTempDir(Path tempDir) { - try { - Files.walkFileTree(tempDir, new SimpleFileVisitor<>() { - @NotNull - @Override - public FileVisitResult visitFile(@NotNull Path file, @NotNull BasicFileAttributes attrs) - throws IOException { - Files.delete(file); - return super.visitFile(file, attrs); - } - - @NotNull - @Override - public FileVisitResult postVisitDirectory(@NotNull Path dir, @Nullable IOException exc) - throws IOException { - if (exc != null) - throw exc; - Files.delete(dir); - return super.postVisitDirectory(dir, exc); - } - }); - log.info("Temp directory has been deleted."); - } - catch (Exception e) { - log.warn("Exception in clean temp directory: {}", e.getMessage()); - } - } - - /** - * Generate unique container name - */ - protected String generateContainerName() { - return this.properties.getContainerNamePrefix() + "_" + System.currentTimeMillis() + "_" - + Thread.currentThread().getName(); - } - -} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/impls/AiSimulationCodeExecutorService.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/impls/AiSimulationCodeExecutorService.java deleted file mode 100644 index 8a073193d..000000000 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/impls/AiSimulationCodeExecutorService.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright 2024-2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code.impls; - -import com.alibaba.cloud.ai.dataagent.service.code.CodePoolExecutorService; -import com.alibaba.cloud.ai.dataagent.service.llm.LlmService; -import lombok.extern.slf4j.Slf4j; - -/** - * 使用AI模拟运行Python代码（便于在无Docker环境测试） - * - * @author vlsmb - * @since 2025/7/30 - */ -@Slf4j -public class AiSimulationCodeExecutorService implements CodePoolExecutorService { - - private static final String SYSTEM_PROMPT = """ - 你将模拟Python的执行，根据我提供的代码和输入数据，并给出最终的数据结果。 - 在模拟运行时，请按照以下要求操作： - 1. 仔细理解代码和输入数据的内容。 - 2. 输出模拟运行结果。 - **要求**：仅输出模拟运行结果，禁止包含任何额外说明或自然语言。 - """; - - private final LlmService llmService; - - public AiSimulationCodeExecutorService(LlmService llmService) { - this.llmService = llmService; - } - - @Override - public TaskResponse runTask(TaskRequest request) { - String userPrompt = String.format(""" - 【代码】 - ```python - %s - ``` - 【标准输入】 - ```json - %s - ``` - """, request.code(), request.input()); - String output = llmService.toStringFlux(llmService.call(SYSTEM_PROMPT, userPrompt)) - .collect(StringBuilder::new, StringBuilder::append) - .map(StringBuilder::toString) - .block(); - return TaskResponse.success(output); - } - -} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/impls/DockerCodePoolExecutorService.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/impls/DockerCodePoolExecutorService.java deleted file mode 100644 index 64e5f8d8d..000000000 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/impls/DockerCodePoolExecutorService.java +++ /dev/null @@ -1,285 +0,0 @@ -/* - * Copyright 2024-2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code.impls; - -import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; -import com.alibaba.cloud.ai.dataagent.service.code.CodePoolExecutorService; -import com.github.dockerjava.api.DockerClient; -import com.github.dockerjava.api.command.CreateContainerResponse; -import com.github.dockerjava.api.command.InspectContainerResponse; -import com.github.dockerjava.api.async.ResultCallback; -import com.github.dockerjava.api.model.AccessMode; -import com.github.dockerjava.api.model.Bind; -import com.github.dockerjava.api.model.Capability; -import com.github.dockerjava.api.model.Frame; -import com.github.dockerjava.api.model.HostConfig; -import com.github.dockerjava.api.model.StreamType; -import com.github.dockerjava.api.model.Volume; -import lombok.extern.slf4j.Slf4j; -import org.springframework.util.StringUtils; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; - -import static com.github.dockerjava.api.model.HostConfig.newHostConfig; - -/** - * 运行Python任务的容器池（Docker实现类） - * - * @author vlsmb - * @since 2025/7/12 - */ -@Slf4j -public class DockerCodePoolExecutorService extends AbstractCodePoolExecutorService implements CodePoolExecutorService { - - private final DockerClient dockerClient; - - private final boolean isRemote; - - private final ConcurrentHashMap containerTempPath; - - public DockerCodePoolExecutorService(CodeExecutorProperties properties, DockerClient dockerClient, - boolean isRemote) { - super(properties); - this.dockerClient = Objects.requireNonNull(dockerClient, "dockerClient"); - this.isRemote = isRemote; - this.containerTempPath = new ConcurrentHashMap<>(); - log.info("Docker Code Pool initialized. Mode: {}", - this.isRemote ? "Remote (Copy Files)" : "Local (Bind Mounts)"); - } - - /** - * Create container's HostConfig - */ - private HostConfig createHostConfig(Path tempDir) { - HostConfig config = newHostConfig().withMemory(this.properties.getLimitMemory() * 1024L * 1024L) - .withCpuCount(this.properties.getCpuCore()) - .withCapDrop(Capability.ALL) - .withAutoRemove(false) - .withTmpFs(Map.of("/tmp", "")) - .withNetworkMode(this.properties.getNetworkMode()); - - if (!this.isRemote) { - List binds = new ArrayList<>(); - binds.add(new Bind(tempDir.resolve("script.py").toAbsolutePath().toString(), new Volume("/app/script.py"), - AccessMode.ro)); - binds.add(new Bind(tempDir.resolve("requirements.txt").toAbsolutePath().toString(), - new Volume("/app/requirements.txt"), AccessMode.ro)); - binds.add(new Bind(tempDir.resolve("input_data.txt").toAbsolutePath().toString(), - new Volume("/app/input_data.txt"), AccessMode.ro)); - config.withBinds(binds.toArray(new Bind[0])); - } - return config; - } - - /** - * Clean up existing container with same name - */ - private void cleanupExistingResources(String containName) { - try { - // Try to delete container with same name - dockerClient.removeContainerCmd(containName).withForce(true).exec(); - log.info("Removed existing container: {}", containName); - } - catch (Exception e) { - log.warn("Failed to remove container {}: {}", containName, e.getMessage()); - } - } - - @Override - protected String createNewContainer() throws Exception { - String containerName = this.generateContainerName(); - // First clean up possibly existing container with same name - this.cleanupExistingResources(containerName); - - // Generate temporary directory and files - Path tempDir = Files.createTempDirectory(containerName); - Files.createFile(tempDir.resolve("requirements.txt")); - Files.createFile(tempDir.resolve("script.py")); - Files.createFile(tempDir.resolve("input_data.txt")); - - // Create container - HostConfig hostConfig = this.createHostConfig(tempDir); - String cmd = this.buildExecutionCommand(tempDir); - - CreateContainerResponse container = dockerClient.createContainerCmd(properties.getImageName()) - .withName(containerName) - .withWorkingDir("/app") - .withHostConfig(hostConfig) - .withCmd("sh", "-c", cmd) - .exec(); - String containerId = container.getId(); - // Save temporary directory object - this.containerTempPath.put(containerId, tempDir); - return containerId; - } - - @Override - protected TaskResponse execTaskInContainer(TaskRequest request, String containerId) { - // Get temporary directory object - Path tempDir = this.containerTempPath.get(containerId); - if (tempDir == null) { - log.error("Container '{}' does not exist work dir", containerId); - return TaskResponse.exception("Container '" + containerId + "' does not exist work dir"); - } - - try { - // 1. Prepare files - this.writeContextFiles(tempDir, request); - this.uploadFilesIfRemote(containerId, tempDir); - - // 2. Start container and wait - dockerClient.startContainerCmd(containerId).exec(); - dockerClient.waitContainerCmd(containerId) - .start() - .awaitCompletion(this.properties.getContainerTimeout(), TimeUnit.SECONDS); - - // 3. Fetch logs - LogResult logs = this.fetchExecutionLogs(containerId, tempDir); - String stdout = logs.stdout; - String stderr = logs.stderr; - - // 4. Check exit code - InspectContainerResponse inspectResponse = dockerClient.inspectContainerCmd(containerId).exec(); - int exitCode = Objects.requireNonNull(inspectResponse.getState().getExitCodeLong()).intValue(); - if (exitCode != 0) { - String errorMessage = "Docker exit code " + exitCode + ". Stderr: " + stderr + ". Stdout: " + stdout; - log.error("Error executing Docker container {}: {}", containerId, errorMessage); - return TaskResponse.failure(stdout, stderr); - } - return TaskResponse.success(stdout); - } - catch (Exception e) { - log.error("Error executing task in container: {}", e.getMessage()); - return TaskResponse.exception(e.getMessage()); - } - } - - // --- Helper Methods --- - - private String buildExecutionCommand(Path tempDir) { - return String.format( - "if [ -s requirements.txt ]; then pip3 install --no-cache-dir -r requirements.txt > /dev/null; fi && timeout -s SIGKILL %s python3 -u script.py < input_data.txt", - properties.getCodeTimeout()); - } - - private void writeContextFiles(Path tempDir, TaskRequest request) throws IOException { - Files.write(tempDir.resolve("script.py"), - StringUtils.hasText(request.code()) ? request.code().getBytes() : "".getBytes()); - Files.write(tempDir.resolve("requirements.txt"), - StringUtils.hasText(request.requirement()) ? request.requirement().getBytes() : "".getBytes()); - Files.write(tempDir.resolve("input_data.txt"), - StringUtils.hasText(request.input()) ? request.input().getBytes() : "".getBytes()); - } - - private void uploadFilesIfRemote(String containerId, Path tempDir) { - if (!this.isRemote) { - return; - } - String[] files = { "script.py", "requirements.txt", "input_data.txt" }; - for (String file : files) { - dockerClient.copyArchiveToContainerCmd(containerId) - .withHostResource(tempDir.resolve(file).toString()) - .withRemotePath("/app/") - .exec(); - } - } - - private record LogResult(String stdout, String stderr) { - } - - private LogResult fetchExecutionLogs(String containerId, Path tempDir) throws InterruptedException { - StringBuilder stdoutBuilder = new StringBuilder(); - StringBuilder stderrBuilder = new StringBuilder(); - - final int MAX_LOG_SIZE = 5 * 1024 * 1024; // 5MB limit - dockerClient.logContainerCmd(containerId) - .withStdOut(true) - .withStdErr(true) - .exec(new ResultCallback.Adapter() { - @Override - public void onNext(Frame item) { - String payload = new String(item.getPayload(), StandardCharsets.UTF_8); - if (item.getStreamType() == StreamType.STDOUT) { - appendWithLimit(stdoutBuilder, payload, MAX_LOG_SIZE); - } - else if (item.getStreamType() == StreamType.STDERR) { - appendWithLimit(stderrBuilder, payload, MAX_LOG_SIZE); - } - } - }) - .awaitCompletion(); - - return new LogResult(stdoutBuilder.toString(), stderrBuilder.toString()); - } - - private void appendWithLimit(StringBuilder builder, String payload, int limit) { - if (builder.length() < limit) { - builder.append(payload); - } - else if (builder.length() == limit) { - builder.append("\n...[Output truncated due to size limit]..."); - builder.append(" "); // Prevent re-entry - } - } - - @Override - protected void stopContainer(String containerId) throws Exception { - try { - this.dockerClient.stopContainerCmd(containerId).exec(); - log.info("Successfully stopped container: {}", containerId); - } - catch (Exception e) { - log.warn("Failed to stop container: {}, message: {}", containerId, e.getMessage()); - } - } - - @Override - protected void removeContainer(String containerId) throws Exception { - try { - this.dockerClient.removeContainerCmd(containerId).withForce(true).exec(); - Path tempDir = containerTempPath.get(containerId); - if (tempDir != null) { - this.clearTempDir(tempDir); - } - containerTempPath.remove(containerId); - log.info("Successfully removed container: {}", containerId); - } - catch (Exception e) { - log.warn("Failed to remove container: {}, message: {}", containerId, e.getMessage()); - } - } - - @Override - protected void shutdownPool() throws Exception { - try { - super.shutdownPool(); - this.dockerClient.close(); - } - catch (IOException ignored) { - - } - } - -} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/impls/LocalCodePoolExecutorService.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/impls/LocalCodePoolExecutorService.java deleted file mode 100644 index a0d5cca57..000000000 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/impls/LocalCodePoolExecutorService.java +++ /dev/null @@ -1,224 +0,0 @@ -/* - * Copyright 2024-2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code.impls; - -import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; -import com.alibaba.cloud.ai.dataagent.service.code.CodePoolExecutorService; -import com.alibaba.cloud.ai.dataagent.service.code.local.ExecutableProgramLocator; -import com.alibaba.cloud.ai.dataagent.service.code.local.ExecutionTimeoutParser; -import com.alibaba.cloud.ai.dataagent.service.code.local.PathExecutableProgramLocator; -import lombok.extern.slf4j.Slf4j; -import org.springframework.util.StringUtils; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.StringWriter; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; - -/** - * 使用本地Python3环境运行代码的实现类，要求本地的Python3需要有pandas等数据分析库。 - * - * @author vlsmb - * @since 2025/8/23 - */ -@Slf4j -public class LocalCodePoolExecutorService extends AbstractCodePoolExecutorService implements CodePoolExecutorService { - - private final ConcurrentHashMap containers; - - private static final List PYTHON_NAMES = List.of("python3", "pypy3", "py3", "python", "pypy", "py"); - - private static final List PIP_NAMES = List.of("pip3", "pip"); - - private final String pythonExecutable; - - private final String pipExecutable; - - private final ExecutionTimeoutParser timeoutParser; - - // 对于本地运行这个实现类，“容器”为临时文件夹 - public LocalCodePoolExecutorService(CodeExecutorProperties properties) { - this(properties, new PathExecutableProgramLocator(), new ExecutionTimeoutParser()); - } - - LocalCodePoolExecutorService(CodeExecutorProperties properties, ExecutableProgramLocator programLocator, - ExecutionTimeoutParser timeoutParser) { - this(properties, timeoutParser, locatePrograms(programLocator)); - } - - private LocalCodePoolExecutorService(CodeExecutorProperties properties, ExecutionTimeoutParser timeoutParser, - LocalPrograms programs) { - super(properties); - this.containers = new ConcurrentHashMap<>(); - this.pythonExecutable = programs.pythonExecutable(); - this.pipExecutable = programs.pipExecutable(); - this.timeoutParser = timeoutParser; - } - - @Override - protected String createNewContainer() throws Exception { - Path container = Files.createTempDirectory(this.properties.getContainerNamePrefix()); - String containerId = container.toString(); - this.containers.put(containerId, container); - return containerId; - } - - @Override - protected TaskResponse execTaskInContainer(TaskRequest request, String containerId) { - Path container = this.containers.get(containerId); - - // 写入Py代码和标准输入 - Path scriptFile = container.resolve("script.py"); - Path stdinFile = container.resolve("stdin.txt"); - Path requirementFile = container.resolve("requirements.txt"); - try { - Files.write(scriptFile, Optional.ofNullable(request.code()).orElse("").getBytes()); - Files.write(stdinFile, Optional.ofNullable(request.input()).orElse("").getBytes()); - Files.write(requirementFile, Optional.ofNullable(request.requirement()).orElse("").getBytes()); - } - catch (Exception e) { - log.error("Create temp file failed: {}", e.getMessage(), e); - return TaskResponse.exception(e.getMessage()); - } - - // 如果有requirements，则先安装依赖 - if (this.pipExecutable != null && StringUtils.hasText(request.requirement())) { - ProcessBuilder pip = new ProcessBuilder(this.pipExecutable, "install", "--no-cache-dir", "-r", - requirementFile.toAbsolutePath().toString()); - pip.redirectOutput(ProcessBuilder.Redirect.DISCARD); - pip.redirectError(ProcessBuilder.Redirect.DISCARD); - Process process = null; - - try { - process = pip.start(); - boolean completed = process.waitFor(this.properties.getContainerTimeout(), TimeUnit.MINUTES); - if (!completed) { - process.destroy(); - if (process.isAlive()) { - process.destroyForcibly(); - } - throw new RuntimeException("Pip command timed out."); - } - } - catch (Exception e) { - // 即使PIP安装失败，仍然尝试运行Python代码 - log.warn("Pip install failed: {}", e.getMessage(), e); - } - finally { - if (process != null && process.isAlive()) { - process.destroyForcibly(); - } - } - } - - // 运行Python代码 - Process process = null; - try { - ProcessBuilder pb = new ProcessBuilder(this.pythonExecutable, scriptFile.toAbsolutePath().toString()); - pb.directory(container.toFile()); - pb.redirectInput(stdinFile.toFile()); - process = pb.start(); - - // 读取stdout和stderr - StringWriter stdoutWriter = new StringWriter(); - StringWriter stderrWriter = new StringWriter(); - try (BufferedReader stdoutReader = new BufferedReader(new InputStreamReader(process.getInputStream())); - BufferedReader stderrReader = new BufferedReader(new InputStreamReader(process.getErrorStream()))) { - CompletableFuture stdoutFuture = CompletableFuture.runAsync(() -> { - try { - stdoutReader.transferTo(stdoutWriter); - } - catch (IOException e) { - stderrWriter.write("Error reading stdout: " + e.getMessage()); - } - }); - CompletableFuture stderrFuture = CompletableFuture.runAsync(() -> { - try { - stderrReader.transferTo(stderrWriter); - } - catch (IOException e) { - stderrWriter.write("Error reading stderr: " + e.getMessage()); - } - }); - - // 等待进程完成，带超时限制 - boolean completed = process.waitFor(this.timeoutParser.parse(this.properties.getCodeTimeout()), - TimeUnit.MILLISECONDS); - if (!completed) { - process.destroy(); - if (process.isAlive()) { - process.destroyForcibly(); - } - return TaskResponse.failure("", "python code timeout, Killed."); - } - - // 等待输出读取完成，给输出读取额外2秒时间 - CompletableFuture.allOf(stdoutFuture, stderrFuture).get(2, TimeUnit.SECONDS); - } - - // 返回结果 - int exitCode = process.exitValue(); - String stdout = stdoutWriter.toString(); - String stderr = stderrWriter.toString(); - if (exitCode != 0) { - return TaskResponse.failure(stdout, stderr); - } - else { - return TaskResponse.success(stdout); - } - - } - catch (Exception e) { - log.error("Python execution failed: {}", e.getMessage(), e); - return TaskResponse.exception(e.getMessage()); - } - finally { - if (process != null && process.isAlive()) { - process.destroyForcibly(); - } - } - } - - @Override - protected void stopContainer(String containerId) throws Exception { - // 临时文件夹没有停止方法 - } - - @Override - protected void removeContainer(String containerId) throws Exception { - Path container = this.containers.remove(containerId); - this.clearTempDir(container); - } - - private static LocalPrograms locatePrograms(ExecutableProgramLocator locator) { - String python = locator.findFirst(PYTHON_NAMES) - .orElseThrow(() -> new IllegalStateException( - "No valid Python interpreter was found in PATH. Please install Python 3.")); - String pip = locator.findFirst(PIP_NAMES).orElse(null); - return new LocalPrograms(python, pip); - } - - private record LocalPrograms(String pythonExecutable, String pipExecutable) { - } - -} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/local/ExecutionTimeoutParser.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/local/ExecutionTimeoutParser.java deleted file mode 100644 index f7d294941..000000000 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/local/ExecutionTimeoutParser.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright 2024-2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code.local; - -import java.util.Locale; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * Parses the compact timeout format used by the code executor. - */ -public final class ExecutionTimeoutParser { - - public static final long DEFAULT_TIMEOUT_MILLIS = 60_000L; - - private static final Pattern TIMEOUT_PATTERN = Pattern.compile("(\\d+)(ms|[smhd])"); - - public long parse(String value) { - if (value == null) { - return DEFAULT_TIMEOUT_MILLIS; - } - Matcher matcher = TIMEOUT_PATTERN.matcher(value.trim().toLowerCase(Locale.ROOT)); - if (!matcher.matches()) { - return DEFAULT_TIMEOUT_MILLIS; - } - - try { - long amount = Long.parseLong(matcher.group(1)); - return Math.multiplyExact(amount, multiplier(matcher.group(2))); - } - catch (ArithmeticException | NumberFormatException exception) { - return DEFAULT_TIMEOUT_MILLIS; - } - } - - private long multiplier(String unit) { - return switch (unit) { - case "ms" -> 1L; - case "s" -> 1_000L; - case "m" -> 60_000L; - case "h" -> 3_600_000L; - case "d" -> 86_400_000L; - default -> throw new IllegalArgumentException("Unsupported timeout unit: " + unit); - }; - } - -} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/local/PathExecutableProgramLocator.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/local/PathExecutableProgramLocator.java deleted file mode 100644 index d59d1761a..000000000 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/local/PathExecutableProgramLocator.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright 2024-2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code.local; - -import java.io.File; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.List; -import java.util.Locale; -import java.util.Optional; -import java.util.regex.Pattern; - -/** - * Locates executables from the process {@code PATH}. - */ -public final class PathExecutableProgramLocator implements ExecutableProgramLocator { - - private final String pathEnvironment; - - private final boolean windows; - - public PathExecutableProgramLocator() { - this(System.getenv("PATH"), System.getProperty("os.name", "").toLowerCase(Locale.ROOT).contains("win")); - } - - PathExecutableProgramLocator(String pathEnvironment, boolean windows) { - this.pathEnvironment = pathEnvironment; - this.windows = windows; - } - - @Override - public Optional findFirst(List programNames) { - if (pathEnvironment == null || pathEnvironment.isBlank() || programNames == null || programNames.isEmpty()) { - return Optional.empty(); - } - - for (String programName : programNames) { - if (programName == null || programName.isBlank()) { - continue; - } - for (String directory : pathEnvironment.split(Pattern.quote(File.pathSeparator))) { - if (directory.isBlank()) { - continue; - } - if (isExecutable(Path.of(directory, programName)) - || windows && isExecutable(Path.of(directory, programName + ".exe"))) { - return Optional.of(programName); - } - } - } - return Optional.empty(); - } - - private boolean isExecutable(Path path) { - return Files.isRegularFile(path) && Files.isExecutable(path); - } - -} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SaaSandboxPythonCodeExecutorService.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SaaSandboxPythonCodeExecutorService.java new file mode 100644 index 000000000..b1fafb809 --- /dev/null +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SaaSandboxPythonCodeExecutorService.java @@ -0,0 +1,104 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.cloud.ai.dataagent.service.code.sandbox; + +import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; +import com.alibaba.cloud.ai.dataagent.service.code.PythonCodeExecutorService; +import com.alibaba.cloud.ai.dataagent.service.code.sandbox.runtime.SaaSandboxTaskRunner; +import jakarta.annotation.PreDestroy; +import org.springframework.stereotype.Service; + +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +@Service +public class SaaSandboxPythonCodeExecutorService implements PythonCodeExecutorService { + + private final CodeExecutorProperties properties; + + private final SaaSandboxTaskRunner taskRunner; + + private final ThreadPoolExecutor executor; + + public SaaSandboxPythonCodeExecutorService(CodeExecutorProperties properties, SaaSandboxTaskRunner taskRunner) { + this.properties = properties; + this.taskRunner = taskRunner; + int concurrency = properties.getSandbox().getMaxConcurrency(); + this.executor = new ThreadPoolExecutor(concurrency, concurrency, 0L, TimeUnit.MILLISECONDS, + new ArrayBlockingQueue<>(properties.getSandbox().getQueueCapacity()), + new ThreadPoolExecutor.AbortPolicy()); + } + + @Override + public TaskResponse runTask(TaskRequest request) { + try { + validateSize("Python code", request.code(), properties.getSandbox().getMaxCodeBytes()); + validateSize("Python input", request.input(), properties.getSandbox().getMaxInputBytes()); + } + catch (IllegalArgumentException ex) { + return TaskResponse.exception(ex.getMessage()); + } + + Future future; + try { + future = executor.submit(() -> taskRunner.run(request)); + } + catch (RuntimeException ex) { + return TaskResponse.exception("Python sandbox capacity is exhausted"); + } + + Duration taskTimeout = properties.getSandbox() + .getDependencyInstallTimeout() + .plus(properties.getCodeTimeout()) + .plusSeconds(30); + try { + return future.get(taskTimeout.toMillis(), TimeUnit.MILLISECONDS); + } + catch (TimeoutException ex) { + future.cancel(true); + return TaskResponse.exception("Python sandbox task timed out"); + } + catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + return TaskResponse.exception("Python sandbox task was interrupted"); + } + catch (ExecutionException ex) { + Throwable cause = ex.getCause(); + return TaskResponse.exception(cause == null ? ex.getMessage() : cause.getMessage()); + } + } + + private void validateSize(String label, String value, int maximumBytes) { + if (value == null) { + throw new IllegalArgumentException(label + " must not be null"); + } + if (value.getBytes(StandardCharsets.UTF_8).length > maximumBytes) { + throw new IllegalArgumentException(label + " exceeds the configured size limit"); + } + } + + @PreDestroy + public void close() { + executor.shutdownNow(); + } + +} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/local/ExecutableProgramLocator.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/dependency/PythonDependencyMetadata.java similarity index 62% rename from data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/local/ExecutableProgramLocator.java rename to data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/dependency/PythonDependencyMetadata.java index 3fa7b0607..cf3de5882 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/local/ExecutableProgramLocator.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/dependency/PythonDependencyMetadata.java @@ -13,17 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.alibaba.cloud.ai.dataagent.service.code.local; +package com.alibaba.cloud.ai.dataagent.service.code.sandbox.dependency; import java.util.List; -import java.util.Optional; -/** - * Finds the first available executable from an ordered candidate list. - */ -@FunctionalInterface -public interface ExecutableProgramLocator { +public record PythonDependencyMetadata(List dependencies, String requiresPython) { + + public static PythonDependencyMetadata empty() { + return new PythonDependencyMetadata(List.of(), null); + } - Optional findFirst(List programNames); + public PythonDependencyMetadata { + dependencies = dependencies == null ? List.of() : List.copyOf(dependencies); + } } diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/dependency/PythonDependencyMetadataParser.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/dependency/PythonDependencyMetadataParser.java new file mode 100644 index 000000000..b2c2b7401 --- /dev/null +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/dependency/PythonDependencyMetadataParser.java @@ -0,0 +1,114 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.cloud.ai.dataagent.service.code.sandbox.dependency; + +import org.springframework.stereotype.Component; +import org.tomlj.Toml; +import org.tomlj.TomlArray; +import org.tomlj.TomlParseResult; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +@Component +public class PythonDependencyMetadataParser { + + private static final String BLOCK_START = "# /// script"; + + private static final String BLOCK_END = "# ///"; + + private static final Set SUPPORTED_KEYS = Set.of("dependencies", "requires-python"); + + private final PythonDependencyPolicy dependencyPolicy; + + public PythonDependencyMetadataParser(PythonDependencyPolicy dependencyPolicy) { + this.dependencyPolicy = dependencyPolicy; + } + + public PythonDependencyMetadata parse(String code) { + String metadata = extractMetadata(code); + if (metadata == null) { + return PythonDependencyMetadata.empty(); + } + dependencyPolicy.validateMetadataSize(metadata); + + TomlParseResult result = Toml.parse(metadata); + if (result.hasErrors()) { + throw new IllegalArgumentException("Invalid PEP 723 dependency metadata: " + result.errors().get(0)); + } + if (!SUPPORTED_KEYS.containsAll(result.keySet())) { + throw new IllegalArgumentException("PEP 723 metadata contains unsupported fields"); + } + + List dependencies = new ArrayList<>(); + TomlArray dependencyArray = result.getArray("dependencies"); + if (dependencyArray != null) { + for (int index = 0; index < dependencyArray.size(); index++) { + Object dependency = dependencyArray.get(index); + if (!(dependency instanceof String dependencyString)) { + throw new IllegalArgumentException("PEP 723 dependencies must be strings"); + } + dependencies.add(dependencyString); + } + } + String requiresPython = result.getString("requires-python"); + return new PythonDependencyMetadata(dependencyPolicy.validateDependencies(dependencies), requiresPython); + } + + private String extractMetadata(String code) { + String[] lines = code.split("\\R", -1); + StringBuilder metadata = null; + boolean insideBlock = false; + boolean foundBlock = false; + for (int lineIndex = 0; lineIndex < lines.length; lineIndex++) { + String line = lines[lineIndex]; + if (BLOCK_START.equals(line.strip())) { + if (insideBlock || foundBlock) { + throw new IllegalArgumentException("Python code must contain at most one PEP 723 script block"); + } + metadata = new StringBuilder(); + insideBlock = true; + foundBlock = true; + continue; + } + if (!insideBlock) { + continue; + } + if (BLOCK_END.equals(line.strip())) { + insideBlock = false; + continue; + } + String stripped = line.stripLeading(); + if (stripped.isEmpty() && lineIndex == lines.length - 1) { + continue; + } + if (!stripped.startsWith("#")) { + throw new IllegalArgumentException("Each PEP 723 metadata line must be a Python comment"); + } + String value = stripped.substring(1); + if (value.startsWith(" ")) { + value = value.substring(1); + } + metadata.append(value).append('\n'); + } + if (insideBlock) { + throw new IllegalArgumentException("PEP 723 script block is not closed"); + } + return foundBlock ? metadata.toString() : null; + } + +} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/dependency/PythonDependencyPolicy.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/dependency/PythonDependencyPolicy.java new file mode 100644 index 000000000..87770ad92 --- /dev/null +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/dependency/PythonDependencyPolicy.java @@ -0,0 +1,56 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.cloud.ai.dataagent.service.code.sandbox.dependency; + +import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; +import org.springframework.stereotype.Component; + +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.regex.Pattern; + +@Component +public class PythonDependencyPolicy { + + private static final Pattern PACKAGE_SPECIFIER = Pattern.compile( + "^[A-Za-z0-9][A-Za-z0-9._-]*(?:\\[[A-Za-z0-9._,-]+])?(?:\\s*[!<>=~]{1,2}\\s*[A-Za-z0-9.*+!_-]+(?:\\s*,\\s*[!<>=~]{1,2}\\s*[A-Za-z0-9.*+!_-]+)*)?$"); + + private final CodeExecutorProperties properties; + + public PythonDependencyPolicy(CodeExecutorProperties properties) { + this.properties = properties; + } + + public void validateMetadataSize(String metadata) { + int bytes = metadata.getBytes(StandardCharsets.UTF_8).length; + if (bytes > properties.getSandbox().getMaxMetadataBytes()) { + throw new IllegalArgumentException("Python dependency metadata exceeds the configured size limit"); + } + } + + public List validateDependencies(List dependencies) { + if (dependencies.size() > properties.getSandbox().getMaxDependencies()) { + throw new IllegalArgumentException("Too many Python dependencies"); + } + for (String dependency : dependencies) { + if (dependency == null || !PACKAGE_SPECIFIER.matcher(dependency).matches()) { + throw new IllegalArgumentException("Unsupported Python dependency specifier: " + dependency); + } + } + return List.copyOf(dependencies); + } + +} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/execution/PythonSandboxBootstrapBuilder.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/execution/PythonSandboxBootstrapBuilder.java new file mode 100644 index 000000000..f5438ccec --- /dev/null +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/execution/PythonSandboxBootstrapBuilder.java @@ -0,0 +1,154 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.cloud.ai.dataagent.service.code.sandbox.execution; + +import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; +import com.alibaba.cloud.ai.dataagent.util.JsonUtil; +import com.fasterxml.jackson.core.JsonProcessingException; +import org.springframework.stereotype.Component; + +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.List; + +@Component +public class PythonSandboxBootstrapBuilder { + + static final String RESULT_MARKER = "__DATAAGENT_RESULT_V1__:"; + + private final CodeExecutorProperties properties; + + public PythonSandboxBootstrapBuilder(CodeExecutorProperties properties) { + this.properties = properties; + } + + public String buildDependencyInstaller(List dependencies) { + String dependenciesJson; + try { + dependenciesJson = JsonUtil.getObjectMapper().writeValueAsString(dependencies); + } + catch (JsonProcessingException ex) { + throw new IllegalArgumentException("Failed to serialize Python dependencies", ex); + } + String encodedDependencies = encode(dependenciesJson); + String encodedIndex = encode(properties.getSandbox().getPackageIndexUrl()); + long timeoutSeconds = properties.getSandbox().getDependencyInstallTimeout().toSeconds(); + int maxOutputCharacters = properties.getSandbox().getMaxOutputBytes(); + int maxErrorCharacters = properties.getSandbox().getMaxErrorBytes(); + return """ + import base64 + import json + import subprocess + import sys + + _dependencies = json.loads(base64.b64decode("%s").decode("utf-8")) + _index_url = base64.b64decode("%s").decode("utf-8") + _envelope = {"success": True, "stdout": "", "stderr": "", "errorType": None, "errorMessage": None} + try: + _command = [ + sys.executable, "-m", "pip", "install", + "--disable-pip-version-check", "--no-input", "--no-cache-dir", + "--target", "/tmp/dataagent-deps", "--index-url", _index_url, + *_dependencies + ] + _completed = subprocess.run( + _command, capture_output=True, text=True, timeout=%d, check=False + ) + _envelope["stdout"] = _completed.stdout[-%d:] + _envelope["stderr"] = _completed.stderr[-%d:] + if _completed.returncode != 0: + _envelope["success"] = False + _envelope["errorType"] = "DependencyInstallError" + _envelope["errorMessage"] = "pip exited with status " + str(_completed.returncode) + except subprocess.TimeoutExpired as _error: + _envelope["success"] = False + _envelope["errorType"] = "DependencyInstallTimeout" + _envelope["errorMessage"] = str(_error) + except BaseException as _error: + _envelope["success"] = False + _envelope["errorType"] = type(_error).__name__ + _envelope["errorMessage"] = str(_error) + _payload = base64.b64encode( + json.dumps(_envelope, ensure_ascii=False).encode("utf-8") + ).decode("ascii") + print("%s" + _payload) + """.formatted(encodedDependencies, encodedIndex, timeoutSeconds, maxOutputCharacters, + maxErrorCharacters, RESULT_MARKER); + } + + public String buildCodeExecution(String code, String input) { + String encodedCode = encode(code); + String encodedInput = encode(input); + long timeoutSeconds = properties.getCodeTimeout().toSeconds(); + int maxOutputCharacters = properties.getSandbox().getMaxOutputBytes(); + int maxErrorCharacters = properties.getSandbox().getMaxErrorBytes(); + return """ + import base64 + import json + import os + import subprocess + import sys + + _code = base64.b64decode("%s").decode("utf-8") + _input = base64.b64decode("%s").decode("utf-8") + _envelope = {"success": True, "stdout": "", "stderr": "", "errorType": None, "errorMessage": None} + + def _dataagent_as_text(_value): + if isinstance(_value, bytes): + return _value.decode("utf-8", errors="replace") + return _value or "" + + try: + _environment = dict(os.environ) + _environment["PYTHONPATH"] = "/tmp/dataagent-deps" + _completed = subprocess.run( + [sys.executable, "-c", _code], + input=_input, + capture_output=True, + text=True, + timeout=%d, + check=False, + env=_environment + ) + _envelope["stdout"] = _completed.stdout[-%d:] + _envelope["stderr"] = _completed.stderr[-%d:] + if _completed.returncode != 0: + _envelope["success"] = False + _envelope["errorType"] = "PythonProcessError" + _envelope["errorMessage"] = "Python exited with status " + str(_completed.returncode) + except subprocess.TimeoutExpired as _error: + _envelope["success"] = False + _envelope["errorType"] = "PythonTimeout" + _envelope["errorMessage"] = "Python code execution exceeded %d seconds" + _envelope["stdout"] = _dataagent_as_text(_error.stdout)[-%d:] + _envelope["stderr"] = _dataagent_as_text(_error.stderr)[-%d:] + except BaseException as _error: + _envelope["success"] = False + _envelope["errorType"] = type(_error).__name__ + _envelope["errorMessage"] = str(_error) + _payload = base64.b64encode( + json.dumps(_envelope, ensure_ascii=False).encode("utf-8") + ).decode("ascii") + print("%s" + _payload) + """.formatted(encodedCode, encodedInput, timeoutSeconds, maxOutputCharacters, maxErrorCharacters, + timeoutSeconds, maxOutputCharacters, maxErrorCharacters, RESULT_MARKER); + } + + private String encode(String value) { + return Base64.getEncoder().encodeToString(value.getBytes(StandardCharsets.UTF_8)); + } + +} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/enums/CodePoolExecutorEnum.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/execution/SandboxExecutionResult.java similarity index 73% rename from data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/enums/CodePoolExecutorEnum.java rename to data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/execution/SandboxExecutionResult.java index c324ba244..fbfc93db0 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/enums/CodePoolExecutorEnum.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/execution/SandboxExecutionResult.java @@ -13,16 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.alibaba.cloud.ai.dataagent.enums; +package com.alibaba.cloud.ai.dataagent.service.code.sandbox.execution; -/** - * 运行Python任务的容器池（实现枚举） - * - * @author vlsmb - * @since 2025/7/28 - */ -public enum CodePoolExecutorEnum { - - DOCKER, CONTAINERD, KATA, AI_SIMULATION, LOCAL; +public record SandboxExecutionResult(boolean success, String stdout, String stderr, String errorType, + String errorMessage) { } diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/execution/SandboxExecutionResultParser.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/execution/SandboxExecutionResultParser.java new file mode 100644 index 000000000..a373d2a4b --- /dev/null +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/execution/SandboxExecutionResultParser.java @@ -0,0 +1,74 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.cloud.ai.dataagent.service.code.sandbox.execution; + +import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; +import com.alibaba.cloud.ai.dataagent.util.JsonUtil; +import com.fasterxml.jackson.core.JsonProcessingException; +import org.springframework.stereotype.Component; + +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +@Component +public class SandboxExecutionResultParser { + + private static final Pattern ENCODED_RESULT = Pattern + .compile(Pattern.quote(PythonSandboxBootstrapBuilder.RESULT_MARKER) + "([A-Za-z0-9+/=]+)"); + + private final CodeExecutorProperties properties; + + public SandboxExecutionResultParser(CodeExecutorProperties properties) { + this.properties = properties; + } + + public SandboxExecutionResult parse(String rawResult) { + if (rawResult == null) { + throw new IllegalArgumentException("Sandbox returned an empty response"); + } + Matcher matcher = ENCODED_RESULT.matcher(rawResult); + String encodedResult = null; + while (matcher.find()) { + encodedResult = matcher.group(1); + } + if (encodedResult == null) { + throw new IllegalArgumentException("Sandbox response does not contain the DataAgent result marker"); + } + + byte[] decoded; + try { + decoded = Base64.getDecoder().decode(encodedResult); + } + catch (IllegalArgumentException ex) { + throw new IllegalArgumentException("Sandbox returned an invalid encoded result", ex); + } + int maximumBytes = properties.getSandbox().getMaxOutputBytes() + properties.getSandbox().getMaxErrorBytes(); + if (decoded.length > maximumBytes + 4096) { + throw new IllegalArgumentException("Sandbox result exceeds the configured size limit"); + } + + try { + return JsonUtil.getObjectMapper() + .readValue(new String(decoded, StandardCharsets.UTF_8), SandboxExecutionResult.class); + } + catch (JsonProcessingException ex) { + throw new IllegalArgumentException("Sandbox returned an invalid result envelope", ex); + } + } + +} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/runtime/SaaSandboxRuntime.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/runtime/SaaSandboxRuntime.java new file mode 100644 index 000000000..d3eee9217 --- /dev/null +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/runtime/SaaSandboxRuntime.java @@ -0,0 +1,111 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.cloud.ai.dataagent.service.code.sandbox.runtime; + +import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; +import io.agentscope.runtime.sandbox.box.BaseSandbox; +import io.agentscope.runtime.sandbox.manager.ManagerConfig; +import io.agentscope.runtime.sandbox.manager.SandboxService; +import io.agentscope.runtime.sandbox.manager.client.container.docker.DockerClientStarter; +import io.agentscope.runtime.sandbox.manager.model.sandbox.SandboxConfig; +import io.agentscope.runtime.sandbox.manager.registry.SandboxRegistryService; +import jakarta.annotation.PreDestroy; +import org.springframework.stereotype.Component; + +import java.util.Map; + +@Component +public class SaaSandboxRuntime { + + private final CodeExecutorProperties properties; + + private volatile SandboxService sandboxService; + + public SaaSandboxRuntime(CodeExecutorProperties properties) { + this.properties = properties; + } + + public SandboxService getOrStart() { + SandboxService current = sandboxService; + if (current != null) { + return current; + } + synchronized (this) { + if (sandboxService == null) { + sandboxService = startSandboxService(); + } + return sandboxService; + } + } + + private SandboxService startSandboxService() { + CodeExecutorProperties.Sandbox sandboxProperties = properties.getSandbox(); + Map runtimeConfig = Map.of("mem_limit", properties.getLimitMemory() + "m", "nano_cpus", + properties.getCpuCore() * 1_000_000_000L, "max_connections", sandboxProperties.getMaxConnections(), + "privileged", false); + SandboxConfig sandboxConfig = new SandboxConfig.Builder().imageName(sandboxProperties.getImageName()) + .sandboxType("base") + .securityLevel("high") + .timeout(sandboxTimeoutSeconds()) + .description("DataAgent generated Python sandbox") + .runtimeConfig(runtimeConfig) + .build(); + SandboxRegistryService.register(BaseSandbox.class, sandboxConfig); + + DockerClientStarter dockerClientStarter = createDockerClientStarter(sandboxProperties.getDockerHost()); + ManagerConfig managerConfig = ManagerConfig.builder() + .containerPrefixKey(sandboxProperties.getContainerPrefix()) + .clientStarter(dockerClientStarter) + .build(); + SandboxService service = new SandboxService(managerConfig); + service.start(); + return service; + } + + private DockerClientStarter createDockerClientStarter(String dockerHost) { + if (dockerHost != null && dockerHost.startsWith("tcp://")) { + String address = dockerHost.substring("tcp://".length()); + int separator = address.lastIndexOf(':'); + if (separator > 0) { + return DockerClientStarter.builder() + .host(address.substring(0, separator)) + .port(Integer.parseInt(address.substring(separator + 1))) + .build(); + } + } + // AgentScope 1.0.2 falls back to docker-java's standard local socket + // discovery when its default TCP endpoint is unavailable. + return DockerClientStarter.builder().build(); + } + + int sandboxTimeoutSeconds() { + return Math.toIntExact(properties.getSandbox() + .getDependencyInstallTimeout() + .plus(properties.getCodeTimeout()) + .plusSeconds(30) + .toSeconds()); + } + + @PreDestroy + public synchronized void close() { + if (sandboxService != null) { + sandboxService.cleanupAllSandboxes(); + sandboxService.close(); + sandboxService = null; + } + } + +} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/runtime/SaaSandboxTaskRunner.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/runtime/SaaSandboxTaskRunner.java new file mode 100644 index 000000000..6333887b3 --- /dev/null +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/runtime/SaaSandboxTaskRunner.java @@ -0,0 +1,85 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.cloud.ai.dataagent.service.code.sandbox.runtime; + +import com.alibaba.cloud.ai.dataagent.service.code.sandbox.execution.PythonSandboxBootstrapBuilder; +import com.alibaba.cloud.ai.dataagent.service.code.sandbox.execution.SandboxExecutionResult; +import com.alibaba.cloud.ai.dataagent.service.code.sandbox.execution.SandboxExecutionResultParser; +import com.alibaba.cloud.ai.dataagent.service.code.PythonCodeExecutorService; +import com.alibaba.cloud.ai.sandbox.tools.base.SaaBasePythonRunner; +import io.agentscope.runtime.sandbox.box.BaseSandbox; +import org.springframework.stereotype.Component; + +import java.util.UUID; + +@Component +public class SaaSandboxTaskRunner { + + private final SaaSandboxRuntime sandboxRuntime; + + private final PythonSandboxBootstrapBuilder bootstrapBuilder; + + private final SandboxExecutionResultParser resultParser; + + public SaaSandboxTaskRunner(SaaSandboxRuntime sandboxRuntime, PythonSandboxBootstrapBuilder bootstrapBuilder, + SandboxExecutionResultParser resultParser) { + this.sandboxRuntime = sandboxRuntime; + this.bootstrapBuilder = bootstrapBuilder; + this.resultParser = resultParser; + } + + public PythonCodeExecutorService.TaskResponse run(PythonCodeExecutorService.TaskRequest request) { + try (BaseSandbox sandbox = new BaseSandbox(sandboxRuntime.getOrStart(), "dataagent", + UUID.randomUUID().toString())) { + SaaBasePythonRunner pythonRunner = new SaaBasePythonRunner(); + pythonRunner.setSandbox(sandbox); + + if (!request.dependencies().isEmpty()) { + SandboxExecutionResult installResult = run(pythonRunner, + bootstrapBuilder.buildDependencyInstaller(request.dependencies())); + if (!installResult.success()) { + return PythonCodeExecutorService.TaskResponse.failure(installResult.stdout(), + formatError(installResult)); + } + } + + SandboxExecutionResult executionResult = run(pythonRunner, + bootstrapBuilder.buildCodeExecution(request.code(), request.input())); + if (executionResult.success()) { + return PythonCodeExecutorService.TaskResponse.success(executionResult.stdout()); + } + return PythonCodeExecutorService.TaskResponse.failure(executionResult.stdout(), + formatError(executionResult)); + } + catch (Exception ex) { + return PythonCodeExecutorService.TaskResponse.exception(ex.getMessage()); + } + } + + private SandboxExecutionResult run(SaaBasePythonRunner pythonRunner, String code) { + SaaBasePythonRunner.RunPythonToolResponse response = pythonRunner + .apply(new SaaBasePythonRunner.RunPythonToolRequest(code), null); + return resultParser.parse(response.output().result()); + } + + private String formatError(SandboxExecutionResult result) { + String errorType = result.errorType() == null ? "PythonError" : result.errorType(); + String errorMessage = result.errorMessage() == null ? "" : ": " + result.errorMessage(); + String stderr = result.stderr() == null ? "" : "\n" + result.stderr(); + return errorType + errorMessage + stderr; + } + +} diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/workflow/node/PythonExecuteNode.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/workflow/node/PythonExecuteNode.java index 5cc1531d5..f1a7e77dc 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/workflow/node/PythonExecuteNode.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/workflow/node/PythonExecuteNode.java @@ -16,13 +16,15 @@ package com.alibaba.cloud.ai.dataagent.workflow.node; import com.alibaba.cloud.ai.dataagent.enums.TextType; +import com.alibaba.cloud.ai.dataagent.service.code.PythonCodeExecutorService; +import com.alibaba.cloud.ai.dataagent.service.code.sandbox.dependency.PythonDependencyMetadata; +import com.alibaba.cloud.ai.dataagent.service.code.sandbox.dependency.PythonDependencyMetadataParser; import com.alibaba.cloud.ai.dataagent.util.JsonParseUtil; import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; import com.alibaba.cloud.ai.graph.GraphResponse; import com.alibaba.cloud.ai.graph.OverAllState; import com.alibaba.cloud.ai.graph.action.NodeAction; import com.alibaba.cloud.ai.graph.streaming.StreamingOutput; -import com.alibaba.cloud.ai.dataagent.service.code.CodePoolExecutorService; import com.alibaba.cloud.ai.dataagent.util.ChatResponseUtil; import com.alibaba.cloud.ai.dataagent.util.FluxUtil; import com.alibaba.cloud.ai.dataagent.util.JsonUtil; @@ -49,7 +51,9 @@ @Component public class PythonExecuteNode implements NodeAction { - private final CodePoolExecutorService codePoolExecutor; + private final PythonCodeExecutorService pythonCodeExecutor; + + private final PythonDependencyMetadataParser dependencyMetadataParser; private final ObjectMapper objectMapper; @@ -57,9 +61,11 @@ public class PythonExecuteNode implements NodeAction { private final CodeExecutorProperties codeExecutorProperties; - public PythonExecuteNode(CodePoolExecutorService codePoolExecutor, JsonParseUtil jsonParseUtil, + public PythonExecuteNode(PythonCodeExecutorService pythonCodeExecutor, + PythonDependencyMetadataParser dependencyMetadataParser, JsonParseUtil jsonParseUtil, CodeExecutorProperties codeExecutorProperties) { - this.codePoolExecutor = codePoolExecutor; + this.pythonCodeExecutor = pythonCodeExecutor; + this.dependencyMetadataParser = dependencyMetadataParser; this.objectMapper = JsonUtil.getObjectMapper(); this.jsonParseUtil = jsonParseUtil; this.codeExecutorProperties = codeExecutorProperties; @@ -77,11 +83,12 @@ public Map apply(OverAllState state) throws Exception { // 检查重试次数 int triesCount = StateUtil.getObjectValue(state, PYTHON_TRIES_COUNT, Integer.class, 0); - CodePoolExecutorService.TaskRequest taskRequest = new CodePoolExecutorService.TaskRequest(pythonCode, - objectMapper.writeValueAsString(sqlResults), null); + PythonDependencyMetadata metadata = dependencyMetadataParser.parse(pythonCode); + PythonCodeExecutorService.TaskRequest taskRequest = new PythonCodeExecutorService.TaskRequest(pythonCode, + objectMapper.writeValueAsString(sqlResults), metadata.dependencies()); // Run Python code - CodePoolExecutorService.TaskResponse taskResponse = this.codePoolExecutor.runTask(taskRequest); + PythonCodeExecutorService.TaskResponse taskResponse = this.pythonCodeExecutor.runTask(taskRequest); if (!taskResponse.isSuccess()) { String errorMsg = "Python Execute Failed!\nStdOut: " + taskResponse.stdOut() + "\nStdErr: " + taskResponse.stdErr() + "\nExceptionMsg: " + taskResponse.exceptionMsg(); diff --git a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/workflow/node/PythonGenerateNode.java b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/workflow/node/PythonGenerateNode.java index 70f0ecb4c..d120ba763 100644 --- a/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/workflow/node/PythonGenerateNode.java +++ b/data-agent-management/src/main/java/com/alibaba/cloud/ai/dataagent/workflow/node/PythonGenerateNode.java @@ -103,7 +103,7 @@ public Map apply(OverAllState state) throws Exception { // Load Python code generation template String systemPrompt = PromptConstant.getPythonGeneratorPromptTemplate() .render(Map.of("python_memory", codeExecutorProperties.getLimitMemory().toString(), "python_timeout", - codeExecutorProperties.getCodeTimeout(), "database_schema", + codeExecutorProperties.getCodeTimeout().toSeconds() + "s", "database_schema", objectMapper.writeValueAsString(schemaDTO), "sample_input", objectMapper.writeValueAsString(sqlResults.stream().limit(SAMPLE_DATA_NUMBER).toList()), "plan_description", objectMapper.writeValueAsString(toolParameters))); diff --git a/data-agent-management/src/main/resources/application-h2.yml b/data-agent-management/src/main/resources/application-h2.yml index 3bf8b2c5e..f76aab97c 100644 --- a/data-agent-management/src/main/resources/application-h2.yml +++ b/data-agent-management/src/main/resources/application-h2.yml @@ -3,6 +3,9 @@ server: # 存储 DataAgent 业务数据的数据库，并非Agent分析数据的来源 spring: + codec: + # Python workflow timelines can contain generated code, result data and the final report. + max-in-memory-size: 10MB datasource: url: jdbc:h2:mem:nl2sql_database;DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=true;MODE=MySQL;DB_CLOSE_ON_EXIT=FALSE username: root @@ -38,10 +41,18 @@ spring: default-topk-limit: 8 default-similarity-threshold: 0.4 code-executor: - # 运行Python代码的环境（生产环境建议使用docker，不建议使用local） - code-pool-executor: local - # Python执行的最大重试次数 python-max-tries-count: 5 + code-timeout: 60s + limit-memory: 500 + cpu-core: 1 + sandbox: + docker-host: ${DATAAGENT_SANDBOX_DOCKER_HOST:unix:///var/run/docker.sock} + image-name: ${DATAAGENT_SANDBOX_IMAGE:agentscope-registry.ap-southeast-1.cr.aliyuncs.com/agentscope/runtime-sandbox-base:latest} + max-concurrency: 4 + queue-capacity: 10 + max-connections: 4096 + package-index-url: ${DATAAGENT_PYPI_INDEX_URL:https://pypi.org/simple} + dependency-install-timeout: 3m file: type: local path-prefix: data-agent diff --git a/data-agent-management/src/main/resources/application.yml b/data-agent-management/src/main/resources/application.yml index a8beeefea..e1be57528 100644 --- a/data-agent-management/src/main/resources/application.yml +++ b/data-agent-management/src/main/resources/application.yml @@ -9,6 +9,9 @@ springdoc: # 存储 DataAgent 业务数据的数据库，并非Agent分析数据的来源 spring: + codec: + # Python workflow timelines can contain generated code, result data and the final report. + max-in-memory-size: 10MB datasource: url: ${DATA_AGENT_DATASOURCE_URL:jdbc:mysql://127.0.0.1:3306/saa_data_agent?useUnicode=true&characterEncoding=utf-8&zeroDateTimeBehavior=convertToNull&transformedBitIsBoolean=true&allowMultiQueries=true&allowPublicKeyRetrieval=true&useSSL=false&serverTimezone=Asia/Shanghai} username: ${DATA_AGENT_DATASOURCE_USERNAME:root} @@ -38,10 +41,18 @@ spring: default-topk-limit: 8 default-similarity-threshold: 0.4 code-executor: - # 运行Python代码的环境（生产环境建议使用docker，不建议使用local） - code-pool-executor: local - # Python执行的最大重试次数 python-max-tries-count: 5 + code-timeout: 60s + limit-memory: 500 + cpu-core: 1 + sandbox: + docker-host: ${DATAAGENT_SANDBOX_DOCKER_HOST:unix:///var/run/docker.sock} + image-name: ${DATAAGENT_SANDBOX_IMAGE:agentscope-registry.ap-southeast-1.cr.aliyuncs.com/agentscope/runtime-sandbox-base:latest} + max-concurrency: 4 + queue-capacity: 10 + max-connections: 4096 + package-index-url: ${DATAAGENT_PYPI_INDEX_URL:https://pypi.org/simple} + dependency-install-timeout: 3m file: type: local path-prefix: data-agent diff --git a/data-agent-management/src/main/resources/prompts/python-generator.txt b/data-agent-management/src/main/resources/prompts/python-generator.txt index c000c8a42..ee54b11e1 100644 --- a/data-agent-management/src/main/resources/prompts/python-generator.txt +++ b/data-agent-management/src/main/resources/prompts/python-generator.txt @@ -16,7 +16,10 @@ traceback.print_exc(file=sys.stderr) sys.exit(1) ``` -5. **依赖限制**：所有使用的库必须是`continuumio/anaconda3`默认安装的库，如`pandas`, `numpy`, `json`, `sys`等。 +5. **动态依赖**： + - 所有非标准库依赖必须在脚本顶部使用 [PEP 723] 的 `# /// script` 元数据声明，执行器会在隔离沙盒中动态安装。 + - `dependencies` 只能填写 PyPI 包名及版本约束，例如 `"pandas>=2,<3"`；禁止 URL、VCS、文件路径、环境标记和 pip 参数。 + - 每一个非标准库 `import` 都必须有对应的依赖声明；只使用标准库时可以省略元数据块。 6. **动态处理**：禁止硬编码列名/值，所有逻辑基于输入数据动态构建。 7. **安全限制**：禁止以下操作： - 任何文件/网络操作（open/requests等）。 @@ -32,6 +35,10 @@ 以下是生成代码的模板，请根据具体需求填充逻辑： ```python +# /// script +# dependencies = ["pandas>=2,<3"] +# /// + import sys import json import traceback @@ -94,6 +101,10 @@ except Exception: 假设用户需求是“统计每个渠道的线索数量和转化率”，生成的代码可能如下： ```python +# /// script +# dependencies = ["pandas>=2,<3"] +# /// + import sys import json import traceback diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/config/DataAgentConfigurationTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/config/DataAgentConfigurationTest.java index da649bc2a..205ca61a1 100644 --- a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/config/DataAgentConfigurationTest.java +++ b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/config/DataAgentConfigurationTest.java @@ -15,7 +15,6 @@ */ package com.alibaba.cloud.ai.dataagent.config; -import com.alibaba.cloud.ai.dataagent.service.code.CodePoolExecutorServiceFactory; import com.alibaba.cloud.ai.dataagent.service.file.FileStorageServiceFactory; import com.alibaba.cloud.ai.graph.CompileConfig; import com.alibaba.cloud.ai.graph.RunnableConfig; @@ -34,9 +33,8 @@ class DataAgentConfigurationTest { @Test - void runtimeServiceFactories_arePlainSelectorsInsteadOfSpringFactoryBeans() { + void fileStorageServiceFactory_isPlainSelectorInsteadOfSpringFactoryBean() { assertThat(FactoryBean.class.isAssignableFrom(FileStorageServiceFactory.class)).isFalse(); - assertThat(FactoryBean.class.isAssignableFrom(CodePoolExecutorServiceFactory.class)).isFalse(); } @Test diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/config/VectorStoreConfigurationTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/config/VectorStoreConfigurationTest.java index 79c1ab129..942c85d21 100644 --- a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/config/VectorStoreConfigurationTest.java +++ b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/config/VectorStoreConfigurationTest.java @@ -15,12 +15,9 @@ */ package com.alibaba.cloud.ai.dataagent.config; -import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; import com.alibaba.cloud.ai.dataagent.properties.FileStorageProperties; import com.alibaba.cloud.ai.dataagent.properties.OssStorageProperties; import com.alibaba.cloud.ai.dataagent.service.aimodelconfig.AiModelRegistry; -import com.alibaba.cloud.ai.dataagent.service.code.docker.DockerExecutorFactory; -import com.alibaba.cloud.ai.dataagent.service.llm.LlmService; import org.junit.jupiter.api.Test; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.context.properties.bind.Binder; @@ -40,7 +37,7 @@ class VectorStoreConfigurationTest { private final YamlPropertySourceLoader loader = new YamlPropertySourceLoader(); @Test - void vectorStoreProfiles_useExplicitSpringAiStarterTypes() throws Exception { + void vectorStoreProfiles_declareExpectedTypes() throws Exception { assertThat(property("application.yml", "spring.ai.vectorstore.type")).isEqualTo("simple"); assertThat(property("application-milvus.yml", "spring.ai.vectorstore.type")).isEqualTo("milvus"); assertThat(property("application-elasticsearch.yml", "spring.ai.vectorstore.type")).isEqualTo("elasticsearch"); @@ -67,8 +64,6 @@ void milvusProfile_bindsConnectionPropertiesUsedBySpringAi() throws Exception { void replaceableRuntimeServices_areConditionalOnMissingBean() throws Exception { assertConditional("llmService", AiModelRegistry.class); assertConditional("fileStorageService", FileStorageProperties.class, OssStorageProperties.class); - assertConditional("codePoolExecutorService", CodeExecutorProperties.class, LlmService.class, - DockerExecutorFactory.class); } private Object property(String resource, String name) throws Exception { diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/enums/CodePoolExecutorEnumTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/enums/CodePoolExecutorEnumTest.java deleted file mode 100644 index 06c949b31..000000000 --- a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/enums/CodePoolExecutorEnumTest.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.enums; - -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.*; - -class CodePoolExecutorEnumTest { - - @Test - void testValues() { - assertEquals(5, CodePoolExecutorEnum.values().length); - } - - @Test - void testValueOf() { - assertEquals(CodePoolExecutorEnum.DOCKER, CodePoolExecutorEnum.valueOf("DOCKER")); - assertEquals(CodePoolExecutorEnum.CONTAINERD, CodePoolExecutorEnum.valueOf("CONTAINERD")); - assertEquals(CodePoolExecutorEnum.KATA, CodePoolExecutorEnum.valueOf("KATA")); - assertEquals(CodePoolExecutorEnum.AI_SIMULATION, CodePoolExecutorEnum.valueOf("AI_SIMULATION")); - assertEquals(CodePoolExecutorEnum.LOCAL, CodePoolExecutorEnum.valueOf("LOCAL")); - } - - @Test - void testValueOf_invalid() { - assertThrows(IllegalArgumentException.class, () -> CodePoolExecutorEnum.valueOf("UNKNOWN")); - } - -} diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/integration/PythonWorkflowIntegrationTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/integration/PythonWorkflowIntegrationTest.java index 0d7408373..a0470cc92 100644 --- a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/integration/PythonWorkflowIntegrationTest.java +++ b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/integration/PythonWorkflowIntegrationTest.java @@ -24,6 +24,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.time.Duration; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -35,7 +36,9 @@ import com.alibaba.cloud.ai.dataagent.common.TestFixtures; import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; -import com.alibaba.cloud.ai.dataagent.service.code.CodePoolExecutorService; +import com.alibaba.cloud.ai.dataagent.service.code.PythonCodeExecutorService; +import com.alibaba.cloud.ai.dataagent.service.code.sandbox.dependency.PythonDependencyMetadata; +import com.alibaba.cloud.ai.dataagent.service.code.sandbox.dependency.PythonDependencyMetadataParser; import com.alibaba.cloud.ai.dataagent.service.llm.LlmService; import com.alibaba.cloud.ai.dataagent.util.ChatResponseUtil; import com.alibaba.cloud.ai.dataagent.util.JsonParseUtil; @@ -55,7 +58,10 @@ class PythonWorkflowIntegrationTest { private LlmService llmService; @Mock - private CodePoolExecutorService codePoolExecutor; + private PythonCodeExecutorService pythonCodeExecutor; + + @Mock + private PythonDependencyMetadataParser dependencyMetadataParser; @Mock private JsonParseUtil jsonParseUtil; @@ -72,11 +78,13 @@ class PythonWorkflowIntegrationTest { @BeforeEach void setUp() { when(codeExecutorProperties.getLimitMemory()).thenReturn(500L); - when(codeExecutorProperties.getCodeTimeout()).thenReturn("60s"); + when(codeExecutorProperties.getCodeTimeout()).thenReturn(Duration.ofSeconds(60)); when(codeExecutorProperties.getPythonMaxTriesCount()).thenReturn(5); + when(dependencyMetadataParser.parse(anyString())).thenReturn(PythonDependencyMetadata.empty()); pythonGenerateNode = new PythonGenerateNode(codeExecutorProperties, llmService); - pythonExecuteNode = new PythonExecuteNode(codePoolExecutor, jsonParseUtil, codeExecutorProperties); + pythonExecuteNode = new PythonExecuteNode(pythonCodeExecutor, dependencyMetadataParser, jsonParseUtil, + codeExecutorProperties); pythonAnalyzeNode = new PythonAnalyzeNode(llmService); } @@ -125,7 +133,8 @@ void endToEnd_pythonAnalysis_generatesExecutesAnalyzes_returnsResults() throws E state.updateState(Map.of(PYTHON_GENERATE_NODE_OUTPUT, generatedCode, PYTHON_TRIES_COUNT, 1)); String executionOutput = "{\"count\": 3, \"mean\": 200.0}"; - when(codePoolExecutor.runTask(any())).thenReturn(CodePoolExecutorService.TaskResponse.success(executionOutput)); + when(pythonCodeExecutor.runTask(any())) + .thenReturn(PythonCodeExecutorService.TaskResponse.success(executionOutput)); when(jsonParseUtil.tryConvertToObject(anyString(), any(Class.class))).thenReturn(null); Map executeResult = pythonExecuteNode.apply(state); @@ -143,7 +152,7 @@ void endToEnd_pythonAnalysis_generatesExecutesAnalyzes_returnsResults() throws E assertTrue(analyzeResult.containsKey(PYTHON_ANALYSIS_NODE_OUTPUT)); verify(llmService).call(anyString(), anyString()); - verify(codePoolExecutor).runTask(any()); + verify(pythonCodeExecutor).runTask(any()); verify(llmService).callSystem(anyString()); } diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/CodePoolExecutorServiceFactoryTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/CodePoolExecutorServiceFactoryTest.java deleted file mode 100644 index 604c4f4a4..000000000 --- a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/CodePoolExecutorServiceFactoryTest.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright 2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code; - -import com.alibaba.cloud.ai.dataagent.enums.CodePoolExecutorEnum; -import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; -import com.alibaba.cloud.ai.dataagent.service.code.docker.DockerExecutorFactory; -import com.alibaba.cloud.ai.dataagent.service.code.impls.AiSimulationCodeExecutorService; -import com.alibaba.cloud.ai.dataagent.service.code.impls.DockerCodePoolExecutorService; -import com.alibaba.cloud.ai.dataagent.service.llm.LlmService; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.Mockito.when; - -@ExtendWith(MockitoExtension.class) -class CodePoolExecutorServiceFactoryTest { - - @Mock - private CodeExecutorProperties properties; - - @Mock - private LlmService llmService; - - @Mock - private DockerExecutorFactory dockerExecutorFactory; - - @Mock - private DockerCodePoolExecutorService dockerExecutor; - - private CodePoolExecutorServiceFactory factory; - - @BeforeEach - void setUp() { - factory = new CodePoolExecutorServiceFactory(properties, llmService, dockerExecutorFactory); - } - - @Test - void testGetObject_docker() { - when(properties.getCodePoolExecutor()).thenReturn(CodePoolExecutorEnum.DOCKER); - when(dockerExecutorFactory.create(properties)).thenReturn(dockerExecutor); - - assertSame(dockerExecutor, factory.getObject()); - } - - @Test - void testGetObject_aiSimulation() { - when(properties.getCodePoolExecutor()).thenReturn(CodePoolExecutorEnum.AI_SIMULATION); - - CodePoolExecutorService result = factory.getObject(); - assertInstanceOf(AiSimulationCodeExecutorService.class, result); - } - - @Test - void testGetObject_unsupported() { - when(properties.getCodePoolExecutor()).thenReturn(CodePoolExecutorEnum.CONTAINERD); - - assertThrows(IllegalStateException.class, () -> factory.getObject()); - } - - @Test - void testGetObjectType() { - assertEquals(CodePoolExecutorService.class, factory.getObjectType()); - } - -} diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/CodePoolExecutorServiceTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/CodePoolExecutorServiceTest.java deleted file mode 100644 index 891256962..000000000 --- a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/CodePoolExecutorServiceTest.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright 2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code; - -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.*; - -class CodePoolExecutorServiceTest { - - @Test - void taskResponse_success_hasCorrectFields() { - CodePoolExecutorService.TaskResponse response = CodePoolExecutorService.TaskResponse.success("output"); - - assertTrue(response.isSuccess()); - assertFalse(response.executionSuccessButResultFailed()); - assertEquals("output", response.stdOut()); - assertNull(response.stdErr()); - assertNull(response.exceptionMsg()); - } - - @Test - void taskResponse_failure_hasCorrectFields() { - CodePoolExecutorService.TaskResponse response = CodePoolExecutorService.TaskResponse.failure("stdout", - "stderr msg"); - - assertFalse(response.isSuccess()); - assertTrue(response.executionSuccessButResultFailed()); - assertEquals("stdout", response.stdOut()); - assertEquals("stderr msg", response.stdErr()); - assertTrue(response.exceptionMsg().contains("stderr msg")); - } - - @Test - void taskResponse_exception_hasCorrectFields() { - CodePoolExecutorService.TaskResponse response = CodePoolExecutorService.TaskResponse - .exception("connection timeout"); - - assertFalse(response.isSuccess()); - assertFalse(response.executionSuccessButResultFailed()); - assertNull(response.stdOut()); - assertNull(response.stdErr()); - assertTrue(response.exceptionMsg().contains("connection timeout")); - } - - @Test - void taskResponse_toString_containsFields() { - CodePoolExecutorService.TaskResponse response = CodePoolExecutorService.TaskResponse.success("hello"); - - String str = response.toString(); - assertTrue(str.contains("isSuccess=true")); - assertTrue(str.contains("hello")); - } - - @Test - void taskRequest_fieldsAccessible() { - CodePoolExecutorService.TaskRequest request = new CodePoolExecutorService.TaskRequest("print('hi')", - "{\"key\": 1}", "analyze data"); - - assertEquals("print('hi')", request.code()); - assertEquals("{\"key\": 1}", request.input()); - assertEquals("analyze data", request.requirement()); - } - - @Test - void taskResponse_success_nullStdErr() { - CodePoolExecutorService.TaskResponse response = CodePoolExecutorService.TaskResponse.success(null); - - assertTrue(response.isSuccess()); - assertNull(response.stdOut()); - } - - @Test - void stateEnum_values() { - assertEquals(3, CodePoolExecutorService.State.values().length); - assertNotNull(CodePoolExecutorService.State.READY); - assertNotNull(CodePoolExecutorService.State.RUNNING); - assertNotNull(CodePoolExecutorService.State.REMOVING); - } - -} diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/PythonCodeExecutorServiceTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/PythonCodeExecutorServiceTest.java new file mode 100644 index 000000000..39c14a676 --- /dev/null +++ b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/PythonCodeExecutorServiceTest.java @@ -0,0 +1,48 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.cloud.ai.dataagent.service.code; + +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +class PythonCodeExecutorServiceTest { + + @Test + void taskRequest_copiesDependenciesAndUsesEmptyListForNull() { + PythonCodeExecutorService.TaskRequest empty = new PythonCodeExecutorService.TaskRequest("print(1)", "[]", null); + PythonCodeExecutorService.TaskRequest populated = new PythonCodeExecutorService.TaskRequest("print(1)", "[]", + List.of("pandas>=2,<3")); + + assertThat(empty.dependencies()).isEmpty(); + assertThat(populated.dependencies()).containsExactly("pandas>=2,<3"); + } + + @Test + void taskResponse_distinguishesPythonFailureFromInfrastructureFailure() { + PythonCodeExecutorService.TaskResponse pythonFailure = PythonCodeExecutorService.TaskResponse.failure("partial", + "ValueError"); + PythonCodeExecutorService.TaskResponse infrastructureFailure = PythonCodeExecutorService.TaskResponse + .exception("Docker unavailable"); + + assertThat(pythonFailure.executionSuccessButResultFailed()).isTrue(); + assertThat(infrastructureFailure.executionSuccessButResultFailed()).isFalse(); + assertThat(infrastructureFailure.exceptionMsg()).contains("Docker unavailable"); + } + +} diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerClientFactoryTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerClientFactoryTest.java deleted file mode 100644 index a59cc651b..000000000 --- a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerClientFactoryTest.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright 2024-2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code.docker; - -import com.github.dockerjava.api.DockerClient; -import com.github.dockerjava.api.command.PingCmd; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.InOrder; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -import java.io.IOException; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertSame; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.inOrder; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -@ExtendWith(MockitoExtension.class) -class DockerClientFactoryTest { - - private static final String PRIMARY = "npipe://./pipe/docker_engine"; - - private static final String FALLBACK = "tcp://localhost:2375"; - - @Mock - private DockerClientConnector connector; - - @Mock - private DockerClient primaryClient; - - @Mock - private DockerClient fallbackClient; - - @Mock - private PingCmd primaryPing; - - @Mock - private PingCmd fallbackPing; - - @Test - void create_primaryCandidateWorks_returnsItWithoutTryingFallback() { - when(connector.connect(PRIMARY)).thenReturn(primaryClient); - when(primaryClient.pingCmd()).thenReturn(primaryPing); - - DockerClient result = new DockerClientFactory(connector).create(List.of(PRIMARY, FALLBACK)); - - assertSame(primaryClient, result); - verify(primaryPing).exec(); - verify(connector, never()).connect(FALLBACK); - } - - @Test - void create_primaryPingFails_closesItThenUsesFallback() throws IOException { - when(connector.connect(PRIMARY)).thenReturn(primaryClient); - when(primaryClient.pingCmd()).thenReturn(primaryPing); - when(primaryPing.exec()).thenThrow(new IllegalStateException("pipe unavailable")); - when(connector.connect(FALLBACK)).thenReturn(fallbackClient); - when(fallbackClient.pingCmd()).thenReturn(fallbackPing); - - DockerClient result = new DockerClientFactory(connector).create(List.of(PRIMARY, FALLBACK)); - - assertSame(fallbackClient, result); - InOrder order = inOrder(primaryClient, connector); - order.verify(primaryClient).close(); - order.verify(connector).connect(FALLBACK); - verify(fallbackPing).exec(); - } - - @Test - void create_allCandidatesFail_reportsEveryHostAndLastCause() { - when(connector.connect(PRIMARY)).thenThrow(new IllegalStateException("pipe unavailable")); - when(connector.connect(FALLBACK)).thenThrow(new IllegalStateException("tcp unavailable")); - - IllegalStateException error = assertThrows(IllegalStateException.class, - () -> new DockerClientFactory(connector).create(List.of(PRIMARY, FALLBACK))); - - assertTrue(error.getMessage().contains(PRIMARY)); - assertTrue(error.getMessage().contains(FALLBACK)); - assertTrue(error.getCause().getMessage().contains("tcp unavailable")); - } - - @Test - void create_withoutCandidates_rejectsInvalidConfiguration() { - IllegalArgumentException error = assertThrows(IllegalArgumentException.class, - () -> new DockerClientFactory(connector).create(List.of())); - - assertTrue(error.getMessage().contains("candidate")); - } - -} diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerExecutorFactoryTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerExecutorFactoryTest.java deleted file mode 100644 index a8e0f480b..000000000 --- a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerExecutorFactoryTest.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright 2024-2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code.docker; - -import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; -import com.alibaba.cloud.ai.dataagent.service.code.docker.DockerClientFactory.DockerConnection; -import com.alibaba.cloud.ai.dataagent.service.code.impls.DockerCodePoolExecutorService; -import com.github.dockerjava.api.DockerClient; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.InOrder; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -import java.io.IOException; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertSame; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.Mockito.inOrder; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -@ExtendWith(MockitoExtension.class) -class DockerExecutorFactoryTest { - - private static final String REMOTE = "tcp://docker.example:2375"; - - private static final String LOCAL = "tcp://localhost:2375"; - - @Mock - private DockerHostResolver hostResolver; - - @Mock - private DockerClientFactory clientFactory; - - @Mock - private DockerImageManager imageManager; - - @Mock - private DockerClient client; - - @Test - void create_remotePrimaryFallsBackToLocal_usesConnectedHostClassification() { - CodeExecutorProperties properties = properties(); - List candidates = List.of(REMOTE, LOCAL); - when(hostResolver.candidates(REMOTE, System.getProperty("os.name"))).thenReturn(candidates); - when(clientFactory.connect(candidates)).thenReturn(new DockerConnection(client, LOCAL)); - when(hostResolver.isRemote(LOCAL)).thenReturn(false); - - DockerCodePoolExecutorService executor = new DockerExecutorFactory(hostResolver, clientFactory, imageManager) - .create(properties); - - verify(hostResolver).isRemote(LOCAL); - executor.close(); - } - - @Test - void create_imagePreparationFails_closesClientAndRethrowsOriginalFailure() throws IOException { - CodeExecutorProperties properties = properties(); - List candidates = List.of(REMOTE); - IllegalStateException imageFailure = new IllegalStateException("pull failed"); - when(hostResolver.candidates(REMOTE, System.getProperty("os.name"))).thenReturn(candidates); - when(clientFactory.connect(candidates)).thenReturn(new DockerConnection(client, REMOTE)); - org.mockito.Mockito.doThrow(imageFailure).when(imageManager).ensureAvailable(client, properties.getImageName()); - - IllegalStateException actual = assertThrows(IllegalStateException.class, - () -> new DockerExecutorFactory(hostResolver, clientFactory, imageManager).create(properties)); - - assertSame(imageFailure, actual); - InOrder order = inOrder(imageManager, client); - order.verify(imageManager).ensureAvailable(client, properties.getImageName()); - order.verify(client).close(); - } - - private CodeExecutorProperties properties() { - CodeExecutorProperties properties = new CodeExecutorProperties(); - properties.setHost(REMOTE); - properties.setTaskQueueSize(2); - properties.setCoreContainerNum(1); - properties.setTempContainerNum(1); - properties.setCoreThreadSize(1); - properties.setMaxThreadSize(1); - properties.setThreadQueueSize(2); - return properties; - } - -} diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerHostResolverTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerHostResolverTest.java deleted file mode 100644 index a119ad22f..000000000 --- a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerHostResolverTest.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright 2024-2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code.docker; - -import org.junit.jupiter.api.Test; - -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -class DockerHostResolverTest { - - private final DockerHostResolver resolver = new DockerHostResolver(); - - @Test - void candidates_explicitUnixHost_keepsOnlyConfiguredHost() { - assertEquals(List.of("tcp://docker.example:2375"), resolver.candidates("tcp://docker.example:2375", "Linux")); - } - - @Test - void candidates_explicitWindowsHost_keepsFallbacksWithoutDuplicates() { - assertEquals(List.of("tcp://docker.example:2375", "npipe://./pipe/docker_engine", "tcp://localhost:2375"), - resolver.candidates("tcp://docker.example:2375", "Windows 11")); - } - - @Test - void candidates_windowsWithoutHost_returnsNamedPipeThenLocalTcp() { - assertEquals(List.of("npipe://./pipe/docker_engine", "tcp://localhost:2375"), - resolver.candidates(null, "Windows 11")); - } - - @Test - void candidates_unixWithoutHost_returnsUnixSocket() { - assertEquals(List.of("unix:///var/run/docker.sock"), resolver.candidates(null, "Mac OS X")); - assertEquals(List.of("unix:///var/run/docker.sock"), resolver.candidates(null, "Linux")); - assertEquals(List.of("unix:///var/run/docker.sock"), resolver.candidates(null, null)); - } - - @Test - void isRemote_distinguishesLocalAndRemoteHosts() { - assertFalse(resolver.isRemote(null)); - assertFalse(resolver.isRemote("")); - assertFalse(resolver.isRemote("unix:///var/run/docker.sock")); - assertFalse(resolver.isRemote("npipe://./pipe/docker_engine")); - assertFalse(resolver.isRemote("tcp://localhost:2375")); - assertFalse(resolver.isRemote("tcp://127.0.0.1:2375")); - assertFalse(resolver.isRemote("tcp://[::1]:2375")); - assertTrue(resolver.isRemote("tcp://docker.example:2375")); - } - - @Test - void isRemote_malformedOrUnsupportedHost_isRejected() { - assertThrows(IllegalArgumentException.class, () -> resolver.isRemote("not a docker uri")); - assertThrows(IllegalArgumentException.class, () -> resolver.isRemote("http://docker.example:2375")); - } - -} diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerImageManagerTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerImageManagerTest.java deleted file mode 100644 index 780681433..000000000 --- a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/docker/DockerImageManagerTest.java +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Copyright 2024-2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code.docker; - -import com.github.dockerjava.api.DockerClient; -import com.github.dockerjava.api.command.ListImagesCmd; -import com.github.dockerjava.api.command.PullImageCmd; -import com.github.dockerjava.api.model.Image; -import com.github.dockerjava.core.command.PullImageResultCallback; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -import java.util.List; -import java.util.function.Supplier; - -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -@ExtendWith(MockitoExtension.class) -class DockerImageManagerTest { - - private static final String IMAGE_NAME = "python:test"; - - @Mock - private DockerClient client; - - @Mock - private ListImagesCmd listImagesCmd; - - @Mock - private PullImageCmd pullImageCmd; - - @Mock - private PullImageResultCallback pullCallback; - - @Mock - private Image image; - - @Mock - private Supplier callbackFactory; - - private DockerImageManager imageManager; - - @BeforeEach - void setUp() { - imageManager = new DockerImageManager(callbackFactory); - when(client.listImagesCmd()).thenReturn(listImagesCmd); - when(listImagesCmd.withImageNameFilter(IMAGE_NAME)).thenReturn(listImagesCmd); - } - - @AfterEach - void clearInterruptedFlag() { - Thread.interrupted(); - } - - @Test - void ensureAvailable_existingExactTag_doesNotPull() { - when(listImagesCmd.exec()).thenReturn(List.of(image)); - when(image.getRepoTags()).thenReturn(new String[] { "other:test", IMAGE_NAME }); - - imageManager.ensureAvailable(client, IMAGE_NAME); - - verify(client, never()).pullImageCmd(anyString()); - } - - @Test - void ensureAvailable_missingTag_pullsAndWaitsForCompletion() throws InterruptedException { - when(listImagesCmd.exec()).thenReturn(List.of(image)); - when(image.getRepoTags()).thenReturn(new String[] { "other:test" }); - stubPull(); - - imageManager.ensureAvailable(client, IMAGE_NAME); - - verify(client).pullImageCmd(IMAGE_NAME); - verify(pullCallback).awaitCompletion(); - } - - @Test - void ensureAvailable_nullTags_areTreatedAsMissing() throws InterruptedException { - when(listImagesCmd.exec()).thenReturn(List.of(image)); - when(image.getRepoTags()).thenReturn(null); - stubPull(); - - imageManager.ensureAvailable(client, IMAGE_NAME); - - verify(pullCallback).awaitCompletion(); - } - - @Test - void ensureAvailable_interruptedPull_restoresInterruptAndThrowsSpecificFailure() throws InterruptedException { - when(listImagesCmd.exec()).thenReturn(List.of()); - stubPull(); - when(pullCallback.awaitCompletion()).thenThrow(new InterruptedException("cancelled")); - - IllegalStateException error = assertThrows(IllegalStateException.class, - () -> imageManager.ensureAvailable(client, IMAGE_NAME)); - - assertTrue(error.getMessage().contains(IMAGE_NAME)); - assertTrue(Thread.currentThread().isInterrupted()); - } - - private void stubPull() { - when(callbackFactory.get()).thenReturn(pullCallback); - when(client.pullImageCmd(IMAGE_NAME)).thenReturn(pullImageCmd); - when(pullImageCmd.exec(pullCallback)).thenReturn(pullCallback); - } - -} diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/impls/AiSimulationCodeExecutorServiceTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/impls/AiSimulationCodeExecutorServiceTest.java deleted file mode 100644 index 44c417c6d..000000000 --- a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/impls/AiSimulationCodeExecutorServiceTest.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright 2024-2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code.impls; - -import com.alibaba.cloud.ai.dataagent.service.code.CodePoolExecutorService.TaskRequest; -import com.alibaba.cloud.ai.dataagent.service.code.CodePoolExecutorService.TaskResponse; -import com.alibaba.cloud.ai.dataagent.service.llm.LlmService; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.springframework.ai.chat.model.ChatResponse; -import reactor.core.publisher.Flux; - -import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.ArgumentMatchers.*; -import static org.mockito.Mockito.*; - -@ExtendWith(MockitoExtension.class) -class AiSimulationCodeExecutorServiceTest { - - private AiSimulationCodeExecutorService service; - - @Mock - private LlmService llmService; - - @BeforeEach - void setUp() { - service = new AiSimulationCodeExecutorService(llmService); - } - - @Test - void runTask_returnsSuccessWithLlmOutput() { - TaskRequest request = new TaskRequest("print('hello')", "{}", "run it"); - - Flux chatResponseFlux = Flux.empty(); - when(llmService.call(anyString(), anyString())).thenReturn(chatResponseFlux); - when(llmService.toStringFlux(chatResponseFlux)).thenReturn(Flux.just("hello")); - - TaskResponse response = service.runTask(request); - - assertTrue(response.isSuccess()); - assertEquals("hello", response.stdOut()); - assertNull(response.stdErr()); - assertNull(response.exceptionMsg()); - verify(llmService).call(anyString(), contains("print('hello')")); - } - - @Test - void runTask_concatenatesMultipleFluxElements() { - TaskRequest request = new TaskRequest("code", "input", "req"); - - Flux chatResponseFlux = Flux.empty(); - when(llmService.call(anyString(), anyString())).thenReturn(chatResponseFlux); - when(llmService.toStringFlux(chatResponseFlux)).thenReturn(Flux.just("part1", " part2", " part3")); - - TaskResponse response = service.runTask(request); - - assertTrue(response.isSuccess()); - assertEquals("part1 part2 part3", response.stdOut()); - } - -} diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/impls/DockerCodePoolExecutorServiceIT.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/impls/DockerCodePoolExecutorServiceIT.java deleted file mode 100644 index 1f383a66b..000000000 --- a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/impls/DockerCodePoolExecutorServiceIT.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright 2024-2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code.impls; - -import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; -import com.alibaba.cloud.ai.dataagent.service.code.docker.DockerClientFactory; -import com.alibaba.cloud.ai.dataagent.service.code.docker.DockerExecutorFactory; -import com.alibaba.cloud.ai.dataagent.service.code.docker.DockerHostResolver; -import com.alibaba.cloud.ai.dataagent.service.code.docker.DockerImageManager; -import com.alibaba.cloud.ai.dataagent.service.code.docker.ZerodepDockerClientConnector; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertNotNull; - -/** - * Opt-in smoke test for a real Docker daemon. Maven Surefire does not select the - * {@code *IT} suffix by default. Run explicitly with - * {@code ./mvnw -pl data-agent-management -Dtest=DockerCodePoolExecutorServiceIT test}. - */ -@Tag("docker-integration") -class DockerCodePoolExecutorServiceIT { - - @Test - void create_withAvailableDaemon_preparesAndClosesExecutor() { - CodeExecutorProperties properties = new CodeExecutorProperties(); - properties.setImageName("alpine:3.20"); - DockerHostResolver resolver = new DockerHostResolver(); - DockerExecutorFactory factory = new DockerExecutorFactory(resolver, - new DockerClientFactory(new ZerodepDockerClientConnector()), new DockerImageManager()); - - try (DockerCodePoolExecutorService service = factory.create(properties)) { - assertNotNull(service); - } - } - -} diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/impls/DockerCodePoolExecutorServiceTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/impls/DockerCodePoolExecutorServiceTest.java deleted file mode 100644 index 061bdd9c5..000000000 --- a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/impls/DockerCodePoolExecutorServiceTest.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright 2024-2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code.impls; - -import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; -import com.alibaba.cloud.ai.dataagent.service.code.CodePoolExecutorService.TaskRequest; -import com.alibaba.cloud.ai.dataagent.service.code.CodePoolExecutorService.TaskResponse; -import com.github.dockerjava.api.DockerClient; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -import java.io.IOException; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; - -@ExtendWith(MockitoExtension.class) -class DockerCodePoolExecutorServiceTest { - - @Mock - private DockerClient dockerClient; - - private CodeExecutorProperties properties; - - @BeforeEach - void setUp() { - properties = new CodeExecutorProperties(); - properties.setContainerNamePrefix("test-docker-"); - properties.setTaskQueueSize(2); - properties.setCoreContainerNum(1); - properties.setTempContainerNum(1); - properties.setCoreThreadSize(1); - properties.setMaxThreadSize(1); - properties.setThreadQueueSize(2); - } - - @Test - void constructor_withInjectedClient_hasNoDockerSideEffects() { - DockerCodePoolExecutorService service = new DockerCodePoolExecutorService(properties, dockerClient, false); - - verifyNoInteractions(dockerClient); - - service.close(); - } - - @Test - void execute_missingContainerWorkDirectory_returnsSpecificFailure() { - DockerCodePoolExecutorService service = new DockerCodePoolExecutorService(properties, dockerClient, false); - - TaskResponse response = service.execTaskInContainer(new TaskRequest("print(1)", "", ""), "missing"); - - assertFalse(response.isSuccess()); - assertEquals("An exception occurred while executing the task: Container 'missing' does not exist work dir", - response.exceptionMsg()); - service.close(); - } - - @Test - void close_calledTwice_closesDockerClientExactlyOnce() throws IOException { - DockerCodePoolExecutorService service = new DockerCodePoolExecutorService(properties, dockerClient, false); - - service.close(); - service.close(); - - verify(dockerClient).close(); - } - -} diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/impls/LocalCodePoolExecutorServiceTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/impls/LocalCodePoolExecutorServiceTest.java deleted file mode 100644 index 4ce92b563..000000000 --- a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/impls/LocalCodePoolExecutorServiceTest.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright 2024-2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code.impls; - -import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; -import com.alibaba.cloud.ai.dataagent.service.code.local.ExecutableProgramLocator; -import com.alibaba.cloud.ai.dataagent.service.code.local.ExecutionTimeoutParser; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -import java.util.Optional; - -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.anyList; -import static org.mockito.Mockito.when; - -@ExtendWith(MockitoExtension.class) -class LocalCodePoolExecutorServiceTest { - - @Mock - private ExecutableProgramLocator programLocator; - - private final ExecutionTimeoutParser timeoutParser = new ExecutionTimeoutParser(); - - @Test - void constructor_pythonAvailable_constructsWithoutInspectingMachinePath() { - when(programLocator.findFirst(anyList())).thenReturn(Optional.of("python-test"), Optional.empty()); - - LocalCodePoolExecutorService service = - new LocalCodePoolExecutorService(properties(), programLocator, timeoutParser); - - assertDoesNotThrow(service::close); - } - - @Test - void constructor_pythonUnavailable_throwsSpecificConfigurationFailure() { - when(programLocator.findFirst(anyList())).thenReturn(Optional.empty()); - - IllegalStateException error = assertThrows(IllegalStateException.class, - () -> new LocalCodePoolExecutorService(properties(), programLocator, timeoutParser)); - - assertTrue(error.getMessage().contains("Python")); - } - - private CodeExecutorProperties properties() { - CodeExecutorProperties properties = new CodeExecutorProperties(); - properties.setContainerNamePrefix("test-local-"); - properties.setCodeTimeout("60s"); - properties.setContainerTimeout(5L); - properties.setTaskQueueSize(2); - properties.setCoreContainerNum(1); - properties.setTempContainerNum(1); - properties.setCoreThreadSize(1); - properties.setMaxThreadSize(1); - properties.setThreadQueueSize(2); - return properties; - } - -} diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/local/ExecutionTimeoutParserTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/local/ExecutionTimeoutParserTest.java deleted file mode 100644 index 66fcf4190..000000000 --- a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/local/ExecutionTimeoutParserTest.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright 2024-2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code.local; - -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.CsvSource; -import org.junit.jupiter.params.provider.NullAndEmptySource; -import org.junit.jupiter.params.provider.ValueSource; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -class ExecutionTimeoutParserTest { - - private static final long DEFAULT_TIMEOUT_MILLIS = 60_000; - - private final ExecutionTimeoutParser parser = new ExecutionTimeoutParser(); - - @ParameterizedTest - @CsvSource({ "500ms,500", "0s,0", "30s,30000", "2m,120000", "1h,3600000", "1d,86400000", "100S,100000" }) - void parse_validValue_returnsMilliseconds(String value, long expected) { - assertEquals(expected, parser.parse(value)); - } - - @ParameterizedTest - @NullAndEmptySource - @ValueSource(strings = { " ", "invalid", "30s trailing", "-1s", "9223372036854775807d" }) - void parse_invalidOrOverflowingValue_returnsDocumentedDefault(String value) { - assertEquals(DEFAULT_TIMEOUT_MILLIS, parser.parse(value)); - } - -} diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/local/PathExecutableProgramLocatorTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/local/PathExecutableProgramLocatorTest.java deleted file mode 100644 index 123c1dad3..000000000 --- a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/local/PathExecutableProgramLocatorTest.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright 2024-2026 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.cloud.ai.dataagent.service.code.local; - -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.List; -import java.util.Optional; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - -class PathExecutableProgramLocatorTest { - - @TempDir - private Path tempDirectory; - - @Test - void findFirst_secondCandidateExists_returnsSecondCandidate() throws IOException { - Path executable = Files.createFile(tempDirectory.resolve("python3")); - assertTrue(executable.toFile().setExecutable(true)); - PathExecutableProgramLocator locator = new PathExecutableProgramLocator(tempDirectory.toString(), false); - - assertEquals(Optional.of("python3"), locator.findFirst(List.of("missing", "python3"))); - } - - @Test - void findFirst_windowsExecutableSuffixExists_returnsProgramName() throws IOException { - Path executable = Files.createFile(tempDirectory.resolve("python.exe")); - assertTrue(executable.toFile().setExecutable(true)); - PathExecutableProgramLocator locator = new PathExecutableProgramLocator(tempDirectory.toString(), true); - - assertEquals(Optional.of("python"), locator.findFirst(List.of("python"))); - } - - @Test - void findFirst_missingPathOrCandidates_returnsEmpty() { - assertFalse(new PathExecutableProgramLocator(null, false).findFirst(List.of("python")).isPresent()); - assertFalse(new PathExecutableProgramLocator(tempDirectory.toString(), false).findFirst(List.of()).isPresent()); - } - -} diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SaaSandboxPythonCodeExecutorServiceTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SaaSandboxPythonCodeExecutorServiceTest.java new file mode 100644 index 000000000..e97f96533 --- /dev/null +++ b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/SaaSandboxPythonCodeExecutorServiceTest.java @@ -0,0 +1,69 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.cloud.ai.dataagent.service.code.sandbox; + +import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; +import com.alibaba.cloud.ai.dataagent.service.code.PythonCodeExecutorService; +import com.alibaba.cloud.ai.dataagent.service.code.sandbox.runtime.SaaSandboxTaskRunner; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class SaaSandboxPythonCodeExecutorServiceTest { + + @Test + void runTask_delegatesAcceptedTaskToSandboxRunner() { + CodeExecutorProperties properties = new CodeExecutorProperties(); + SaaSandboxTaskRunner taskRunner = mock(SaaSandboxTaskRunner.class); + PythonCodeExecutorService.TaskRequest request = new PythonCodeExecutorService.TaskRequest("print(1)", "[]", + List.of()); + when(taskRunner.run(request)).thenReturn(PythonCodeExecutorService.TaskResponse.success("1")); + + try (TestExecutor service = new TestExecutor(properties, taskRunner)) { + assertThat(service.runTask(request).stdOut()).isEqualTo("1"); + } + + verify(taskRunner).run(request); + } + + @Test + void runTask_rejectsOversizedCodeBeforeCreatingSandbox() { + CodeExecutorProperties properties = new CodeExecutorProperties(); + properties.getSandbox().setMaxCodeBytes(3); + SaaSandboxTaskRunner taskRunner = mock(SaaSandboxTaskRunner.class); + + try (TestExecutor service = new TestExecutor(properties, taskRunner)) { + PythonCodeExecutorService.TaskResponse response = service + .runTask(new PythonCodeExecutorService.TaskRequest("print(1)", "[]", List.of())); + assertThat(response.isSuccess()).isFalse(); + assertThat(response.exceptionMsg()).contains("size limit"); + } + } + + private static final class TestExecutor extends SaaSandboxPythonCodeExecutorService implements AutoCloseable { + + private TestExecutor(CodeExecutorProperties properties, SaaSandboxTaskRunner taskRunner) { + super(properties, taskRunner); + } + + } + +} diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/dependency/PythonDependencyMetadataParserTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/dependency/PythonDependencyMetadataParserTest.java new file mode 100644 index 000000000..3ce42e7f4 --- /dev/null +++ b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/dependency/PythonDependencyMetadataParserTest.java @@ -0,0 +1,77 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.cloud.ai.dataagent.service.code.sandbox.dependency; + +import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class PythonDependencyMetadataParserTest { + + private PythonDependencyMetadataParser parser; + + @BeforeEach + void setUp() { + CodeExecutorProperties properties = new CodeExecutorProperties(); + parser = new PythonDependencyMetadataParser(new PythonDependencyPolicy(properties)); + } + + @Test + void parse_readsCanonicalPep723ScriptBlock() { + PythonDependencyMetadata metadata = parser.parse(""" + # /// script + # requires-python = ">=3.10" + # dependencies = ["pandas>=2,<3", "numpy==2.2.1"] + # /// + import pandas + """); + + assertThat(metadata.requiresPython()).isEqualTo(">=3.10"); + assertThat(metadata.dependencies()).containsExactly("pandas>=2,<3", "numpy==2.2.1"); + } + + @Test + void parse_returnsEmptyMetadataWhenBlockIsAbsent() { + assertThat(parser.parse("import json").dependencies()).isEmpty(); + } + + @Test + void parse_rejectsUrlsPathsMarkersAndPipFlags() { + for (String dependency : new String[] { "pkg @ https://example.com/pkg.whl", "../local-package", + "pandas; python_version > '3.10'", "--extra-index-url=https://example.com" }) { + String code = "# /// script\n# dependencies = [\"" + dependency + "\"]\n# ///\n"; + assertThatThrownBy(() -> parser.parse(code)).isInstanceOf(IllegalArgumentException.class); + } + } + + @Test + void parse_rejectsMultipleOrUnclosedBlocks() { + assertThatThrownBy(() -> parser.parse(""" + # /// script + # dependencies = [] + # /// + # /// script + # dependencies = [] + # /// + """)).isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> parser.parse("# /// script\n# dependencies = []\n")) + .isInstanceOf(IllegalArgumentException.class); + } + +} diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/execution/PythonSandboxBootstrapBuilderTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/execution/PythonSandboxBootstrapBuilderTest.java new file mode 100644 index 000000000..da4cd6a0d --- /dev/null +++ b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/execution/PythonSandboxBootstrapBuilderTest.java @@ -0,0 +1,54 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.cloud.ai.dataagent.service.code.sandbox.execution; + +import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +class PythonSandboxBootstrapBuilderTest { + + private final PythonSandboxBootstrapBuilder builder = new PythonSandboxBootstrapBuilder( + new CodeExecutorProperties()); + + @Test + void dependencyInstaller_usesFixedArgumentListAndDoesNotInterpolateModelText() { + String dependency = "safe-package==1.0"; + String installer = builder.buildDependencyInstaller(List.of(dependency)); + + assertThat(installer).contains("sys.executable, \"-m\", \"pip\", \"install\"") + .contains("\"--target\", \"/tmp/dataagent-deps\"") + .doesNotContain(dependency) + .contains(PythonSandboxBootstrapBuilder.RESULT_MARKER); + } + + @Test + void codeExecution_encodesGeneratedCodeAndRunsItInChildProcess() { + String code = "print('sensitive generated code')"; + String bootstrap = builder.buildCodeExecution(code, "[]"); + + assertThat(bootstrap).doesNotContain(code) + .contains("[sys.executable, \"-c\", _code]") + .contains("input=_input") + .contains("timeout=60") + .contains("_environment[\"PYTHONPATH\"] = \"/tmp/dataagent-deps\"") + .contains(PythonSandboxBootstrapBuilder.RESULT_MARKER); + } + +} diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/execution/SandboxExecutionResultParserTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/execution/SandboxExecutionResultParserTest.java new file mode 100644 index 000000000..862f3d499 --- /dev/null +++ b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/execution/SandboxExecutionResultParserTest.java @@ -0,0 +1,51 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.cloud.ai.dataagent.service.code.sandbox.execution; + +import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; +import com.alibaba.cloud.ai.dataagent.util.JsonUtil; +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.util.Base64; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class SandboxExecutionResultParserTest { + + private final SandboxExecutionResultParser parser = new SandboxExecutionResultParser(new CodeExecutorProperties()); + + @Test + void parse_extractsLastEnvelopeFromEscapedSandboxProtocol() throws Exception { + SandboxExecutionResult expected = new SandboxExecutionResult(true, "{\"ok\":true}", "", null, null); + String json = JsonUtil.getObjectMapper().writeValueAsString(expected); + String encoded = Base64.getEncoder().encodeToString(json.getBytes(StandardCharsets.UTF_8)); + String response = "{\"content\":[{\"text\":\"cell output\\\\n" + PythonSandboxBootstrapBuilder.RESULT_MARKER + + encoded + "\\\\n\"}]}"; + + SandboxExecutionResult actual = parser.parse(response); + + assertThat(actual).isEqualTo(expected); + } + + @Test + void parse_rejectsResponsesWithoutResultMarker() { + assertThatThrownBy(() -> parser.parse("{\"isError\":true}")).isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("result marker"); + } + +} diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/runtime/SaaSandboxRuntimeTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/runtime/SaaSandboxRuntimeTest.java new file mode 100644 index 000000000..840340cb6 --- /dev/null +++ b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/runtime/SaaSandboxRuntimeTest.java @@ -0,0 +1,38 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.cloud.ai.dataagent.service.code.sandbox.runtime; + +import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; +import org.junit.jupiter.api.Test; + +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; + +class SaaSandboxRuntimeTest { + + @Test + void shouldKeepSandboxAliveForDependencyInstallationAndCodeExecution() { + CodeExecutorProperties properties = new CodeExecutorProperties(); + properties.setCodeTimeout(Duration.ofSeconds(45)); + properties.getSandbox().setDependencyInstallTimeout(Duration.ofMinutes(2)); + + SaaSandboxRuntime runtime = new SaaSandboxRuntime(properties); + + assertThat(runtime.sandboxTimeoutSeconds()).isEqualTo(195); + } + +} diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/runtime/SaaSandboxTaskRunnerIT.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/runtime/SaaSandboxTaskRunnerIT.java new file mode 100644 index 000000000..d5275c6e9 --- /dev/null +++ b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/service/code/sandbox/runtime/SaaSandboxTaskRunnerIT.java @@ -0,0 +1,59 @@ +/* + * Copyright 2024-2026 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.cloud.ai.dataagent.service.code.sandbox.runtime; + +import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; +import com.alibaba.cloud.ai.dataagent.service.code.PythonCodeExecutorService; +import com.alibaba.cloud.ai.dataagent.service.code.sandbox.execution.PythonSandboxBootstrapBuilder; +import com.alibaba.cloud.ai.dataagent.service.code.sandbox.execution.SandboxExecutionResultParser; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Requires a running local Docker daemon and the configured AgentScope base image. + */ +class SaaSandboxTaskRunnerIT { + + @Test + void run_installsDeclaredDependencyAndExecutesPythonInSaaSandbox() { + CodeExecutorProperties properties = new CodeExecutorProperties(); + SaaSandboxRuntime runtime = new SaaSandboxRuntime(properties); + try { + PythonSandboxBootstrapBuilder builder = new PythonSandboxBootstrapBuilder(properties); + SandboxExecutionResultParser parser = new SandboxExecutionResultParser(properties); + SaaSandboxTaskRunner runner = new SaaSandboxTaskRunner(runtime, builder, parser); + PythonCodeExecutorService.TaskRequest request = new PythonCodeExecutorService.TaskRequest(""" + import json + import six + import sys + input_data = json.load(sys.stdin) + print(json.dumps({"six_version": six.__version__, "rows": len(input_data)})) + """, "[{\"id\":1}]", List.of("six==1.17.0")); + + PythonCodeExecutorService.TaskResponse response = runner.run(request); + + assertThat(response.isSuccess()).as(response.toString()).isTrue(); + assertThat(response.stdOut()).contains("\"six_version\": \"1.17.0\"").contains("\"rows\": 1"); + } + finally { + runtime.close(); + } + } + +} diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/workflow/node/python/PythonExecuteNodeTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/workflow/node/python/PythonExecuteNodeTest.java index 258d571f9..9519332a1 100644 --- a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/workflow/node/python/PythonExecuteNodeTest.java +++ b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/workflow/node/python/PythonExecuteNodeTest.java @@ -29,12 +29,15 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; +import org.mockito.ArgumentCaptor; import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoSettings; import org.mockito.quality.Strictness; import com.alibaba.cloud.ai.dataagent.properties.CodeExecutorProperties; -import com.alibaba.cloud.ai.dataagent.service.code.CodePoolExecutorService; +import com.alibaba.cloud.ai.dataagent.service.code.PythonCodeExecutorService; +import com.alibaba.cloud.ai.dataagent.service.code.sandbox.dependency.PythonDependencyMetadata; +import com.alibaba.cloud.ai.dataagent.service.code.sandbox.dependency.PythonDependencyMetadataParser; import com.alibaba.cloud.ai.dataagent.util.JsonParseUtil; import com.alibaba.cloud.ai.dataagent.workflow.node.PythonExecuteNode; import com.alibaba.cloud.ai.graph.OverAllState; @@ -45,7 +48,10 @@ class PythonExecuteNodeTest { @Mock - private CodePoolExecutorService codePoolExecutor; + private PythonCodeExecutorService pythonCodeExecutor; + + @Mock + private PythonDependencyMetadataParser dependencyMetadataParser; @Mock private JsonParseUtil jsonParseUtil; @@ -58,7 +64,9 @@ class PythonExecuteNodeTest { @BeforeEach void setUp() { when(codeExecutorProperties.getPythonMaxTriesCount()).thenReturn(5); - pythonExecuteNode = new PythonExecuteNode(codePoolExecutor, jsonParseUtil, codeExecutorProperties); + when(dependencyMetadataParser.parse(anyString())).thenReturn(PythonDependencyMetadata.empty()); + pythonExecuteNode = new PythonExecuteNode(pythonCodeExecutor, dependencyMetadataParser, jsonParseUtil, + codeExecutorProperties); } private OverAllState createTestState() { @@ -81,7 +89,8 @@ void apply_validCode_executesSuccessfully() throws Exception { OverAllState state = createTestState(); setupBasicState(state); - when(codePoolExecutor.runTask(any())).thenReturn(CodePoolExecutorService.TaskResponse.success("hello world")); + when(pythonCodeExecutor.runTask(any())) + .thenReturn(PythonCodeExecutorService.TaskResponse.success("hello world")); when(jsonParseUtil.tryConvertToObject(anyString(), any(Class.class))).thenReturn(null); Map result = pythonExecuteNode.apply(state); @@ -96,7 +105,7 @@ void apply_jsonOutput_parsesCorrectly() throws Exception { state.updateState(Map.of(PYTHON_GENERATE_NODE_OUTPUT, "import json\nprint(json.dumps({'key': 'value'}))")); String jsonOutput = "{\"key\": \"value\"}"; - when(codePoolExecutor.runTask(any())).thenReturn(CodePoolExecutorService.TaskResponse.success(jsonOutput)); + when(pythonCodeExecutor.runTask(any())).thenReturn(PythonCodeExecutorService.TaskResponse.success(jsonOutput)); Map parsed = Map.of("key", "value"); when(jsonParseUtil.tryConvertToObject(anyString(), any(Class.class))).thenReturn(parsed); @@ -113,8 +122,8 @@ void apply_executionError_setsFailureState() throws Exception { setupBasicState(state); state.updateState(Map.of(PYTHON_TRIES_COUNT, 1)); - when(codePoolExecutor.runTask(any())) - .thenReturn(CodePoolExecutorService.TaskResponse.failure("", "NameError: name 'x' is not defined")); + when(pythonCodeExecutor.runTask(any())) + .thenReturn(PythonCodeExecutorService.TaskResponse.failure("", "NameError: name 'x' is not defined")); Map result = pythonExecuteNode.apply(state); assertNotNull(result); @@ -128,8 +137,8 @@ void apply_maxRetryExceeded_setsFallbackMode() throws Exception { setupBasicState(state); state.updateState(Map.of(PYTHON_TRIES_COUNT, 6)); - when(codePoolExecutor.runTask(any())) - .thenReturn(CodePoolExecutorService.TaskResponse.failure("", "SyntaxError")); + when(pythonCodeExecutor.runTask(any())) + .thenReturn(PythonCodeExecutorService.TaskResponse.failure("", "SyntaxError")); Map result = pythonExecuteNode.apply(state); assertNotNull(result); @@ -143,7 +152,7 @@ void apply_jsonParseFailure_usesRawOutput() throws Exception { setupBasicState(state); String rawOutput = "not json content"; - when(codePoolExecutor.runTask(any())).thenReturn(CodePoolExecutorService.TaskResponse.success(rawOutput)); + when(pythonCodeExecutor.runTask(any())).thenReturn(PythonCodeExecutorService.TaskResponse.success(rawOutput)); when(jsonParseUtil.tryConvertToObject(anyString(), any(Class.class))).thenReturn(null); Map result = pythonExecuteNode.apply(state); @@ -158,7 +167,8 @@ void apply_unicodeInOutput_handlesCorrectly() throws Exception { state.updateState(Map.of(PYTHON_GENERATE_NODE_OUTPUT, "print('你好世界')")); String unicodeOutput = "{\"message\": \"\\u4f60\\u597d\\u4e16\\u754c\"}"; - when(codePoolExecutor.runTask(any())).thenReturn(CodePoolExecutorService.TaskResponse.success(unicodeOutput)); + when(pythonCodeExecutor.runTask(any())) + .thenReturn(PythonCodeExecutorService.TaskResponse.success(unicodeOutput)); Map parsed = Map.of("message", "你好世界"); when(jsonParseUtil.tryConvertToObject(anyString(), any(Class.class))).thenReturn(parsed); @@ -174,7 +184,7 @@ void apply_emptyOutput_setsEmptyResult() throws Exception { OverAllState state = createTestState(); setupBasicState(state); - when(codePoolExecutor.runTask(any())).thenReturn(CodePoolExecutorService.TaskResponse.success("")); + when(pythonCodeExecutor.runTask(any())).thenReturn(PythonCodeExecutorService.TaskResponse.success("")); when(jsonParseUtil.tryConvertToObject(anyString(), any(Class.class))).thenReturn(null); Map result = pythonExecuteNode.apply(state); @@ -192,7 +202,7 @@ void apply_withSqlResultsAsCSV_passesCorrectly() throws Exception { sqlResults.add(Map.of("name", "Bob", "amount", "200")); state.updateState(Map.of(SQL_RESULT_LIST_MEMORY, sqlResults)); - when(codePoolExecutor.runTask(any())).thenReturn(CodePoolExecutorService.TaskResponse.success("processed")); + when(pythonCodeExecutor.runTask(any())).thenReturn(PythonCodeExecutorService.TaskResponse.success("processed")); when(jsonParseUtil.tryConvertToObject(anyString(), any(Class.class))).thenReturn(null); Map result = pythonExecuteNode.apply(state); @@ -211,8 +221,8 @@ void apply_largeOutput_handlesMemoryPressure() throws Exception { largeOutput.append("line ").append(i).append(": data_value_").append(i).append("\n"); } - when(codePoolExecutor.runTask(any())) - .thenReturn(CodePoolExecutorService.TaskResponse.success(largeOutput.toString())); + when(pythonCodeExecutor.runTask(any())) + .thenReturn(PythonCodeExecutorService.TaskResponse.success(largeOutput.toString())); when(jsonParseUtil.tryConvertToObject(anyString(), any(Class.class))).thenReturn(null); Map result = pythonExecuteNode.apply(state); @@ -221,4 +231,27 @@ void apply_largeOutput_handlesMemoryPressure() throws Exception { assertNotNull(result.get(PYTHON_EXECUTE_NODE_OUTPUT)); } + @Test + void apply_pep723Dependencies_passesStructuredDependenciesToSandbox() throws Exception { + OverAllState state = createTestState(); + String code = """ + # /// script + # dependencies = ["pandas>=2,<3"] + # /// + print("{}") + """; + state.updateState(Map.of(PYTHON_GENERATE_NODE_OUTPUT, code)); + when(dependencyMetadataParser.parse(code)) + .thenReturn(new PythonDependencyMetadata(List.of("pandas>=2,<3"), null)); + when(pythonCodeExecutor.runTask(any())).thenReturn(PythonCodeExecutorService.TaskResponse.success("{}")); + when(jsonParseUtil.tryConvertToObject(anyString(), any(Class.class))).thenReturn(null); + + pythonExecuteNode.apply(state); + + ArgumentCaptor requestCaptor = ArgumentCaptor + .forClass(PythonCodeExecutorService.TaskRequest.class); + org.mockito.Mockito.verify(pythonCodeExecutor).runTask(requestCaptor.capture()); + assertEquals(List.of("pandas>=2,<3"), requestCaptor.getValue().dependencies()); + } + } diff --git a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/workflow/node/python/PythonGenerateNodeTest.java b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/workflow/node/python/PythonGenerateNodeTest.java index c3e649cdc..a4064b939 100644 --- a/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/workflow/node/python/PythonGenerateNodeTest.java +++ b/data-agent-management/src/test/java/com/alibaba/cloud/ai/dataagent/workflow/node/python/PythonGenerateNodeTest.java @@ -24,6 +24,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.time.Duration; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -98,7 +99,7 @@ class PythonGenerateNodeTest { @BeforeEach void setUp() { when(codeExecutorProperties.getLimitMemory()).thenReturn(500L); - when(codeExecutorProperties.getCodeTimeout()).thenReturn("60s"); + when(codeExecutorProperties.getCodeTimeout()).thenReturn(Duration.ofSeconds(60)); pythonGenerateNode = new PythonGenerateNode(codeExecutorProperties, llmService); } diff --git a/docs/ADVANCED_FEATURES-en.md b/docs/ADVANCED_FEATURES-en.md index 8980214de..161c3cf98 100644 --- a/docs/ADVANCED_FEATURES-en.md +++ b/docs/ADVANCED_FEATURES-en.md @@ -383,15 +383,10 @@ WHERE u.name = 'Zhang San' ## Python Execution Environment Configuration -### Executor Types +Python always runs through Spring AI Alibaba Sandbox in a task-scoped container. Host-local, +legacy Docker-pool, and AI Simulation executors are no longer available. -The system supports three Python executors: - -1. **Docker Executor** (Recommended) -2. **Local Executor** -3. **AI Simulation Executor** - -### Docker Executor Configuration +### SAA Sandbox Configuration ```yaml spring: @@ -399,41 +394,28 @@ spring: alibaba: data-agent: code-executor: - type: docker - docker: - image: continuumio/anaconda3:latest - timeout: 300000 # 5 minute timeout - memory-limit: 512m - cpu-limit: 1.0 + code-timeout: 60s + limit-memory: 500 + cpu-core: 1 + sandbox: + image-name: ${DATAAGENT_SANDBOX_IMAGE} + max-concurrency: 4 + queue-capacity: 10 + package-index-url: ${DATAAGENT_PYPI_INDEX_URL:https://pypi.org/simple} + dependency-install-timeout: 3m ``` -### Local Executor Configuration +### Dynamic Dependencies -```yaml -spring: - ai: - alibaba: - data-agent: - code-executor: - type: local - local: - python-path: /usr/bin/python3 - timeout: 300000 - work-dir: /tmp/dataagent +```python +# /// script +# dependencies = ["pandas>=2,<3"] +# /// ``` -### AI Simulation Executor - -Used for testing environments, doesn't actually execute Python code, but simulates execution results through AI: - -```yaml -spring: - ai: - alibaba: - data-agent: - code-executor: - type: ai-simulation -``` +Dependencies are installed inside the task sandbox by a fixed backend command. The model cannot +specify repositories, URLs, file paths, pip options, or shell commands. Production should use an +enterprise private PyPI proxy. ## Advanced Configuration Options diff --git a/docs/ADVANCED_FEATURES.md b/docs/ADVANCED_FEATURES.md index 8f729b518..2daab80de 100644 --- a/docs/ADVANCED_FEATURES.md +++ b/docs/ADVANCED_FEATURES.md @@ -382,15 +382,10 @@ WHERE u.name = '张三' ## 🐍 Python 执行环境配置 -### 执行器类型 +Python 统一通过 Spring AI Alibaba Sandbox 在任务级独立容器中执行，不再提供宿主机 +Local、旧 Docker 容器池或 AI Simulation 执行器。 -系统支持三种Python执行器： - -1. **Docker Executor** (推荐) -2. **Local Executor** -3. **AI Simulation Executor** - -### Docker 执行器配置 +### SAA 沙盒配置 ```yaml spring: @@ -398,41 +393,27 @@ spring: alibaba: data-agent: code-executor: - type: docker - docker: - image: continuumio/anaconda3:latest - timeout: 300000 # 5分钟超时 - memory-limit: 512m - cpu-limit: 1.0 + code-timeout: 60s + limit-memory: 500 + cpu-core: 1 + sandbox: + image-name: ${DATAAGENT_SANDBOX_IMAGE} + max-concurrency: 4 + queue-capacity: 10 + package-index-url: ${DATAAGENT_PYPI_INDEX_URL:https://pypi.org/simple} + dependency-install-timeout: 3m ``` -### Local 执行器配置 +### 动态依赖 -```yaml -spring: - ai: - alibaba: - data-agent: - code-executor: - type: local - local: - python-path: /usr/bin/python3 - timeout: 300000 - work-dir: /tmp/dataagent +```python +# /// script +# dependencies = ["pandas>=2,<3"] +# /// ``` -### AI 模拟执行器 - -用于测试环境，不实际执行Python代码，而是通过AI模拟执行结果： - -```yaml -spring: - ai: - alibaba: - data-agent: - code-executor: - type: ai-simulation -``` +依赖由固定的后端命令在当前任务沙盒内安装。模型不能指定仓库、URL、文件路径、pip 参数 +或 shell 命令。生产环境应将包索引配置为企业私有 PyPI 代理。 ## ⚙️ 高级配置选项 diff --git a/docs/ARCHITECTURE-en.md b/docs/ARCHITECTURE-en.md index 1a14d063b..cba9cce03 100644 --- a/docs/ARCHITECTURE-en.md +++ b/docs/ARCHITECTURE-en.md @@ -32,7 +32,7 @@ flowchart LR ModelRegistry[AiModelRegistry] VectorSvc[AgentVectorStoreService] Hybrid[HybridRetrievalStrategy] - CodePool[CodePoolExecutorService] + CodeExec[PythonCodeExecutorService] McpSvc[McpServerService] end @@ -657,8 +657,9 @@ sequenceDiagram #### Key Points - **Code Generation**: `PythonGenerateNode` generates Python based on plan and SQL results -- **Code Execution**: `PythonExecuteNode` uses `CodePoolExecutorService` (Docker/Local/AI simulation) -- **Execution Configuration**: `spring.ai.alibaba.data-agent.code-executor.*` (default Docker image `continuumio/anaconda3:latest`) +- **Code Execution**: `PythonExecuteNode` uses `PythonCodeExecutorService` and always runs in a task-scoped SAA sandbox +- **Dynamic Dependencies**: Generated code declares dependencies with PEP 723; a fixed bootstrap installs them inside the sandbox +- **Execution Configuration**: `spring.ai.alibaba.data-agent.code-executor.*` - **Result Return**: Execution results are written back to `PYTHON_EXECUTE_NODE_OUTPUT`, `PythonAnalyzeNode` summarizes and writes to `SQL_EXECUTE_NODE_OUTPUT` for final report #### Architecture Diagram @@ -666,14 +667,13 @@ sequenceDiagram ```mermaid flowchart LR PyGen[PythonGenerateNode] --> PyExec[PythonExecuteNode] - PyExec --> ExecSvc[CodePoolExecutorService] - ExecSvc --> Queue[Task Queue] - ExecSvc --> Pool[Container Pool] - Pool --> Docker[Docker Executor] - Pool --> Local[Local Executor] - Pool --> AISim[AI Simulation Executor] - Docker --> TempFiles[Temp Files] - TempFiles --> StdIO[Stdout Stderr] + PyExec --> Parser[PEP 723 Parser] + Parser --> ExecSvc[PythonCodeExecutorService] + ExecSvc --> Queue[Bounded Queue] + Queue --> SAA[SAA SandboxService] + SAA --> Install[Controlled Dependency Install] + Install --> Run[Python Subprocess] + Run --> StdIO[Stdout Stderr] StdIO --> JsonParse[JsonParseUtil] JsonParse --> PyAnalyze[PythonAnalyzeNode] PyAnalyze --> Report[ReportGeneratorNode] @@ -683,8 +683,8 @@ flowchart LR classDef data fill:#FEF3C7,stroke:#F59E0B,stroke-width:1px,color:#1F2937; class PyGen,PyExec,PyAnalyze,Report service - class ExecSvc,Pool,Docker,Local,AISim exec - class Queue,TempFiles,StdIO,JsonParse data + class ExecSvc,SAA,Install,Run exec + class Parser,Queue,StdIO,JsonParse data ``` #### Flow Diagram @@ -697,19 +697,20 @@ sequenceDiagram participant G as PythonGenerateNode participant L as LlmService participant E as PythonExecuteNode - participant CP as CodePoolExecutorService - participant D as Docker Executor + participant CP as PythonCodeExecutorService + participant S as SAA Sandbox participant J as JsonParseUtil participant A as PythonAnalyzeNode participant R as ReportGeneratorNode P->>G: Enter Python step with instructions G->>L: generate python code - L-->>G: python code + L-->>G: python code + PEP 723 G->>E: pass code and sql results E->>CP: runTask - CP->>D: execute in container - D-->>CP: stdout stderr + CP->>S: create sandbox and install dependencies + S->>S: execute with stdin + S-->>CP: stdout stderr CP-->>E: task response E->>J: parse stdout json J-->>E: normalized output diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 31926e860..83653792d 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -32,7 +32,7 @@ flowchart LR ModelRegistry[AiModelRegistry] VectorSvc[AgentVectorStoreService] Hybrid[HybridRetrievalStrategy] - CodePool[CodePoolExecutorService] + CodeExec[PythonCodeExecutorService] McpSvc[McpServerService] LangfuseSvc[LangfuseService] end @@ -666,8 +666,9 @@ sequenceDiagram #### 说明要点 - **代码生成**: `PythonGenerateNode` 根据计划与 SQL 结果生成 Python -- **代码执行**: `PythonExecuteNode` 使用 `CodePoolExecutorService`（Docker/Local/AI 模拟） -- **执行配置**: `spring.ai.alibaba.data-agent.code-executor.*`（默认 Docker 镜像 `continuumio/anaconda3:latest`） +- **代码执行**: `PythonExecuteNode` 使用 `PythonCodeExecutorService`，统一进入任务级 SAA 沙盒 +- **动态依赖**: 生成代码通过 PEP 723 声明依赖，由固定 bootstrap 在沙盒内受控安装 +- **执行配置**: `spring.ai.alibaba.data-agent.code-executor.*` - **结果回传**: 执行结果写回 `PYTHON_EXECUTE_NODE_OUTPUT`，`PythonAnalyzeNode` 汇总后写入 `SQL_EXECUTE_NODE_OUTPUT`，用于最终报告 #### 架构图 @@ -675,14 +676,13 @@ sequenceDiagram ```mermaid flowchart LR PyGen[PythonGenerateNode] --> PyExec[PythonExecuteNode] - PyExec --> ExecSvc[CodePoolExecutorService] - ExecSvc --> Queue[Task Queue] - ExecSvc --> Pool[Container Pool] - Pool --> Docker[Docker Executor] - Pool --> Local[Local Executor] - Pool --> AISim[AI Simulation Executor] - Docker --> TempFiles[Temp Files] - TempFiles --> StdIO[Stdout Stderr] + PyExec --> Parser[PEP 723 Parser] + Parser --> ExecSvc[PythonCodeExecutorService] + ExecSvc --> Queue[Bounded Queue] + Queue --> SAA[SAA SandboxService] + SAA --> Install[Controlled Dependency Install] + Install --> Run[Python Subprocess] + Run --> StdIO[Stdout Stderr] StdIO --> JsonParse[JsonParseUtil] JsonParse --> PyAnalyze[PythonAnalyzeNode] PyAnalyze --> Report[ReportGeneratorNode] @@ -692,8 +692,8 @@ flowchart LR classDef data fill:#FEF3C7,stroke:#F59E0B,stroke-width:1px,color:#1F2937; class PyGen,PyExec,PyAnalyze,Report service - class ExecSvc,Pool,Docker,Local,AISim exec - class Queue,TempFiles,StdIO,JsonParse data + class ExecSvc,SAA,Install,Run exec + class Parser,Queue,StdIO,JsonParse data ``` #### 流程图 @@ -706,19 +706,20 @@ sequenceDiagram participant G as PythonGenerateNode participant L as LlmService participant E as PythonExecuteNode - participant CP as CodePoolExecutorService - participant D as Docker Executor + participant CP as PythonCodeExecutorService + participant S as SAA Sandbox participant J as JsonParseUtil participant A as PythonAnalyzeNode participant R as ReportGeneratorNode P->>G: 进入Python步骤并传入指令 G->>L: generate python code - L-->>G: python code + L-->>G: python code + PEP 723 G->>E: pass code and sql results E->>CP: runTask - CP->>D: execute in container - D-->>CP: stdout stderr + CP->>S: create sandbox and install dependencies + S->>S: execute with stdin + S-->>CP: stdout stderr CP-->>E: task response E->>J: parse stdout json J-->>E: normalized output diff --git a/docs/DEVELOPER_GUIDE-en.md b/docs/DEVELOPER_GUIDE-en.md index 3cb13f4ff..46783523d 100644 --- a/docs/DEVELOPER_GUIDE-en.md +++ b/docs/DEVELOPER_GUIDE-en.md @@ -300,19 +300,18 @@ Configuration prefix: `spring.ai.alibaba.data-agent.code-executor` | Configuration Item | Description | Default Value | |-------------------|-------------|---------------| -| `code-pool-executor` | Executor type (DOCKER/LOCAL) | DOCKER (default is local in application.yml) | -| `image-name` | Docker image name | continuumio/anaconda3:latest | -| `container-name-prefix` | Container name prefix | nl2sql-python-exec- | -| `host` | Service host address | null | -| `task-queue-size` | Task blocking queue size | 5 | -| `core-container-num` | Maximum core container count | 2 | -| `temp-container-num` | Maximum temporary container count | 2 | -| `core-thread-size` | Thread pool core thread count | 5 | -| `max-thread-size` | Thread pool maximum thread count | 5 | | `code-timeout` | Python code execution timeout | 60s | -| `container-timeout` | Maximum container runtime | 3000 (ms) | | `limit-memory` | Container memory limit (MB) | 500 | | `cpu-core` | Container CPU cores | 1 | +| `python-max-tries-count` | Maximum Python execution retries | 5 | +| `sandbox.image-name` | SAA base image | AgentScope runtime sandbox base | +| `sandbox.max-concurrency` | Maximum concurrent sandboxes | 4 | +| `sandbox.queue-capacity` | Bounded wait queue size | 10 | +| `sandbox.package-index-url` | Dynamic package index | https://pypi.org/simple | +| `sandbox.dependency-install-timeout` | Dependency installation timeout | 3m | + +Third-party dependencies must be declared in the generated script's PEP 723 `dependencies`. +Host-local, legacy Docker pool, and AI Simulation executors are no longer available. ### 6. File Storage Configuration diff --git a/docs/DEVELOPER_GUIDE.md b/docs/DEVELOPER_GUIDE.md index 8369fd36d..15ed810e7 100644 --- a/docs/DEVELOPER_GUIDE.md +++ b/docs/DEVELOPER_GUIDE.md @@ -353,19 +353,18 @@ public class AgentVectorStoreService { | 配置项 | 说明 | 默认值 | |--------|------|--------| -| `code-pool-executor` | 执行器类型 (DOCKER/LOCAL) | DOCKER (application.yml中默认为local) | -| `image-name` | Docker镜像名称 | continuumio/anaconda3:latest | -| `container-name-prefix` | 容器名称前缀 | nl2sql-python-exec- | -| `host` | 服务主机地址 | null | -| `task-queue-size` | 任务阻塞队列大小 | 5 | -| `core-container-num` | 核心容器数量最大值 | 2 | -| `temp-container-num` | 临时容器数量最大值 | 2 | -| `core-thread-size` | 线程池核心线程数 | 5 | -| `max-thread-size` | 线程池最大线程数 | 5 | | `code-timeout` | Python代码执行超时时间 | 60s | -| `container-timeout` | 容器最大运行时长 | 3000 (ms) | | `limit-memory` | 容器内存限制 (MB) | 500 | | `cpu-core` | 容器CPU核数 | 1 | +| `python-max-tries-count` | Python执行最大重试次数 | 5 | +| `sandbox.image-name` | SAA基础镜像 | AgentScope runtime sandbox base | +| `sandbox.max-concurrency` | 最大并发沙盒数 | 4 | +| `sandbox.queue-capacity` | 有界等待队列大小 | 10 | +| `sandbox.package-index-url` | 动态依赖包索引 | https://pypi.org/simple | +| `sandbox.dependency-install-timeout` | 依赖安装超时 | 3m | + +第三方依赖必须在生成脚本的 PEP 723 `dependencies` 中声明。系统不再提供宿主机 Local、 +旧 Docker 容器池或 AI Simulation 执行器。 ### 6. 文件存储配置 (File Storage) diff --git a/docs/superpowers/mock-strategy-guide.md b/docs/superpowers/mock-strategy-guide.md index 229814387..b98f9c416 100644 --- a/docs/superpowers/mock-strategy-guide.md +++ b/docs/superpowers/mock-strategy-guide.md @@ -441,7 +441,8 @@ class PythonGenerateNodeTest { ### PythonExecuteNode **Dependencies:** -- `CodePoolExecutorService codePoolExecutor` - Code execution +- `PythonCodeExecutorService pythonCodeExecutor` - SAA Sandbox execution +- `PythonDependencyMetadataParser dependencyMetadataParser` - PEP 723 dependency parsing - `JsonParseUtil jsonParseUtil` - JSON parsing - `CodeExecutorProperties codeExecutorProperties` - Config @@ -453,7 +454,10 @@ class PythonGenerateNodeTest { class PythonExecuteNodeTest { @Mock - private CodePoolExecutorService codePoolExecutor; + private PythonCodeExecutorService pythonCodeExecutor; + + @Mock + private PythonDependencyMetadataParser dependencyMetadataParser; @Mock private JsonParseUtil jsonParseUtil; @@ -465,8 +469,10 @@ class PythonExecuteNodeTest { @BeforeEach void setUp() { + when(dependencyMetadataParser.parse(anyString())) + .thenReturn(PythonDependencyMetadata.empty()); pythonExecuteNode = new PythonExecuteNode( - codePoolExecutor, jsonParseUtil, codeExecutorProperties); + pythonCodeExecutor, dependencyMetadataParser, jsonParseUtil, codeExecutorProperties); } @Test @@ -481,11 +487,11 @@ class PythonExecuteNodeTest { PYTHON_TRIES_COUNT, 0 )); - // Mock code pool executor - successful execution + // Mock SAA Sandbox execution String jsonOutput = "{\"result\": \"hello\"}"; - CodePoolExecutorService.TaskResponse taskResponse = - new CodePoolExecutorService.TaskResponse(true, jsonOutput, "", ""); - when(codePoolExecutor.runTask(any())).thenReturn(taskResponse); + PythonCodeExecutorService.TaskResponse taskResponse = + PythonCodeExecutorService.TaskResponse.success(jsonOutput); + when(pythonCodeExecutor.runTask(any())).thenReturn(taskResponse); when(codeExecutorProperties.getPythonMaxTriesCount()).thenReturn(3); when(jsonParseUtil.tryConvertToObject(any(), any())).thenReturn(Map.of("result", "hello")); diff --git a/docs/superpowers/nodes-dependency-analysis.md b/docs/superpowers/nodes-dependency-analysis.md index 75869d27d..f13b73b9c 100644 --- a/docs/superpowers/nodes-dependency-analysis.md +++ b/docs/superpowers/nodes-dependency-analysis.md @@ -14,7 +14,7 @@ | PlannerNode | LlmService | TABLE_RELATION_OUTPUT, EVIDENCE, GENEGRATED_SEMANTIC_MODEL_PROMPT, PLAN_VALIDATION_ERROR, IS_ONLY_NL2SQL, QUERY_ENHANCE_NODE_OUTPUT | PLANNER_NODE_OUTPUT | Medium | | EvidenceRecallNode | LlmService, AgentVectorStoreService, JsonParseUtil, AgentKnowledgeMapper | INPUT_KEY, AGENT_ID, MULTI_TURN_CONTEXT | EVIDENCE | High | | PythonGenerateNode | CodeExecutorProperties, LlmService | TABLE_RELATION_OUTPUT, SQL_RESULT_LIST_MEMORY, PYTHON_IS_SUCCESS, PYTHON_TRIES_COUNT, QUERY_ENHANCE_NODE_OUTPUT | PYTHON_GENERATE_NODE_OUTPUT, PYTHON_TRIES_COUNT | Medium | -| PythonExecuteNode | CodePoolExecutorService, JsonParseUtil, CodeExecutorProperties | PYTHON_GENERATE_NODE_OUTPUT, SQL_RESULT_LIST_MEMORY, PYTHON_TRIES_COUNT | PYTHON_EXECUTE_NODE_OUTPUT, PYTHON_IS_SUCCESS, PYTHON_FALLBACK_MODE | Medium | +| PythonExecuteNode | PythonCodeExecutorService, PythonDependencyMetadataParser, JsonParseUtil, CodeExecutorProperties | PYTHON_GENERATE_NODE_OUTPUT, SQL_RESULT_LIST_MEMORY, PYTHON_TRIES_COUNT | PYTHON_EXECUTE_NODE_OUTPUT, PYTHON_IS_SUCCESS, PYTHON_FALLBACK_MODE | Medium | | PythonAnalyzeNode | LlmService | PYTHON_EXECUTE_NODE_OUTPUT, SQL_EXECUTE_NODE_OUTPUT, PLAN_CURRENT_STEP, PYTHON_FALLBACK_MODE, QUERY_ENHANCE_NODE_OUTPUT | SQL_EXECUTE_NODE_OUTPUT, PLAN_CURRENT_STEP, PYTHON_ANALYSIS_NODE_OUTPUT | Low | | SemanticConsistencyNode | Nl2SqlService | EVIDENCE, TABLE_RELATION_OUTPUT, DB_DIALECT_TYPE, SQL_GENERATE_OUTPUT, QUERY_ENHANCE_NODE_OUTPUT | SEMANTIC_CONSISTENCY_NODE_OUTPUT, SQL_REGENERATE_REASON | Low | | PlanExecutorNode | None | PLANNER_NODE_OUTPUT, PLAN_CURRENT_STEP, IS_ONLY_NL2SQL, PLAN_REPAIR_COUNT, HUMAN_REVIEW_ENABLED | PLAN_VALIDATION_STATUS, PLAN_VALIDATION_ERROR, PLAN_NEXT_NODE, PLAN_CURRENT_STEP, PLAN_REPAIR_COUNT | Very Low | @@ -95,16 +95,17 @@ - Timeout handling #### PythonExecuteNode -- **Purpose**: Execute Python code via code pool -- **Dependencies**: 3 - - `CodePoolExecutorService.runTask(TaskRequest)` - Returns `TaskResponse` +- **Purpose**: Parse PEP 723 dependencies and execute Python in SAA Sandbox +- **Dependencies**: 4 + - `PythonCodeExecutorService.runTask(TaskRequest)` - Returns `TaskResponse` + - `PythonDependencyMetadataParser.parse()` - Validates dynamic dependencies - `JsonParseUtil.tryConvertToObject()` - For parsing JSON output - `CodeExecutorProperties.getPythonMaxTriesCount()` - Max retries - **Logic Flow**: 1. Get code from `PYTHON_GENERATE_NODE_OUTPUT` 2. Get SQL results from `SQL_RESULT_LIST_MEMORY` 3. Check retry count - 4. Execute via code pool + 4. Parse dependencies and execute via SAA Sandbox 5. Parse JSON output (handle Unicode escape) 6. Return streaming generator with output markers - **Test Scenarios**: diff --git a/docs/superpowers/specs/2026-07-28-saa-python-sandbox-integration-design.md b/docs/superpowers/specs/2026-07-28-saa-python-sandbox-integration-design.md new file mode 100644 index 000000000..c2910fb34 --- /dev/null +++ b/docs/superpowers/specs/2026-07-28-saa-python-sandbox-integration-design.md @@ -0,0 +1,327 @@ +# SAA 1.1.2.2 Python 沙盒接入方案 + +## 1. 结论与范围 + +本次只改造 Python 生成与执行链路： + +- Spring AI Alibaba 升级到 `1.1.2.2` +- Spring AI 对齐到 `1.1.2` +- Python 统一通过 `spring-ai-alibaba-sandbox` 执行 +- 支持模型用 PEP 723 声明依赖，并在任务沙盒内动态安装 +- 删除本地进程、自建 Docker 容器池和 AI Simulation 执行器 +- SQL 生成、校验和执行链路不做改动 + +不存在本地执行或旧 Docker 执行器的回退。SAA 启动、依赖安装或代码执行失败时，结果直接 +进入现有 Python 重试与降级分支。 + +## 2. 改造后的链路 + +```mermaid +flowchart LR + PG["PythonGenerateNode
生成代码和 PEP 723 元数据"] + PE["PythonExecuteNode
解析依赖并组装 stdin JSON"] + PARSER["PythonDependencyMetadataParser"] + POLICY["PythonDependencyPolicy"] + EXEC["PythonCodeExecutorService
并发、队列、大小和总超时"] + RUNTIME["SaaSandboxRuntime
SandboxService"] + SB["一次任务一个 BaseSandbox"] + INSTALL["固定 bootstrap
python -m pip install"] + RUN["固定 bootstrap
python -c generated.py"] + RESULT["结构化结果 envelope"] + EXISTING["现有重试 / 降级 / SSE / 报告"] + + PG --> PE + PE --> PARSER --> POLICY + POLICY --> EXEC + PE --> EXEC + EXEC --> RUNTIME --> SB + SB --> INSTALL --> RUN --> RESULT + RESULT --> PE --> EXISTING +``` + +保持不变的业务协议： + +1. 生成代码仍通过 `json.load(sys.stdin)` 读取 SQL 结果 JSON。 +2. 业务结果仍从 stdout 返回，并由 `PythonExecuteNode` 做现有 JSON 解析。 +3. Python 失败仍进入现有重试，超过次数仍走现有降级逻辑。 +4. StateGraph 节点关系和 SSE 事件协议不变。 + +### 2.1 包边界 + +`sandbox` 顶层只保留对外执行服务，内部按变化原因拆分： + +```text +service/code/sandbox +├── SaaSandboxPythonCodeExecutorService.java +├── dependency +│ ├── PythonDependencyMetadata.java +│ ├── PythonDependencyMetadataParser.java +│ └── PythonDependencyPolicy.java +├── execution +│ ├── PythonSandboxBootstrapBuilder.java +│ ├── SandboxExecutionResult.java +│ └── SandboxExecutionResultParser.java +└── runtime + ├── SaaSandboxRuntime.java + └── SaaSandboxTaskRunner.java +``` + +- `dependency` 只负责 PEP 723 和依赖策略，不感知 SAA。 +- `execution` 只负责固定 Python bootstrap 与结果协议，不管理容器。 +- `runtime` 只负责 SAA 服务、任务 Sandbox 和 Runner 生命周期。 +- 顶层服务只负责输入限制、并发队列和任务总超时。 + +## 3. 动态依赖协议 + +生成代码使用标准 PEP 723 `script` 元数据块： + +```python +# /// script +# dependencies = ["pandas>=2,<3", "numpy>=1.24,<3"] +# /// + +import json +import sys +import pandas as pd +``` + +处理规则： + +1. 脚本最多包含一个 `# /// script` 块。 +2. 后端使用 TOML Parser 读取 `dependencies` 和 `requires-python`。 +3. 最多允许 20 个直接依赖，元数据最大 8 KiB。 +4. 允许 PyPI 包名、extras 和版本约束。 +5. 拒绝 URL、VCS、文件路径、`@`、环境标记和任何 pip 参数。 +6. 模型不能提供 index、证书、命令、安装目录或 shell 片段。 + +依赖声明只是一组结构化包规格，不是命令字符串： + +```text +TaskRequest +├── code +├── input +└── dependencies: List +``` + +## 4. 受控动态安装 + +依赖和业务代码在同一个任务级 Sandbox 中分两阶段执行： + +```mermaid +sequenceDiagram + participant E as PythonCodeExecutorService + participant S as SAA BaseSandbox + participant P as Fixed Installer Bootstrap + participant C as Fixed Code Bootstrap + + E->>S: 创建独立 Sandbox + E->>P: Base64 依赖清单 + 固定 index + P->>P: python -m pip install --target /tmp/dataagent-deps + P-->>E: 安装结果 envelope + E->>C: Base64 代码 + Base64 stdin + C->>C: PYTHONPATH=/tmp/dataagent-deps + C->>C: python -c code，stdin=input + C-->>E: stdout/stderr/error envelope + E->>S: close，停止并删除容器 +``` + +固定安装命令由后端生成： + +```text +python -m pip install + --disable-pip-version-check + --no-input + --no-cache-dir + --target /tmp/dataagent-deps + --index-url <服务端配置> + <校验后的依赖清单> +``` + +关键边界： + +- 不使用 shell。 +- 代码、stdin、依赖和 index 均先 Base64 编码，避免字符串拼接注入。 +- 安装目录只存在于本次容器，任务结束即删除。 +- 安装超时和代码超时分开控制。 +- 安装失败不会继续执行业务代码。 +- 每次调用创建新 Sandbox，不跨 Agent、会话或重试复用可写状态。 + +## 5. SAA 框架接入 + +直接使用 1.1.2.2 提供的对象： + +- `SandboxService` +- `ManagerConfig` +- `DockerClientStarter` +- `BaseSandbox` +- `SaaBasePythonRunner` +- `RunPythonToolRequest` / `RunPythonToolResponse` + +`SaaSandboxRuntime` 懒启动单例 `SandboxService`，避免普通 Spring 单元测试要求 Docker +在线；应用关闭时统一调用 `cleanupAllSandboxes()` 和 `close()`。 + +每次任务创建新的 `BaseSandbox` 和新的 `SaaBasePythonRunner`。Runner 持有可变 Sandbox +引用，不能作为单例跨线程复用。 + +当前 SAA 1.1.2.2 的 Docker runtime config 可落地： + +| 参数 | 默认值 | +|---|---:| +| 内存 | 500 MiB | +| CPU | 1 核 | +| nofile | 4096 | +| privileged | `false` | +| 代码超时 | 60 秒 | +| 依赖安装超时 | 3 分钟 | +| 最大并发 | 4 | +| 等待队列 | 10 | + +`nofile` 不能沿用原方案的 256。官方基础镜像中的 supervisord 要求至少 1024；本方案使用 +4096，并通过真实容器启动测试验证。 + +SAA 的 `SandboxConfig.timeout` 同时约束依赖安装和代码执行时的 HTTP 调用，不能只设置为 +`code-timeout`。本方案按 `dependency-install-timeout + code-timeout + 30 秒通信余量` +计算 Sandbox 请求超时，避免 pandas 等较大依赖仍在安装时连接被提前关闭。 + +## 6. 输入输出协议 + +SAA Runner 只有 `code` 字段，没有独立 stdin 字段，因此使用固定 bootstrap 适配： + +1. Java 校验代码和输入大小。 +2. Java 对代码和 stdin 做 UTF-8 Base64。 +3. Sandbox 内用固定 `subprocess.run([python, "-c", code])` 执行。 +4. stdin 通过 subprocess 的 `input` 参数传入。 +5. stdout、stderr、异常类型和错误信息封装为 JSON，再以固定 marker + Base64 返回。 +6. Java 从 SAA 的原始响应中定位 marker 并反序列化。 + +结果协议： + +```json +{ + "success": true, + "stdout": "{\"result\":42}\n", + "stderr": "", + "errorType": null, + "errorMessage": null +} +``` + +映射规则： + +| Sandbox 结果 | 工作流结果 | +|---|---| +| `success=true` | `TaskResponse.success(stdout)` | +| Python 非零退出 | `TaskResponse.failure(stdout, stderr)` | +| 依赖安装失败 | `TaskResponse.failure(install stdout, install stderr)` | +| 响应协议错误、Docker 或通信异常 | `TaskResponse.exception(...)` | +| 外层任务超时 | 取消任务并返回 timeout exception | + +## 7. 大小与并发限制 + +| 内容 | 默认上限 | +|---|---:| +| Python 源码 | 256 KiB | +| stdin JSON | 10 MiB | +| stdout | 1 MiB | +| stderr | 256 KiB | +| PEP 723 元数据 | 8 KiB | +| 直接依赖 | 20 | +| 并发 Sandbox | 4 | +| 等待队列 | 10 | + +代码和 stdin 超限时不创建 Sandbox。stdout/stderr 在返回 envelope 前截断，Java 再校验 +envelope 总大小。队列满时快速失败，不无限堆积请求。 + +完整工作流时间线会包含生成代码、安装日志和执行结果。WebFlux 请求缓冲上限设置为 +`spring.codec.max-in-memory-size: 10MB`，保证浏览器保存这一时间线时不会因默认缓冲上限 +触发 HTTP 413；Python stdout/stderr 仍受上表的独立沙盒限制。 + +## 8. 配置 + +```yaml +spring: + ai: + alibaba: + data-agent: + code-executor: + python-max-tries-count: 5 + code-timeout: 60s + limit-memory: 500 + cpu-core: 1 + sandbox: + docker-host: ${DATAAGENT_SANDBOX_DOCKER_HOST:unix:///var/run/docker.sock} + image-name: ${DATAAGENT_SANDBOX_IMAGE:agentscope-registry.ap-southeast-1.cr.aliyuncs.com/agentscope/runtime-sandbox-base:latest} + max-concurrency: 4 + queue-capacity: 10 + package-index-url: ${DATAAGENT_PYPI_INDEX_URL:https://pypi.org/simple} + dependency-install-timeout: 3m +``` + +开发环境可以使用公网 PyPI 完成端到端验证。生产环境必须通过环境变量切到企业私有 PyPI +代理和固定镜像 digest。 + +## 9. 删除项 + +以下实现不再保留： + +- `CodePoolExecutorServiceFactory` +- `CodePoolExecutorEnum` +- `AbstractCodePoolExecutorService` +- `LocalCodePoolExecutorService` +- `DockerCodePoolExecutorService` +- `AiSimulationCodeExecutorService` +- 自建 docker-java client、image、host 和 pool 辅助类 +- 对应单元测试和旧 Docker 集成测试 + +新的稳定边界为 `PythonCodeExecutorService`，当前只有 +`SaaSandboxPythonCodeExecutorService` 一个实现。 + +## 10. 安全边界与生产加固 + +当前版本已经具备： + +- 独立任务容器 +- 非 privileged +- CPU、内存、nofile、超时、并发和输入输出限制 +- 依赖声明策略 +- 固定 pip 命令 +- 任务完成后删除容器 +- 不回退到宿主机执行 + +需要明确：动态安装要求 Sandbox 在安装阶段能访问包仓库，而 SAA 1.1.2.2 没有公开 +“安装后立即断开容器网络”的 API。因此当前实现适用于本地和受控测试环境；生产上线还需 +补齐基础设施级网络控制： + +1. 仅允许访问私有 PyPI 代理，禁止访问其他公网和业务内网。 +2. 固定基础镜像 digest，镜像内使用非 root 用户。 +3. 不向 Sandbox 注入数据库、模型、OSS 或 Docker Socket 等业务凭据。 +4. 私有代理增加恶意包、CVE 和许可证扫描。 +5. 后续可增加 wheel-only、hash lock 和只读 wheel 缓存；这些是生产加固项，不作为 + 当前代码已实现能力描述。 + +## 11. 验收 + +| 场景 | 验收条件 | +|---|---| +| 动态依赖 | PEP 723 声明的包在任务 Sandbox 内安装成功 | +| stdin | 生成代码能通过 `json.load(sys.stdin)` 读取 SQL 结果 | +| stdout | JSON 结果保持现有工作流协议 | +| Python 异常 | 进入现有重试或降级分支 | +| 超时/超限 | 返回明确失败并释放 Sandbox | +| 隔离 | 两次执行不复用可写运行态 | +| 清理 | 成功和失败后均无残留 `dataagent-sandbox-*` 容器 | +| 端到端 | 浏览器发起真实分析后出现最终报告且 SSE 收到 `event:complete` | + +验证分层： + +1. Parser、依赖策略、bootstrap、响应映射和执行器单元测试。 +2. 真实 Docker 集成测试：SAA 创建容器、动态安装 `six==1.17.0`、读取 stdin、输出 + JSON、停止并删除容器。 +3. 完整后端测试与格式检查。 +4. 本地启动 Milvus、后端和前端，使用浏览器完成真实 StateGraph 请求。 + +## 12. 参考 + +- [Spring AI Alibaba v1.1.2.2 releases](https://github.com/alibaba/spring-ai-alibaba/releases) +- [Spring AI Alibaba Sandbox example](https://github.com/spring-ai-alibaba/examples/tree/main/spring-ai-alibaba-sandbox-example/sandbox-simple-tool) +- [PEP 723 – Inline script metadata](https://peps.python.org/pep-0723/) diff --git a/pom.xml b/pom.xml index 81140106b..426d0ea2a 100644 --- a/pom.xml +++ b/pom.xml @@ -21,7 +21,7 @@ - 1.1.0.0 + 1.1.2.2 1.0.0-SNAPSHOT @@ -35,7 +35,7 @@ ${java.version} - 1.1.0 + 1.1.2 3.4.8 2.8.8 0.0.39 @@ -49,7 +49,8 @@ 3.11.0 3.1.0 - 3.5.3 + 1.1.1 + 33.4.8-jre 3.0.0 1.2.22 42.4.1 @@ -87,18 +88,16 @@ pom import - - com.github.docker-java - docker-java-bom - ${docker-java.version} - pom - import - com.aliyun gpdb20160503 ${gpdb.version} + + com.google.guava + guava + ${guava.version} + com.alibaba druid