Skip to content

Commit 8f26600

Browse files
Use bespokelabs-sandbox to run code in sandboxes (#716)
* Replace code execution backends with bespokelabs-sandbox Remove the four built-in execution backends (multiprocessing, docker, ray, e2b) and replace them with a single unified backend powered by bespokelabs-sandbox. This delegates sandbox lifecycle, isolation, and multi-provider support to the sandbox package while keeping curator's orchestration layer (caching, retries, request/response files) intact. - Add SandboxCodeExecutionBackend using bespokelabs.sandbox.Sandbox - bespokelabs-sandbox is a required dep; "local" backend works out of the box - code_execution extra installs docker/ray/e2b SDKs for non-local backends - Preserve legacy docker_image and base_url config params for backwards compat - Tar archives retain directory structure (os.path.relpath, not basename) - Regenerate poetry.lock for the new dependency graph - Update tests and examples for new backend names Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * Pin bespokelabs-sandbox to 0.1.1 Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * Remove redundant docker/ray/e2b deps and code_execution extra Backend SDKs are managed by bespokelabs-sandbox's own extras (e.g. pip install bespokelabs-sandbox[docker]). Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * Remove code_execution extra from CI workflow The code_execution extra was removed from pyproject.toml since backend SDKs are now managed by bespokelabs-sandbox's own extras. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * Always return message=success for completed executions Non-zero exit codes are valid outcomes (e.g. code-verification workloads). Setting error on non-zero exits caused validate_existing_response_file to discard and re-run those responses on every resume. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix: Auto-format base_backend.py and curator_response.py with ruff Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * Add unit tests for CuratorResponse and misc to meet 80% coverage - Test all dataclass add/append methods on TokenUsage, CostInfo, RequestStats, PerformanceStats - Test CuratorResponse save/load, to_dict, get_failed_requests, __getattr__ deprecation, update_tracker_stats offline path - Test safe_model_dump normal and fallback paths Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * Enforce per-execution timeout via shell timeout command The sandbox lifecycle timeout (timeout_secs) is a safety net, not the per-execution limit. Use `timeout N` in the shell command to kill the python process after exactly N seconds, matching the old subprocess.run timeout behavior. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix: Format test_misc.py with ruff Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * Improve sandbox backend error handling and add unit tests - Distinguish exit code 124 (timeout) with message="timeout" - Format non-zero exit code errors with stderr details - Preserve stdout/stderr/files in exception handler - Add unit tests with fake sandbox for all error paths Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent cb11a56 commit 8f26600

22 files changed

Lines changed: 544 additions & 857 deletions

File tree

‎.github/workflows/lint-and-test.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ jobs:
4343
- name: Install dependencies
4444
run: |
4545
pip install poetry
46-
poetry install --with dev --extras "vllm code_execution"
46+
poetry install --with dev --extras "vllm"
4747
4848
- name: Run ruff
4949
run: |

‎examples/code-execution/chart-generation/charts.py‎

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -108,16 +108,11 @@ def code_output(self, row, execution_output):
108108
tar.extractall(path=temp_dir)
109109

110110
# read the chart image
111-
# it is in workspace/chart.png for 'docker' backend and chart.png for 'multiprocessing' backend
112111
try:
113-
with open(os.path.join(temp_dir, "workspace/chart.png"), "rb") as f:
112+
with open(os.path.join(temp_dir, "chart.png"), "rb") as f:
114113
row["chart_image"] = f.read()
115114
except Exception:
116-
try:
117-
with open(os.path.join(temp_dir, "chart.png"), "rb") as f:
118-
row["chart_image"] = f.read()
119-
except Exception:
120-
row["chart_image"] = None
115+
row["chart_image"] = None
121116

122117
return row
123118

@@ -126,7 +121,7 @@ def code_output(self, row, execution_output):
126121
# This section demonstrates how to use the ChartCodeExecutor
127122

128123
# Initialize executor with the multiprocessing backend (runs code in a separate process)
129-
executor = ChartCodeExecutor(backend="multiprocessing") # or ray, docker (with matplotlib image)
124+
executor = ChartCodeExecutor(backend="local") # or docker, e2b, modal, daytona
130125

131126
# Create a simple dataset with one example
132127
dataset = Dataset.from_list([{"id": 1}])

‎examples/code-execution/code-verification/code_verify.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
"""This example demonstrates how to use Curator's code execution functionality with Ray backend for executing code on large datasets.
1+
"""This example demonstrates how to use Curator's code execution functionality for executing code on large datasets.
22
33
It shows how to verify code solutions against expected outputs.
44
@@ -60,7 +60,7 @@ def code_output(self, row, execution_output):
6060

6161

6262
if __name__ == "__main__":
63-
executor = TACOCodeExecutor(backend="ray") # or multiprocessing, docker, e2b
63+
executor = TACOCodeExecutor(backend="local") # or docker, e2b, modal, daytona
6464
dataset = load_dataset("bespokelabs/sky-t1-taco-test-rejection-sampled-shreyas")
6565
execution_output = executor(dataset["train"].select(range(10)))
6666

‎examples/code-execution/math-animation/execute_code.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -75,8 +75,8 @@ def code_output(self, input: Dict, output) -> Dict:
7575
with tarfile.open(fileobj=bytes_io) as tar:
7676
tar.extractall(path=temp_dir)
7777

78-
# get the video file
79-
video_file = os.path.join(temp_dir, "workspace/media/videos/temp_animation/720p30/video.mp4")
78+
# get the video file (path relative to sandbox workspace)
79+
video_file = os.path.join(temp_dir, "media/videos/temp_animation/720p30/video.mp4")
8080
if os.path.exists(video_file):
8181
# Copy the video file to the output dataset
8282
input["video"] = open(video_file, "rb").read()
@@ -111,7 +111,7 @@ def execute_manim_code(dataset_name="pimpalgaonkar/manim_codes_10k", output_data
111111
executor = ManimCodeExecutor(
112112
backend="docker",
113113
backend_params={
114-
"docker_image": "manimcommunity/manim:latest",
114+
"image": "manimcommunity/manim:latest",
115115
},
116116
)
117117

‎poetry.lock‎

Lines changed: 22 additions & 57 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pyproject.toml‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -35,17 +35,14 @@ aiofiles = ">=22.0,<26.0"
3535
anthropic = "^0.84.0"
3636
vertexai = "1.71.1"
3737
vllm = { version = "^0.6.3", optional = true }
38-
aiodocker = { version = "^0.24.0", optional = true }
39-
ray = { version = "^2.41.0", optional = true }
40-
e2b-code-interpreter = { version = "1.0.3", optional = true }
38+
bespokelabs-sandbox = "0.1.1"
4139
tinker = { version = "^0.8.0", optional = true, python = ">=3.11" }
4240

4341
posthog = "^3.11.0"
4442
mistralai = "^1.5.1"
4543

4644
[tool.poetry.extras]
4745
vllm = ["vllm"]
48-
code_execution = ["ray", "aiodocker", "e2b-code-interpreter"]
4946
finetune = ["tinker"]
5047

5148
[tool.poetry.group.dev.dependencies]
Lines changed: 23 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,31 +1,35 @@
1+
"""Factory for creating code execution backends."""
2+
13
from typing import Optional
24

35
from bespokelabs.curator.code_executor.types import CodeExecutionBackendConfig
46

7+
# Map old backend names to sandbox backend names
8+
_BACKEND_ALIASES = {
9+
"multiprocessing": "local",
10+
}
11+
512

613
class _CodeExecutionBackendFactory:
7-
@classmethod
8-
def create(cls, backend: str, backend_params: Optional[CodeExecutionBackendConfig] = None):
9-
if backend_params is None:
10-
backend_params = {}
14+
"""Factory for creating code execution backends."""
1115

12-
if backend == "multiprocessing":
13-
from bespokelabs.curator.code_executor.code_execution_backend.multiprocessing_backend import MultiprocessingCodeExecutionBackend
16+
@classmethod
17+
def create(cls, backend: str, backend_params: Optional[dict] = None):
18+
"""Create a code execution backend.
1419
15-
_code_executor = MultiprocessingCodeExecutionBackend(CodeExecutionBackendConfig(**backend_params))
16-
elif backend == "docker":
17-
from bespokelabs.curator.code_executor.code_execution_backend.docker_backend import DockerCodeExecutionBackend
20+
Args:
21+
backend: Backend name (e.g. "local", "docker", "e2b", "modal", "daytona")
22+
backend_params: Configuration parameters for the backend
1823
19-
_code_executor = DockerCodeExecutionBackend(CodeExecutionBackendConfig(**backend_params))
20-
elif backend == "ray":
21-
from bespokelabs.curator.code_executor.code_execution_backend.ray_backend import RayCodeExecutionBackend
24+
Returns:
25+
SandboxCodeExecutionBackend instance
26+
"""
27+
if backend_params is None:
28+
backend_params = {}
2229

23-
_code_executor = RayCodeExecutionBackend(CodeExecutionBackendConfig(**backend_params))
24-
elif backend == "e2b":
25-
from bespokelabs.curator.code_executor.code_execution_backend.e2b_backend import E2BCodeExecutionBackend
30+
backend_name = _BACKEND_ALIASES.get(backend, backend)
31+
config = CodeExecutionBackendConfig(**backend_params)
2632

27-
_code_executor = E2BCodeExecutionBackend(CodeExecutionBackendConfig(**backend_params))
28-
else:
29-
raise ValueError(f"Unsupported backend: {backend}")
33+
from bespokelabs.curator.code_executor.code_execution_backend.sandbox_backend import SandboxCodeExecutionBackend
3034

31-
return _code_executor
35+
return SandboxCodeExecutionBackend(config, backend_name=backend_name)

‎src/bespokelabs/curator/code_executor/code_execution_backend/base_backend.py‎

Lines changed: 0 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,9 @@
22

33
import asyncio
44
import glob
5-
import io
65
import json
76
import os
87
import resource
9-
import tarfile
108
import time
119
from abc import abstractmethod
1210
from typing import TYPE_CHECKING, List, Optional
@@ -656,47 +654,3 @@ async def append_execution_response(self, data: dict, filename: str) -> None:
656654
async with aiofiles.open(filename, "a") as f:
657655
await f.write(json_string + "\n")
658656
logger.debug(f"Successfully appended response to {filename}")
659-
660-
@classmethod
661-
def _create_temp_file(cls, content: str, execution_directory: str) -> str:
662-
"""Create a temporary file with the given content.
663-
664-
Args:
665-
content: Content to write to temp file
666-
execution_directory: Directory to create the temp file in
667-
668-
Returns:
669-
Path to the created temp file
670-
"""
671-
# create execution directory if it doesn't exist
672-
os.makedirs(execution_directory, exist_ok=True)
673-
674-
temp_file_path = os.path.join(execution_directory, "program.py")
675-
with open(temp_file_path, "w", encoding="utf-8") as temp_file:
676-
temp_file.write(content)
677-
return temp_file_path
678-
679-
@classmethod
680-
def _get_created_files(cls, program_dir: str) -> bytes:
681-
"""Get any files created during code execution.
682-
683-
Args:
684-
program_dir: Directory containing the executed program and created files
685-
686-
Returns:
687-
Bytes containing a zip archive of any created files
688-
"""
689-
# Create a BytesIO object to store the zip data
690-
tar_buffer = io.BytesIO()
691-
692-
# Create a zip archive containing all files in program_dir
693-
with tarfile.open(fileobj=tar_buffer, mode="w") as tar:
694-
# Add all files in program_dir to the archive
695-
for filename in os.listdir(program_dir):
696-
file_path = os.path.join(program_dir, filename)
697-
if os.path.isfile(file_path):
698-
tar.add(file_path, arcname=filename)
699-
700-
# Get the bytes from the buffer
701-
tar_buffer.seek(0)
702-
return str(tar_buffer.getvalue())

0 commit comments

Comments
 (0)