Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions langfuse/_client/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -2667,6 +2667,7 @@ def run_experiment(
Each run evaluator receives all item_results and can compute aggregate metrics.
Useful for calculating averages, distributions, or cross-item comparisons.
max_concurrency: Maximum number of concurrent task executions (default: 50).
Must be greater than 0, including for empty datasets.
Controls the number of items processed simultaneously. Adjust based on
API rate limits and system resources.
metadata: Optional metadata dictionary to attach to all experiment traces.
Expand Down Expand Up @@ -2825,6 +2826,9 @@ async def _run_experiment_async(
metadata: Optional[Dict[str, Any]] = None,
dataset_version: Optional[datetime] = None,
) -> ExperimentResult:
if max_concurrency <= 0:
raise ValueError("max_concurrency must be greater than 0")

langfuse_logger.debug(
"Starting experiment '%s' run '%s' with %s items", name, run_name, len(data)
)
Expand Down
1 change: 1 addition & 0 deletions langfuse/_client/datasets.py
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,7 @@ def run_experiment(
run_evaluators: List of functions to evaluate the entire experiment run.
Useful for computing aggregate statistics across all dataset items.
max_concurrency: Maximum number of concurrent task executions (default: 50).
Must be greater than 0, including for empty datasets.
Adjust based on API rate limits and system resources.
metadata: Optional metadata to attach to the experiment run and all traces.
Will be combined with individual item metadata.
Expand Down
62 changes: 62 additions & 0 deletions tests/unit/test_experiment.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
"""Tests for ``langfuse.experiment`` — ``RunnerContext`` and ``RegressionError``."""

import asyncio
import inspect
import typing
from datetime import datetime
Expand Down Expand Up @@ -524,3 +525,64 @@ def failing_evaluator(**kwargs):
"failed_evaluator_count": 1,
"skipped_evaluator_count": 1,
}


class TestExperimentConcurrency:
@pytest.mark.timeout(5)
@pytest.mark.parametrize("max_concurrency", [0, -1])
def test_public_api_rejects_nonpositive_concurrency(
self, langfuse_memory_client, max_concurrency
):
task = MagicMock()
with pytest.raises(ValueError, match="max_concurrency must be greater than 0"):
langfuse_memory_client.run_experiment(
name="invalid-concurrency",
data=[{"input": "question"}],
task=task,
max_concurrency=max_concurrency,
)
task.assert_not_called()

@pytest.mark.asyncio
@pytest.mark.parametrize("max_concurrency", [0, -1])
@pytest.mark.parametrize("data", [[], [{"input": "question"}]])
async def test_nonpositive_concurrency_raises(
self, langfuse_memory_client, max_concurrency, data
):
task = MagicMock()
# Bound the regression: before validation, zero permits never release.
with pytest.raises(ValueError, match="max_concurrency must be greater than 0"):
await asyncio.wait_for(
langfuse_memory_client._run_experiment_async(
name="invalid-concurrency",
run_name="invalid-concurrency-run",
description=None,
data=data,
task=task,
evaluators=[],
composite_evaluator=None,
run_evaluators=[],
max_concurrency=max_concurrency,
),
timeout=1,
)
task.assert_not_called()

@pytest.mark.parametrize("max_concurrency", [1, 2])
@pytest.mark.parametrize("data", [[], [{"input": "a"}, {"input": "b"}]])
def test_positive_concurrency_preserves_results(
self, langfuse_memory_client, max_concurrency, data
):
async def task(*, item):
await asyncio.sleep(0)
return item["input"].upper()

result = langfuse_memory_client.run_experiment(
name="valid-concurrency",
data=data,
task=task,
max_concurrency=max_concurrency,
)
assert [item.output for item in result.item_results] == [
item["input"].upper() for item in data
]