From 50f80918d0bbc80e9c2dff05c2320095ec174b1b Mon Sep 17 00:00:00 2001 From: "Gareth.Y" <91742523+linjiaye929@users.noreply.github.com> Date: Sun, 11 Oct 2026 10:19:56 -0700 Subject: [PATCH] fix(experiments): reject nonpositive concurrency limits --- langfuse/_client/client.py | 4 +++ langfuse/_client/datasets.py | 1 + tests/unit/test_experiment.py | 62 +++++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+) diff --git a/langfuse/_client/client.py b/langfuse/_client/client.py index af06e69ef..dcb4cc53f 100644 --- a/langfuse/_client/client.py +++ b/langfuse/_client/client.py @@ -2667,6 +2667,7 @@ def run_experiment( Each run evaluator receives all item_results and can compute aggregate metrics. Useful for calculating averages, distributions, or cross-item comparisons. max_concurrency: Maximum number of concurrent task executions (default: 50). + Must be greater than 0, including for empty datasets. Controls the number of items processed simultaneously. Adjust based on API rate limits and system resources. metadata: Optional metadata dictionary to attach to all experiment traces. @@ -2825,6 +2826,9 @@ async def _run_experiment_async( metadata: Optional[Dict[str, Any]] = None, dataset_version: Optional[datetime] = None, ) -> ExperimentResult: + if max_concurrency <= 0: + raise ValueError("max_concurrency must be greater than 0") + langfuse_logger.debug( "Starting experiment '%s' run '%s' with %s items", name, run_name, len(data) ) diff --git a/langfuse/_client/datasets.py b/langfuse/_client/datasets.py index e28ddbebf..18a1ae0f8 100644 --- a/langfuse/_client/datasets.py +++ b/langfuse/_client/datasets.py @@ -124,6 +124,7 @@ def run_experiment( run_evaluators: List of functions to evaluate the entire experiment run. Useful for computing aggregate statistics across all dataset items. max_concurrency: Maximum number of concurrent task executions (default: 50). + Must be greater than 0, including for empty datasets. Adjust based on API rate limits and system resources. metadata: Optional metadata to attach to the experiment run and all traces. Will be combined with individual item metadata. diff --git a/tests/unit/test_experiment.py b/tests/unit/test_experiment.py index 3eb55ad40..30820b79b 100644 --- a/tests/unit/test_experiment.py +++ b/tests/unit/test_experiment.py @@ -1,5 +1,6 @@ """Tests for ``langfuse.experiment`` — ``RunnerContext`` and ``RegressionError``.""" +import asyncio import inspect import typing from datetime import datetime @@ -524,3 +525,64 @@ def failing_evaluator(**kwargs): "failed_evaluator_count": 1, "skipped_evaluator_count": 1, } + + +class TestExperimentConcurrency: + @pytest.mark.timeout(5) + @pytest.mark.parametrize("max_concurrency", [0, -1]) + def test_public_api_rejects_nonpositive_concurrency( + self, langfuse_memory_client, max_concurrency + ): + task = MagicMock() + with pytest.raises(ValueError, match="max_concurrency must be greater than 0"): + langfuse_memory_client.run_experiment( + name="invalid-concurrency", + data=[{"input": "question"}], + task=task, + max_concurrency=max_concurrency, + ) + task.assert_not_called() + + @pytest.mark.asyncio + @pytest.mark.parametrize("max_concurrency", [0, -1]) + @pytest.mark.parametrize("data", [[], [{"input": "question"}]]) + async def test_nonpositive_concurrency_raises( + self, langfuse_memory_client, max_concurrency, data + ): + task = MagicMock() + # Bound the regression: before validation, zero permits never release. + with pytest.raises(ValueError, match="max_concurrency must be greater than 0"): + await asyncio.wait_for( + langfuse_memory_client._run_experiment_async( + name="invalid-concurrency", + run_name="invalid-concurrency-run", + description=None, + data=data, + task=task, + evaluators=[], + composite_evaluator=None, + run_evaluators=[], + max_concurrency=max_concurrency, + ), + timeout=1, + ) + task.assert_not_called() + + @pytest.mark.parametrize("max_concurrency", [1, 2]) + @pytest.mark.parametrize("data", [[], [{"input": "a"}, {"input": "b"}]]) + def test_positive_concurrency_preserves_results( + self, langfuse_memory_client, max_concurrency, data + ): + async def task(*, item): + await asyncio.sleep(0) + return item["input"].upper() + + result = langfuse_memory_client.run_experiment( + name="valid-concurrency", + data=data, + task=task, + max_concurrency=max_concurrency, + ) + assert [item.output for item in result.item_results] == [ + item["input"].upper() for item in data + ]