diff --git a/AGENTS.md b/AGENTS.md index 42f4612da..3551ffb24 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -102,6 +102,12 @@ Minimum verification matrix: - This suite uses real provider calls and runs as one dedicated CI suite. CI skips it on fork PRs, which have no provider secrets. - Any test that calls a model provider belongs here, never in `tests/e2e`, so fork PRs stay green on the e2e shards. + +### Quarantine + +- `QUARANTINED_NODEIDS` in `tests/conftest.py` lists tests that fail intermittently for reasons outside the SDK. They still run but are marked non-strict `xfail`, so CI reports XFAIL/XPASS instead of failing. +- Quarantine only with evidence of flakiness, such as the same commit both passing and failing in CI. Do not quarantine deterministic failures. +- Remove the entry once the cause is fixed. - Do not split or shard `tests/live_provider` into separate smoke and extended jobs unless the team explicitly changes that policy. - Keep assertions focused on stable provider-facing behavior rather than brittle observation counts. diff --git a/tests/conftest.py b/tests/conftest.py index 97a78d212..aa4896b34 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -22,6 +22,23 @@ "tests/e2e/test_media.py::test_replace_media_reference_string_in_object", } +# Tests that fail intermittently for reasons outside the SDK. They still run and +# report XFAIL/XPASS, but cannot fail CI. Remove an entry once its cause is fixed. +QUARANTINED_NODEIDS = { + "tests/live_provider/test_openai.py::test_audio_input_and_output": ( + "flaky: OpenAI audio responses fail intermittently" + ), + "tests/live_provider/test_openai.py::test_base_64_image_input": ( + "flaky: OpenAI image input fails intermittently" + ), + "tests/live_provider/test_langchain.py::test_multimodal": ( + "flaky: OpenAI multimodal call intermittently hangs until timeout" + ), +} +# Signal-based so a hang raises inside the test and counts as XFAIL; the +# suite's thread-based timeout kills the xdist worker instead. +QUARANTINE_TIMEOUT_SECONDS = 60 + class InMemorySpanExporter(SpanExporter): """Simple in-memory exporter to collect spans for deterministic tests.""" @@ -51,6 +68,13 @@ def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: for item in items: test_group = Path(str(item.fspath)).parent.name + quarantine_reason = QUARANTINED_NODEIDS.get(item.nodeid) + if quarantine_reason is not None: + item.add_marker(pytest.mark.xfail(reason=quarantine_reason, strict=False)) + item.add_marker( + pytest.mark.timeout(QUARANTINE_TIMEOUT_SECONDS, method="signal") + ) + if test_group == "unit": item.add_marker(pytest.mark.unit) continue