test: quarantine flaky live-provider tests (#2044)

Co-authored-by: Cursor Agent <cursoragent@cursor.com>
Co-authored-by: Hassieb Pakzad <hassiebp@users.noreply.github.com>

Author: hassiebp

AGENTS.md

@@ -102,6 +102,12 @@ Minimum verification matrix:
 
 - This suite uses real provider calls and runs as one dedicated CI suite. CI skips it on fork PRs, which have no provider secrets.
 - Any test that calls a model provider belongs here, never in `tests/e2e`, so fork PRs stay green on the e2e shards.
+
+### Quarantine
+
+- `QUARANTINED_NODEIDS` in `tests/conftest.py` lists tests that fail intermittently for reasons outside the SDK. They still run but are marked non-strict `xfail`, so CI reports XFAIL/XPASS instead of failing.
+- Quarantine only with evidence of flakiness, such as the same commit both passing and failing in CI. Do not quarantine deterministic failures.
+- Remove the entry once the cause is fixed.
 - Do not split or shard `tests/live_provider` into separate smoke and extended jobs unless the team explicitly changes that policy.
 - Keep assertions focused on stable provider-facing behavior rather than brittle observation counts.
 

tests/conftest.py

@@ -22,6 +22,23 @@
     "tests/e2e/test_media.py::test_replace_media_reference_string_in_object",
 }
 
+# Tests that fail intermittently for reasons outside the SDK. They still run and
+# report XFAIL/XPASS, but cannot fail CI. Remove an entry once its cause is fixed.
+QUARANTINED_NODEIDS = {
+    "tests/live_provider/test_openai.py::test_audio_input_and_output": (
+        "flaky: OpenAI audio responses fail intermittently"
+    ),
+    "tests/live_provider/test_openai.py::test_base_64_image_input": (
+        "flaky: OpenAI image input fails intermittently"
+    ),
+    "tests/live_provider/test_langchain.py::test_multimodal": (
+        "flaky: OpenAI multimodal call intermittently hangs until timeout"
+    ),
+}
+# Signal-based so a hang raises inside the test and counts as XFAIL; the
+# suite's thread-based timeout kills the xdist worker instead.
+QUARANTINE_TIMEOUT_SECONDS = 60
+
 
 class InMemorySpanExporter(SpanExporter):
     """Simple in-memory exporter to collect spans for deterministic tests."""
@@ -51,6 +68,13 @@ def pytest_collection_modifyitems(items: list[pytest.Item]) -> None:
     for item in items:
         test_group = Path(str(item.fspath)).parent.name
 
+        quarantine_reason = QUARANTINED_NODEIDS.get(item.nodeid)
+        if quarantine_reason is not None:
+            item.add_marker(pytest.mark.xfail(reason=quarantine_reason, strict=False))
+            item.add_marker(
+                pytest.mark.timeout(QUARANTINE_TIMEOUT_SECONDS, method="signal")
+            )
+
         if test_group == "unit":
             item.add_marker(pytest.mark.unit)
             continue