From 618247b55f7f8a1ac4bf17e6980a31c3f5381f56 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 20 May 2026 02:10:43 +0000 Subject: [PATCH] test(ci): mark Railway-dependent proxy e2e tests as flaky MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four CircleCI tests intermittently fail with the upstream Railway-hosted fake openai endpoint (https://exampleopenaiendpoint-production.up.railway.app) returning 'Application not found' (cold-start) or timing out: - tests/spend_tracking_tests/test_spend_accuracy_tests.py::test_basic_spend_accuracy - tests/store_model_in_db_tests/test_callbacks_in_db.py::test_e2e_langfuse_callbacks_in_db - tests/test_fallbacks.py::test_chat_completion_with_retries - tests/test_fallbacks.py::test_chat_completion_with_fallbacks Both staging and main are affected (same flake on both branches), so this is not a code regression — it's the upstream test fixture being slow to wake. Add @pytest.mark.flaky(reruns=2, reruns_delay=5) so transient upstream failures don't fail the build, matching the pattern already used in tests/test_callbacks_on_proxy.py for similar Railway-dependent tests. Co-authored-by: Krrish Dholakia --- tests/spend_tracking_tests/test_spend_accuracy_tests.py | 1 + tests/store_model_in_db_tests/test_callbacks_in_db.py | 1 + tests/test_fallbacks.py | 2 ++ 3 files changed, 4 insertions(+) diff --git a/tests/spend_tracking_tests/test_spend_accuracy_tests.py b/tests/spend_tracking_tests/test_spend_accuracy_tests.py index be071f2f0f8..3fc9d12a918 100644 --- a/tests/spend_tracking_tests/test_spend_accuracy_tests.py +++ b/tests/spend_tracking_tests/test_spend_accuracy_tests.py @@ -206,6 +206,7 @@ async def fail_with_diagnostics(session, stage: str, expected: float, observed: ) +@pytest.mark.flaky(reruns=2, reruns_delay=5) @pytest.mark.asyncio async def test_basic_spend_accuracy(): """ diff --git a/tests/store_model_in_db_tests/test_callbacks_in_db.py b/tests/store_model_in_db_tests/test_callbacks_in_db.py index 4a851251a3e..f6bd4196af8 100644 --- a/tests/store_model_in_db_tests/test_callbacks_in_db.py +++ b/tests/store_model_in_db_tests/test_callbacks_in_db.py @@ -71,6 +71,7 @@ async def make_chat_completions_request() -> ChatCompletion: return response +@pytest.mark.flaky(reruns=2, reruns_delay=5) @pytest.mark.asyncio async def test_e2e_langfuse_callbacks_in_db(): diff --git a/tests/test_fallbacks.py b/tests/test_fallbacks.py index bc9aa4c64c8..e2792ac40c4 100644 --- a/tests/test_fallbacks.py +++ b/tests/test_fallbacks.py @@ -132,6 +132,7 @@ async def test_chat_completion_client_fallbacks(has_access): pytest.fail("Expected this to work: {}".format(str(e))) +@pytest.mark.flaky(reruns=2, reruns_delay=5) @pytest.mark.asyncio async def test_chat_completion_with_retries(): """ @@ -156,6 +157,7 @@ async def test_chat_completion_with_retries(): assert headers["x-litellm-max-retries"] == "50" +@pytest.mark.flaky(reruns=2, reruns_delay=5) @pytest.mark.asyncio async def test_chat_completion_with_fallbacks(): """