From 5f5c8212df404f9c801c7ee67d16c8d0543f264c Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Sat, 1 Aug 2026 23:23:52 -0700 Subject: [PATCH] test(e2e): skip the flaky apply_guardrail and datadog messages tests pending LIT-5120 / LIT-5121 apply_guardrail_blocks_banned_and_allows_clean 404s when the request round-robins onto a backend pod that has not yet synced the guardrail the test just created; 6 of ~21 stage runs since Jul 28 failed with the identical signature. LIT-5120 tracks fixing the propagation race, which is customer-visible beyond the test. test_messages_emits_one_log_event missed the Datadog poll window for the first time on the Aug 2 03:01 UTC run. One miss reads as ingest latency, but the rust namespace fails the streamed variant chronically, so LIT-5121 investigates delivery before the test comes back. Both skips name their ticket; the coverage registry returns the cells to the gap list while they are in place. --- tests/e2e/guardrails/test_apply_guardrail_e2e.py | 6 ++++++ tests/e2e/logging/test_datadog_log_e2e.py | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/tests/e2e/guardrails/test_apply_guardrail_e2e.py b/tests/e2e/guardrails/test_apply_guardrail_e2e.py index ee691db22da..efbdd6ca97e 100644 --- a/tests/e2e/guardrails/test_apply_guardrail_e2e.py +++ b/tests/e2e/guardrails/test_apply_guardrail_e2e.py @@ -18,6 +18,12 @@ pytestmark = pytest.mark.e2e class TestApplyGuardrailEndpoint: + @pytest.mark.skip( + reason=( + "LIT-5120: flaky - /apply_guardrail round-robins onto a pod that has not synced the " + "guardrail the test just created and 404s (6 of ~21 stage runs since Jul 28)" + ) + ) @pytest.mark.covers( "guardrail.litellm_content_filter.apply_endpoint.blocks", "guardrail.litellm_content_filter.apply_endpoint.allows", diff --git a/tests/e2e/logging/test_datadog_log_e2e.py b/tests/e2e/logging/test_datadog_log_e2e.py index 94811c6217e..368fb187424 100644 --- a/tests/e2e/logging/test_datadog_log_e2e.py +++ b/tests/e2e/logging/test_datadog_log_e2e.py @@ -141,6 +141,12 @@ class TestDataDogLogDelivery: events, model_group=CHEAP_ANTHROPIC_MODEL, call_type="acompletion", cost_anchor=outcome.response_cost ) + @pytest.mark.skip( + reason=( + "LIT-5121: flaky - the /v1/messages log event missed the Datadog poll window on the " + "2026-08-02 stage run; skipped while delivery vs ingest-latency is investigated" + ) + ) @pytest.mark.covers("logging.datadog.success.exports_metric", exercised_on=["messages"]) def test_messages_emits_one_log_event( self, client: LoggingClient, dd_logs: DdLogsReader, resources: ResourceManager