From 71d9fe65a06b8aeae6dda9708475fc38a2a3e560 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sat, 12 Sep 2026 14:23:53 -0700 Subject: [PATCH] test: classify Bedrock stream outages and interrupted transfers --- tests/e2e/CLAUDE.md | 2 +- tests/e2e/CONTRIBUTING.md | 2 +- tests/e2e/bedrock_eventstream.py | 2 + tests/e2e/conftest.py | 4 +- tests/e2e/e2e_http.py | 37 ++++++++++++-- tests/e2e/provider_diagnostics.py | 27 ++++++++++- tests/e2e/test_bedrock_eventstream.py | 70 +++++++++++++++++++++++++-- tests/e2e/test_provider_retry.py | 38 ++++++++++++++- 8 files changed, 168 insertions(+), 14 deletions(-) diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index 9cfd37320a1..98fb1d5bb1e 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -100,7 +100,7 @@ Bedrock tests use `bedrock_config.bedrock_params()` for Chat Completions, Respon The recording workflow uses the existing `e2e-changed` environment and OIDC secrets reader (`E2E_AWS_ROLE_TO_ASSUME`, secret `litellm-e2e-changed-provider-keys`), and fails before starting the proxy if any required provider credential is missing. Recording requires approval under that environment's existing protection policy, including scheduled runs. Replay supplies only test credentials and includes the four supported Bedrock endpoints in the egress sentinel -HTTP assertions classify a Bedrock 503 as `ProviderUnavailableError` only when AWS headers with Bedrock route context, or a recognized Bedrock error envelope support that attribution. The default live lane allows one test rerun after ten seconds for that error or a transport failure. Generic proxy 5xx, authentication errors and assertion failures do not qualify. Persistent provider failures remain failed, and JUnit properties retain AWS and proxy request IDs from the failed attempt even when the rerun passes. Record and replay continue to use `--reruns 0` +HTTP assertions classify a Bedrock 503 as `ProviderUnavailableError` only when AWS headers with Bedrock route context, or a recognized Bedrock error envelope support that attribution. A decoded native `serviceUnavailableException` receives the same classification even when the committed HTTP status is 200, with `stream_event` recorded as its evidence. Interrupted binary or SSE transfers become modeled network failures with the received status and request IDs preserved. The default live lane allows one test rerun after ten seconds for an evidenced availability error or a transport failure. Generic proxy 5xx, authentication errors and assertion failures do not qualify. Persistent provider failures remain failed, and JUnit properties retain AWS and proxy request IDs from the failed attempt even when the rerun passes. Record and replay continue to use `--reruns 0` Current limits: deployments baked into the proxy's config file cannot be edge-wired (only `/model/new` registrations can carry the edge api_base), and a file upload routed by `custom_llm_provider` through the proxy's `files_settings` block never passes a deployment at all, so the batches `model_param` and `provider_fallback` scenarios keep uploading live in every mode diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index e77d0489c73..a21812e7377 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -101,7 +101,7 @@ A couple of logging destinations are configured on the proxy rather than by the ### The pull request check -Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite and both JWT suites as canaries, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Keycloak, Jaeger, and TLS cluster-mode Valkey. Realm-only edits also trigger these canaries. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, and `guardrails/test_presidio_masking_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start. The Redis chaos test under `load/` needs a proxy it can pause the Redis of on the same host (`gateway/redis_chaos_ci_config.yml`), which `.github/workflows/test-e2e-redis-chaos.yml` boots, and which the Buildkite `e2e-redis-chaos` step in project-releaser runs co-located with Postgres and Valkey in one pod; it is deselected unless `E2E_REDIS_CHAOS` is set +Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite and both JWT suites as canaries, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's single rerun for network failures and evidenced Bedrock availability errors (see `pytest.ini`) applies on every pass, including outages reported inside native streams. Generic proxy 5xx and failed behavior assertions do not qualify, and persistent failures still fail the check. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Keycloak, Jaeger, and TLS cluster-mode Valkey. Realm-only edits also trigger these canaries. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, and `guardrails/test_presidio_masking_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start. The Redis chaos test under `load/` needs a proxy it can pause the Redis of on the same host (`gateway/redis_chaos_ci_config.yml`), which `.github/workflows/test-e2e-redis-chaos.yml` boots, and which the Buildkite `e2e-redis-chaos` step in project-releaser runs co-located with Postgres and Valkey in one pod; it is deselected unless `E2E_REDIS_CHAOS` is set Every selected file must execute at least one passing test in each pass, and any test failure, collection error, or entirely skipped or deselected file fails the check. A file whose tests are all marked skip therefore cannot pass this check, so unskip at least one of them, or add the file to `UNSUPPORTED` in `select_tests.py` with the reason, before changing one. A failed pass stops the run. The public log prints pytest's one-line summary for each pass, including the rerun count, and names each failed or errored test as `classname::name`, so a retried network error or a failing test is visible without the raw output. The final `e2e-changed-tests` job succeeds only when no supported test files changed or the approved run completed all three passes. Fork PRs with selected tests fail this gate until a maintainer brings the reviewed change onto a same-repository branch diff --git a/tests/e2e/bedrock_eventstream.py b/tests/e2e/bedrock_eventstream.py index a980d994413..021146feb32 100644 --- a/tests/e2e/bedrock_eventstream.py +++ b/tests/e2e/bedrock_eventstream.py @@ -21,6 +21,7 @@ class _InvokeChunk(BaseModel): class BedrockEvent: payload: str error: str | None = None + error_code: str | None = None _JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) @@ -32,6 +33,7 @@ def _decode_event(event: EventStreamMessage) -> BedrockEvent: return BedrockEvent( payload=event.payload.decode(), error=f"{headers.exception_type or headers.message_type}: {event.payload.decode()}", + error_code=headers.exception_type or headers.message_type, ) if headers.event_type == "chunk": encoded: Final = _InvokeChunk.model_validate_json(event.payload) diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 9ec2bf7ef1a..96276eb5f37 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -29,7 +29,7 @@ from idp import Identity, Keycloak, keycloak_from_env from junit_properties import attach_result_properties from lifecycle import ProxyClientProvider, ResourceManager from models import TeamNewBody, UserNewBody, UserNewResponse -from provider_diagnostics import ProviderUnavailableError +from provider_diagnostics import NetworkFailureError, ProviderUnavailableError from provider_edge import replay_leftover_error from proxy_client import ProxyClient, build_proxy_client @@ -183,7 +183,7 @@ def pytest_runtest_makereport( report = yield if report.when == "call": item.stash[_CALL_PASSED] = report.passed - if call.excinfo is not None and isinstance(call.excinfo.value, ProviderUnavailableError): + if call.excinfo is not None and isinstance(call.excinfo.value, (ProviderUnavailableError, NetworkFailureError)): failure: Final = call.excinfo.value.failure item.user_properties.extend(failure.junit_properties()) report.user_properties = list(item.user_properties) diff --git a/tests/e2e/e2e_http.py b/tests/e2e/e2e_http.py index f2eb4fecd09..5bf26433a32 100644 --- a/tests/e2e/e2e_http.py +++ b/tests/e2e/e2e_http.py @@ -23,7 +23,7 @@ from typing import Final, Generator, Generic, Iterator, Literal, NewType, Protoc import pytest import requests from bedrock_eventstream import decode_bedrock_stream -from provider_diagnostics import raise_if_provider_unavailable +from provider_diagnostics import NetworkFailureError, provider_failure, raise_if_provider_unavailable from pydantic import BaseModel, ConfigDict, Field URL = NewType("URL", str) @@ -165,11 +165,13 @@ class StreamingResponse(BaseModel): # quota) arrive as SSE error events inside an otherwise-successful response; # the consumed body is elided, so this is the only place they surface. stream_error: str | None = None + stream_error_code: str | None = None + network_error: NetworkError | None = None stream_done: bool = False @property def ok(self) -> bool: - return 200 <= self.status_code < 300 + return 200 <= self.status_code < 300 and self.network_error is None @property def is_streaming(self) -> bool: @@ -259,9 +261,19 @@ def is_ok[R: BaseModel](result: Result[R]) -> bool: def require_successful_call(result: StreamingResponse, *, expected_provider: Literal["bedrock"] | None = None) -> None: """A call that should have succeeded but didn't is a hard failure, never a skip: if the proxy can't make a call it's expected to, the test must fail.""" + if result.network_error is not None: + raise NetworkFailureError( + provider_failure(result.status_code, "", result.headers), result.network_error.message + ) + raise_if_provider_unavailable( + result.status_code, + result.stream_error or result.body, + result.headers, + expected_provider=expected_provider, + stream_error_code=result.stream_error_code, + ) if result.ok: return - raise_if_provider_unavailable(result.status_code, result.body, result.headers, expected_provider=expected_provider) pytest.fail( f"upstream call failed (status {result.status_code}); headers={result.headers}; body={result.body[:1000]}" ) @@ -611,6 +623,22 @@ def _is_stream_error_line(line: bytes) -> bool: def streaming_outcome( resp: SseResponse, stream: bool, *, sent_at: float, clock: Callable[[], float] = time.monotonic +) -> StreamingResponse: + try: + return _streaming_outcome(resp, stream, sent_at=sent_at, clock=clock) + except requests.RequestException as exc: + return StreamingResponse( + status_code=resp.status_code, + call_id=_hdr(resp, "x-litellm-call-id"), + content_type=_hdr(resp, "content-type"), + headers={name.lower(): value for name, value in resp.headers.items()}, + body=str(exc), + network_error=NetworkError(message=str(exc)), + ) + + +def _streaming_outcome( + resp: SseResponse, stream: bool, *, sent_at: float, clock: Callable[[], float] ) -> StreamingResponse: call_id: Final = _hdr(resp, "x-litellm-call-id") response_cost: Final = _parse_response_cost(resp) @@ -640,6 +668,7 @@ def streaming_outcome( stream_events=[event.payload for event, _ in bedrock_events], stream_event_arrivals=[arrived for _, arrived in bedrock_events], stream_error=next((event.error for event, _ in bedrock_events if event.error is not None), None), + stream_error_code=next((event.error_code for event, _ in bedrock_events if event.error is not None), None), ) stamped: Final = tuple((line, clock() - sent_at) for line in resp.iter_lines() if line) payloads: Final = tuple( @@ -692,7 +721,7 @@ def send( ) ) except requests.RequestException as exc: - return StreamingResponse(status_code=-1, body=str(exc)) + return StreamingResponse(status_code=-1, body=str(exc), network_error=NetworkError(message=str(exc))) return streaming_outcome(resp, stream, sent_at=sent_at) diff --git a/tests/e2e/provider_diagnostics.py b/tests/e2e/provider_diagnostics.py index cec71cf3eed..63035db8413 100644 --- a/tests/e2e/provider_diagnostics.py +++ b/tests/e2e/provider_diagnostics.py @@ -10,7 +10,7 @@ class ProviderFailure(BaseModel): error_code: str | None = None request_id: str | None = None call_id: str | None = None - evidence: Literal["headers", "body"] | None = None + evidence: Literal["headers", "body", "stream_event"] | None = None def junit_properties(self) -> tuple[tuple[str, str], ...]: fields: Final = ( @@ -30,6 +30,12 @@ class ProviderUnavailableError(AssertionError): super().__init__(f"{failure.model_dump_json(exclude_none=True)}; body={body[:1000]}") +class NetworkFailureError(AssertionError): + def __init__(self, failure: ProviderFailure, message: str) -> None: + self.failure: Final = failure + super().__init__(f"kind='network'; {failure.model_dump_json(exclude_none=True)}; body={message[:1000]}") + + class _ErrorMessage(BaseModel): message: str = "" @@ -67,11 +73,25 @@ def provider_failure( headers: Mapping[str, str], *, expected_provider: Literal["bedrock"] | None = None, + stream_error_code: str | None = None, ) -> ProviderFailure: normalized: Final = {key.lower(): value for key, value in headers.items()} request_id: Final = normalized.get("llm_provider-x-amzn-requestid") or normalized.get("x-amzn-requestid") error_code: Final = normalized.get("llm_provider-x-amzn-errortype") or normalized.get("x-amzn-errortype") call_id: Final = normalized.get("x-litellm-call-id") + if ( + 200 <= status_code < 300 + and expected_provider == "bedrock" + and stream_error_code == "serviceUnavailableException" + ): + return ProviderFailure( + status_code=status_code, + provider="bedrock", + error_code="ServiceUnavailableException", + request_id=request_id, + call_id=call_id, + evidence="stream_event", + ) if status_code == 503: bedrock_body: Final = _bedrock_unavailable_body(body, expected_provider) if ( @@ -106,7 +126,10 @@ def raise_if_provider_unavailable( headers: Mapping[str, str], *, expected_provider: Literal["bedrock"] | None = None, + stream_error_code: str | None = None, ) -> None: - failure: Final = provider_failure(status_code, body, headers, expected_provider=expected_provider) + failure: Final = provider_failure( + status_code, body, headers, expected_provider=expected_provider, stream_error_code=stream_error_code + ) if failure.provider is not None: raise ProviderUnavailableError(failure, body) diff --git a/tests/e2e/test_bedrock_eventstream.py b/tests/e2e/test_bedrock_eventstream.py index e6152de215a..34d2cc216e1 100644 --- a/tests/e2e/test_bedrock_eventstream.py +++ b/tests/e2e/test_bedrock_eventstream.py @@ -4,14 +4,17 @@ import struct import zlib from collections.abc import Iterator from dataclasses import dataclass +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from threading import Thread from types import MappingProxyType from typing import Final import pytest from bedrock_eventstream import decode_bedrock_stream from botocore.eventstream import ChecksumMismatch -from e2e_http import StreamingResponse, streaming_outcome +from e2e_http import URL, NoBody, StreamingResponse, require_successful_call, send, streaming_outcome from llm_translation.bedrock_stream import assert_converse_stream, assert_invoke_stream +from provider_diagnostics import NetworkFailureError, ProviderUnavailableError from pydantic import JsonValue, TypeAdapter _JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) @@ -52,7 +55,13 @@ class _BinaryResponse: chunks: tuple[bytes, ...] status_code: int = 200 text: str = "" - headers = MappingProxyType({"content-type": "application/vnd.amazon.eventstream"}) + headers = MappingProxyType( + { + "content-type": "application/vnd.amazon.eventstream", + "x-amzn-requestid": "stream-request", + "x-litellm-call-id": "stream-call", + } + ) def iter_content(self, chunk_size: int | None = 1) -> Iterator[bytes]: return iter(self.chunks) @@ -96,8 +105,20 @@ class TestBedrockEventStream: wire: Final = _frame('{"message":"unavailable"}', "serviceUnavailableException", "exception") result: Final = streaming_outcome(_BinaryResponse((wire,)), True, sent_at=0.0) assert result.stream_error == 'serviceUnavailableException: {"message":"unavailable"}' - with pytest.raises(AssertionError, match="serviceUnavailableException"): + with pytest.raises(ProviderUnavailableError) as caught: assert_converse_stream(result) + assert caught.value.failure.status_code == 200 + assert caught.value.failure.evidence == "stream_event" + assert caught.value.failure.request_id == "stream-request" + assert caught.value.failure.call_id == "stream-call" + + @pytest.mark.parametrize("code", ["validationException", "throttlingException", "internalServerException"]) + def test_other_stream_exceptions_do_not_qualify_for_the_availability_retry(self, code: str) -> None: + wire: Final = _frame('{"message":"provider rejected the request"}', code, "exception") + result: Final = streaming_outcome(_BinaryResponse((wire,)), True, sent_at=0.0) + with pytest.raises(AssertionError, match=code) as caught: + assert_converse_stream(result) + assert not isinstance(caught.value, ProviderUnavailableError) def test_corrupt_binary_frame_is_rejected(self) -> None: wire: Final = _frame('{"role":"assistant"}', "messageStart") @@ -125,3 +146,46 @@ class TestNativeStreamAssertions: def test_converse_rejects_content_after_the_stop_event(self) -> None: with pytest.raises(AssertionError, match="event order"): assert_converse_stream(_stream((_CONVERSE[0], _CONVERSE[3], _CONVERSE[1], _CONVERSE[4]))) + + +class _InterruptedStream(BaseHTTPRequestHandler): + def do_POST(self) -> None: + _ = self.rfile.read(int(self.headers.get("content-length", "0"))) + binary: Final = self.path == "/binary" + data: Final = ( + _frame('{"role":"assistant"}', "messageStart") if binary else b'data: {"text":"Hello"}\n\n' + ) + self.send_response(200) + self.send_header("content-type", "application/vnd.amazon.eventstream" if binary else "text/event-stream") + self.send_header("transfer-encoding", "chunked") + self.send_header("x-amzn-requestid", "interrupted-request") + self.send_header("x-litellm-call-id", "interrupted-call") + self.end_headers() + _ = self.wfile.write(f"{len(data):x}\r\n".encode() + data + b"\r\n") + self.wfile.flush() + self.close_connection = True + + def log_message(self, format: str, *args: object) -> None: + pass + + +@pytest.mark.parametrize("path", ["binary", "sse"]) +def test_interrupted_http_stream_preserves_network_classification_and_ids(path: str) -> None: + server: Final = ThreadingHTTPServer(("127.0.0.1", 0), _InterruptedStream) + worker: Final = Thread(target=server.serve_forever, daemon=True) + worker.start() + try: + result: Final = send( + URL(f"http://127.0.0.1:{server.server_port}/{path}"), headers=NoBody(), json=NoBody(), stream=True, timeout=5 + ) + assert result.status_code == 200 and not result.ok + assert result.network_error is not None and result.network_error.kind == "network" + with pytest.raises(NetworkFailureError, match="kind='network'") as caught: + require_successful_call(result) + assert caught.value.failure.provider is None + assert caught.value.failure.request_id == "interrupted-request" + assert caught.value.failure.call_id == "interrupted-call" + finally: + server.shutdown() + server.server_close() + worker.join(timeout=5) diff --git a/tests/e2e/test_provider_retry.py b/tests/e2e/test_provider_retry.py index 7f2c789261b..a064b6aef22 100644 --- a/tests/e2e/test_provider_retry.py +++ b/tests/e2e/test_provider_retry.py @@ -53,6 +53,29 @@ def test_native(): require_successful_call(StreamingResponse(status_code=503, body="unavailable", headers={ "x-amzn-requestid": "native-request", "x-amzn-errortype": "ServiceUnavailableException", }), expected_provider="bedrock") + +def streamed_unavailable(): + require_successful_call(StreamingResponse( + status_code=200, body="", stream_error="serviceUnavailableException: unavailable", + stream_error_code="serviceUnavailableException", headers={ + "x-amzn-requestid": "stream-request", "x-litellm-call-id": "stream-call", + }, + ), expected_provider="bedrock") + +def test_stream_recovers(): + if attempt("stream_recovers") == 1: + streamed_unavailable() + +def test_stream_persistent(): + attempt("stream_persistent") + streamed_unavailable() + +def test_stream_transport(): + attempt("stream_transport") + require_successful_call(StreamingResponse( + status_code=200, body="connection interrupted", network_error=NetworkError(message="connection interrupted"), + headers={"x-amzn-requestid": "transport-request", "x-litellm-call-id": "transport-call"}, + )) """, encoding="utf-8", ) @@ -84,7 +107,7 @@ def test_native(): check=False, ) assert result.returncode == 1, result.stdout + result.stderr - assert "5 failed, 1 passed, 4 rerun" in result.stdout, result.stdout + assert "7 failed, 2 passed, 7 rerun" in result.stdout, result.stdout assert {path.stem: int(path.read_text()) for path in tmp_path.glob("*.attempts")} == { "recovers": 2, "persistent": 2, @@ -92,6 +115,9 @@ def test_native(): "auth": 1, "network": 2, "native": 2, + "stream_recovers": 2, + "stream_persistent": 2, + "stream_transport": 2, } cases: Final = ET.parse(tmp_path / "results.xml").getroot() recovered: Final = cases.find(".//testcase[@name='test_recovers']") @@ -99,3 +125,13 @@ def test_native(): assert recovered.find("failure") is None assert recovered.find("properties/property[@name='upstream_request_id'][@value='aws-first-attempt']") is not None assert recovered.find("properties/property[@name='upstream_call_id'][@value='proxy-first-attempt']") is not None + streamed: Final = cases.find(".//testcase[@name='test_stream_recovers']") + assert streamed is not None and streamed.find("failure") is None + assert streamed.find("properties/property[@name='upstream_status_code'][@value='200']") is not None + assert streamed.find("properties/property[@name='upstream_evidence'][@value='stream_event']") is not None + assert streamed.find("properties/property[@name='upstream_request_id'][@value='stream-request']") is not None + assert streamed.find("properties/property[@name='upstream_call_id'][@value='stream-call']") is not None + transport: Final = cases.find(".//testcase[@name='test_stream_transport']") + assert transport is not None and transport.find("failure") is not None + assert transport.find("properties/property[@name='upstream_request_id'][@value='transport-request']") is not None + assert transport.find("properties/property[@name='upstream_call_id'][@value='transport-call']") is not None