From c4e7d093be323199bcd3f22c41f18b935837ca36 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sat, 12 Sep 2026 13:02:52 -0700 Subject: [PATCH 1/7] test: strengthen Bedrock e2e diagnostics and replay coverage --- .github/workflows/e2e_record_replay.yml | 64 ++++++++- tests/e2e/CLAUDE.md | 10 +- tests/e2e/CONTRIBUTING.md | 2 +- tests/e2e/bedrock_config.py | 29 ++++ tests/e2e/bedrock_edge.py | 77 +++++++++++ tests/e2e/bedrock_eventstream.py | 47 +++++++ tests/e2e/conftest.py | 5 + tests/e2e/e2e_http.py | 49 +++++-- tests/e2e/llm_translation/bedrock_stream.py | 111 +++++++++++++++ tests/e2e/llm_translation/endpoints_client.py | 4 +- .../test_bedrock_native_e2e.py | 26 ++-- .../test_chat_completions_regression_e2e.py | 23 ++-- .../e2e/llm_translation/test_responses_e2e.py | 25 ++-- tests/e2e/provider_diagnostics.py | 106 +++++++++++++++ tests/e2e/provider_edge.py | 41 +++--- tests/e2e/pytest.ini | 2 +- tests/e2e/test_bedrock_edge.py | 121 +++++++++++++++++ tests/e2e/test_bedrock_eventstream.py | 127 ++++++++++++++++++ tests/e2e/test_e2e_http.py | 4 + tests/e2e/test_provider_diagnostics.py | 83 ++++++++++++ tests/e2e/test_provider_edge.py | 13 ++ tests/e2e/test_provider_retry.py | 99 ++++++++++++++ 22 files changed, 986 insertions(+), 82 deletions(-) create mode 100644 tests/e2e/bedrock_config.py create mode 100644 tests/e2e/bedrock_edge.py create mode 100644 tests/e2e/bedrock_eventstream.py create mode 100644 tests/e2e/llm_translation/bedrock_stream.py create mode 100644 tests/e2e/provider_diagnostics.py create mode 100644 tests/e2e/test_bedrock_edge.py create mode 100644 tests/e2e/test_bedrock_eventstream.py create mode 100644 tests/e2e/test_provider_diagnostics.py create mode 100644 tests/e2e/test_provider_retry.py diff --git a/.github/workflows/e2e_record_replay.yml b/.github/workflows/e2e_record_replay.yml index ca52f0b4d81..2136676fd01 100644 --- a/.github/workflows/e2e_record_replay.yml +++ b/.github/workflows/e2e_record_replay.yml @@ -26,6 +26,10 @@ jobs: (github.event_name == 'workflow_dispatch' && github.event.inputs.mode == 'record')) runs-on: ubuntu-latest timeout-minutes: 45 + environment: e2e-changed + permissions: + contents: read + id-token: write services: postgres: image: postgres:16.6 @@ -44,8 +48,7 @@ jobs: DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm LITELLM_MASTER_KEY: sk-e2e-record-replay LITELLM_LOCAL_MODEL_COST_MAP: "True" - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + E2E_BEDROCK_REGION: us-east-1 steps: - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: @@ -54,7 +57,7 @@ jobs: - name: Set up Python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: - python-version: "3.12" + python-version: "3.13" - name: Set up uv uses: ./.github/actions/setup-uv-with-retries @@ -66,7 +69,7 @@ jobs: - name: Install dependencies run: | - .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy + .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --group e2e-dev --extra proxy --extra proxy-runtime - name: Cache Prisma binaries uses: ./.github/actions/cache-prisma-binaries @@ -75,9 +78,47 @@ jobs: run: | uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma + - name: Validate recording configuration + env: + ROLE: ${{ vars.E2E_AWS_ROLE_TO_ASSUME }} + run: test -n "${ROLE}" || { echo "::error::Set E2E_AWS_ROLE_TO_ASSUME to the existing e2e secrets reader role"; exit 1; } + + - name: Configure the existing e2e secrets reader + id: aws + uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0 + with: + role-to-assume: ${{ vars.E2E_AWS_ROLE_TO_ASSUME }} + aws-region: us-east-1 + role-session-name: litellm-e2e-record-${{ github.run_id }} + role-duration-seconds: 900 + output-env-credentials: false + output-credentials: true + + - name: Fetch provider credentials from AWS Secrets Manager + env: + AWS_ACCESS_KEY_ID: ${{ steps.aws.outputs.aws-access-key-id }} + AWS_SECRET_ACCESS_KEY: ${{ steps.aws.outputs.aws-secret-access-key }} + AWS_SESSION_TOKEN: ${{ steps.aws.outputs.aws-session-token }} + AWS_DEFAULT_REGION: us-east-1 + run: | + umask 077 + aws secretsmanager get-secret-value --secret-id litellm-e2e-changed-provider-keys \ + --query SecretString --output text \ + | uv run --no-sync python .github/e2e-stack/secrets_to_env.py tests/e2e/.env + + - name: Require all recording credentials before starting the proxy + run: | + uv run --no-sync --env-file tests/e2e/.env python - <<'PY' + import os + required = ("OPENAI_API_KEY", "ANTHROPIC_API_KEY", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY") + missing = [key for key in required if not os.environ.get(key)] + if missing: + raise SystemExit("Missing recording credentials: " + ", ".join(missing)) + PY + - name: Start the proxy run: | - nohup uv run --no-sync litellm --config tests/e2e/gateway/record_replay_ci_config.yml --port 4000 > proxy.log 2>&1 & + nohup uv run --no-sync --env-file tests/e2e/.env litellm --config tests/e2e/gateway/record_replay_ci_config.yml --port 4000 > proxy.log 2>&1 & for _ in $(seq 1 90); do if curl -fs http://localhost:4000/health/liveliness > /dev/null; then exit 0 @@ -144,6 +185,11 @@ jobs: GH_TOKEN: ${{ github.token }} OPENAI_API_KEY: sk-replay-must-never-reach-a-provider ANTHROPIC_API_KEY: sk-ant-replay-must-never-reach-a-provider + AWS_ACCESS_KEY_ID: e2e-bedrock-replay + AWS_SECRET_ACCESS_KEY: e2e-bedrock-replay-secret + AWS_REGION: us-east-1 + E2E_BEDROCK_REGION: us-east-1 + AWS_EC2_METADATA_DISABLED: "true" steps: - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 with: @@ -152,7 +198,7 @@ jobs: - name: Set up Python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: - python-version: "3.12" + python-version: "3.13" - name: Set up uv uses: ./.github/actions/setup-uv-with-retries @@ -164,7 +210,7 @@ jobs: - name: Install dependencies run: | - .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy + .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --group e2e-dev --extra proxy --extra proxy-runtime - name: Cache Prisma binaries uses: ./.github/actions/cache-prisma-binaries @@ -202,6 +248,10 @@ jobs: sudo python3 .github/scripts/e2e_egress_sentinel.py serve \ --host api.openai.com \ --host api.anthropic.com \ + --host bedrock-runtime.us-east-1.amazonaws.com \ + --host bedrock-runtime.us-east-2.amazonaws.com \ + --host bedrock-runtime.us-west-1.amazonaws.com \ + --host bedrock-runtime.us-west-2.amazonaws.com \ --hits-file "${RUNNER_TEMP}/egress-hits.jsonl" \ --ready-file "${RUNNER_TEMP}/egress-ready" \ --pid-file "${RUNNER_TEMP}/egress.pid" \ diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index a58c13d6a1c..cf4793f1810 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -77,7 +77,7 @@ Mark live tests with `@pytest.mark.e2e` (on the class or the module). Pure cover The seam is `provider_edge.py`: `start_provider_edge` boots an in-process HTTP server (one shared instance per pytest process, `e2e_config.provider_edge_base` is the accessor) that mounts each supported provider under a path prefix (`EDGE_MOUNTS`: `/openai` -> `https://api.openai.com`, `/anthropic` -> `https://api.anthropic.com`). A test participates by registering its deployment with `api_base=provider_edge_base("openai")` plus the provider's path suffix; `quota_management/spend_tracking/test_provider_edge_spend_e2e.py` is the reference. In live mode the accessor returns None and the deployment defaults to the real provider, so an edge-wired test runs in all three modes unchanged. Non-wired tests hit their providers live in every mode. The edge binds `E2E_PROVIDER_EDGE_BIND_HOST` (default 127.0.0.1) and advertises `E2E_PROVIDER_EDGE_ADVERTISE_HOST` in the api_base it hands out, for proxies running in containers -A bundle (default `tests/e2e/.fixtures`, override with `E2E_FIXTURE_DIR`) is a directory: `manifest.json` carries the record timestamp, harness git version, and format version, and each test gets a subdirectory holding one JSON file per provider call in call order (`0000-post-openai-v1-chat-completions.json`). Request headers are never stored (provider credentials never touch disk), non-JSON request bodies store a canonicalized sha256 digest instead of the bytes, `multipart/form-data` bodies store their ordinary fields plus a JSON list of the uploaded parts' `[field, filename, content-type]` triples and a digest of their content, so the per-request random boundary and the envelope never reach the key, and responses store status, filtered headers, and the verbatim body base64-encoded, which is part of why bundles are gitignored. Responses come in two shapes told apart by a `kind` tag: an ordinary one holding a single base64 body, and, for a response the provider streamed (`content-type: text/event-stream`), one holding its transfer chunks in order plus why the stream ended early if it did, so replay reproduces the split points the provider chose instead of one coalesced body. `fixture_bundle.py` owns the format, and `BUNDLE_FORMAT_VERSION` is checked on load, so a bundle recorded under older rules is refused by name rather than partially read. Record serves the proxy the same filtered stored response replay will serve later, chunk for chunk on a stream, so the two modes are byte-identical from the proxy's side of the socket. Replay does not reproduce the provider's inter-chunk timing (chunks go out as fast as the socket takes them), so a test that judges streaming on the clock, such as the `stream_event_arrivals` lead between the first content delta and `message_stop`, gates that assertion on `provider_paces_stream()` and proves only the event grammar in replay +A bundle (default `tests/e2e/.fixtures`, override with `E2E_FIXTURE_DIR`) is a directory: `manifest.json` carries the record timestamp, harness git version, and format version, and each test gets a subdirectory holding one JSON file per provider call in call order (`0000-post-openai-v1-chat-completions.json`). Request headers are never stored (provider credentials never touch disk), non-JSON request bodies store a canonicalized sha256 digest instead of the bytes, `multipart/form-data` bodies store their ordinary fields plus a JSON list of the uploaded parts' `[field, filename, content-type]` triples and a digest of their content, so the per-request random boundary and the envelope never reach the key, and responses store status, filtered headers, and the verbatim body base64-encoded, which is part of why bundles are gitignored. Responses come in two shapes told apart by a `kind` tag: an ordinary one holding a single base64 body, and, for a response the provider streamed (`content-type: text/event-stream` or `application/vnd.amazon.eventstream`), one holding its transfer chunks in order plus why the stream ended early if it did, so replay reproduces the split points the provider chose instead of one coalesced body. `fixture_bundle.py` owns the format, and `BUNDLE_FORMAT_VERSION` is checked on load, so a bundle recorded under older rules is refused by name rather than partially read. Record serves the proxy the same filtered stored response replay will serve later, chunk for chunk on a stream, so the two modes are byte-identical from the proxy's side of the socket. Replay does not reproduce the provider's inter-chunk timing (chunks go out as fast as the socket takes them), so a test that judges streaming on the clock, such as the `stream_event_arrivals` lead between the first content delta and `message_stop`, gates that assertion on `provider_paces_stream()` and proves only the event grammar in replay Multipart identity is the fiddly corner, and the rules exist because each one had a collision behind it. A part counts as an upload when it carries a filename or declares its own content type, and everything else is an ordinary field. Field names get a `name[n]` suffix on repeats, with a literal `[` doubled first, so a form that repeats `purpose` never keys the same as one that literally sends `purpose[1]`. A field whose name reads as a credential is stored as ``, which stays key-preserving because the key is recomputed from the stored request rather than saved alongside it, so the live request carrying the real value still matches its redacted fixture. A field value that is not UTF-8 is stored as a base64 sha256 digest, base64 and not hex because the canonicalizer rewrites any 64-character hex run to `` and would fold every binary value onto one key. The uploaded parts contribute a JSON list rather than a `field:filename` string, so a separator inside a filename cannot impersonate a field boundary, and their byte length is stored for a reader's benefit but deliberately left out of the key, since the canonicalizer absorbs timestamp and id drift inside a file that changes its length @@ -96,7 +96,13 @@ E2E_FIXTURE_MODE=replay E2E_FIXTURE_DIR=/tmp/e2e-fixtures E2E_RESET_SPEND_LOGS=1 Point the proxy at bogus provider credentials for the replay run and it still has to pass: that is the whole proof that nothing left the process. Bundles are never committed. `tests/e2e/.fixtures` is gitignored because a bundle holds verbatim provider response bodies and hard-fails after seven days. CI records and replays this lane on a schedule in `.github/workflows/e2e_record_replay.yml`, publishing the bundle as a private `e2e-fixtures-bundle` artifact instead of committing it, selecting the tests with the `@pytest.mark.replayable` marker, and proving the bogus-credentials replay hermetic by counting provider egress with `.github/scripts/e2e_egress_sentinel.py` -Current limits: Bedrock cannot be mounted (SigV4 signs the Host header, so a rewritten api_base fails signature verification), deployments baked into the proxy's config file cannot be edge-wired (only `/model/new` registrations can carry the edge api_base), and a file upload routed by `custom_llm_provider` through the proxy's `files_settings` block never passes a deployment at all, so the batches `model_param` and `provider_fallback` scenarios keep uploading live in every mode +Bedrock tests use `bedrock_config.bedrock_params()` for Chat Completions, Responses, Converse and Invoke. Live mode retains the normal AWS credential path. Record and replay use a `/bedrock-us-east-1` mount by default (`E2E_BEDROCK_REGION` selects one of `us-east-1`, `us-east-2`, `us-west-1`, `us-west-2`; use the same region in both runs). The proxy signs each edge request with public test credentials. The edge verifies that signature over the host, path and body before accepting it. Recording re-signs the unchanged payload for the AWS endpoint using the runner's `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and optional `AWS_SESSION_TOKEN`. Replay requires no real AWS credentials and never forwards a miss. This checks request signing and protocol behavior in replay; live runs remain necessary to check real IAM access and model availability. Binary AWS EventStream responses preserve transfer chunks in the bundle, and native stream assertions decode events with botocore and require text, ordered completion and usage + +The recording workflow uses the existing `e2e-changed` environment and OIDC secrets reader (`E2E_AWS_ROLE_TO_ASSUME`, secret `litellm-e2e-changed-provider-keys`), and fails before starting the proxy if any required provider credential is missing. Replay supplies only test credentials and includes the four supported Bedrock endpoints in the egress sentinel + +HTTP assertions classify a Bedrock 503 as `ProviderUnavailableError` only when AWS headers or a recognized Bedrock error envelope support that attribution. The default live lane allows one test rerun after ten seconds for that error or a transport failure. Generic proxy 5xx, authentication errors and assertion failures do not qualify. Persistent provider failures remain failed, and JUnit properties retain AWS and proxy request IDs from the failed attempt even when the rerun passes. Record and replay continue to use `--reruns 0` + +Current limits: deployments baked into the proxy's config file cannot be edge-wired (only `/model/new` registrations can carry the edge api_base), and a file upload routed by `custom_llm_provider` through the proxy's `files_settings` block never passes a deployment at all, so the batches `model_param` and `provider_fallback` scenarios keep uploading live in every mode ## Typing diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index 44564a51e26..e77d0489c73 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -130,7 +130,7 @@ One sharp edge: a replayed response reuses the recorded provider response id, an Another sharp edge, same root: record and replay derive every per-test token deterministically (the model name included, so a replay regenerates the exact requests the record run sent), which means an edge-wired deployment left in the database by an interrupted earlier run carries the same model name as the fresh one the current run registers. The proxy then holds two deployments under one model group and load-balances across both, and because the leftover's `api_base` points at the earlier run's edge process, which is gone, the calls that land on it fail with a connection error that reads like a transport bug rather than the stale row it is. Give each record or replay run a fresh database, or let a run finish so its own teardown deletes what it registered, and never reuse one long-lived proxy across back-to-back record/replay sessions. CI hands every job its own empty database and its own proxy, so it never sees this -Replay answers any provider call that drifted from the recording with an HTTP 599 whose body names the computed and closest recorded keys, so the test fails loudly instead of silently going live, and a bundle older than seven days fails at collection time naming its age; either way the fix is to re-record. Only tests that register edge-wired deployments participate: everything else hits its provider live in every mode, so record exactly the suite you replay. If the proxy runs in a container, set `E2E_PROVIDER_EDGE_ADVERTISE_HOST` (e.g. `host.docker.internal`) so the api_base the proxy stores can reach the edge on the pytest host, and `E2E_PROVIDER_EDGE_BIND_HOST=0.0.0.0` so the edge accepts it. The suites wired to the edge today are `quota_management/spend_tracking/test_provider_edge_spend_e2e.py`, `llm_translation/test_chat_completions_contract_e2e.py`, the OpenAI registrations in `llm_translation/test_embeddings_endpoint_e2e.py`, the Anthropic tests in `llm_translation/test_messages_e2e.py`, streamed and not, and the OpenAI batch deployment behind `batches/`. A streamed response replays as the chunk sequence the provider sent rather than one buffered body. See `CLAUDE.md` in this directory for the bundle format, the edge design, and the current limits (Bedrock). The scheduled CI record/replay lane is described above +Replay answers any provider call that drifted from the recording with an HTTP 599 whose body names the computed and closest recorded keys, so the test fails loudly instead of silently going live, and a bundle older than seven days fails at collection time naming its age; either way the fix is to re-record. Only tests that register edge-wired deployments participate: everything else hits its provider live in every mode, so record exactly the suite you replay. If the proxy runs in a container, set `E2E_PROVIDER_EDGE_ADVERTISE_HOST` (e.g. `host.docker.internal`) so the api_base the proxy stores can reach the edge on the pytest host, and `E2E_PROVIDER_EDGE_BIND_HOST=0.0.0.0` so the edge accepts it. The suites wired to the edge today are `quota_management/spend_tracking/test_provider_edge_spend_e2e.py`, `llm_translation/test_chat_completions_contract_e2e.py`, the OpenAI registrations in `llm_translation/test_embeddings_endpoint_e2e.py`, the Anthropic tests in `llm_translation/test_messages_e2e.py`, streamed and not, the OpenAI batch deployment behind `batches/`, and the Bedrock Chat Completions, Responses and native Converse/Invoke tests. Bedrock recording requires AWS credentials in the runner; replay validates signatures made with public test credentials. Use the same `E2E_BEDROCK_REGION` in both modes (default `us-east-1`). A streamed response replays as the chunk sequence the provider sent rather than one buffered body. See `CLAUDE.md` in this directory for the bundle format, the edge design, and the remaining limits. The scheduled CI record/replay lane is described above Tests marked `@pytest.mark.e2e` hard-fail when no proxy answers `/health/liveliness`, so a run that goes red with `No live proxy` at setup means the proxy isn't up; they never skip for a missing proxy, so an absent proxy can't be mistaken for a pass diff --git a/tests/e2e/bedrock_config.py b/tests/e2e/bedrock_config.py new file mode 100644 index 00000000000..4a00651133d --- /dev/null +++ b/tests/e2e/bedrock_config.py @@ -0,0 +1,29 @@ +import os +from typing import Final + +from bedrock_edge import TEST_ACCESS_KEY, TEST_SECRET_KEY, recording_credentials +from e2e_config import FIXTURE_MODE_RAW, provider_edge_base +from fixture_mode import parse_fixture_mode +from models import LiteLLMParamsBody + +BEDROCK_BACKEND: Final = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" + + +def bedrock_params() -> LiteLLMParamsBody: + if parse_fixture_mode(FIXTURE_MODE_RAW) == "live": + return LiteLLMParamsBody( + model=BEDROCK_BACKEND, + aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID", + aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY", + aws_region_name="os.environ/AWS_REGION", + ) + if parse_fixture_mode(FIXTURE_MODE_RAW) == "record": + recording_credentials() + region: Final = os.environ.get("E2E_BEDROCK_REGION", "us-east-1") + return LiteLLMParamsBody( + model=BEDROCK_BACKEND, + api_base=provider_edge_base(f"bedrock-{region}"), + aws_access_key_id=TEST_ACCESS_KEY, + aws_secret_access_key=TEST_SECRET_KEY, + aws_region_name=region, + ) diff --git a/tests/e2e/bedrock_edge.py b/tests/e2e/bedrock_edge.py new file mode 100644 index 00000000000..8f47b68d220 --- /dev/null +++ b/tests/e2e/bedrock_edge.py @@ -0,0 +1,77 @@ +import hashlib +import hmac +import os +import re +from collections.abc import Mapping +from typing import Final + +from botocore.auth import SigV4Auth +from botocore.awsrequest import AWSRequest +from botocore.credentials import Credentials + +TEST_ACCESS_KEY: Final = "e2e-bedrock-replay" +TEST_SECRET_KEY: Final = "e2e-bedrock-replay-secret" +_MOUNT: Final = re.compile(r"bedrock-(us-(?:east|west)-[12])") +_AUTH: Final = re.compile( + r"AWS4-HMAC-SHA256 Credential=([^/]+)/([0-9]{8})/([^/]+)/bedrock/aws4_request, " + r"SignedHeaders=([a-z0-9;-]+), Signature=([0-9a-f]{64})" +) + + +def bedrock_region(mount: str) -> str | None: + match: Final = _MOUNT.fullmatch(mount) + return match.group(1) if match else None + + +def bedrock_upstream(mount: str) -> str | None: + region: Final = bedrock_region(mount) + return f"https://bedrock-runtime.{region}.amazonaws.com" if region else None + + +def valid_bedrock_signature( + method: str, raw_path: str, headers: Mapping[str, str], body: bytes | None, region: str +) -> bool: + normalized: Final = {key.lower(): value for key, value in headers.items()} + auth: Final = _AUTH.fullmatch(normalized.get("authorization", "")) + if auth is None or auth.group(1) != TEST_ACCESS_KEY or auth.group(3) != region: + return False + timestamp: Final = normalized.get("x-amz-date", "") + signed: Final = frozenset(auth.group(4).split(";")) + if not {"host", "x-amz-date"} <= signed or not signed <= normalized.keys() or auth.group(2) != timestamp[:8]: + return False + payload_hash: Final = normalized.get("x-amz-content-sha256") + if payload_hash is not None and payload_hash != hashlib.sha256(body or b"").hexdigest(): + return False + request: Final = AWSRequest( + method=method, + url=f"http://{normalized['host']}{raw_path}", + data=body, + headers={key: normalized[key] for key in signed}, + ) + request.context["timestamp"] = timestamp + signer: Final = SigV4Auth(Credentials(TEST_ACCESS_KEY, TEST_SECRET_KEY), "bedrock", region) + canonical: Final = signer.canonical_request(request) + expected: Final = signer.signature(signer.string_to_sign(request, canonical), request) + return hmac.compare_digest(expected, auth.group(5)) + + +def recording_credentials() -> Credentials: + access_key: Final = os.environ.get("AWS_ACCESS_KEY_ID", "") + secret_key: Final = os.environ.get("AWS_SECRET_ACCESS_KEY", "") + if not access_key or not secret_key: + raise ValueError("Bedrock recording requires AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in the runner") + return Credentials(access_key, secret_key, os.environ.get("AWS_SESSION_TOKEN") or None) + + +def sign_bedrock_forward( + method: str, url: str, headers: Mapping[str, str], body: bytes | None, region: str, credentials: Credentials +) -> dict[str, str]: + forwarded: Final = { + key: value + for key, value in headers.items() + if key.lower() not in {"authorization", "host", "content-length", "connection", "accept-encoding"} + and not key.lower().startswith("x-amz-") + } + request: Final = AWSRequest(method=method, url=url, data=body, headers=forwarded) + SigV4Auth(credentials, "bedrock", region).add_auth(request) + return dict(request.headers.items()) diff --git a/tests/e2e/bedrock_eventstream.py b/tests/e2e/bedrock_eventstream.py new file mode 100644 index 00000000000..a980d994413 --- /dev/null +++ b/tests/e2e/bedrock_eventstream.py @@ -0,0 +1,47 @@ +import base64 +from collections.abc import Iterable, Iterator +from dataclasses import dataclass +from typing import Final + +from botocore.eventstream import EventStreamBuffer, EventStreamMessage +from pydantic import BaseModel, Field, JsonValue, TypeAdapter + + +class _EventHeaders(BaseModel): + message_type: str = Field(alias=":message-type") + event_type: str = Field(default="", alias=":event-type") + exception_type: str = Field(default="", alias=":exception-type") + + +class _InvokeChunk(BaseModel): + bytes: str + + +@dataclass(frozen=True, slots=True) +class BedrockEvent: + payload: str + error: str | None = None + + +_JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) + + +def _decode_event(event: EventStreamMessage) -> BedrockEvent: + headers: Final = _EventHeaders.model_validate(event.headers) + if headers.message_type != "event": + return BedrockEvent( + payload=event.payload.decode(), + error=f"{headers.exception_type or headers.message_type}: {event.payload.decode()}", + ) + if headers.event_type == "chunk": + encoded: Final = _InvokeChunk.model_validate_json(event.payload) + return BedrockEvent(payload=base64.b64decode(encoded.bytes, validate=True).decode()) + return BedrockEvent(payload=_JSON.dump_json({headers.event_type: _JSON.validate_json(event.payload)}).decode()) + + +def decode_bedrock_stream(chunks: Iterable[bytes]) -> Iterator[BedrockEvent]: + buffer: Final = EventStreamBuffer() + for chunk in chunks: + buffer.add_data(chunk) + for event in buffer: + yield _decode_event(event) diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 36569896125..9ec2bf7ef1a 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -29,6 +29,7 @@ from idp import Identity, Keycloak, keycloak_from_env from junit_properties import attach_result_properties from lifecycle import ProxyClientProvider, ResourceManager from models import TeamNewBody, UserNewBody, UserNewResponse +from provider_diagnostics import ProviderUnavailableError from provider_edge import replay_leftover_error from proxy_client import ProxyClient, build_proxy_client @@ -182,6 +183,10 @@ def pytest_runtest_makereport( report = yield if report.when == "call": item.stash[_CALL_PASSED] = report.passed + if call.excinfo is not None and isinstance(call.excinfo.value, ProviderUnavailableError): + failure: Final = call.excinfo.value.failure + item.user_properties.extend(failure.junit_properties()) + report.user_properties = list(item.user_properties) return report diff --git a/tests/e2e/e2e_http.py b/tests/e2e/e2e_http.py index ce069720c6e..f2eb4fecd09 100644 --- a/tests/e2e/e2e_http.py +++ b/tests/e2e/e2e_http.py @@ -22,6 +22,8 @@ from typing import Final, Generator, Generic, Iterator, Literal, NewType, Protoc import pytest import requests +from bedrock_eventstream import decode_bedrock_stream +from provider_diagnostics import raise_if_provider_unavailable from pydantic import BaseModel, ConfigDict, Field URL = NewType("URL", str) @@ -106,15 +108,11 @@ class UnknownApiError(BaseModel): kind: Literal["unknown"] = "unknown" status_code: int body: str + headers: dict[str, str] = {} type Result[R: BaseModel] = ( - Success[R] - | NetworkError - | UnauthorizedError - | RateLimitedError - | ValidationError - | UnknownApiError + Success[R] | NetworkError | UnauthorizedError | RateLimitedError | ValidationError | UnknownApiError ) @@ -216,6 +214,8 @@ class SseResponse(Protocol): def iter_lines(self) -> Iterator[bytes]: ... + def iter_content(self, chunk_size: int | None = 1) -> Iterator[bytes]: ... + def _hdr(resp: SseResponse, name: str) -> str | None: value = resp.headers.get(name) @@ -226,6 +226,9 @@ def unwrap[R: BaseModel](result: Result[R]) -> R: match result: case Success(data=data): return data + case UnknownApiError(status_code=status, body=body, headers=headers): + raise_if_provider_unavailable(status, body, headers) + raise AssertionError(result) case _: raise AssertionError(result) @@ -238,6 +241,9 @@ def unwrap_status[R: BaseModel](result: Result[R], expected_status: int) -> R: return data case Success(status_code=status_code): raise AssertionError(f"expected HTTP {expected_status}, got {status_code}") + case UnknownApiError(status_code=status, body=body, headers=headers): + raise_if_provider_unavailable(status, body, headers) + raise AssertionError(result) case _: raise AssertionError(result) @@ -250,13 +256,14 @@ def is_ok[R: BaseModel](result: Result[R]) -> bool: return False -def require_successful_call(result: StreamingResponse) -> None: +def require_successful_call(result: StreamingResponse, *, expected_provider: Literal["bedrock"] | None = None) -> None: """A call that should have succeeded but didn't is a hard failure, never a skip: if the proxy can't make a call it's expected to, the test must fail.""" if result.ok: return + raise_if_provider_unavailable(result.status_code, result.body, result.headers, expected_provider=expected_provider) pytest.fail( - f"upstream call failed (status {result.status_code}); body={result.body[:300]}" + f"upstream call failed (status {result.status_code}); headers={result.headers}; body={result.body[:1000]}" ) @@ -271,6 +278,7 @@ def assert_auth_denied(result: StreamingResponse, context: str) -> None: f"{context}: expected 401/403, got {result.status_code}: {result.body[:300]}" ) + def wire_body(json: BaseModel) -> dict[str, object]: if isinstance(json, PartialBody): return json.model_dump(by_alias=True, exclude_unset=True) @@ -349,6 +357,9 @@ class ClassifiableResponse(Protocol): @property def content(self) -> bytes: ... + @property + def headers(self) -> Mapping[str, str]: ... + def json(self) -> object: ... @@ -358,7 +369,11 @@ def classify[R: BaseModel](resp: ClassifiableResponse, response_type: type[R]) - if resp.status_code == 429: return RateLimitedError(body=resp.text) if not resp.ok: - return UnknownApiError(status_code=resp.status_code, body=resp.text) + return UnknownApiError( + status_code=resp.status_code, + body=resp.text, + headers={key.lower(): value for key, value in resp.headers.items()}, + ) try: payload: Final[object] = resp.json() if resp.content else {} return Success(status_code=resp.status_code, data=response_type.model_validate(payload)) @@ -610,6 +625,22 @@ def streaming_outcome( headers=headers, body=resp.text, ) + if "application/vnd.amazon.eventstream" in (content_type or "").lower(): + bedrock_events: Final = tuple( + (event, clock() - sent_at) for event in decode_bedrock_stream(resp.iter_content(chunk_size=None)) + ) + return StreamingResponse( + status_code=resp.status_code, + call_id=call_id, + response_cost=response_cost, + content_type=content_type, + headers=headers, + body="", + chunks=len(bedrock_events), + stream_events=[event.payload for event, _ in bedrock_events], + stream_event_arrivals=[arrived for _, arrived in bedrock_events], + stream_error=next((event.error for event, _ in bedrock_events if event.error is not None), None), + ) stamped: Final = tuple((line, clock() - sent_at) for line in resp.iter_lines() if line) payloads: Final = tuple( (line.removeprefix(_SSE_DATA_PREFIX).decode(errors="replace"), arrived) diff --git a/tests/e2e/llm_translation/bedrock_stream.py b/tests/e2e/llm_translation/bedrock_stream.py new file mode 100644 index 00000000000..67512ae293d --- /dev/null +++ b/tests/e2e/llm_translation/bedrock_stream.py @@ -0,0 +1,111 @@ +from typing import Final + +from e2e_http import StreamingResponse, require_successful_call +from pydantic import BaseModel + + +class _TextDelta(BaseModel): + text: str = "" + stop_reason: str | None = None + + +class _ContentBlockDelta(BaseModel): + delta: _TextDelta + + +class _MessageStart(BaseModel): + role: str + + +class _MessageStop(BaseModel): + stopReason: str + + +class _ConverseUsage(BaseModel): + inputTokens: int + outputTokens: int + totalTokens: int + + +class _Metadata(BaseModel): + usage: _ConverseUsage + + +class _ConverseEvent(BaseModel): + messageStart: _MessageStart | None = None + contentBlockDelta: _ContentBlockDelta | None = None + messageStop: _MessageStop | None = None + metadata: _Metadata | None = None + + +class _InvokeUsage(BaseModel): + input_tokens: int = 0 + output_tokens: int = 0 + + +class _InvokeMessage(BaseModel): + role: str + usage: _InvokeUsage + + +class _InvokeEvent(BaseModel): + type: str + message: _InvokeMessage | None = None + delta: _TextDelta | None = None + usage: _InvokeUsage | None = None + + +def _require_native_stream(result: StreamingResponse) -> None: + require_successful_call(result, expected_provider="bedrock") + assert "application/vnd.amazon.eventstream" in (result.content_type or "").lower(), result.content_type + assert result.stream_error is None, result.stream_error + assert result.stream_events, "Bedrock stream returned no decoded events" + + +def assert_converse_stream(result: StreamingResponse) -> None: + _require_native_stream(result) + events: Final = tuple(_ConverseEvent.model_validate_json(event) for event in result.stream_events) + starts: Final = tuple(i for i, event in enumerate(events) if event.messageStart is not None) + deltas: Final = tuple(i for i, event in enumerate(events) if event.contentBlockDelta is not None) + stops: Final = tuple(i for i, event in enumerate(events) if event.messageStop is not None) + metadata: Final = tuple(i for i, event in enumerate(events) if event.metadata is not None) + assert len(starts) == len(stops) == len(metadata) == 1, "Converse requires messageStart, messageStop and metadata" + assert deltas, "Converse stream returned no content deltas" + assert starts[0] < deltas[0] <= deltas[-1] < stops[0] < metadata[0] == len(events) - 1, "Converse event order" + start: Final = events[starts[0]].messageStart + stop: Final = events[stops[0]].messageStop + tail: Final = events[metadata[0]].metadata + assert start is not None and start.role == "assistant", "Converse stream has no assistant" + text: Final = "".join(event.contentBlockDelta.delta.text for event in events if event.contentBlockDelta is not None) + assert text.strip(), "Converse stream returned empty text" + assert stop is not None and stop.stopReason in {"end_turn", "max_tokens"}, ( + "Converse stream has no valid stop reason" + ) + assert tail is not None and tail.usage.inputTokens > 0 and tail.usage.outputTokens > 0, "Converse usage is missing" + assert tail.usage.totalTokens == tail.usage.inputTokens + tail.usage.outputTokens, "Converse usage does not add up" + + +def assert_invoke_stream(result: StreamingResponse) -> None: + _require_native_stream(result) + events: Final = tuple(_InvokeEvent.model_validate_json(event) for event in result.stream_events) + starts: Final = tuple(i for i, event in enumerate(events) if event.type == "message_start") + deltas: Final = tuple(i for i, event in enumerate(events) if event.type == "content_block_delta") + stops: Final = tuple(i for i, event in enumerate(events) if event.type == "message_stop") + finishes: Final = tuple( + i + for i, event in enumerate(events) + if event.type == "message_delta" and event.delta is not None and event.delta.stop_reason is not None + ) + assert len(starts) == len(stops) == len(finishes) == 1, "Invoke requires message_start, finish and message_stop" + assert deltas, "Invoke stream returned no content deltas" + assert starts[0] < deltas[0] <= deltas[-1] < finishes[0] < stops[0] == len(events) - 1, "Invoke event order" + message: Final = events[starts[0]].message + assert message is not None and message.role == "assistant", "Invoke stream has no assistant" + assert message.usage.input_tokens > 0, "Invoke input usage is missing" + text: Final = "".join( + event.delta.text for event in events if event.type == "content_block_delta" and event.delta is not None + ) + assert text.strip(), "Invoke stream returned empty text" + finish: Final = events[finishes[0]] + assert finish.delta is not None and finish.delta.stop_reason in {"end_turn", "max_tokens"}, "Invoke stop reason" + assert finish.usage is not None and finish.usage.output_tokens > 0, "Invoke output usage is missing" diff --git a/tests/e2e/llm_translation/endpoints_client.py b/tests/e2e/llm_translation/endpoints_client.py index 4d2c73e7078..d900b1a6edf 100644 --- a/tests/e2e/llm_translation/endpoints_client.py +++ b/tests/e2e/llm_translation/endpoints_client.py @@ -74,6 +74,7 @@ class ResponsesRequest(BaseModel): instructions: str | None = None stream: bool = False tools: list[ResponsesFunctionTool] | None = None + tool_choice: str | None = None guardrails: list[str] | None = None cache: dict[str, bool] | None = {"no-cache": True} @@ -351,7 +352,7 @@ class EndpointsClient: ) def responses_with_tools( - self, key: str, model: str, text: str, tools: list[ResponsesFunctionTool] + self, key: str, model: str, text: str, tools: list[ResponsesFunctionTool], *, tool_choice: str | None = None ) -> StreamingResponse: return self._send( "/v1/responses", @@ -361,6 +362,7 @@ class EndpointsClient: input=text, instructions="You are a helpful assistant", tools=tools, + tool_choice=tool_choice, ), ) diff --git a/tests/e2e/llm_translation/test_bedrock_native_e2e.py b/tests/e2e/llm_translation/test_bedrock_native_e2e.py index 19c1be7b6db..a193bc71927 100644 --- a/tests/e2e/llm_translation/test_bedrock_native_e2e.py +++ b/tests/e2e/llm_translation/test_bedrock_native_e2e.py @@ -7,19 +7,18 @@ missing messages and invalid model handling without crashing the proxy. from __future__ import annotations import pytest +from bedrock_config import bedrock_params +from bedrock_stream import assert_converse_stream, assert_invoke_stream from e2e_config import unique_marker from e2e_http import ( assert_client_error, require_successful_call, ) from lifecycle import ResourceManager -from models import LiteLLMParamsBody from proxy_client import ProxyClient from pydantic import BaseModel -pytestmark = pytest.mark.e2e - -BEDROCK_BACKEND = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" +pytestmark = [pytest.mark.e2e, pytest.mark.replayable] class ConverseContent(BaseModel): @@ -71,12 +70,7 @@ def _register(proxy: ProxyClient, resources: ResourceManager) -> tuple[str, str] model = f"e2e-bedrock-native-{unique_marker()}" model_id = proxy.create_model( model, - LiteLLMParamsBody( - model=BEDROCK_BACKEND, - aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID", - aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY", - aws_region_name="os.environ/AWS_REGION", - ), + bedrock_params(), ) resources.defer(lambda: proxy.delete_model(model_id)) return model, resources.key() @@ -107,7 +101,7 @@ class TestBedrockNative: headers=proxy.transport.bearer(key), json=_default_converse(), ) - require_successful_call(result) + require_successful_call(result, expected_provider="bedrock") response = ConverseResponse.model_validate_json(result.body) assert response.output.message.role == "assistant" assert any(part.text.strip() for part in response.output.message.content) @@ -121,9 +115,7 @@ class TestBedrockNative: json=_default_converse(), stream=True, ) - require_successful_call(result) - assert result.stream_error is None, result.stream_error - assert result.chunks > 0, "converse-stream returned no events" + assert_converse_stream(result) @pytest.mark.covers("llm.bedrock_native.bedrock_invoke.basic.nonstream.works") def test_invoke_returns_message(self, proxy: ProxyClient, resources: ResourceManager) -> None: @@ -133,7 +125,7 @@ class TestBedrockNative: headers=proxy.transport.bearer(key), json=_default_invoke(), ) - require_successful_call(result) + require_successful_call(result, expected_provider="bedrock") response = InvokeResponse.model_validate_json(result.body) assert any(part.text.strip() for part in response.content) @@ -146,9 +138,7 @@ class TestBedrockNative: json=_default_invoke(), stream=True, ) - require_successful_call(result) - assert result.stream_error is None, result.stream_error - assert result.chunks > 0, "invoke stream returned no events" + assert_invoke_stream(result) @pytest.mark.covers("llm.bedrock_native.bedrock_converse.input_validation.nonstream.works") def test_converse_missing_messages_returns_error(self, proxy: ProxyClient, resources: ResourceManager) -> None: diff --git a/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py b/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py index 87bd32d8dab..f5662348751 100644 --- a/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py +++ b/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py @@ -22,10 +22,11 @@ from pathlib import Path from typing import Final import pytest +from bedrock_config import bedrock_params from pydantic import BaseModel from e2e_config import unique_marker -from e2e_http import StreamingResponse, unwrap +from e2e_http import StreamingResponse, require_successful_call, unwrap from lifecycle import ResourceManager from models import ( ChatBody, @@ -47,7 +48,6 @@ COHERE_BACKEND = "cohere/command-r-08-2024" GEMINI_BACKEND = "gemini/gemini-2.5-flash" OPENAI_BACKEND = "openai/gpt-5.6" ANTHROPIC_BACKEND = "anthropic/claude-haiku-4-5-20251001" -BEDROCK_CONVERSE_BACKEND = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" class _StreamToolCallFunction(BaseModel): @@ -66,6 +66,7 @@ class _StreamDelta(BaseModel): class _StreamChoice(BaseModel): delta: _StreamDelta = _StreamDelta() + finish_reason: str | None = None class _StreamChunk(BaseModel): @@ -124,21 +125,16 @@ def _streamed_text(events: list[str]) -> str: def _assert_streamed_completion(result: StreamingResponse) -> None: """A streamed /chat/completions must deliver real content, not a clean-but-empty stream (the #28991 class on the streaming path).""" - assert result.ok and result.is_streaming, f"stream was not established: {result}" + require_successful_call(result) + assert result.is_streaming, f"stream was not established: {result}" assert result.stream_error is None, f"stream carried an error event: {result.stream_error}" assert len(result.stream_events) > 1, f"stream did not deliver multiple data events: {result}" assert _streamed_text(result.stream_events).strip(), ( f"stream completed with no content deltas: {result.stream_events[:3]}" ) - - -def _bedrock_params() -> LiteLLMParamsBody: - return LiteLLMParamsBody( - model=BEDROCK_CONVERSE_BACKEND, - aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID", - aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY", - aws_region_name="os.environ/AWS_REGION", - ) + assert result.stream_done, "chat stream ended without [DONE]" + chunks: Final = tuple(_StreamChunk.model_validate_json(event) for event in result.stream_events) + assert any(choice.finish_reason for chunk in chunks for choice in chunk.choices), "chat stream has no finish reason" class _WeatherArgs(BaseModel): @@ -621,6 +617,7 @@ class TestOpenAIChatCompletions: assert args.location.strip(), f"streamed tool call arguments missing location: {arguments!r}" +@pytest.mark.replayable class TestBedrockConverseChatCompletions: """Bedrock Converse via /chat/completions, the customer's AWS stack. A non-OpenAI provider must return real content on both the non-streamed and streamed paths. @@ -628,7 +625,7 @@ class TestBedrockConverseChatCompletions: def _register(self, client: PassthroughClient, resources: ResourceManager, prefix: str) -> str: model = f"{prefix}-{unique_marker()}" - model_id = client.proxy.create_model(model, _bedrock_params()) + model_id = client.proxy.create_model(model, bedrock_params()) resources.defer(lambda: client.proxy.delete_model(model_id)) return model diff --git a/tests/e2e/llm_translation/test_responses_e2e.py b/tests/e2e/llm_translation/test_responses_e2e.py index 3fcf2d1ac05..dd40e816e14 100644 --- a/tests/e2e/llm_translation/test_responses_e2e.py +++ b/tests/e2e/llm_translation/test_responses_e2e.py @@ -11,6 +11,7 @@ import json from typing import cast import pytest +from bedrock_config import bedrock_params from e2e_config import unique_marker from e2e_http import ( assert_client_error, @@ -38,8 +39,6 @@ class _OptionalResponsesBody(BaseModel): max_output_tokens: int | None = None -BEDROCK_CONVERSE_BACKEND = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" - WEATHER_TOOL = ResponsesFunctionTool( name="get_weather", description="Get the weather for a location", @@ -50,15 +49,6 @@ WEATHER_TOOL = ResponsesFunctionTool( ) -def _bedrock_params() -> LiteLLMParamsBody: - return LiteLLMParamsBody( - model=BEDROCK_CONVERSE_BACKEND, - aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID", - aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY", - aws_region_name="os.environ/AWS_REGION", - ) - - class WeatherArguments(BaseModel): location: str @@ -260,12 +250,13 @@ class TestResponses: arguments = WeatherArguments.model_validate(raw_arguments) assert arguments.location, f"function call arguments missing location: {function_call.arguments}" + @pytest.mark.replayable @pytest.mark.covers("llm.responses.bedrock_converse.basic.nonstream.works") def test_responses_bedrock_returns_completion( self, endpoints_client: EndpointsClient, resources: ResourceManager ) -> None: model = f"e2e-responses-{unique_marker()}" - model_id = endpoints_client.create_model(model, _bedrock_params()) + model_id = endpoints_client.create_model(model, bedrock_params()) resources.defer(lambda: endpoints_client.delete_model(model_id)) key = resources.key() @@ -274,17 +265,22 @@ class TestResponses: parsed = ResponsesResult.model_validate_json(result.body) assert parsed.text.strip(), f"/responses over bedrock returned no output text: {result.body[:300]}" + @pytest.mark.replayable @pytest.mark.covers("llm.responses.bedrock_converse.tool_use.nonstream.works") def test_responses_bedrock_returns_function_call( self, endpoints_client: EndpointsClient, resources: ResourceManager ) -> None: model = f"e2e-responses-{unique_marker()}" - model_id = endpoints_client.create_model(model, _bedrock_params()) + model_id = endpoints_client.create_model(model, bedrock_params()) resources.defer(lambda: endpoints_client.delete_model(model_id)) key = resources.key() result = endpoints_client.responses_with_tools( - key, model, "What is the weather in San Francisco? Use the get_weather tool.", [WEATHER_TOOL] + key, + model, + "What is the weather in San Francisco? Use the get_weather tool.", + [WEATHER_TOOL], + tool_choice="required", ) require_successful_call(result) parsed = ResponsesResult.model_validate_json(result.body) @@ -344,6 +340,7 @@ class TestResponses: ) assert_client_error(result, "responses empty input") + def _parse_stream_event( event: str, ) -> ResponsesOutputTextDeltaEvent | None: diff --git a/tests/e2e/provider_diagnostics.py b/tests/e2e/provider_diagnostics.py new file mode 100644 index 00000000000..65d72d393a8 --- /dev/null +++ b/tests/e2e/provider_diagnostics.py @@ -0,0 +1,106 @@ +from collections.abc import Mapping +from typing import Final, Literal + +from pydantic import BaseModel, ValidationError + + +class ProviderFailure(BaseModel): + status_code: int + provider: Literal["bedrock"] | None = None + error_code: str | None = None + request_id: str | None = None + call_id: str | None = None + evidence: Literal["headers", "body"] | None = None + + def junit_properties(self) -> tuple[tuple[str, str], ...]: + fields: Final = ( + ("status_code", self.status_code), + ("provider", self.provider), + ("error_code", self.error_code), + ("request_id", self.request_id), + ("call_id", self.call_id), + ("evidence", self.evidence), + ) + return tuple((f"upstream_{key}", str(value)) for key, value in fields if value is not None) + + +class ProviderUnavailableError(AssertionError): + def __init__(self, failure: ProviderFailure, body: str) -> None: + self.failure: Final = failure + super().__init__(f"{failure.model_dump_json(exclude_none=True)}; body={body[:1000]}") + + +class _ErrorMessage(BaseModel): + message: str = "" + + +class _NativeError(BaseModel): + error: str = "" + + +class _ErrorEnvelope(BaseModel): + error: _ErrorMessage | None = None + detail: _NativeError | None = None + + +_BEDROCK_UNAVAILABLE: Final = "Bedrock is unable to process your request." + + +def _bedrock_unavailable_body(body: str, expected_provider: Literal["bedrock"] | None) -> bool: + try: + envelope: Final = _ErrorEnvelope.model_validate_json(body) + if envelope.error is not None: + message: Final = envelope.error.message + return message.startswith("litellm.ServiceUnavailableError: BedrockException - ") and ( + _BEDROCK_UNAVAILABLE in message + ) + if expected_provider == "bedrock" and envelope.detail is not None: + return _ErrorMessage.model_validate_json(envelope.detail.error).message == _BEDROCK_UNAVAILABLE + except ValidationError: + return False + return False + + +def provider_failure( + status_code: int, + body: str, + headers: Mapping[str, str], + *, + expected_provider: Literal["bedrock"] | None = None, +) -> ProviderFailure: + normalized: Final = {key.lower(): value for key, value in headers.items()} + request_id: Final = normalized.get("llm_provider-x-amzn-requestid") or normalized.get("x-amzn-requestid") + error_code: Final = normalized.get("llm_provider-x-amzn-errortype") or normalized.get("x-amzn-errortype") + call_id: Final = normalized.get("x-litellm-call-id") + if status_code == 503: + if request_id and error_code and error_code.split(":", 1)[0] == "ServiceUnavailableException": + return ProviderFailure( + status_code=status_code, + provider="bedrock", + error_code="ServiceUnavailableException", + request_id=request_id, + call_id=call_id, + evidence="headers", + ) + if _bedrock_unavailable_body(body, expected_provider): + return ProviderFailure( + status_code=status_code, + provider="bedrock", + error_code="ServiceUnavailableException", + request_id=request_id, + call_id=call_id, + evidence="body", + ) + return ProviderFailure(status_code=status_code, request_id=request_id, call_id=call_id, error_code=error_code) + + +def raise_if_provider_unavailable( + status_code: int, + body: str, + headers: Mapping[str, str], + *, + expected_provider: Literal["bedrock"] | None = None, +) -> None: + failure: Final = provider_failure(status_code, body, headers, expected_provider=expected_provider) + if failure.provider is not None: + raise ProviderUnavailableError(failure, body) diff --git a/tests/e2e/provider_edge.py b/tests/e2e/provider_edge.py index 6c87c7ef7ac..6be42074948 100644 --- a/tests/e2e/provider_edge.py +++ b/tests/e2e/provider_edge.py @@ -55,6 +55,13 @@ from types import MappingProxyType from typing import Final, Literal, assert_never from urllib.parse import parse_qsl, urlsplit +from bedrock_edge import ( + bedrock_region, + bedrock_upstream, + recording_credentials, + sign_bedrock_forward, + valid_bedrock_signature, +) from e2e_http import ( NetworkError, StreamChunk, @@ -631,7 +638,8 @@ def _is_streamed(headers: Mapping[str, str]) -> bool: move nearly every recording to the streamed shape for no gain. The content type is the header that says "consume this as it arrives", and it is already how the harness defines streaming everywhere else.""" - return "text/event-stream" in _header_value(headers, "content-type").lower() + content_type: Final = _header_value(headers, "content-type").lower() + return any(kind in content_type for kind in ("text/event-stream", "application/vnd.amazon.eventstream")) def _upstream_url(upstream_base: str, upstream_path: str, query: str) -> str: @@ -783,26 +791,29 @@ def handle_edge_request( Socket-free so unit tests exercise every branch without a server.""" split: Final = urlsplit(raw_path) mount, _, upstream_path = split.path.lstrip("/").partition("/") - upstream_base: Final = mounts.get(mount) + upstream_base: Final = mounts.get(mount) or bedrock_upstream(mount) if upstream_base is None: - return _text_reply( - 404, f"unknown provider mount {mount!r}; known mounts: {', '.join(sorted(mounts))}" - ) - request: Final = edge_request( - method, split.path, split.query, body, _header_value(headers, "content-type") + return _text_reply(404, f"unknown provider mount {mount!r}; known mounts: {', '.join(sorted(mounts))}") + request: Final = edge_request(method, split.path, split.query, body, _header_value(headers, "content-type")) + region: Final = bedrock_region(mount) + if region is not None and not valid_bedrock_signature(method, raw_path, headers, body, region): + return _text_reply(403, "Bedrock provider edge rejected the test SigV4 signature") + url: Final = _upstream_url(upstream_base, upstream_path, split.query) + forwarded_headers: Final = ( + sign_bedrock_forward(method, url, headers, body, region, recording_credentials()) + if region is not None and not isinstance(backend, ReplayEdge) + else headers ) match backend: case LiveEdge(): - return _handle_live( - method, _upstream_url(upstream_base, upstream_path, split.query), headers, body, timeout - ) + return _handle_live(method, url, forwarded_headers, body, timeout) case RecordEdge(): return _handle_record( backend, request, method=method, - url=_upstream_url(upstream_base, upstream_path, split.query), - headers=headers, + url=url, + headers=forwarded_headers, body=body, timeout=timeout, ) @@ -1020,10 +1031,8 @@ def provider_edge_api_base( case "live": return None case "record" | "replay": - if mount not in EDGE_MOUNTS: - raise ValueError( - f"unknown provider mount {mount!r}; known mounts: {', '.join(sorted(EDGE_MOUNTS))}" - ) + if mount not in EDGE_MOUNTS and bedrock_upstream(mount) is None: + raise ValueError(f"unknown provider mount {mount!r}; known mounts: {', '.join(sorted(EDGE_MOUNTS))}") return _shared_edge(mode, bundle_dir, bind_host, advertise_host, forward_timeout).api_base(mount) case _: assert_never(mode) diff --git a/tests/e2e/pytest.ini b/tests/e2e/pytest.ini index 774d9644497..d488f531660 100644 --- a/tests/e2e/pytest.ini +++ b/tests/e2e/pytest.ini @@ -2,7 +2,7 @@ # Config when any e2e suite under tests/e2e/ is run directly, e.g. # uv run pytest tests/e2e/quota_management/spend_tracking/ -v # The e2e marker is also registered in conftest.py for runs rooted elsewhere. -addopts = --strict-markers --strict-config --reruns 1 --only-rerun "kind='network'" --only-rerun "status_code=5[0-9][0-9]" +addopts = --strict-markers --strict-config --reruns 1 --reruns-delay 10 --only-rerun "kind='network'" --only-rerun "ProviderUnavailableError" markers = e2e: live test that requires a running proxy and real provider keys replayable: edge-wired test whose provider traffic replays from a fixture bundle, so it makes zero provider calls in replay mode; the record/replay CI lane selects it with -m replayable diff --git a/tests/e2e/test_bedrock_edge.py b/tests/e2e/test_bedrock_edge.py new file mode 100644 index 00000000000..5d61b7c11ea --- /dev/null +++ b/tests/e2e/test_bedrock_edge.py @@ -0,0 +1,121 @@ +import base64 +import threading +from pathlib import Path +from typing import Final + +import pytest +from bedrock_edge import ( + TEST_ACCESS_KEY, + TEST_SECRET_KEY, + bedrock_upstream, + sign_bedrock_forward, + valid_bedrock_signature, +) +from botocore.auth import SigV4Auth +from botocore.awsrequest import AWSRequest +from botocore.credentials import Credentials +from fixture_bundle import BundleRecorder, LoadedBundle, RecordedHttpResponse, load_bundle, prepare_bundle +from fixture_mode import current_test_key +from provider_edge import EdgeReply, RecordEdge, ReplayEdge, ReplaySource, edge_request, handle_edge_request + +_PATH: Final = "/bedrock-us-east-1/model/us.anthropic.claude-haiku-4-5-20251001-v1:0/converse" +_BODY: Final = b'{"messages":[{"role":"user","content":[{"text":"Hello"}]}]}' + + +def _signed_headers() -> dict[str, str]: + request: Final = AWSRequest( + method="POST", + url=f"http://127.0.0.1:1234{_PATH}", + data=_BODY, + headers={"content-type": "application/json", "host": "127.0.0.1:1234"}, + ) + SigV4Auth(Credentials(TEST_ACCESS_KEY, TEST_SECRET_KEY), "bedrock", "us-east-1").add_auth(request) + return {key.lower(): value for key, value in request.headers.items()} + + +class TestBedrockSigningBoundary: + def test_accepts_a_valid_signature_from_the_proxy(self) -> None: + assert valid_bedrock_signature("POST", _PATH, _signed_headers(), _BODY, "us-east-1") + + @pytest.mark.parametrize("change", ["body", "path", "region", "host", "authorization"]) + def test_rejects_changes_to_signed_request_inputs(self, change: str) -> None: + headers: Final = _signed_headers() + assert not valid_bedrock_signature( + "POST", + _PATH + "/tampered" if change == "path" else _PATH, + {**headers, change: "tampered"} if change in {"host", "authorization"} else headers, + b"{}" if change == "body" else _BODY, + "us-west-2" if change == "region" else "us-east-1", + ) + + def test_forward_is_resigned_for_aws_and_preserves_the_body_and_feature_headers(self) -> None: + url: Final = "https://bedrock-runtime.us-east-1.amazonaws.com/model/example/converse" + credentials: Final = Credentials("recording-access", "recording-secret", "recording-session") + incoming: Final = {**_signed_headers(), "anthropic-beta": "feature-flag"} + forwarded: Final = sign_bedrock_forward("POST", url, incoming, _BODY, "us-east-1", credentials) + assert "Credential=recording-access/" in forwarded["Authorization"] + assert TEST_ACCESS_KEY not in forwarded["Authorization"] + assert forwarded["X-Amz-Security-Token"] == "recording-session" + assert forwarded["anthropic-beta"] == "feature-flag" + assert "127.0.0.1" not in str(forwarded) + + @pytest.mark.parametrize( + "mount", ["bedrock-localhost", "bedrock-us-east-1.evil.test", "bedrock-us-east-1/anything"] + ) + def test_mount_cannot_select_an_arbitrary_upstream_host(self, mount: str) -> None: + assert bedrock_upstream(mount) is None + + def test_bad_signature_cannot_reach_aws_or_write_a_recording(self, tmp_path: Path) -> None: + recorder: Final = prepare_bundle(tmp_path / "bundle") + assert isinstance(recorder, BundleRecorder) + result: Final = handle_edge_request( + RecordEdge(recorder=recorder, lock=threading.Lock()), + {}, + "POST", + _PATH, + _signed_headers(), + b"tampered", + timeout=0.1, + ) + assert isinstance(result, EdgeReply) and result.status_code == 403 + bundle: Final = load_bundle(tmp_path / "bundle") + assert isinstance(bundle, LoadedBundle) and not bundle.interactions + + def test_replay_validates_signature_before_consuming_the_recording(self, tmp_path: Path) -> None: + root: Final = tmp_path / "bundle" + recorder: Final = prepare_bundle(root) + assert isinstance(recorder, BundleRecorder) + recorder.record( + test_key=current_test_key(), + request=edge_request("POST", _PATH, "", _BODY, "application/json"), + response=RecordedHttpResponse( + status_code=200, + headers={"content-type": "application/json"}, + body_b64=base64.b64encode(b'{"output":{"message":{"role":"assistant"}}}').decode(), + ), + ) + loaded: Final = load_bundle(root) + assert isinstance(loaded, LoadedBundle) + source: Final = ReplaySource(loaded) + rejected: Final = handle_edge_request( + ReplayEdge(source), + {}, + "POST", + _PATH, + {}, + _BODY, + timeout=0.1, + ) + assert isinstance(rejected, EdgeReply) and rejected.status_code == 403 + result: Final = handle_edge_request( + ReplayEdge(source), + {}, + "POST", + _PATH, + _signed_headers(), + _BODY, + timeout=0.1, + ) + assert isinstance(result, EdgeReply) and result.status_code == 200 + assert result.body == b'{"output":{"message":{"role":"assistant"}}}' + assert source.leftover_error(current_test_key()) is None diff --git a/tests/e2e/test_bedrock_eventstream.py b/tests/e2e/test_bedrock_eventstream.py new file mode 100644 index 00000000000..e6152de215a --- /dev/null +++ b/tests/e2e/test_bedrock_eventstream.py @@ -0,0 +1,127 @@ +import base64 +import json +import struct +import zlib +from collections.abc import Iterator +from dataclasses import dataclass +from types import MappingProxyType +from typing import Final + +import pytest +from bedrock_eventstream import decode_bedrock_stream +from botocore.eventstream import ChecksumMismatch +from e2e_http import StreamingResponse, streaming_outcome +from llm_translation.bedrock_stream import assert_converse_stream, assert_invoke_stream +from pydantic import JsonValue, TypeAdapter + +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) + + +def _frame(payload: str, event_type: str, message_type: str = "event") -> bytes: + headers: Final = { + ":message-type": message_type, + ":event-type" if message_type == "event" else ":exception-type": event_type, + ":content-type": "application/json", + } + encoded_headers: Final = b"".join( + bytes([len(key)]) + key.encode() + b"\x07" + struct.pack("!H", len(value)) + value.encode() + for key, value in headers.items() + ) + prelude: Final = struct.pack("!II", 16 + len(encoded_headers) + len(payload.encode()), len(encoded_headers)) + message: Final = prelude + struct.pack("!I", zlib.crc32(prelude)) + encoded_headers + payload.encode() + return message + struct.pack("!I", zlib.crc32(message)) + + +_CONVERSE: Final = ( + '{"messageStart":{"role":"assistant"}}', + '{"contentBlockDelta":{"delta":{"text":"Hello"},"contentBlockIndex":0}}', + '{"contentBlockStop":{"contentBlockIndex":0}}', + '{"messageStop":{"stopReason":"end_turn"}}', + '{"metadata":{"usage":{"inputTokens":10,"outputTokens":1,"totalTokens":11}}}', +) +_INVOKE: Final = ( + '{"type":"message_start","message":{"role":"assistant","usage":{"input_tokens":10,"output_tokens":0}}}', + '{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hello"}}', + '{"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":1}}', + '{"type":"message_stop"}', +) + + +@dataclass(frozen=True, slots=True) +class _BinaryResponse: + chunks: tuple[bytes, ...] + status_code: int = 200 + text: str = "" + headers = MappingProxyType({"content-type": "application/vnd.amazon.eventstream"}) + + def iter_content(self, chunk_size: int | None = 1) -> Iterator[bytes]: + return iter(self.chunks) + + def iter_lines(self) -> Iterator[bytes]: + raise AssertionError("binary Bedrock streams must not be consumed as text lines") + + +def _stream(events: tuple[str, ...]) -> StreamingResponse: + return StreamingResponse( + status_code=200, + content_type="application/vnd.amazon.eventstream", + body="", + stream_events=list(events), + chunks=len(events), + ) + + +class TestBedrockEventStream: + @pytest.mark.parametrize("size", [1, 7, 4096]) + def test_converse_binary_frames_survive_arbitrary_transport_splits(self, size: int) -> None: + wire: Final = b"".join( + _frame(json.dumps(value), key) + for payload in _CONVERSE + for key, value in _JSON_OBJECT.validate_json(payload).items() + ) + chunks: Final = tuple(wire[i : i + size] for i in range(0, len(wire), size)) + result: Final = streaming_outcome(_BinaryResponse(chunks), True, sent_at=0.0, clock=lambda: 1.0) + assert result.chunks == len(_CONVERSE) + assert_converse_stream(result) + + def test_invoke_payload_is_decoded_from_its_base64_envelope(self) -> None: + wire: Final = b"".join( + _frame(json.dumps({"bytes": base64.b64encode(event.encode()).decode()}), "chunk") for event in _INVOKE + ) + result: Final = streaming_outcome(_BinaryResponse((wire,)), True, sent_at=0.0) + assert result.stream_events == list(_INVOKE) + assert_invoke_stream(result) + + def test_binary_provider_exception_is_not_counted_as_a_successful_stream(self) -> None: + wire: Final = _frame('{"message":"unavailable"}', "serviceUnavailableException", "exception") + result: Final = streaming_outcome(_BinaryResponse((wire,)), True, sent_at=0.0) + assert result.stream_error == 'serviceUnavailableException: {"message":"unavailable"}' + with pytest.raises(AssertionError, match="serviceUnavailableException"): + assert_converse_stream(result) + + def test_corrupt_binary_frame_is_rejected(self) -> None: + wire: Final = _frame('{"role":"assistant"}', "messageStart") + with pytest.raises(ChecksumMismatch): + tuple(decode_bedrock_stream((wire[:-1] + bytes([wire[-1] ^ 1]),))) + + +class TestNativeStreamAssertions: + @pytest.mark.parametrize("missing", [0, 1, 3, 4]) + def test_converse_rejects_missing_content_or_completion(self, missing: int) -> None: + with pytest.raises(AssertionError): + assert_converse_stream(_stream(tuple(event for i, event in enumerate(_CONVERSE) if i != missing))) + + @pytest.mark.parametrize("missing", [0, 1, 2, 3]) + def test_invoke_rejects_missing_content_or_completion(self, missing: int) -> None: + with pytest.raises(AssertionError): + assert_invoke_stream(_stream(tuple(event for i, event in enumerate(_INVOKE) if i != missing))) + + @pytest.mark.parametrize("events", [_CONVERSE, _INVOKE]) + def test_empty_text_does_not_pass_because_events_arrived(self, events: tuple[str, ...]) -> None: + assertion: Final = assert_converse_stream if events == _CONVERSE else assert_invoke_stream + with pytest.raises(AssertionError, match="empty text"): + assertion(_stream(tuple(event.replace("Hello", " ") for event in events))) + + def test_converse_rejects_content_after_the_stop_event(self) -> None: + with pytest.raises(AssertionError, match="event order"): + assert_converse_stream(_stream((_CONVERSE[0], _CONVERSE[3], _CONVERSE[1], _CONVERSE[4]))) diff --git a/tests/e2e/test_e2e_http.py b/tests/e2e/test_e2e_http.py index 81cd6c8d3d1..fa744db66e2 100644 --- a/tests/e2e/test_e2e_http.py +++ b/tests/e2e/test_e2e_http.py @@ -105,6 +105,9 @@ class FakeSseResponse: def iter_lines(self) -> Iterator[bytes]: return iter(self.lines) + def iter_content(self, chunk_size: int | None = 1) -> Iterator[bytes]: + return iter(self.lines) + def _ticking_clock(start: float, step: float) -> Callable[[], float]: ticks: Final = iter(range(10_000)) @@ -182,6 +185,7 @@ class FakeJsonResponse: status_code: int content: bytes + headers: Mapping[str, str] = MappingProxyType({}) @property def ok(self) -> bool: diff --git a/tests/e2e/test_provider_diagnostics.py b/tests/e2e/test_provider_diagnostics.py new file mode 100644 index 00000000000..e57888e4347 --- /dev/null +++ b/tests/e2e/test_provider_diagnostics.py @@ -0,0 +1,83 @@ +import json +from types import MappingProxyType +from typing import Final + +import pytest +from e2e_http import StreamingResponse, UnknownApiError, require_successful_call, unwrap, unwrap_status +from provider_diagnostics import ProviderUnavailableError, provider_failure + +_BODY: Final = json.dumps( + { + "error": { + "message": ( + 'litellm.ServiceUnavailableError: BedrockException - {"message":"Bedrock is unable to process your request."}' + ) + } + } +) +_HEADERS: Final = MappingProxyType( + { + "LLM_Provider-X-Amzn-RequestId": "aws-request-1", + "LLM_Provider-X-Amzn-ErrorType": "ServiceUnavailableException:http://internal.amazon.com/", + "X-Litellm-Call-Id": "proxy-call-1", + } +) + + +def _assert_success(helper: str) -> None: + match helper: + case "unwrap": + unwrap(UnknownApiError(status_code=503, body=_BODY, headers=dict(_HEADERS))) + case "unwrap_status": + unwrap_status(UnknownApiError(status_code=503, body=_BODY, headers=dict(_HEADERS)), 200) + case _: + require_successful_call(StreamingResponse(status_code=503, body=_BODY, headers=dict(_HEADERS))) + + +class TestProviderFailure: + def test_aws_headers_preserve_attribution_and_both_request_ids(self) -> None: + result: Final = provider_failure(503, "unavailable", _HEADERS) + assert result.provider == "bedrock" + assert result.error_code == "ServiceUnavailableException" + assert result.request_id == "aws-request-1" + assert result.call_id == "proxy-call-1" + assert result.evidence == "headers" + + def test_proxy_bedrock_error_body_remains_identifiable_without_headers(self) -> None: + result: Final = provider_failure(503, _BODY, {}) + assert result.provider == "bedrock" and result.evidence == "body" + assert result.request_id is None + + def test_native_error_requires_explicit_bedrock_route_context(self) -> None: + body: Final = json.dumps( + {"detail": {"error": json.dumps({"message": "Bedrock is unable to process your request."})}} + ) + assert provider_failure(503, body, {}).provider is None + result: Final = StreamingResponse(status_code=503, body=body) + with pytest.raises(ProviderUnavailableError) as caught: + require_successful_call(result, expected_provider="bedrock") + assert caught.value.failure.evidence == "body" + + @pytest.mark.parametrize( + "status,body", + [ + (503, '{"error":{"message":"proxy overloaded"}}'), + (503, "Bedrock is unable to process your request."), + (503, '{"error":{"message":"Bedrock is unable to process your request."}}'), + (503, '{"detail":{"error":"not JSON"}}'), + (429, _BODY), + (403, _BODY), + (500, _BODY), + ], + ) + def test_ambiguous_and_non_availability_errors_are_not_retryable(self, status: int, body: str) -> None: + assert provider_failure(status, body, {}, expected_provider="bedrock").provider is None + with pytest.raises(AssertionError) as caught: + unwrap(UnknownApiError(status_code=status, body=body)) + assert not isinstance(caught.value, ProviderUnavailableError) + + @pytest.mark.parametrize("helper", ["unwrap", "unwrap_status", "raw"]) + def test_all_assertion_paths_emit_the_same_provider_failure(self, helper: str) -> None: + with pytest.raises(ProviderUnavailableError) as caught: + _assert_success(helper) + assert caught.value.failure == provider_failure(503, _BODY, _HEADERS) diff --git a/tests/e2e/test_provider_edge.py b/tests/e2e/test_provider_edge.py index 18f72ac0e7a..771ba1b3b0f 100644 --- a/tests/e2e/test_provider_edge.py +++ b/tests/e2e/test_provider_edge.py @@ -996,6 +996,19 @@ class TestStreamingFidelity: provider actually sent, not as one coalesced body. The unit of fidelity is the HTTP transfer chunk, so every assertion here is made at the transfer layer.""" + def test_binary_eventstream_record_and_replay_preserve_transfer_chunks(self, tmp_path: Path) -> None: + root: Final = tmp_path / "bundle" + binary_chunks: Final = (b"\x00\x00\x01\xff:message-", b"type\x00\xfe\r\n", b"\x00\xff\x00\x80") + with chunked_provider(chunks=binary_chunks, content_type="application/vnd.amazon.eventstream") as provider: + with running_edge(record_backend(root), {"openai": provider_url(provider)}) as edge: + recorded_head, recorded_chunks, recorded_ending = raw_stream_post(edge.port, STREAM_PATH, STREAM_BODY) + replayed_head, replayed_chunks, replayed_ending = replay_stream(root) + assert response_header(recorded_head, "content-type") == "application/vnd.amazon.eventstream" + assert response_header(replayed_head, "content-type") == "application/vnd.amazon.eventstream" + assert recorded_chunks == replayed_chunks == list(binary_chunks) + assert stream_chunks(recorded_stream(root)) == list(binary_chunks) + assert recorded_ending == replayed_ending == "terminated" + def test_a_streamed_response_records_its_chunk_boundaries(self, tmp_path: Path) -> None: root = tmp_path / "bundle" record_stream(root) diff --git a/tests/e2e/test_provider_retry.py b/tests/e2e/test_provider_retry.py new file mode 100644 index 00000000000..4536a5248c2 --- /dev/null +++ b/tests/e2e/test_provider_retry.py @@ -0,0 +1,99 @@ +import os +import subprocess +import sys +import xml.etree.ElementTree as ET +from pathlib import Path +from typing import Final + + +def test_retry_policy_preserves_failures_and_first_attempt_evidence(tmp_path: Path) -> None: + suite: Final = Path(__file__).resolve().parent + scenario: Final = tmp_path / "test_scenarios.py" + scenario.write_text( + """ +from pathlib import Path +from e2e_http import NetworkError, StreamingResponse, UnknownApiError, require_successful_call, unwrap + +def attempt(name): + path = Path(__file__).with_name(name + ".attempts") + count = int(path.read_text()) + 1 if path.exists() else 1 + path.write_text(str(count)) + return count + +def unavailable(): + unwrap(UnknownApiError(status_code=503, body="unavailable", headers={ + "x-amzn-requestid": "aws-first-attempt", "x-amzn-errortype": "ServiceUnavailableException", + "x-litellm-call-id": "proxy-first-attempt", + })) + +def test_recovers(): + if attempt("recovers") == 1: + unavailable() + +def test_persistent(): + attempt("persistent") + unavailable() + +def test_proxy_503(): + attempt("proxy_503") + unwrap(UnknownApiError(status_code=503, body="proxy overloaded")) + +def test_auth(): + attempt("auth") + unwrap(UnknownApiError(status_code=403, body="invalid credentials")) + +def test_network(): + attempt("network") + unwrap(NetworkError(message="connection reset")) + +def test_native(): + attempt("native") + require_successful_call(StreamingResponse(status_code=503, body="unavailable", headers={ + "x-amzn-requestid": "native-request", "x-amzn-errortype": "ServiceUnavailableException", + }), expected_provider="bedrock") +""", + encoding="utf-8", + ) + result: Final = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + str(scenario), + "-c", + str(suite / "pytest.ini"), + "--rootdir", + str(tmp_path), + "--confcutdir", + str(tmp_path), + "-p", + "conftest", + "--reruns-delay", + "0", + "--junitxml", + str(tmp_path / "results.xml"), + "-q", + ], + cwd=tmp_path, + env={**os.environ, "PYTHONPATH": str(suite), "PYTEST_ADDOPTS": "", "E2E_FIXTURE_MODE": "live"}, + capture_output=True, + text=True, + timeout=60, + check=False, + ) + assert result.returncode == 1, result.stdout + result.stderr + assert "5 failed, 1 passed, 4 rerun" in result.stdout, result.stdout + assert {path.stem: int(path.read_text()) for path in tmp_path.glob("*.attempts")} == { + "recovers": 2, + "persistent": 2, + "proxy_503": 1, + "auth": 1, + "network": 2, + "native": 2, + } + cases: Final = ET.parse(tmp_path / "results.xml").getroot() + recovered: Final = cases.find(".//testcase[@name='test_recovers']") + assert recovered is not None + assert recovered.find("failure") is None + assert recovered.find("properties/property[@name='upstream_request_id'][@value='aws-first-attempt']") is not None + assert recovered.find("properties/property[@name='upstream_call_id'][@value='proxy-first-attempt']") is not None From c373dd9ae761ccc613257a368f48b68a41d1dc4b Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sat, 12 Sep 2026 13:11:49 -0700 Subject: [PATCH 2/7] test: distinguish Bedrock validation and availability failures --- tests/e2e/CLAUDE.md | 4 ++-- .../llm_translation/test_bedrock_native_e2e.py | 17 +++++++++++------ tests/e2e/provider_diagnostics.py | 10 ++++++++-- tests/e2e/test_provider_diagnostics.py | 7 ++++++- tests/e2e/test_provider_retry.py | 4 +++- 5 files changed, 30 insertions(+), 12 deletions(-) diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index cf4793f1810..98c3a45b3b1 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -98,9 +98,9 @@ Point the proxy at bogus provider credentials for the replay run and it still ha Bedrock tests use `bedrock_config.bedrock_params()` for Chat Completions, Responses, Converse and Invoke. Live mode retains the normal AWS credential path. Record and replay use a `/bedrock-us-east-1` mount by default (`E2E_BEDROCK_REGION` selects one of `us-east-1`, `us-east-2`, `us-west-1`, `us-west-2`; use the same region in both runs). The proxy signs each edge request with public test credentials. The edge verifies that signature over the host, path and body before accepting it. Recording re-signs the unchanged payload for the AWS endpoint using the runner's `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and optional `AWS_SESSION_TOKEN`. Replay requires no real AWS credentials and never forwards a miss. This checks request signing and protocol behavior in replay; live runs remain necessary to check real IAM access and model availability. Binary AWS EventStream responses preserve transfer chunks in the bundle, and native stream assertions decode events with botocore and require text, ordered completion and usage -The recording workflow uses the existing `e2e-changed` environment and OIDC secrets reader (`E2E_AWS_ROLE_TO_ASSUME`, secret `litellm-e2e-changed-provider-keys`), and fails before starting the proxy if any required provider credential is missing. Replay supplies only test credentials and includes the four supported Bedrock endpoints in the egress sentinel +The recording workflow uses the existing `e2e-changed` environment and OIDC secrets reader (`E2E_AWS_ROLE_TO_ASSUME`, secret `litellm-e2e-changed-provider-keys`), and fails before starting the proxy if any required provider credential is missing. Recording requires approval under that environment's existing protection policy, including scheduled runs. Replay supplies only test credentials and includes the four supported Bedrock endpoints in the egress sentinel -HTTP assertions classify a Bedrock 503 as `ProviderUnavailableError` only when AWS headers or a recognized Bedrock error envelope support that attribution. The default live lane allows one test rerun after ten seconds for that error or a transport failure. Generic proxy 5xx, authentication errors and assertion failures do not qualify. Persistent provider failures remain failed, and JUnit properties retain AWS and proxy request IDs from the failed attempt even when the rerun passes. Record and replay continue to use `--reruns 0` +HTTP assertions classify a Bedrock 503 as `ProviderUnavailableError` only when AWS headers with Bedrock route context, or a recognized Bedrock error envelope support that attribution. The default live lane allows one test rerun after ten seconds for that error or a transport failure. Generic proxy 5xx, authentication errors and assertion failures do not qualify. Persistent provider failures remain failed, and JUnit properties retain AWS and proxy request IDs from the failed attempt even when the rerun passes. Record and replay continue to use `--reruns 0` Current limits: deployments baked into the proxy's config file cannot be edge-wired (only `/model/new` registrations can carry the edge api_base), and a file upload routed by `custom_llm_provider` through the proxy's `files_settings` block never passes a deployment at all, so the batches `model_param` and `provider_fallback` scenarios keep uploading live in every mode diff --git a/tests/e2e/llm_translation/test_bedrock_native_e2e.py b/tests/e2e/llm_translation/test_bedrock_native_e2e.py index a193bc71927..892266fa8cb 100644 --- a/tests/e2e/llm_translation/test_bedrock_native_e2e.py +++ b/tests/e2e/llm_translation/test_bedrock_native_e2e.py @@ -11,7 +11,7 @@ from bedrock_config import bedrock_params from bedrock_stream import assert_converse_stream, assert_invoke_stream from e2e_config import unique_marker from e2e_http import ( - assert_client_error, + StreamingResponse, require_successful_call, ) from lifecycle import ResourceManager @@ -76,6 +76,11 @@ def _register(proxy: ProxyClient, resources: ResourceManager) -> tuple[str, str] return model, resources.key() +def _assert_input_error(result: StreamingResponse, field: str) -> None: + assert result.status_code == 400, f"expected Bedrock validation error, got {result.status_code}: {result.body[:500]}" + assert field in result.body.lower(), f"validation error did not name {field}: {result.body[:500]}" + + def _default_converse() -> ConverseBody: return ConverseBody( messages=[ConverseMessage(role="user", content=[ConverseContent(text="Hello")])], @@ -148,7 +153,7 @@ class TestBedrockNative: headers=proxy.transport.bearer(key), json=ConverseBody(inferenceConfig=ConverseInferenceConfig()), ) - assert_client_error(result, "converse missing messages") + _assert_input_error(result, "messages") @pytest.mark.covers("llm.bedrock_native.bedrock_converse.input_validation.nonstream.works") def test_converse_empty_messages_returns_client_error(self, proxy: ProxyClient, resources: ResourceManager) -> None: @@ -158,7 +163,7 @@ class TestBedrockNative: headers=proxy.transport.bearer(key), json=ConverseBody(messages=[]), ) - assert_client_error(result, "converse empty messages") + _assert_input_error(result, "messages") @pytest.mark.covers("llm.bedrock_native.bedrock_converse.input_validation.nonstream.works") def test_converse_invalid_model_returns_error(self, proxy: ProxyClient, resources: ResourceManager) -> None: @@ -180,7 +185,7 @@ class TestBedrockNative: headers=proxy.transport.bearer(key), json=InvokeBody(anthropic_version="bedrock-2023-05-31", max_tokens=50), ) - assert_client_error(result, "invoke missing messages") + _assert_input_error(result, "messages") @pytest.mark.covers("llm.bedrock_native.bedrock_invoke.input_validation.nonstream.works") def test_invoke_missing_max_tokens_returns_error(self, proxy: ProxyClient, resources: ResourceManager) -> None: @@ -193,7 +198,7 @@ class TestBedrockNative: messages=[InvokeMessage(role="user", content="Hello")], ), ) - assert_client_error(result, "invoke missing max_tokens") + _assert_input_error(result, "max_tokens") @pytest.mark.covers("llm.bedrock_native.bedrock_invoke.input_validation.nonstream.works") def test_invoke_invalid_temperature_returns_client_error( @@ -210,4 +215,4 @@ class TestBedrockNative: temperature=5.0, ), ) - assert_client_error(result, "invoke invalid temperature") + _assert_input_error(result, "temperature") diff --git a/tests/e2e/provider_diagnostics.py b/tests/e2e/provider_diagnostics.py index 65d72d393a8..cec71cf3eed 100644 --- a/tests/e2e/provider_diagnostics.py +++ b/tests/e2e/provider_diagnostics.py @@ -73,7 +73,13 @@ def provider_failure( error_code: Final = normalized.get("llm_provider-x-amzn-errortype") or normalized.get("x-amzn-errortype") call_id: Final = normalized.get("x-litellm-call-id") if status_code == 503: - if request_id and error_code and error_code.split(":", 1)[0] == "ServiceUnavailableException": + bedrock_body: Final = _bedrock_unavailable_body(body, expected_provider) + if ( + request_id + and error_code + and error_code.split(":", 1)[0] == "ServiceUnavailableException" + and (expected_provider == "bedrock" or bedrock_body) + ): return ProviderFailure( status_code=status_code, provider="bedrock", @@ -82,7 +88,7 @@ def provider_failure( call_id=call_id, evidence="headers", ) - if _bedrock_unavailable_body(body, expected_provider): + if bedrock_body: return ProviderFailure( status_code=status_code, provider="bedrock", diff --git a/tests/e2e/test_provider_diagnostics.py b/tests/e2e/test_provider_diagnostics.py index e57888e4347..8544182834a 100644 --- a/tests/e2e/test_provider_diagnostics.py +++ b/tests/e2e/test_provider_diagnostics.py @@ -36,13 +36,18 @@ def _assert_success(helper: str) -> None: class TestProviderFailure: def test_aws_headers_preserve_attribution_and_both_request_ids(self) -> None: - result: Final = provider_failure(503, "unavailable", _HEADERS) + result: Final = provider_failure(503, "unavailable", _HEADERS, expected_provider="bedrock") assert result.provider == "bedrock" assert result.error_code == "ServiceUnavailableException" assert result.request_id == "aws-request-1" assert result.call_id == "proxy-call-1" assert result.evidence == "headers" + def test_another_aws_service_is_not_mislabeled_as_bedrock(self) -> None: + result: Final = provider_failure(503, "service unavailable", _HEADERS) + assert result.provider is None + assert result.request_id == "aws-request-1" + def test_proxy_bedrock_error_body_remains_identifiable_without_headers(self) -> None: result: Final = provider_failure(503, _BODY, {}) assert result.provider == "bedrock" and result.evidence == "body" diff --git a/tests/e2e/test_provider_retry.py b/tests/e2e/test_provider_retry.py index 4536a5248c2..7f2c789261b 100644 --- a/tests/e2e/test_provider_retry.py +++ b/tests/e2e/test_provider_retry.py @@ -12,6 +12,7 @@ def test_retry_policy_preserves_failures_and_first_attempt_evidence(tmp_path: Pa scenario.write_text( """ from pathlib import Path +import json from e2e_http import NetworkError, StreamingResponse, UnknownApiError, require_successful_call, unwrap def attempt(name): @@ -21,7 +22,8 @@ def attempt(name): return count def unavailable(): - unwrap(UnknownApiError(status_code=503, body="unavailable", headers={ + body = json.dumps({"error":{"message": "litellm.ServiceUnavailableError: BedrockException - Bedrock is unable to process your request."}}) + unwrap(UnknownApiError(status_code=503, body=body, headers={ "x-amzn-requestid": "aws-first-attempt", "x-amzn-errortype": "ServiceUnavailableException", "x-litellm-call-id": "proxy-first-attempt", })) From 4139f3c7a2718991e54433dac1b30b9d81692d07 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sat, 12 Sep 2026 13:25:41 -0700 Subject: [PATCH 3/7] test: isolate replay credentials and restore recording dependencies --- .github/workflows/e2e_record_replay.yml | 32 ++++++++++- tests/e2e/CLAUDE.md | 2 +- tests/e2e/bedrock_edge.py | 40 ++++++++++--- tests/e2e/gateway/record_replay_ci_config.yml | 7 +++ tests/e2e/test_bedrock_edge.py | 56 +++++++++++++++++++ 5 files changed, 126 insertions(+), 11 deletions(-) diff --git a/.github/workflows/e2e_record_replay.yml b/.github/workflows/e2e_record_replay.yml index 2136676fd01..80525971729 100644 --- a/.github/workflows/e2e_record_replay.yml +++ b/.github/workflows/e2e_record_replay.yml @@ -31,6 +31,15 @@ jobs: contents: read id-token: write services: + redis: + image: redis:7.4.2@sha256:fbdbaea47b9ae4ecc2082ecdb4e1cea81e32176ffb1dcf643d422ad07427e5d9 + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 5s + --health-timeout 5s + --health-retries 10 postgres: image: postgres:16.6 env: @@ -108,17 +117,22 @@ jobs: - name: Require all recording credentials before starting the proxy run: | - uv run --no-sync --env-file tests/e2e/.env python - <<'PY' + PYTHONPATH=tests/e2e uv run --no-sync --env-file tests/e2e/.env python - <<'PY' import os + from bedrock_edge import recording_credentials required = ("OPENAI_API_KEY", "ANTHROPIC_API_KEY", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY") missing = [key for key in required if not os.environ.get(key)] if missing: raise SystemExit("Missing recording credentials: " + ", ".join(missing)) + recording_credentials().get_frozen_credentials() PY - name: Start the proxy run: | - nohup uv run --no-sync --env-file tests/e2e/.env litellm --config tests/e2e/gateway/record_replay_ci_config.yml --port 4000 > proxy.log 2>&1 & + nohup uv run --no-sync --env-file tests/e2e/.env env -u AWS_ROLE_NAME -u AWS_SESSION_NAME -u AWS_PROFILE_NAME \ + -u AWS_WEB_IDENTITY_TOKEN -u AWS_SESSION_TOKEN -u AWS_BEARER_TOKEN_BEDROCK \ + AWS_ACCESS_KEY_ID=e2e-bedrock-replay AWS_SECRET_ACCESS_KEY=e2e-bedrock-replay-secret \ + AWS_EC2_METADATA_DISABLED=true litellm --config tests/e2e/gateway/record_replay_ci_config.yml --port 4000 > proxy.log 2>&1 & for _ in $(seq 1 90); do if curl -fs http://localhost:4000/health/liveliness > /dev/null; then exit 0 @@ -165,6 +179,15 @@ jobs: contents: read actions: read services: + redis: + image: redis:7.4.2@sha256:fbdbaea47b9ae4ecc2082ecdb4e1cea81e32176ffb1dcf643d422ad07427e5d9 + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 5s + --health-timeout 5s + --health-retries 10 postgres: image: postgres:16.6 env: @@ -231,7 +254,10 @@ jobs: - name: Start the proxy run: | - nohup uv run --no-sync litellm --config tests/e2e/gateway/record_replay_ci_config.yml --port 4000 > proxy.log 2>&1 & + nohup uv run --no-sync env -u AWS_ROLE_NAME -u AWS_SESSION_NAME -u AWS_PROFILE_NAME \ + -u AWS_WEB_IDENTITY_TOKEN -u AWS_SESSION_TOKEN -u AWS_BEARER_TOKEN_BEDROCK \ + AWS_ACCESS_KEY_ID=e2e-bedrock-replay AWS_SECRET_ACCESS_KEY=e2e-bedrock-replay-secret \ + AWS_EC2_METADATA_DISABLED=true litellm --config tests/e2e/gateway/record_replay_ci_config.yml --port 4000 > proxy.log 2>&1 & for _ in $(seq 1 90); do if curl -fs http://localhost:4000/health/liveliness > /dev/null; then exit 0 diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index 98c3a45b3b1..9cfd37320a1 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -96,7 +96,7 @@ E2E_FIXTURE_MODE=replay E2E_FIXTURE_DIR=/tmp/e2e-fixtures E2E_RESET_SPEND_LOGS=1 Point the proxy at bogus provider credentials for the replay run and it still has to pass: that is the whole proof that nothing left the process. Bundles are never committed. `tests/e2e/.fixtures` is gitignored because a bundle holds verbatim provider response bodies and hard-fails after seven days. CI records and replays this lane on a schedule in `.github/workflows/e2e_record_replay.yml`, publishing the bundle as a private `e2e-fixtures-bundle` artifact instead of committing it, selecting the tests with the `@pytest.mark.replayable` marker, and proving the bogus-credentials replay hermetic by counting provider egress with `.github/scripts/e2e_egress_sentinel.py` -Bedrock tests use `bedrock_config.bedrock_params()` for Chat Completions, Responses, Converse and Invoke. Live mode retains the normal AWS credential path. Record and replay use a `/bedrock-us-east-1` mount by default (`E2E_BEDROCK_REGION` selects one of `us-east-1`, `us-east-2`, `us-west-1`, `us-west-2`; use the same region in both runs). The proxy signs each edge request with public test credentials. The edge verifies that signature over the host, path and body before accepting it. Recording re-signs the unchanged payload for the AWS endpoint using the runner's `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and optional `AWS_SESSION_TOKEN`. Replay requires no real AWS credentials and never forwards a miss. This checks request signing and protocol behavior in replay; live runs remain necessary to check real IAM access and model availability. Binary AWS EventStream responses preserve transfer chunks in the bundle, and native stream assertions decode events with botocore and require text, ordered completion and usage +Bedrock tests use `bedrock_config.bedrock_params()` for Chat Completions, Responses, Converse and Invoke. Live mode retains the normal AWS credential path. Record and replay use a `/bedrock-us-east-1` mount by default (`E2E_BEDROCK_REGION` selects one of `us-east-1`, `us-east-2`, `us-west-1`, `us-west-2`; use the same region in both runs). The proxy signs each edge request with public test credentials. The edge verifies that signature over the host, path and body before accepting it. Recording re-signs the unchanged payload for the AWS endpoint using the runner's `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and optional `AWS_SESSION_TOKEN`, assuming `AWS_ROLE_NAME` with the SDK when configured. Start the proxy without ambient `AWS_ROLE_NAME`, `AWS_PROFILE_NAME`, web identity or Bedrock bearer credentials in these modes so its public test credentials cannot trigger STS or bypass signature verification. The workflow isolates that proxy environment while retaining real credentials and role settings in the recording runner. Replay requires no real AWS credentials and never forwards a miss. This checks request signing and protocol behavior in replay; live runs remain necessary to check real IAM access and model availability. Binary AWS EventStream responses preserve transfer chunks in the bundle, and native stream assertions decode events with botocore and require text, ordered completion and usage The recording workflow uses the existing `e2e-changed` environment and OIDC secrets reader (`E2E_AWS_ROLE_TO_ASSUME`, secret `litellm-e2e-changed-provider-keys`), and fails before starting the proxy if any required provider credential is missing. Recording requires approval under that environment's existing protection policy, including scheduled runs. Replay supplies only test credentials and includes the four supported Bedrock endpoints in the egress sentinel diff --git a/tests/e2e/bedrock_edge.py b/tests/e2e/bedrock_edge.py index 8f47b68d220..88a1bf29888 100644 --- a/tests/e2e/bedrock_edge.py +++ b/tests/e2e/bedrock_edge.py @@ -2,12 +2,15 @@ import hashlib import hmac import os import re -from collections.abc import Mapping +from collections.abc import Callable, Mapping +from functools import lru_cache from typing import Final from botocore.auth import SigV4Auth from botocore.awsrequest import AWSRequest -from botocore.credentials import Credentials +from botocore.client import BaseClient +from botocore.credentials import AssumeRoleCredentialFetcher, Credentials, DeferredRefreshableCredentials +from botocore.session import get_session TEST_ACCESS_KEY: Final = "e2e-bedrock-replay" TEST_SECRET_KEY: Final = "e2e-bedrock-replay-secret" @@ -55,12 +58,35 @@ def valid_bedrock_signature( return hmac.compare_digest(expected, auth.group(5)) -def recording_credentials() -> Credentials: - access_key: Final = os.environ.get("AWS_ACCESS_KEY_ID", "") - secret_key: Final = os.environ.get("AWS_SECRET_ACCESS_KEY", "") +def recording_credentials_from(environ: Mapping[str, str], client_creator: Callable[..., BaseClient]) -> Credentials: + access_key: Final = environ.get("AWS_ACCESS_KEY_ID", "") + secret_key: Final = environ.get("AWS_SECRET_ACCESS_KEY", "") if not access_key or not secret_key: raise ValueError("Bedrock recording requires AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in the runner") - return Credentials(access_key, secret_key, os.environ.get("AWS_SESSION_TOKEN") or None) + source: Final = Credentials(access_key, secret_key, environ.get("AWS_SESSION_TOKEN") or None) + role: Final = environ.get("AWS_ROLE_NAME") + if not role: + return source + extra_args: Final = { + key: value + for key, value in ( + ("RoleSessionName", environ.get("AWS_SESSION_NAME", "e2e-bedrock-record")), + ("ExternalId", environ.get("AWS_EXTERNAL_ID")), + ) + if value + } + fetcher: Final = AssumeRoleCredentialFetcher( + client_creator=client_creator, + source_credentials=source, + role_arn=role, + extra_args=extra_args, + ) + return DeferredRefreshableCredentials(refresh_using=fetcher.fetch_credentials, method="assume-role") + + +@lru_cache(maxsize=1) +def recording_credentials() -> Credentials: + return recording_credentials_from(os.environ, get_session().create_client) def sign_bedrock_forward( @@ -73,5 +99,5 @@ def sign_bedrock_forward( and not key.lower().startswith("x-amz-") } request: Final = AWSRequest(method=method, url=url, data=body, headers=forwarded) - SigV4Auth(credentials, "bedrock", region).add_auth(request) + SigV4Auth(credentials.get_frozen_credentials(), "bedrock", region).add_auth(request) return dict(request.headers.items()) diff --git a/tests/e2e/gateway/record_replay_ci_config.yml b/tests/e2e/gateway/record_replay_ci_config.yml index 08972969cf0..2aa26eb9f2c 100644 --- a/tests/e2e/gateway/record_replay_ci_config.yml +++ b/tests/e2e/gateway/record_replay_ci_config.yml @@ -1,3 +1,10 @@ general_settings: master_key: os.environ/LITELLM_MASTER_KEY store_model_in_db: true + +litellm_settings: + cache: true + cache_params: + type: redis + host: 127.0.0.1 + port: 6379 diff --git a/tests/e2e/test_bedrock_edge.py b/tests/e2e/test_bedrock_edge.py index 5d61b7c11ea..7acbdda04ab 100644 --- a/tests/e2e/test_bedrock_edge.py +++ b/tests/e2e/test_bedrock_edge.py @@ -1,5 +1,6 @@ import base64 import threading +from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Final @@ -8,12 +9,16 @@ from bedrock_edge import ( TEST_ACCESS_KEY, TEST_SECRET_KEY, bedrock_upstream, + recording_credentials_from, sign_bedrock_forward, valid_bedrock_signature, ) from botocore.auth import SigV4Auth from botocore.awsrequest import AWSRequest +from botocore.client import BaseClient from botocore.credentials import Credentials +from botocore.session import get_session +from botocore.stub import Stubber from fixture_bundle import BundleRecorder, LoadedBundle, RecordedHttpResponse, load_bundle, prepare_bundle from fixture_mode import current_test_key from provider_edge import EdgeReply, RecordEdge, ReplayEdge, ReplaySource, edge_request, handle_edge_request @@ -34,6 +39,57 @@ def _signed_headers() -> dict[str, str]: class TestBedrockSigningBoundary: + def test_recording_resolves_the_real_role_once_without_using_proxy_test_credentials(self) -> None: + client: Final = get_session().create_client( + "sts", + region_name="us-east-1", + aws_access_key_id="recording-access", + aws_secret_access_key="recording-secret", + ) + + def client_creator(service: str, **credentials: str | None) -> BaseClient: + assert service == "sts" + assert credentials == { + "aws_access_key_id": "recording-access", + "aws_secret_access_key": "recording-secret", + "aws_session_token": "source-session", + } + return client + + with Stubber(client) as stub: + stub.add_response( + "assume_role", + { + "Credentials": { + "AccessKeyId": "ASIA1234567890123456", + "SecretAccessKey": "role-secret", + "SessionToken": "role-session", + "Expiration": datetime.now(timezone.utc) + timedelta(hours=1), + }, + }, + { + "RoleArn": "arn:aws:iam::123456789012:role/e2e-test", + "RoleSessionName": "test-recording", + "ExternalId": "external", + }, + ) + result: Final = recording_credentials_from( + { + "AWS_ACCESS_KEY_ID": "recording-access", + "AWS_SECRET_ACCESS_KEY": "recording-secret", + "AWS_SESSION_TOKEN": "source-session", + "AWS_ROLE_NAME": "arn:aws:iam::123456789012:role/e2e-test", + "AWS_SESSION_NAME": "test-recording", + "AWS_EXTERNAL_ID": "external", + }, + client_creator, + ) + first: Final = result.get_frozen_credentials() + assert first.access_key == "ASIA1234567890123456" + assert first.token == "role-session" + assert result.get_frozen_credentials() == first + stub.assert_no_pending_responses() + def test_accepts_a_valid_signature_from_the_proxy(self) -> None: assert valid_bedrock_signature("POST", _PATH, _signed_headers(), _BODY, "us-east-1") From 42d5bdac7aa98211891225326767656119a2ab2d Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sat, 12 Sep 2026 13:42:50 -0700 Subject: [PATCH 4/7] test: assert the Bedrock empty conversation validation detail --- tests/e2e/llm_translation/test_bedrock_native_e2e.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/e2e/llm_translation/test_bedrock_native_e2e.py b/tests/e2e/llm_translation/test_bedrock_native_e2e.py index 892266fa8cb..75cc9b993d8 100644 --- a/tests/e2e/llm_translation/test_bedrock_native_e2e.py +++ b/tests/e2e/llm_translation/test_bedrock_native_e2e.py @@ -76,9 +76,9 @@ def _register(proxy: ProxyClient, resources: ResourceManager) -> tuple[str, str] return model, resources.key() -def _assert_input_error(result: StreamingResponse, field: str) -> None: +def _assert_input_error(result: StreamingResponse, expected_detail: str) -> None: assert result.status_code == 400, f"expected Bedrock validation error, got {result.status_code}: {result.body[:500]}" - assert field in result.body.lower(), f"validation error did not name {field}: {result.body[:500]}" + assert expected_detail in result.body.lower(), f"validation error did not name {expected_detail}: {result.body[:500]}" def _default_converse() -> ConverseBody: @@ -163,7 +163,7 @@ class TestBedrockNative: headers=proxy.transport.bearer(key), json=ConverseBody(messages=[]), ) - _assert_input_error(result, "messages") + _assert_input_error(result, "user message") @pytest.mark.covers("llm.bedrock_native.bedrock_converse.input_validation.nonstream.works") def test_converse_invalid_model_returns_error(self, proxy: ProxyClient, resources: ResourceManager) -> None: From fc6e694fd8ca5789f8f2db9a0dea8332858de1f9 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sat, 12 Sep 2026 14:04:14 -0700 Subject: [PATCH 5/7] ci: filter fixture artifact lookup by name --- .github/scripts/e2e_fetch_fixture_bundle.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/e2e_fetch_fixture_bundle.sh b/.github/scripts/e2e_fetch_fixture_bundle.sh index b76ced74b3b..89b18bfd23f 100755 --- a/.github/scripts/e2e_fetch_fixture_bundle.sh +++ b/.github/scripts/e2e_fetch_fixture_bundle.sh @@ -14,7 +14,7 @@ trap 'rm -rf "${WORKDIR}"' EXIT echo "resolving newest non-expired '${ARTIFACT_NAME}' artifact on ${REPO}@${BASE_BRANCH}" SELECTED="$( - gh api "repos/${REPO}/actions/artifacts" -X GET -f per_page=100 --paginate \ + gh api "repos/${REPO}/actions/artifacts" -X GET -f name="${ARTIFACT_NAME}" -f per_page=100 --paginate \ --jq ".artifacts[] | select(.name == \"${ARTIFACT_NAME}\" and .expired == false and .workflow_run.head_branch == \"${BASE_BRANCH}\") | {id, digest, created_at, run_id: .workflow_run.id, run_number: .workflow_run.run_number}" \ | jq -s 'sort_by(.created_at) | reverse | .[0] // empty' )" From 71d9fe65a06b8aeae6dda9708475fc38a2a3e560 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sat, 12 Sep 2026 14:23:53 -0700 Subject: [PATCH 6/7] test: classify Bedrock stream outages and interrupted transfers --- tests/e2e/CLAUDE.md | 2 +- tests/e2e/CONTRIBUTING.md | 2 +- tests/e2e/bedrock_eventstream.py | 2 + tests/e2e/conftest.py | 4 +- tests/e2e/e2e_http.py | 37 ++++++++++++-- tests/e2e/provider_diagnostics.py | 27 ++++++++++- tests/e2e/test_bedrock_eventstream.py | 70 +++++++++++++++++++++++++-- tests/e2e/test_provider_retry.py | 38 ++++++++++++++- 8 files changed, 168 insertions(+), 14 deletions(-) diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index 9cfd37320a1..98fb1d5bb1e 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -100,7 +100,7 @@ Bedrock tests use `bedrock_config.bedrock_params()` for Chat Completions, Respon The recording workflow uses the existing `e2e-changed` environment and OIDC secrets reader (`E2E_AWS_ROLE_TO_ASSUME`, secret `litellm-e2e-changed-provider-keys`), and fails before starting the proxy if any required provider credential is missing. Recording requires approval under that environment's existing protection policy, including scheduled runs. Replay supplies only test credentials and includes the four supported Bedrock endpoints in the egress sentinel -HTTP assertions classify a Bedrock 503 as `ProviderUnavailableError` only when AWS headers with Bedrock route context, or a recognized Bedrock error envelope support that attribution. The default live lane allows one test rerun after ten seconds for that error or a transport failure. Generic proxy 5xx, authentication errors and assertion failures do not qualify. Persistent provider failures remain failed, and JUnit properties retain AWS and proxy request IDs from the failed attempt even when the rerun passes. Record and replay continue to use `--reruns 0` +HTTP assertions classify a Bedrock 503 as `ProviderUnavailableError` only when AWS headers with Bedrock route context, or a recognized Bedrock error envelope support that attribution. A decoded native `serviceUnavailableException` receives the same classification even when the committed HTTP status is 200, with `stream_event` recorded as its evidence. Interrupted binary or SSE transfers become modeled network failures with the received status and request IDs preserved. The default live lane allows one test rerun after ten seconds for an evidenced availability error or a transport failure. Generic proxy 5xx, authentication errors and assertion failures do not qualify. Persistent provider failures remain failed, and JUnit properties retain AWS and proxy request IDs from the failed attempt even when the rerun passes. Record and replay continue to use `--reruns 0` Current limits: deployments baked into the proxy's config file cannot be edge-wired (only `/model/new` registrations can carry the edge api_base), and a file upload routed by `custom_llm_provider` through the proxy's `files_settings` block never passes a deployment at all, so the batches `model_param` and `provider_fallback` scenarios keep uploading live in every mode diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index e77d0489c73..a21812e7377 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -101,7 +101,7 @@ A couple of logging destinations are configured on the proxy rather than by the ### The pull request check -Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite and both JWT suites as canaries, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Keycloak, Jaeger, and TLS cluster-mode Valkey. Realm-only edits also trigger these canaries. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, and `guardrails/test_presidio_masking_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start. The Redis chaos test under `load/` needs a proxy it can pause the Redis of on the same host (`gateway/redis_chaos_ci_config.yml`), which `.github/workflows/test-e2e-redis-chaos.yml` boots, and which the Buildkite `e2e-redis-chaos` step in project-releaser runs co-located with Postgres and Valkey in one pod; it is deselected unless `E2E_REDIS_CHAOS` is set +Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite and both JWT suites as canaries, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's single rerun for network failures and evidenced Bedrock availability errors (see `pytest.ini`) applies on every pass, including outages reported inside native streams. Generic proxy 5xx and failed behavior assertions do not qualify, and persistent failures still fail the check. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Keycloak, Jaeger, and TLS cluster-mode Valkey. Realm-only edits also trigger these canaries. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, and `guardrails/test_presidio_masking_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start. The Redis chaos test under `load/` needs a proxy it can pause the Redis of on the same host (`gateway/redis_chaos_ci_config.yml`), which `.github/workflows/test-e2e-redis-chaos.yml` boots, and which the Buildkite `e2e-redis-chaos` step in project-releaser runs co-located with Postgres and Valkey in one pod; it is deselected unless `E2E_REDIS_CHAOS` is set Every selected file must execute at least one passing test in each pass, and any test failure, collection error, or entirely skipped or deselected file fails the check. A file whose tests are all marked skip therefore cannot pass this check, so unskip at least one of them, or add the file to `UNSUPPORTED` in `select_tests.py` with the reason, before changing one. A failed pass stops the run. The public log prints pytest's one-line summary for each pass, including the rerun count, and names each failed or errored test as `classname::name`, so a retried network error or a failing test is visible without the raw output. The final `e2e-changed-tests` job succeeds only when no supported test files changed or the approved run completed all three passes. Fork PRs with selected tests fail this gate until a maintainer brings the reviewed change onto a same-repository branch diff --git a/tests/e2e/bedrock_eventstream.py b/tests/e2e/bedrock_eventstream.py index a980d994413..021146feb32 100644 --- a/tests/e2e/bedrock_eventstream.py +++ b/tests/e2e/bedrock_eventstream.py @@ -21,6 +21,7 @@ class _InvokeChunk(BaseModel): class BedrockEvent: payload: str error: str | None = None + error_code: str | None = None _JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) @@ -32,6 +33,7 @@ def _decode_event(event: EventStreamMessage) -> BedrockEvent: return BedrockEvent( payload=event.payload.decode(), error=f"{headers.exception_type or headers.message_type}: {event.payload.decode()}", + error_code=headers.exception_type or headers.message_type, ) if headers.event_type == "chunk": encoded: Final = _InvokeChunk.model_validate_json(event.payload) diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 9ec2bf7ef1a..96276eb5f37 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -29,7 +29,7 @@ from idp import Identity, Keycloak, keycloak_from_env from junit_properties import attach_result_properties from lifecycle import ProxyClientProvider, ResourceManager from models import TeamNewBody, UserNewBody, UserNewResponse -from provider_diagnostics import ProviderUnavailableError +from provider_diagnostics import NetworkFailureError, ProviderUnavailableError from provider_edge import replay_leftover_error from proxy_client import ProxyClient, build_proxy_client @@ -183,7 +183,7 @@ def pytest_runtest_makereport( report = yield if report.when == "call": item.stash[_CALL_PASSED] = report.passed - if call.excinfo is not None and isinstance(call.excinfo.value, ProviderUnavailableError): + if call.excinfo is not None and isinstance(call.excinfo.value, (ProviderUnavailableError, NetworkFailureError)): failure: Final = call.excinfo.value.failure item.user_properties.extend(failure.junit_properties()) report.user_properties = list(item.user_properties) diff --git a/tests/e2e/e2e_http.py b/tests/e2e/e2e_http.py index f2eb4fecd09..5bf26433a32 100644 --- a/tests/e2e/e2e_http.py +++ b/tests/e2e/e2e_http.py @@ -23,7 +23,7 @@ from typing import Final, Generator, Generic, Iterator, Literal, NewType, Protoc import pytest import requests from bedrock_eventstream import decode_bedrock_stream -from provider_diagnostics import raise_if_provider_unavailable +from provider_diagnostics import NetworkFailureError, provider_failure, raise_if_provider_unavailable from pydantic import BaseModel, ConfigDict, Field URL = NewType("URL", str) @@ -165,11 +165,13 @@ class StreamingResponse(BaseModel): # quota) arrive as SSE error events inside an otherwise-successful response; # the consumed body is elided, so this is the only place they surface. stream_error: str | None = None + stream_error_code: str | None = None + network_error: NetworkError | None = None stream_done: bool = False @property def ok(self) -> bool: - return 200 <= self.status_code < 300 + return 200 <= self.status_code < 300 and self.network_error is None @property def is_streaming(self) -> bool: @@ -259,9 +261,19 @@ def is_ok[R: BaseModel](result: Result[R]) -> bool: def require_successful_call(result: StreamingResponse, *, expected_provider: Literal["bedrock"] | None = None) -> None: """A call that should have succeeded but didn't is a hard failure, never a skip: if the proxy can't make a call it's expected to, the test must fail.""" + if result.network_error is not None: + raise NetworkFailureError( + provider_failure(result.status_code, "", result.headers), result.network_error.message + ) + raise_if_provider_unavailable( + result.status_code, + result.stream_error or result.body, + result.headers, + expected_provider=expected_provider, + stream_error_code=result.stream_error_code, + ) if result.ok: return - raise_if_provider_unavailable(result.status_code, result.body, result.headers, expected_provider=expected_provider) pytest.fail( f"upstream call failed (status {result.status_code}); headers={result.headers}; body={result.body[:1000]}" ) @@ -611,6 +623,22 @@ def _is_stream_error_line(line: bytes) -> bool: def streaming_outcome( resp: SseResponse, stream: bool, *, sent_at: float, clock: Callable[[], float] = time.monotonic +) -> StreamingResponse: + try: + return _streaming_outcome(resp, stream, sent_at=sent_at, clock=clock) + except requests.RequestException as exc: + return StreamingResponse( + status_code=resp.status_code, + call_id=_hdr(resp, "x-litellm-call-id"), + content_type=_hdr(resp, "content-type"), + headers={name.lower(): value for name, value in resp.headers.items()}, + body=str(exc), + network_error=NetworkError(message=str(exc)), + ) + + +def _streaming_outcome( + resp: SseResponse, stream: bool, *, sent_at: float, clock: Callable[[], float] ) -> StreamingResponse: call_id: Final = _hdr(resp, "x-litellm-call-id") response_cost: Final = _parse_response_cost(resp) @@ -640,6 +668,7 @@ def streaming_outcome( stream_events=[event.payload for event, _ in bedrock_events], stream_event_arrivals=[arrived for _, arrived in bedrock_events], stream_error=next((event.error for event, _ in bedrock_events if event.error is not None), None), + stream_error_code=next((event.error_code for event, _ in bedrock_events if event.error is not None), None), ) stamped: Final = tuple((line, clock() - sent_at) for line in resp.iter_lines() if line) payloads: Final = tuple( @@ -692,7 +721,7 @@ def send( ) ) except requests.RequestException as exc: - return StreamingResponse(status_code=-1, body=str(exc)) + return StreamingResponse(status_code=-1, body=str(exc), network_error=NetworkError(message=str(exc))) return streaming_outcome(resp, stream, sent_at=sent_at) diff --git a/tests/e2e/provider_diagnostics.py b/tests/e2e/provider_diagnostics.py index cec71cf3eed..63035db8413 100644 --- a/tests/e2e/provider_diagnostics.py +++ b/tests/e2e/provider_diagnostics.py @@ -10,7 +10,7 @@ class ProviderFailure(BaseModel): error_code: str | None = None request_id: str | None = None call_id: str | None = None - evidence: Literal["headers", "body"] | None = None + evidence: Literal["headers", "body", "stream_event"] | None = None def junit_properties(self) -> tuple[tuple[str, str], ...]: fields: Final = ( @@ -30,6 +30,12 @@ class ProviderUnavailableError(AssertionError): super().__init__(f"{failure.model_dump_json(exclude_none=True)}; body={body[:1000]}") +class NetworkFailureError(AssertionError): + def __init__(self, failure: ProviderFailure, message: str) -> None: + self.failure: Final = failure + super().__init__(f"kind='network'; {failure.model_dump_json(exclude_none=True)}; body={message[:1000]}") + + class _ErrorMessage(BaseModel): message: str = "" @@ -67,11 +73,25 @@ def provider_failure( headers: Mapping[str, str], *, expected_provider: Literal["bedrock"] | None = None, + stream_error_code: str | None = None, ) -> ProviderFailure: normalized: Final = {key.lower(): value for key, value in headers.items()} request_id: Final = normalized.get("llm_provider-x-amzn-requestid") or normalized.get("x-amzn-requestid") error_code: Final = normalized.get("llm_provider-x-amzn-errortype") or normalized.get("x-amzn-errortype") call_id: Final = normalized.get("x-litellm-call-id") + if ( + 200 <= status_code < 300 + and expected_provider == "bedrock" + and stream_error_code == "serviceUnavailableException" + ): + return ProviderFailure( + status_code=status_code, + provider="bedrock", + error_code="ServiceUnavailableException", + request_id=request_id, + call_id=call_id, + evidence="stream_event", + ) if status_code == 503: bedrock_body: Final = _bedrock_unavailable_body(body, expected_provider) if ( @@ -106,7 +126,10 @@ def raise_if_provider_unavailable( headers: Mapping[str, str], *, expected_provider: Literal["bedrock"] | None = None, + stream_error_code: str | None = None, ) -> None: - failure: Final = provider_failure(status_code, body, headers, expected_provider=expected_provider) + failure: Final = provider_failure( + status_code, body, headers, expected_provider=expected_provider, stream_error_code=stream_error_code + ) if failure.provider is not None: raise ProviderUnavailableError(failure, body) diff --git a/tests/e2e/test_bedrock_eventstream.py b/tests/e2e/test_bedrock_eventstream.py index e6152de215a..34d2cc216e1 100644 --- a/tests/e2e/test_bedrock_eventstream.py +++ b/tests/e2e/test_bedrock_eventstream.py @@ -4,14 +4,17 @@ import struct import zlib from collections.abc import Iterator from dataclasses import dataclass +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from threading import Thread from types import MappingProxyType from typing import Final import pytest from bedrock_eventstream import decode_bedrock_stream from botocore.eventstream import ChecksumMismatch -from e2e_http import StreamingResponse, streaming_outcome +from e2e_http import URL, NoBody, StreamingResponse, require_successful_call, send, streaming_outcome from llm_translation.bedrock_stream import assert_converse_stream, assert_invoke_stream +from provider_diagnostics import NetworkFailureError, ProviderUnavailableError from pydantic import JsonValue, TypeAdapter _JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) @@ -52,7 +55,13 @@ class _BinaryResponse: chunks: tuple[bytes, ...] status_code: int = 200 text: str = "" - headers = MappingProxyType({"content-type": "application/vnd.amazon.eventstream"}) + headers = MappingProxyType( + { + "content-type": "application/vnd.amazon.eventstream", + "x-amzn-requestid": "stream-request", + "x-litellm-call-id": "stream-call", + } + ) def iter_content(self, chunk_size: int | None = 1) -> Iterator[bytes]: return iter(self.chunks) @@ -96,8 +105,20 @@ class TestBedrockEventStream: wire: Final = _frame('{"message":"unavailable"}', "serviceUnavailableException", "exception") result: Final = streaming_outcome(_BinaryResponse((wire,)), True, sent_at=0.0) assert result.stream_error == 'serviceUnavailableException: {"message":"unavailable"}' - with pytest.raises(AssertionError, match="serviceUnavailableException"): + with pytest.raises(ProviderUnavailableError) as caught: assert_converse_stream(result) + assert caught.value.failure.status_code == 200 + assert caught.value.failure.evidence == "stream_event" + assert caught.value.failure.request_id == "stream-request" + assert caught.value.failure.call_id == "stream-call" + + @pytest.mark.parametrize("code", ["validationException", "throttlingException", "internalServerException"]) + def test_other_stream_exceptions_do_not_qualify_for_the_availability_retry(self, code: str) -> None: + wire: Final = _frame('{"message":"provider rejected the request"}', code, "exception") + result: Final = streaming_outcome(_BinaryResponse((wire,)), True, sent_at=0.0) + with pytest.raises(AssertionError, match=code) as caught: + assert_converse_stream(result) + assert not isinstance(caught.value, ProviderUnavailableError) def test_corrupt_binary_frame_is_rejected(self) -> None: wire: Final = _frame('{"role":"assistant"}', "messageStart") @@ -125,3 +146,46 @@ class TestNativeStreamAssertions: def test_converse_rejects_content_after_the_stop_event(self) -> None: with pytest.raises(AssertionError, match="event order"): assert_converse_stream(_stream((_CONVERSE[0], _CONVERSE[3], _CONVERSE[1], _CONVERSE[4]))) + + +class _InterruptedStream(BaseHTTPRequestHandler): + def do_POST(self) -> None: + _ = self.rfile.read(int(self.headers.get("content-length", "0"))) + binary: Final = self.path == "/binary" + data: Final = ( + _frame('{"role":"assistant"}', "messageStart") if binary else b'data: {"text":"Hello"}\n\n' + ) + self.send_response(200) + self.send_header("content-type", "application/vnd.amazon.eventstream" if binary else "text/event-stream") + self.send_header("transfer-encoding", "chunked") + self.send_header("x-amzn-requestid", "interrupted-request") + self.send_header("x-litellm-call-id", "interrupted-call") + self.end_headers() + _ = self.wfile.write(f"{len(data):x}\r\n".encode() + data + b"\r\n") + self.wfile.flush() + self.close_connection = True + + def log_message(self, format: str, *args: object) -> None: + pass + + +@pytest.mark.parametrize("path", ["binary", "sse"]) +def test_interrupted_http_stream_preserves_network_classification_and_ids(path: str) -> None: + server: Final = ThreadingHTTPServer(("127.0.0.1", 0), _InterruptedStream) + worker: Final = Thread(target=server.serve_forever, daemon=True) + worker.start() + try: + result: Final = send( + URL(f"http://127.0.0.1:{server.server_port}/{path}"), headers=NoBody(), json=NoBody(), stream=True, timeout=5 + ) + assert result.status_code == 200 and not result.ok + assert result.network_error is not None and result.network_error.kind == "network" + with pytest.raises(NetworkFailureError, match="kind='network'") as caught: + require_successful_call(result) + assert caught.value.failure.provider is None + assert caught.value.failure.request_id == "interrupted-request" + assert caught.value.failure.call_id == "interrupted-call" + finally: + server.shutdown() + server.server_close() + worker.join(timeout=5) diff --git a/tests/e2e/test_provider_retry.py b/tests/e2e/test_provider_retry.py index 7f2c789261b..a064b6aef22 100644 --- a/tests/e2e/test_provider_retry.py +++ b/tests/e2e/test_provider_retry.py @@ -53,6 +53,29 @@ def test_native(): require_successful_call(StreamingResponse(status_code=503, body="unavailable", headers={ "x-amzn-requestid": "native-request", "x-amzn-errortype": "ServiceUnavailableException", }), expected_provider="bedrock") + +def streamed_unavailable(): + require_successful_call(StreamingResponse( + status_code=200, body="", stream_error="serviceUnavailableException: unavailable", + stream_error_code="serviceUnavailableException", headers={ + "x-amzn-requestid": "stream-request", "x-litellm-call-id": "stream-call", + }, + ), expected_provider="bedrock") + +def test_stream_recovers(): + if attempt("stream_recovers") == 1: + streamed_unavailable() + +def test_stream_persistent(): + attempt("stream_persistent") + streamed_unavailable() + +def test_stream_transport(): + attempt("stream_transport") + require_successful_call(StreamingResponse( + status_code=200, body="connection interrupted", network_error=NetworkError(message="connection interrupted"), + headers={"x-amzn-requestid": "transport-request", "x-litellm-call-id": "transport-call"}, + )) """, encoding="utf-8", ) @@ -84,7 +107,7 @@ def test_native(): check=False, ) assert result.returncode == 1, result.stdout + result.stderr - assert "5 failed, 1 passed, 4 rerun" in result.stdout, result.stdout + assert "7 failed, 2 passed, 7 rerun" in result.stdout, result.stdout assert {path.stem: int(path.read_text()) for path in tmp_path.glob("*.attempts")} == { "recovers": 2, "persistent": 2, @@ -92,6 +115,9 @@ def test_native(): "auth": 1, "network": 2, "native": 2, + "stream_recovers": 2, + "stream_persistent": 2, + "stream_transport": 2, } cases: Final = ET.parse(tmp_path / "results.xml").getroot() recovered: Final = cases.find(".//testcase[@name='test_recovers']") @@ -99,3 +125,13 @@ def test_native(): assert recovered.find("failure") is None assert recovered.find("properties/property[@name='upstream_request_id'][@value='aws-first-attempt']") is not None assert recovered.find("properties/property[@name='upstream_call_id'][@value='proxy-first-attempt']") is not None + streamed: Final = cases.find(".//testcase[@name='test_stream_recovers']") + assert streamed is not None and streamed.find("failure") is None + assert streamed.find("properties/property[@name='upstream_status_code'][@value='200']") is not None + assert streamed.find("properties/property[@name='upstream_evidence'][@value='stream_event']") is not None + assert streamed.find("properties/property[@name='upstream_request_id'][@value='stream-request']") is not None + assert streamed.find("properties/property[@name='upstream_call_id'][@value='stream-call']") is not None + transport: Final = cases.find(".//testcase[@name='test_stream_transport']") + assert transport is not None and transport.find("failure") is not None + assert transport.find("properties/property[@name='upstream_request_id'][@value='transport-request']") is not None + assert transport.find("properties/property[@name='upstream_call_id'][@value='transport-call']") is not None From 3b3c071d20ea90d7fee98e42b7698d2fdec997a8 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Sat, 12 Sep 2026 17:06:50 -0700 Subject: [PATCH 7/7] test: preserve Bedrock coverage and prevent retry masking --- tests/e2e/CLAUDE.md | 4 +- tests/e2e/CONTRIBUTING.md | 2 +- tests/e2e/bedrock_eventstream.py | 29 ++++- .../e2e/llm_translation/test_responses_e2e.py | 27 +++- tests/e2e/provider_diagnostics.py | 2 +- tests/e2e/pytest.ini | 2 +- tests/e2e/test_bedrock_eventstream.py | 23 +++- tests/e2e/test_provider_retry.py | 115 +++++++++++++++--- 8 files changed, 176 insertions(+), 28 deletions(-) diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index 98fb1d5bb1e..bc644b55da5 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -96,11 +96,11 @@ E2E_FIXTURE_MODE=replay E2E_FIXTURE_DIR=/tmp/e2e-fixtures E2E_RESET_SPEND_LOGS=1 Point the proxy at bogus provider credentials for the replay run and it still has to pass: that is the whole proof that nothing left the process. Bundles are never committed. `tests/e2e/.fixtures` is gitignored because a bundle holds verbatim provider response bodies and hard-fails after seven days. CI records and replays this lane on a schedule in `.github/workflows/e2e_record_replay.yml`, publishing the bundle as a private `e2e-fixtures-bundle` artifact instead of committing it, selecting the tests with the `@pytest.mark.replayable` marker, and proving the bogus-credentials replay hermetic by counting provider egress with `.github/scripts/e2e_egress_sentinel.py` -Bedrock tests use `bedrock_config.bedrock_params()` for Chat Completions, Responses, Converse and Invoke. Live mode retains the normal AWS credential path. Record and replay use a `/bedrock-us-east-1` mount by default (`E2E_BEDROCK_REGION` selects one of `us-east-1`, `us-east-2`, `us-west-1`, `us-west-2`; use the same region in both runs). The proxy signs each edge request with public test credentials. The edge verifies that signature over the host, path and body before accepting it. Recording re-signs the unchanged payload for the AWS endpoint using the runner's `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and optional `AWS_SESSION_TOKEN`, assuming `AWS_ROLE_NAME` with the SDK when configured. Start the proxy without ambient `AWS_ROLE_NAME`, `AWS_PROFILE_NAME`, web identity or Bedrock bearer credentials in these modes so its public test credentials cannot trigger STS or bypass signature verification. The workflow isolates that proxy environment while retaining real credentials and role settings in the recording runner. Replay requires no real AWS credentials and never forwards a miss. This checks request signing and protocol behavior in replay; live runs remain necessary to check real IAM access and model availability. Binary AWS EventStream responses preserve transfer chunks in the bundle, and native stream assertions decode events with botocore and require text, ordered completion and usage +Bedrock tests use `bedrock_config.bedrock_params()` for Chat Completions, Responses, Converse and Invoke. Live mode retains the normal AWS credential path. Record and replay use a `/bedrock-us-east-1` mount by default (`E2E_BEDROCK_REGION` selects one of `us-east-1`, `us-east-2`, `us-west-1`, `us-west-2`; use the same region in both runs). The proxy signs each edge request with public test credentials. The edge verifies that signature over the host, path and body before accepting it. Recording re-signs the unchanged payload for the AWS endpoint using the runner's `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and optional `AWS_SESSION_TOKEN`, assuming `AWS_ROLE_NAME` with the SDK when configured. Start the proxy without ambient `AWS_ROLE_NAME`, `AWS_PROFILE_NAME`, web identity or Bedrock bearer credentials in these modes so its public test credentials cannot trigger STS or bypass signature verification. The workflow isolates that proxy environment while retaining real credentials and role settings in the recording runner. Replay requires no real AWS credentials and never forwards a miss. This checks request signing and protocol behavior in replay; live runs remain necessary to check real IAM access and model availability. Binary AWS EventStream responses preserve transfer chunks in the bundle, and native stream assertions decode events with botocore and require complete frames, text, ordered completion and usage. The original Responses tool test keeps default tool selection, and a separate test covers `tool_choice="required"` The recording workflow uses the existing `e2e-changed` environment and OIDC secrets reader (`E2E_AWS_ROLE_TO_ASSUME`, secret `litellm-e2e-changed-provider-keys`), and fails before starting the proxy if any required provider credential is missing. Recording requires approval under that environment's existing protection policy, including scheduled runs. Replay supplies only test credentials and includes the four supported Bedrock endpoints in the egress sentinel -HTTP assertions classify a Bedrock 503 as `ProviderUnavailableError` only when AWS headers with Bedrock route context, or a recognized Bedrock error envelope support that attribution. A decoded native `serviceUnavailableException` receives the same classification even when the committed HTTP status is 200, with `stream_event` recorded as its evidence. Interrupted binary or SSE transfers become modeled network failures with the received status and request IDs preserved. The default live lane allows one test rerun after ten seconds for an evidenced availability error or a transport failure. Generic proxy 5xx, authentication errors and assertion failures do not qualify. Persistent provider failures remain failed, and JUnit properties retain AWS and proxy request IDs from the failed attempt even when the rerun passes. Record and replay continue to use `--reruns 0` +HTTP assertions classify a Bedrock 503 as `ProviderUnavailableError` only when AWS headers with Bedrock route context, or a recognized Bedrock error envelope support that attribution. A decoded native `serviceUnavailableException` receives the same classification even when the committed HTTP status is 200, with `stream_event` recorded as its evidence. Interrupted binary or SSE transfers produce `NetworkFailureError` with the received status and request IDs preserved. Both diagnostic exceptions hard-fail without a test rerun, including when their messages contain a legacy retry marker. Incomplete binary frames hard-fail even after a valid completion event. The existing typed `NetworkError` retry remains, with its filter anchored to that assertion rather than matching a marker embedded in another response, header or assertion message. No newly classified failure becomes eligible for a rerun. The exercised proxy 5xx, authentication and behavior failures do not qualify. This retains the existing text-based retry convention: an arbitrary assertion that deliberately imitates the exact typed network-error prefix can still match. JUnit retains AWS and proxy request IDs from the failed attempt. Record and replay continue to use `--reruns 0` Current limits: deployments baked into the proxy's config file cannot be edge-wired (only `/model/new` registrations can carry the edge api_base), and a file upload routed by `custom_llm_provider` through the proxy's `files_settings` block never passes a deployment at all, so the batches `model_param` and `provider_fallback` scenarios keep uploading live in every mode diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index a21812e7377..4df51e98d93 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -101,7 +101,7 @@ A couple of logging destinations are configured on the proxy rather than by the ### The pull request check -Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite and both JWT suites as canaries, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's single rerun for network failures and evidenced Bedrock availability errors (see `pytest.ini`) applies on every pass, including outages reported inside native streams. Generic proxy 5xx and failed behavior assertions do not qualify, and persistent failures still fail the check. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Keycloak, Jaeger, and TLS cluster-mode Valkey. Realm-only edits also trigger these canaries. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, and `guardrails/test_presidio_masking_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start. The Redis chaos test under `load/` needs a proxy it can pause the Redis of on the same host (`gateway/redis_chaos_ci_config.yml`), which `.github/workflows/test-e2e-redis-chaos.yml` boots, and which the Buildkite `e2e-redis-chaos` step in project-releaser runs co-located with Postgres and Valkey in one pod; it is deselected unless `E2E_REDIS_CHAOS` is set +Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite and both JWT suites as canaries, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's existing single rerun for typed network results applies on every pass (see `pytest.ini`). Bedrock availability diagnostics and interrupted HTTP streams hard-fail without a rerun. The exercised proxy 5xx and behavior failures do not qualify either. The legacy filter matches assertion text, so custom assertions must not imitate the exact typed network-error prefix. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Keycloak, Jaeger, and TLS cluster-mode Valkey. Realm-only edits also trigger these canaries. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, and `guardrails/test_presidio_masking_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start. The Redis chaos test under `load/` needs a proxy it can pause the Redis of on the same host (`gateway/redis_chaos_ci_config.yml`), which `.github/workflows/test-e2e-redis-chaos.yml` boots, and which the Buildkite `e2e-redis-chaos` step in project-releaser runs co-located with Postgres and Valkey in one pod; it is deselected unless `E2E_REDIS_CHAOS` is set Every selected file must execute at least one passing test in each pass, and any test failure, collection error, or entirely skipped or deselected file fails the check. A file whose tests are all marked skip therefore cannot pass this check, so unskip at least one of them, or add the file to `UNSUPPORTED` in `select_tests.py` with the reason, before changing one. A failed pass stops the run. The public log prints pytest's one-line summary for each pass, including the rerun count, and names each failed or errored test as `classname::name`, so a retried network error or a failing test is visible without the raw output. The final `e2e-changed-tests` job succeeds only when no supported test files changed or the approved run completed all three passes. Fork PRs with selected tests fail this gate until a maintainer brings the reviewed change onto a same-repository branch diff --git a/tests/e2e/bedrock_eventstream.py b/tests/e2e/bedrock_eventstream.py index 021146feb32..8fedcf9da50 100644 --- a/tests/e2e/bedrock_eventstream.py +++ b/tests/e2e/bedrock_eventstream.py @@ -17,6 +17,10 @@ class _InvokeChunk(BaseModel): bytes: str +class _PreludeLength(BaseModel): + total_length: int + + @dataclass(frozen=True, slots=True) class BedrockEvent: payload: str @@ -41,9 +45,24 @@ def _decode_event(event: EventStreamMessage) -> BedrockEvent: return BedrockEvent(payload=_JSON.dump_json({headers.event_type: _JSON.validate_json(event.payload)}).decode()) -def decode_bedrock_stream(chunks: Iterable[bytes]) -> Iterator[BedrockEvent]: - buffer: Final = EventStreamBuffer() - for chunk in chunks: - buffer.add_data(chunk) - for event in buffer: +class _CompleteEventStream: + def __init__(self) -> None: + self._buffer: Final = EventStreamBuffer() + self._pending_bytes: int = 0 + + def feed(self, chunk: bytes) -> Iterator[BedrockEvent]: + self._pending_bytes += len(chunk) + self._buffer.add_data(chunk) + for event in self._buffer: + self._pending_bytes -= _PreludeLength.model_validate(event.prelude, from_attributes=True).total_length yield _decode_event(event) + + def finish(self) -> None: + assert self._pending_bytes == 0, f"incomplete Bedrock frame: {self._pending_bytes} trailing bytes" + + +def decode_bedrock_stream(chunks: Iterable[bytes]) -> Iterator[BedrockEvent]: + stream: Final = _CompleteEventStream() + for chunk in chunks: + yield from stream.feed(chunk) + stream.finish() diff --git a/tests/e2e/llm_translation/test_responses_e2e.py b/tests/e2e/llm_translation/test_responses_e2e.py index dd40e816e14..40558fdb43c 100644 --- a/tests/e2e/llm_translation/test_responses_e2e.py +++ b/tests/e2e/llm_translation/test_responses_e2e.py @@ -280,7 +280,6 @@ class TestResponses: model, "What is the weather in San Francisco? Use the get_weather tool.", [WEATHER_TOOL], - tool_choice="required", ) require_successful_call(result) parsed = ResponsesResult.model_validate_json(result.body) @@ -291,6 +290,32 @@ class TestResponses: arguments = WeatherArguments.model_validate(raw_arguments) assert arguments.location, f"function call arguments missing location: {function_call.arguments}" + @pytest.mark.replayable + @pytest.mark.covers("llm.responses.bedrock_converse.tool_use.nonstream.works") + def test_responses_bedrock_required_tool_returns_function_call( + self, endpoints_client: EndpointsClient, resources: ResourceManager + ) -> None: + model = f"e2e-responses-required-{unique_marker()}" + model_id = endpoints_client.create_model(model, bedrock_params()) + resources.defer(lambda: endpoints_client.delete_model(model_id)) + key = resources.key() + + result = endpoints_client.responses_with_tools( + key, + model, + "What is the weather in San Francisco? Use the get_weather tool.", + [WEATHER_TOOL], + tool_choice="required", + ) + require_successful_call(result) + parsed = ResponsesResult.model_validate_json(result.body) + function_call = next((call for call in parsed.function_calls if call.name == "get_weather"), None) + assert function_call is not None, f"no required get_weather function call over bedrock: {result.body[:500]}" + assert function_call.arguments is not None + raw_arguments = cast(object, json.loads(function_call.arguments)) + arguments = WeatherArguments.model_validate(raw_arguments) + assert arguments.location, f"function call arguments missing location: {function_call.arguments}" + @pytest.mark.skip(reason="stage red: product gap, /v1/responses 500s (aresponses TypeError) on missing input instead of 400") @pytest.mark.covers("llm.responses.openai.input_validation.nonstream.works") def test_missing_input_returns_error( diff --git a/tests/e2e/provider_diagnostics.py b/tests/e2e/provider_diagnostics.py index 63035db8413..d3d336dd638 100644 --- a/tests/e2e/provider_diagnostics.py +++ b/tests/e2e/provider_diagnostics.py @@ -33,7 +33,7 @@ class ProviderUnavailableError(AssertionError): class NetworkFailureError(AssertionError): def __init__(self, failure: ProviderFailure, message: str) -> None: self.failure: Final = failure - super().__init__(f"kind='network'; {failure.model_dump_json(exclude_none=True)}; body={message[:1000]}") + super().__init__(f"HTTP transfer failed; {failure.model_dump_json(exclude_none=True)}; body={message[:1000]}") class _ErrorMessage(BaseModel): diff --git a/tests/e2e/pytest.ini b/tests/e2e/pytest.ini index d488f531660..05ac8f73cb7 100644 --- a/tests/e2e/pytest.ini +++ b/tests/e2e/pytest.ini @@ -2,7 +2,7 @@ # Config when any e2e suite under tests/e2e/ is run directly, e.g. # uv run pytest tests/e2e/quota_management/spend_tracking/ -v # The e2e marker is also registered in conftest.py for runs rooted elsewhere. -addopts = --strict-markers --strict-config --reruns 1 --reruns-delay 10 --only-rerun "kind='network'" --only-rerun "ProviderUnavailableError" +addopts = --strict-markers --strict-config --reruns 1 --only-rerun "^AssertionError: kind='network' message=" --rerun-except "ProviderUnavailableError" --rerun-except "NetworkFailureError" markers = e2e: live test that requires a running proxy and real provider keys replayable: edge-wired test whose provider traffic replays from a fixture bundle, so it makes zero provider calls in replay mode; the record/replay CI lane selects it with -m replayable diff --git a/tests/e2e/test_bedrock_eventstream.py b/tests/e2e/test_bedrock_eventstream.py index 34d2cc216e1..b53cf18876f 100644 --- a/tests/e2e/test_bedrock_eventstream.py +++ b/tests/e2e/test_bedrock_eventstream.py @@ -125,6 +125,27 @@ class TestBedrockEventStream: with pytest.raises(ChecksumMismatch): tuple(decode_bedrock_stream((wire[:-1] + bytes([wire[-1] ^ 1]),))) + @pytest.mark.parametrize("invoke", [False, True]) + @pytest.mark.parametrize("cut", [1, 7, 11, 12, 30, -1]) + def test_incomplete_frame_after_valid_completion_is_rejected(self, invoke: bool, cut: int) -> None: + wire: Final = ( + b"".join( + _frame(json.dumps({"bytes": base64.b64encode(event.encode()).decode()}), "chunk") + for event in _INVOKE + ) + if invoke + else b"".join( + _frame(json.dumps(value), key) + for payload in _CONVERSE + for key, value in _JSON_OBJECT.validate_json(payload).items() + ) + ) + assertion: Final = assert_invoke_stream if invoke else assert_converse_stream + assertion(streaming_outcome(_BinaryResponse((wire,)), True, sent_at=0.0)) + extra: Final = _frame('{"message":"unavailable"}', "serviceUnavailableException", "exception")[:cut] + with pytest.raises(AssertionError, match="incomplete Bedrock frame"): + assertion(streaming_outcome(_BinaryResponse((wire, extra)), True, sent_at=0.0)) + class TestNativeStreamAssertions: @pytest.mark.parametrize("missing", [0, 1, 3, 4]) @@ -180,7 +201,7 @@ def test_interrupted_http_stream_preserves_network_classification_and_ids(path: ) assert result.status_code == 200 and not result.ok assert result.network_error is not None and result.network_error.kind == "network" - with pytest.raises(NetworkFailureError, match="kind='network'") as caught: + with pytest.raises(NetworkFailureError, match="HTTP transfer failed") as caught: require_successful_call(result) assert caught.value.failure.provider is None assert caught.value.failure.request_id == "interrupted-request" diff --git a/tests/e2e/test_provider_retry.py b/tests/e2e/test_provider_retry.py index a064b6aef22..aca1574898c 100644 --- a/tests/e2e/test_provider_retry.py +++ b/tests/e2e/test_provider_retry.py @@ -6,14 +6,19 @@ from pathlib import Path from typing import Final -def test_retry_policy_preserves_failures_and_first_attempt_evidence(tmp_path: Path) -> None: +def test_retry_policy_does_not_turn_new_diagnostics_into_recovered_passes(tmp_path: Path) -> None: suite: Final = Path(__file__).resolve().parent scenario: Final = tmp_path / "test_scenarios.py" scenario.write_text( """ from pathlib import Path import json -from e2e_http import NetworkError, StreamingResponse, UnknownApiError, require_successful_call, unwrap +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from threading import Thread +import pytest +from e2e_http import URL, NoBody, NetworkError, StreamingResponse, UnknownApiError, require_successful_call, send, unwrap +from test_bedrock_eventstream import _CONVERSE, _frame +from llm_translation.bedrock_stream import assert_converse_stream def attempt(name): path = Path(__file__).with_name(name + ".attempts") @@ -71,11 +76,79 @@ def test_stream_persistent(): streamed_unavailable() def test_stream_transport(): - attempt("stream_transport") - require_successful_call(StreamingResponse( - status_code=200, body="connection interrupted", network_error=NetworkError(message="connection interrupted"), - headers={"x-amzn-requestid": "transport-request", "x-litellm-call-id": "transport-call"}, - )) + if attempt("stream_transport") == 1: + require_successful_call(StreamingResponse( + status_code=200, body="connection interrupted", network_error=NetworkError(message="kind='network'"), + headers={"x-amzn-requestid": "transport-request", "x-litellm-call-id": "transport-call"}, + )) + +def test_connection_recovers(): + if attempt("connection_recovers") == 1: + require_successful_call(StreamingResponse( + status_code=-1, body="connection refused", network_error=NetworkError(message="kind='network'"), + )) + +def test_connection_direct_recovers(): + if attempt("connection_direct_recovers") == 1: + result = StreamingResponse(status_code=-1, body="connection refused", network_error=NetworkError(message="connection refused")) + assert result.ok, f"call failed: {result}" + +def test_header_marker_recovers(): + if attempt("header_marker_recovers") == 1: + require_successful_call(StreamingResponse(status_code=403, body="invalid credentials", headers={"x-request-id": "kind='network'"})) + +def test_behavior_recovers(): + assert attempt("behavior_recovers") > 1, "completion returned no content" + +def test_healthy_control(): + require_successful_call(StreamingResponse(status_code=200, body="{}")) + +class InterruptedOnce(BaseHTTPRequestHandler): + def do_POST(self): + self.rfile.read(int(self.headers.get("content-length", "0"))) + binary = self.path.endswith("-binary") + wire = ( + b"".join(_frame(json.dumps(value), key) for payload in _CONVERSE for key, value in json.loads(payload).items()) + if binary else b'data: {"text":"Hello"}\\n\\ndata: [DONE]\\n\\n' + ) + interrupted = attempt("http_" + self.path.lstrip("/")) == 1 and not self.path.startswith("/healthy-") + data = wire[:30] if interrupted else wire + self.send_response(200) + self.send_header("content-type", "application/vnd.amazon.eventstream" if binary else "text/event-stream") + self.send_header("transfer-encoding", "chunked") + self.send_header("x-litellm-call-id", "controlled-disconnect") + self.end_headers() + self.wfile.write(f"{len(data):x}\\r\\n".encode() + data + b"\\r\\n") + if not interrupted: + self.wfile.write(b"0\\r\\n\\r\\n") + self.wfile.flush() + self.close_connection = True + + def log_message(self, *args): + pass + +@pytest.fixture(scope="session") +def endpoint(): + server = ThreadingHTTPServer(("127.0.0.1", 0), InterruptedOnce) + worker = Thread(target=server.serve_forever, daemon=True) + worker.start() + yield f"http://127.0.0.1:{server.server_port}" + server.shutdown() + server.server_close() + worker.join(timeout=5) + +@pytest.mark.parametrize("path", ["binary", "sse"]) +@pytest.mark.parametrize("mode", ["helper", "direct", "healthy"]) +def test_http_stream_would_recover_on_a_second_request(endpoint, path, mode): + result = send(URL(f"{endpoint}/{mode}-{path}"), headers=NoBody(), json=NoBody(), stream=True, timeout=5) + if mode == "direct": + assert result.ok, f"stream was not established: {result}" + else: + require_successful_call(result) + if path == "binary": + assert_converse_stream(result) + else: + assert result.stream_done """, encoding="utf-8", ) @@ -107,26 +180,36 @@ def test_stream_transport(): check=False, ) assert result.returncode == 1, result.stdout + result.stderr - assert "7 failed, 2 passed, 7 rerun" in result.stdout, result.stdout + assert "17 failed, 3 passed, 1 rerun" in result.stdout, result.stdout assert {path.stem: int(path.read_text()) for path in tmp_path.glob("*.attempts")} == { - "recovers": 2, - "persistent": 2, + "recovers": 1, + "persistent": 1, "proxy_503": 1, "auth": 1, "network": 2, - "native": 2, - "stream_recovers": 2, - "stream_persistent": 2, - "stream_transport": 2, + "native": 1, + "stream_recovers": 1, + "stream_persistent": 1, + "stream_transport": 1, + "connection_recovers": 1, + "behavior_recovers": 1, + "connection_direct_recovers": 1, + "header_marker_recovers": 1, + "http_helper-binary": 1, + "http_helper-sse": 1, + "http_direct-binary": 1, + "http_direct-sse": 1, + "http_healthy-binary": 1, + "http_healthy-sse": 1, } cases: Final = ET.parse(tmp_path / "results.xml").getroot() recovered: Final = cases.find(".//testcase[@name='test_recovers']") assert recovered is not None - assert recovered.find("failure") is None + assert recovered.find("failure") is not None assert recovered.find("properties/property[@name='upstream_request_id'][@value='aws-first-attempt']") is not None assert recovered.find("properties/property[@name='upstream_call_id'][@value='proxy-first-attempt']") is not None streamed: Final = cases.find(".//testcase[@name='test_stream_recovers']") - assert streamed is not None and streamed.find("failure") is None + assert streamed is not None and streamed.find("failure") is not None assert streamed.find("properties/property[@name='upstream_status_code'][@value='200']") is not None assert streamed.find("properties/property[@name='upstream_evidence'][@value='stream_event']") is not None assert streamed.find("properties/property[@name='upstream_request_id'][@value='stream-request']") is not None