This commit is contained in:
yuneng-jiang 2026-09-14 23:47:53 +08:00 • committed by GitHub
commit 5ebd71a103
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
24 changed files with 1432 additions and 93 deletions

View file

@ -14,7 +14,7 @@ trap 'rm -rf "${WORKDIR}"' EXIT
echo "resolving newest non-expired '${ARTIFACT_NAME}' artifact on ${REPO}@${BASE_BRANCH}"
SELECTED="$(
gh api "repos/${REPO}/actions/artifacts" -X GET -f per_page=100 --paginate \
gh api "repos/${REPO}/actions/artifacts" -X GET -f name="${ARTIFACT_NAME}" -f per_page=100 --paginate \
--jq ".artifacts[] | select(.name == \"${ARTIFACT_NAME}\" and .expired == false and .workflow_run.head_branch == \"${BASE_BRANCH}\") | {id, digest, created_at, run_id: .workflow_run.id, run_number: .workflow_run.run_number}" \
| jq -s 'sort_by(.created_at) | reverse | .[0] // empty'
)"

View file

@ -26,7 +26,20 @@ jobs:
(github.event_name == 'workflow_dispatch' && github.event.inputs.mode == 'record'))
runs-on: ubuntu-latest
timeout-minutes: 45
environment: e2e-changed
permissions:
contents: read
id-token: write
services:
redis:
image: redis:7.4.2@sha256:fbdbaea47b9ae4ecc2082ecdb4e1cea81e32176ffb1dcf643d422ad07427e5d9
ports:
- 6379:6379
options: >-
--health-cmd "redis-cli ping"
--health-interval 5s
--health-timeout 5s
--health-retries 10
postgres:
image: postgres:16.6
env:
@ -44,8 +57,7 @@ jobs:
DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
LITELLM_MASTER_KEY: sk-e2e-record-replay
LITELLM_LOCAL_MODEL_COST_MAP: "True"
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
E2E_BEDROCK_REGION: us-east-1
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
@ -54,7 +66,7 @@ jobs:
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
python-version: "3.13"
- name: Set up uv
uses: ./.github/actions/setup-uv-with-retries
@ -66,7 +78,7 @@ jobs:
- name: Install dependencies
run: |
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --group e2e-dev --extra proxy --extra proxy-runtime
- name: Cache Prisma binaries
uses: ./.github/actions/cache-prisma-binaries
@ -75,9 +87,52 @@ jobs:
run: |
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
- name: Validate recording configuration
env:
ROLE: ${{ vars.E2E_AWS_ROLE_TO_ASSUME }}
run: test -n "${ROLE}" || { echo "::error::Set E2E_AWS_ROLE_TO_ASSUME to the existing e2e secrets reader role"; exit 1; }
- name: Configure the existing e2e secrets reader
id: aws
uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0
with:
role-to-assume: ${{ vars.E2E_AWS_ROLE_TO_ASSUME }}
aws-region: us-east-1
role-session-name: litellm-e2e-record-${{ github.run_id }}
role-duration-seconds: 900
output-env-credentials: false
output-credentials: true
- name: Fetch provider credentials from AWS Secrets Manager
env:
AWS_ACCESS_KEY_ID: ${{ steps.aws.outputs.aws-access-key-id }}
AWS_SECRET_ACCESS_KEY: ${{ steps.aws.outputs.aws-secret-access-key }}
AWS_SESSION_TOKEN: ${{ steps.aws.outputs.aws-session-token }}
AWS_DEFAULT_REGION: us-east-1
run: |
umask 077
aws secretsmanager get-secret-value --secret-id litellm-e2e-changed-provider-keys \
--query SecretString --output text \
| uv run --no-sync python .github/e2e-stack/secrets_to_env.py tests/e2e/.env
- name: Require all recording credentials before starting the proxy
run: |
PYTHONPATH=tests/e2e uv run --no-sync --env-file tests/e2e/.env python - <<'PY'
import os
from bedrock_edge import recording_credentials
required = ("OPENAI_API_KEY", "ANTHROPIC_API_KEY", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY")
missing = [key for key in required if not os.environ.get(key)]
if missing:
raise SystemExit("Missing recording credentials: " + ", ".join(missing))
recording_credentials().get_frozen_credentials()
PY
- name: Start the proxy
run: |
nohup uv run --no-sync litellm --config tests/e2e/gateway/record_replay_ci_config.yml --port 4000 > proxy.log 2>&1 &
nohup uv run --no-sync --env-file tests/e2e/.env env -u AWS_ROLE_NAME -u AWS_SESSION_NAME -u AWS_PROFILE_NAME \
-u AWS_WEB_IDENTITY_TOKEN -u AWS_SESSION_TOKEN -u AWS_BEARER_TOKEN_BEDROCK \
AWS_ACCESS_KEY_ID=e2e-bedrock-replay AWS_SECRET_ACCESS_KEY=e2e-bedrock-replay-secret \
AWS_EC2_METADATA_DISABLED=true litellm --config tests/e2e/gateway/record_replay_ci_config.yml --port 4000 > proxy.log 2>&1 &
for _ in $(seq 1 90); do
if curl -fs http://localhost:4000/health/liveliness > /dev/null; then
exit 0
@ -124,6 +179,15 @@ jobs:
contents: read
actions: read
services:
redis:
image: redis:7.4.2@sha256:fbdbaea47b9ae4ecc2082ecdb4e1cea81e32176ffb1dcf643d422ad07427e5d9
ports:
- 6379:6379
options: >-
--health-cmd "redis-cli ping"
--health-interval 5s
--health-timeout 5s
--health-retries 10
postgres:
image: postgres:16.6
env:
@ -144,6 +208,11 @@ jobs:
GH_TOKEN: ${{ github.token }}
OPENAI_API_KEY: sk-replay-must-never-reach-a-provider
ANTHROPIC_API_KEY: sk-ant-replay-must-never-reach-a-provider
AWS_ACCESS_KEY_ID: e2e-bedrock-replay
AWS_SECRET_ACCESS_KEY: e2e-bedrock-replay-secret
AWS_REGION: us-east-1
E2E_BEDROCK_REGION: us-east-1
AWS_EC2_METADATA_DISABLED: "true"
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
@ -152,7 +221,7 @@ jobs:
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
python-version: "3.13"
- name: Set up uv
uses: ./.github/actions/setup-uv-with-retries
@ -164,7 +233,7 @@ jobs:
- name: Install dependencies
run: |
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --group e2e-dev --extra proxy --extra proxy-runtime
- name: Cache Prisma binaries
uses: ./.github/actions/cache-prisma-binaries
@ -185,7 +254,10 @@ jobs:
- name: Start the proxy
run: |
nohup uv run --no-sync litellm --config tests/e2e/gateway/record_replay_ci_config.yml --port 4000 > proxy.log 2>&1 &
nohup uv run --no-sync env -u AWS_ROLE_NAME -u AWS_SESSION_NAME -u AWS_PROFILE_NAME \
-u AWS_WEB_IDENTITY_TOKEN -u AWS_SESSION_TOKEN -u AWS_BEARER_TOKEN_BEDROCK \
AWS_ACCESS_KEY_ID=e2e-bedrock-replay AWS_SECRET_ACCESS_KEY=e2e-bedrock-replay-secret \
AWS_EC2_METADATA_DISABLED=true litellm --config tests/e2e/gateway/record_replay_ci_config.yml --port 4000 > proxy.log 2>&1 &
for _ in $(seq 1 90); do
if curl -fs http://localhost:4000/health/liveliness > /dev/null; then
exit 0
@ -202,6 +274,10 @@ jobs:
sudo python3 .github/scripts/e2e_egress_sentinel.py serve \
--host api.openai.com \
--host api.anthropic.com \
--host bedrock-runtime.us-east-1.amazonaws.com \
--host bedrock-runtime.us-east-2.amazonaws.com \
--host bedrock-runtime.us-west-1.amazonaws.com \
--host bedrock-runtime.us-west-2.amazonaws.com \
--hits-file "${RUNNER_TEMP}/egress-hits.jsonl" \
--ready-file "${RUNNER_TEMP}/egress-ready" \
--pid-file "${RUNNER_TEMP}/egress.pid" \

View file

@ -77,7 +77,7 @@ Mark live tests with `@pytest.mark.e2e` (on the class or the module). Use `scope
The seam is `provider_edge.py`: `start_provider_edge` boots an in-process HTTP server (one shared instance per pytest process, `e2e_config.provider_edge_base` is the accessor) that mounts each supported provider under a path prefix (`EDGE_MOUNTS`: `/openai` -> `https://api.openai.com`, `/anthropic` -> `https://api.anthropic.com`). A test participates by registering its deployment with `api_base=provider_edge_base("openai")` plus the provider's path suffix; `quota_management/spend_tracking/test_provider_edge_spend_e2e.py` is the reference. In live mode the accessor returns None and the deployment defaults to the real provider, so an edge-wired test runs in all three modes unchanged. Non-wired tests hit their providers live in every mode. The edge binds `E2E_PROVIDER_EDGE_BIND_HOST` (default 127.0.0.1) and advertises `E2E_PROVIDER_EDGE_ADVERTISE_HOST` in the api_base it hands out, for proxies running in containers
A bundle (default `tests/e2e/.fixtures`, override with `E2E_FIXTURE_DIR`) is a directory: `manifest.json` carries the record timestamp, harness git version, and format version, and each test gets a subdirectory holding one JSON file per provider call in call order (`0000-post-openai-v1-chat-completions.json`). Request headers are never stored (provider credentials never touch disk), non-JSON request bodies store a canonicalized sha256 digest instead of the bytes, `multipart/form-data` bodies store their ordinary fields plus a JSON list of the uploaded parts' `[field, filename, content-type]` triples and a digest of their content, so the per-request random boundary and the envelope never reach the key, and responses store status, filtered headers, and the verbatim body base64-encoded, which is part of why bundles are gitignored. Responses come in two shapes told apart by a `kind` tag: an ordinary one holding a single base64 body, and, for a response the provider streamed (`content-type: text/event-stream`), one holding its transfer chunks in order plus why the stream ended early if it did, so replay reproduces the split points the provider chose instead of one coalesced body. `fixture_bundle.py` owns the format, and `BUNDLE_FORMAT_VERSION` is checked on load, so a bundle recorded under older rules is refused by name rather than partially read. Record serves the proxy the same filtered stored response replay will serve later, chunk for chunk on a stream, so the two modes are byte-identical from the proxy's side of the socket. Replay does not reproduce the provider's inter-chunk timing (chunks go out as fast as the socket takes them), so a test that judges streaming on the clock, such as the `stream_event_arrivals` lead between the first content delta and `message_stop`, gates that assertion on `provider_paces_stream()` and proves only the event grammar in replay
A bundle (default `tests/e2e/.fixtures`, override with `E2E_FIXTURE_DIR`) is a directory: `manifest.json` carries the record timestamp, harness git version, and format version, and each test gets a subdirectory holding one JSON file per provider call in call order (`0000-post-openai-v1-chat-completions.json`). Request headers are never stored (provider credentials never touch disk), non-JSON request bodies store a canonicalized sha256 digest instead of the bytes, `multipart/form-data` bodies store their ordinary fields plus a JSON list of the uploaded parts' `[field, filename, content-type]` triples and a digest of their content, so the per-request random boundary and the envelope never reach the key, and responses store status, filtered headers, and the verbatim body base64-encoded, which is part of why bundles are gitignored. Responses come in two shapes told apart by a `kind` tag: an ordinary one holding a single base64 body, and, for a response the provider streamed (`content-type: text/event-stream` or `application/vnd.amazon.eventstream`), one holding its transfer chunks in order plus why the stream ended early if it did, so replay reproduces the split points the provider chose instead of one coalesced body. `fixture_bundle.py` owns the format, and `BUNDLE_FORMAT_VERSION` is checked on load, so a bundle recorded under older rules is refused by name rather than partially read. Record serves the proxy the same filtered stored response replay will serve later, chunk for chunk on a stream, so the two modes are byte-identical from the proxy's side of the socket. Replay does not reproduce the provider's inter-chunk timing (chunks go out as fast as the socket takes them), so a test that judges streaming on the clock, such as the `stream_event_arrivals` lead between the first content delta and `message_stop`, gates that assertion on `provider_paces_stream()` and proves only the event grammar in replay
Multipart identity is the fiddly corner, and the rules exist because each one had a collision behind it. A part counts as an upload when it carries a filename or declares its own content type, and everything else is an ordinary field. Field names get a `name[n]` suffix on repeats, with a literal `[` doubled first, so a form that repeats `purpose` never keys the same as one that literally sends `purpose[1]`. A field whose name reads as a credential is stored as `<secret>`, which stays key-preserving because the key is recomputed from the stored request rather than saved alongside it, so the live request carrying the real value still matches its redacted fixture. A field value that is not UTF-8 is stored as a base64 sha256 digest, base64 and not hex because the canonicalizer rewrites any 64-character hex run to `<sha256>` and would fold every binary value onto one key. The uploaded parts contribute a JSON list rather than a `field:filename` string, so a separator inside a filename cannot impersonate a field boundary, and their byte length is stored for a reader's benefit but deliberately left out of the key, since the canonicalizer absorbs timestamp and id drift inside a file that changes its length
@ -96,7 +96,13 @@ E2E_FIXTURE_MODE=replay E2E_FIXTURE_DIR=/tmp/e2e-fixtures E2E_RESET_SPEND_LOGS=1
Point the proxy at bogus provider credentials for the replay run and it still has to pass: that is the whole proof that nothing left the process. Bundles are never committed. `tests/e2e/.fixtures` is gitignored because a bundle holds verbatim provider response bodies and hard-fails after seven days. CI records and replays this lane on a schedule in `.github/workflows/e2e_record_replay.yml`, publishing the bundle as a private `e2e-fixtures-bundle` artifact instead of committing it, selecting the tests with the `@pytest.mark.replayable` marker, and proving the bogus-credentials replay hermetic by counting provider egress with `.github/scripts/e2e_egress_sentinel.py`
Current limits: Bedrock cannot be mounted (SigV4 signs the Host header, so a rewritten api_base fails signature verification), deployments baked into the proxy's config file cannot be edge-wired (only `/model/new` registrations can carry the edge api_base), and a file upload routed by `custom_llm_provider` through the proxy's `files_settings` block never passes a deployment at all, so the batches `model_param` and `provider_fallback` scenarios keep uploading live in every mode
Bedrock tests use `bedrock_config.bedrock_params()` for Chat Completions, Responses, Converse and Invoke. Live mode retains the normal AWS credential path. Record and replay use a `/bedrock-us-east-1` mount by default (`E2E_BEDROCK_REGION` selects one of `us-east-1`, `us-east-2`, `us-west-1`, `us-west-2`; use the same region in both runs). The proxy signs each edge request with public test credentials. The edge verifies that signature over the host, path and body before accepting it. Recording re-signs the unchanged payload for the AWS endpoint using the runner's `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and optional `AWS_SESSION_TOKEN`, assuming `AWS_ROLE_NAME` with the SDK when configured. Start the proxy without ambient `AWS_ROLE_NAME`, `AWS_PROFILE_NAME`, web identity or Bedrock bearer credentials in these modes so its public test credentials cannot trigger STS or bypass signature verification. The workflow isolates that proxy environment while retaining real credentials and role settings in the recording runner. Replay requires no real AWS credentials and never forwards a miss. This checks request signing and protocol behavior in replay; live runs remain necessary to check real IAM access and model availability. Binary AWS EventStream responses preserve transfer chunks in the bundle, and native stream assertions decode events with botocore and require complete frames, text, ordered completion and usage. The original Responses tool test keeps default tool selection, and a separate test covers `tool_choice="required"`
The recording workflow uses the existing `e2e-changed` environment and OIDC secrets reader (`E2E_AWS_ROLE_TO_ASSUME`, secret `litellm-e2e-changed-provider-keys`), and fails before starting the proxy if any required provider credential is missing. Recording requires approval under that environment's existing protection policy, including scheduled runs. Replay supplies only test credentials and includes the four supported Bedrock endpoints in the egress sentinel
HTTP assertions classify a Bedrock 503 as `ProviderUnavailableError` only when AWS headers with Bedrock route context, or a recognized Bedrock error envelope support that attribution. A decoded native `serviceUnavailableException` receives the same classification even when the committed HTTP status is 200, with `stream_event` recorded as its evidence. Interrupted binary or SSE transfers produce `NetworkFailureError` with the received status and request IDs preserved. Both diagnostic exceptions hard-fail without a test rerun, including when their messages contain a legacy retry marker. Incomplete binary frames hard-fail even after a valid completion event. The existing typed `NetworkError` retry remains, with its filter anchored to that assertion rather than matching a marker embedded in another response, header or assertion message. No newly classified failure becomes eligible for a rerun. The exercised proxy 5xx, authentication and behavior failures do not qualify. This retains the existing text-based retry convention: an arbitrary assertion that deliberately imitates the exact typed network-error prefix can still match. JUnit retains AWS and proxy request IDs from the failed attempt. Record and replay continue to use `--reruns 0`
Current limits: deployments baked into the proxy's config file cannot be edge-wired (only `/model/new` registrations can carry the edge api_base), and a file upload routed by `custom_llm_provider` through the proxy's `files_settings` block never passes a deployment at all, so the batches `model_param` and `provider_fallback` scenarios keep uploading live in every mode
## Typing

View file

@ -101,7 +101,7 @@ A couple of logging destinations are configured on the proxy rather than by the
### The pull request check
Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite and both JWT suites as canaries, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Keycloak, Jaeger, and TLS cluster-mode Valkey. Realm-only edits also trigger these canaries. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, and `guardrails/test_presidio_masking_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start. The Redis chaos test under `load/` needs a proxy it can pause the Redis of on the same host (`gateway/redis_chaos_ci_config.yml`), which `.github/workflows/test-e2e-redis-chaos.yml` boots, and which the Buildkite `e2e-redis-chaos` step in project-releaser runs co-located with Postgres and Valkey in one pod; it is deselected unless `E2E_REDIS_CHAOS` is set
Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite and both JWT suites as canaries, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's existing single rerun for typed network results applies on every pass (see `pytest.ini`). Bedrock availability diagnostics and interrupted HTTP streams hard-fail without a rerun. The exercised proxy 5xx and behavior failures do not qualify either. The legacy filter matches assertion text, so custom assertions must not imitate the exact typed network-error prefix. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Keycloak, Jaeger, and TLS cluster-mode Valkey. Realm-only edits also trigger these canaries. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, and `guardrails/test_presidio_masking_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start. The Redis chaos test under `load/` needs a proxy it can pause the Redis of on the same host (`gateway/redis_chaos_ci_config.yml`), which `.github/workflows/test-e2e-redis-chaos.yml` boots, and which the Buildkite `e2e-redis-chaos` step in project-releaser runs co-located with Postgres and Valkey in one pod; it is deselected unless `E2E_REDIS_CHAOS` is set
Every selected file must execute at least one passing test in each pass, and any test failure, collection error, or entirely skipped or deselected file fails the check. A file whose tests are all marked skip therefore cannot pass this check, so unskip at least one of them, or add the file to `UNSUPPORTED` in `select_tests.py` with the reason, before changing one. A failed pass stops the run. The public log prints pytest's one-line summary for each pass, including the rerun count, and names each failed or errored test as `classname::name`, so a retried network error or a failing test is visible without the raw output. The final `e2e-changed-tests` job succeeds only when no supported test files changed or the approved run completed all three passes. Fork PRs with selected tests fail this gate until a maintainer brings the reviewed change onto a same-repository branch
@ -130,7 +130,7 @@ One sharp edge: a replayed response reuses the recorded provider response id, an
Another sharp edge, same root: record and replay derive every per-test token deterministically (the model name included, so a replay regenerates the exact requests the record run sent), which means an edge-wired deployment left in the database by an interrupted earlier run carries the same model name as the fresh one the current run registers. The proxy then holds two deployments under one model group and load-balances across both, and because the leftover's `api_base` points at the earlier run's edge process, which is gone, the calls that land on it fail with a connection error that reads like a transport bug rather than the stale row it is. Give each record or replay run a fresh database, or let a run finish so its own teardown deletes what it registered, and never reuse one long-lived proxy across back-to-back record/replay sessions. CI hands every job its own empty database and its own proxy, so it never sees this
Replay answers any provider call that drifted from the recording with an HTTP 599 whose body names the computed and closest recorded keys, so the test fails loudly instead of silently going live, and a bundle older than seven days fails at collection time naming its age; either way the fix is to re-record. Only tests that register edge-wired deployments participate: everything else hits its provider live in every mode, so record exactly the suite you replay. If the proxy runs in a container, set `E2E_PROVIDER_EDGE_ADVERTISE_HOST` (e.g. `host.docker.internal`) so the api_base the proxy stores can reach the edge on the pytest host, and `E2E_PROVIDER_EDGE_BIND_HOST=0.0.0.0` so the edge accepts it. The suites wired to the edge today are `quota_management/spend_tracking/test_provider_edge_spend_e2e.py`, `llm_translation/test_chat_completions_contract_e2e.py`, the OpenAI registrations in `llm_translation/test_embeddings_endpoint_e2e.py`, the Anthropic tests in `llm_translation/test_messages_e2e.py`, streamed and not, and the OpenAI batch deployment behind `batches/`. A streamed response replays as the chunk sequence the provider sent rather than one buffered body. See `CLAUDE.md` in this directory for the bundle format, the edge design, and the current limits (Bedrock). The scheduled CI record/replay lane is described above
Replay answers any provider call that drifted from the recording with an HTTP 599 whose body names the computed and closest recorded keys, so the test fails loudly instead of silently going live, and a bundle older than seven days fails at collection time naming its age; either way the fix is to re-record. Only tests that register edge-wired deployments participate: everything else hits its provider live in every mode, so record exactly the suite you replay. If the proxy runs in a container, set `E2E_PROVIDER_EDGE_ADVERTISE_HOST` (e.g. `host.docker.internal`) so the api_base the proxy stores can reach the edge on the pytest host, and `E2E_PROVIDER_EDGE_BIND_HOST=0.0.0.0` so the edge accepts it. The suites wired to the edge today are `quota_management/spend_tracking/test_provider_edge_spend_e2e.py`, `llm_translation/test_chat_completions_contract_e2e.py`, the OpenAI registrations in `llm_translation/test_embeddings_endpoint_e2e.py`, the Anthropic tests in `llm_translation/test_messages_e2e.py`, streamed and not, the OpenAI batch deployment behind `batches/`, and the Bedrock Chat Completions, Responses and native Converse/Invoke tests. Bedrock recording requires AWS credentials in the runner; replay validates signatures made with public test credentials. Use the same `E2E_BEDROCK_REGION` in both modes (default `us-east-1`). A streamed response replays as the chunk sequence the provider sent rather than one buffered body. See `CLAUDE.md` in this directory for the bundle format, the edge design, and the remaining limits. The scheduled CI record/replay lane is described above
Tests marked `@pytest.mark.e2e` hard-fail when no proxy answers `/health/liveliness`, so a run that goes red with `No live proxy` at setup means the proxy isn't up; they never skip for a missing proxy, so an absent proxy can't be mistaken for a pass

View file

@ -0,0 +1,29 @@
import os
from typing import Final
from bedrock_edge import TEST_ACCESS_KEY, TEST_SECRET_KEY, recording_credentials
from e2e_config import FIXTURE_MODE_RAW, provider_edge_base
from fixture_mode import parse_fixture_mode
from models import LiteLLMParamsBody
BEDROCK_BACKEND: Final = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"
def bedrock_params() -> LiteLLMParamsBody:
if parse_fixture_mode(FIXTURE_MODE_RAW) == "live":
return LiteLLMParamsBody(
model=BEDROCK_BACKEND,
aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID",
aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY",
aws_region_name="os.environ/AWS_REGION",
)
if parse_fixture_mode(FIXTURE_MODE_RAW) == "record":
recording_credentials()
region: Final = os.environ.get("E2E_BEDROCK_REGION", "us-east-1")
return LiteLLMParamsBody(
model=BEDROCK_BACKEND,
api_base=provider_edge_base(f"bedrock-{region}"),
aws_access_key_id=TEST_ACCESS_KEY,
aws_secret_access_key=TEST_SECRET_KEY,
aws_region_name=region,
)

103
tests/e2e/bedrock_edge.py Normal file
View file

@ -0,0 +1,103 @@
import hashlib
import hmac
import os
import re
from collections.abc import Callable, Mapping
from functools import lru_cache
from typing import Final
from botocore.auth import SigV4Auth
from botocore.awsrequest import AWSRequest
from botocore.client import BaseClient
from botocore.credentials import AssumeRoleCredentialFetcher, Credentials, DeferredRefreshableCredentials
from botocore.session import get_session
TEST_ACCESS_KEY: Final = "e2e-bedrock-replay"
TEST_SECRET_KEY: Final = "e2e-bedrock-replay-secret"
_MOUNT: Final = re.compile(r"bedrock-(us-(?:east|west)-[12])")
_AUTH: Final = re.compile(
r"AWS4-HMAC-SHA256 Credential=([^/]+)/([0-9]{8})/([^/]+)/bedrock/aws4_request, "
r"SignedHeaders=([a-z0-9;-]+), Signature=([0-9a-f]{64})"
)
def bedrock_region(mount: str) -> str | None:
match: Final = _MOUNT.fullmatch(mount)
return match.group(1) if match else None
def bedrock_upstream(mount: str) -> str | None:
region: Final = bedrock_region(mount)
return f"https://bedrock-runtime.{region}.amazonaws.com" if region else None
def valid_bedrock_signature(
method: str, raw_path: str, headers: Mapping[str, str], body: bytes | None, region: str
) -> bool:
normalized: Final = {key.lower(): value for key, value in headers.items()}
auth: Final = _AUTH.fullmatch(normalized.get("authorization", ""))
if auth is None or auth.group(1) != TEST_ACCESS_KEY or auth.group(3) != region:
return False
timestamp: Final = normalized.get("x-amz-date", "")
signed: Final = frozenset(auth.group(4).split(";"))
if not {"host", "x-amz-date"} <= signed or not signed <= normalized.keys() or auth.group(2) != timestamp[:8]:
return False
payload_hash: Final = normalized.get("x-amz-content-sha256")
if payload_hash is not None and payload_hash != hashlib.sha256(body or b"").hexdigest():
return False
request: Final = AWSRequest(
method=method,
url=f"http://{normalized['host']}{raw_path}",
data=body,
headers={key: normalized[key] for key in signed},
)
request.context["timestamp"] = timestamp
signer: Final = SigV4Auth(Credentials(TEST_ACCESS_KEY, TEST_SECRET_KEY), "bedrock", region)
canonical: Final = signer.canonical_request(request)
expected: Final = signer.signature(signer.string_to_sign(request, canonical), request)
return hmac.compare_digest(expected, auth.group(5))
def recording_credentials_from(environ: Mapping[str, str], client_creator: Callable[..., BaseClient]) -> Credentials:
access_key: Final = environ.get("AWS_ACCESS_KEY_ID", "")
secret_key: Final = environ.get("AWS_SECRET_ACCESS_KEY", "")
if not access_key or not secret_key:
raise ValueError("Bedrock recording requires AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in the runner")
source: Final = Credentials(access_key, secret_key, environ.get("AWS_SESSION_TOKEN") or None)
role: Final = environ.get("AWS_ROLE_NAME")
if not role:
return source
extra_args: Final = {
key: value
for key, value in (
("RoleSessionName", environ.get("AWS_SESSION_NAME", "e2e-bedrock-record")),
("ExternalId", environ.get("AWS_EXTERNAL_ID")),
)
if value
}
fetcher: Final = AssumeRoleCredentialFetcher(
client_creator=client_creator,
source_credentials=source,
role_arn=role,
extra_args=extra_args,
)
return DeferredRefreshableCredentials(refresh_using=fetcher.fetch_credentials, method="assume-role")
@lru_cache(maxsize=1)
def recording_credentials() -> Credentials:
return recording_credentials_from(os.environ, get_session().create_client)
def sign_bedrock_forward(
method: str, url: str, headers: Mapping[str, str], body: bytes | None, region: str, credentials: Credentials
) -> dict[str, str]:
forwarded: Final = {
key: value
for key, value in headers.items()
if key.lower() not in {"authorization", "host", "content-length", "connection", "accept-encoding"}
and not key.lower().startswith("x-amz-")
}
request: Final = AWSRequest(method=method, url=url, data=body, headers=forwarded)
SigV4Auth(credentials.get_frozen_credentials(), "bedrock", region).add_auth(request)
return dict(request.headers.items())

View file

@ -0,0 +1,68 @@
import base64
from collections.abc import Iterable, Iterator
from dataclasses import dataclass
from typing import Final
from botocore.eventstream import EventStreamBuffer, EventStreamMessage
from pydantic import BaseModel, Field, JsonValue, TypeAdapter
class _EventHeaders(BaseModel):
message_type: str = Field(alias=":message-type")
event_type: str = Field(default="", alias=":event-type")
exception_type: str = Field(default="", alias=":exception-type")
class _InvokeChunk(BaseModel):
bytes: str
class _PreludeLength(BaseModel):
total_length: int
@dataclass(frozen=True, slots=True)
class BedrockEvent:
payload: str
error: str | None = None
error_code: str | None = None
_JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
def _decode_event(event: EventStreamMessage) -> BedrockEvent:
headers: Final = _EventHeaders.model_validate(event.headers)
if headers.message_type != "event":
return BedrockEvent(
payload=event.payload.decode(),
error=f"{headers.exception_type or headers.message_type}: {event.payload.decode()}",
error_code=headers.exception_type or headers.message_type,
)
if headers.event_type == "chunk":
encoded: Final = _InvokeChunk.model_validate_json(event.payload)
return BedrockEvent(payload=base64.b64decode(encoded.bytes, validate=True).decode())
return BedrockEvent(payload=_JSON.dump_json({headers.event_type: _JSON.validate_json(event.payload)}).decode())
class _CompleteEventStream:
def __init__(self) -> None:
self._buffer: Final = EventStreamBuffer()
self._pending_bytes: int = 0
def feed(self, chunk: bytes) -> Iterator[BedrockEvent]:
self._pending_bytes += len(chunk)
self._buffer.add_data(chunk)
for event in self._buffer:
self._pending_bytes -= _PreludeLength.model_validate(event.prelude, from_attributes=True).total_length
yield _decode_event(event)
def finish(self) -> None:
assert self._pending_bytes == 0, f"incomplete Bedrock frame: {self._pending_bytes} trailing bytes"
def decode_bedrock_stream(chunks: Iterable[bytes]) -> Iterator[BedrockEvent]:
stream: Final = _CompleteEventStream()
for chunk in chunks:
yield from stream.feed(chunk)
stream.finish()

View file

@ -29,6 +29,7 @@ from idp import Identity, Keycloak, keycloak_from_env
from junit_properties import attach_result_properties
from lifecycle import ProxyClientProvider, ResourceManager
from models import TeamNewBody, UserNewBody, UserNewResponse
from provider_diagnostics import NetworkFailureError, ProviderUnavailableError
from provider_edge import replay_leftover_error
from proxy_client import ProxyClient, build_proxy_client
@ -182,6 +183,10 @@ def pytest_runtest_makereport(
report = yield
if report.when == "call":
item.stash[_CALL_PASSED] = report.passed
if call.excinfo is not None and isinstance(call.excinfo.value, (ProviderUnavailableError, NetworkFailureError)):
failure: Final = call.excinfo.value.failure
item.user_properties.extend(failure.junit_properties())
report.user_properties = list(item.user_properties)
return report

View file

@ -22,6 +22,8 @@ from typing import Final, Generator, Generic, Iterator, Literal, NewType, Protoc
import pytest
import requests
from bedrock_eventstream import decode_bedrock_stream
from provider_diagnostics import NetworkFailureError, provider_failure, raise_if_provider_unavailable
from pydantic import BaseModel, ConfigDict, Field
URL = NewType("URL", str)
@ -106,15 +108,11 @@ class UnknownApiError(BaseModel):
kind: Literal["unknown"] = "unknown"
status_code: int
body: str
headers: dict[str, str] = {}
type Result[R: BaseModel] = (
Success[R]
| NetworkError
| UnauthorizedError
| RateLimitedError
| ValidationError
| UnknownApiError
Success[R] | NetworkError | UnauthorizedError | RateLimitedError | ValidationError | UnknownApiError
)
@ -167,11 +165,13 @@ class StreamingResponse(BaseModel):
# quota) arrive as SSE error events inside an otherwise-successful response;
# the consumed body is elided, so this is the only place they surface.
stream_error: str | None = None
stream_error_code: str | None = None
network_error: NetworkError | None = None
stream_done: bool = False
@property
def ok(self) -> bool:
return 200 <= self.status_code < 300
return 200 <= self.status_code < 300 and self.network_error is None
@property
def is_streaming(self) -> bool:
@ -216,6 +216,8 @@ class SseResponse(Protocol):
def iter_lines(self) -> Iterator[bytes]: ...
def iter_content(self, chunk_size: int | None = 1) -> Iterator[bytes]: ...
def _hdr(resp: SseResponse, name: str) -> str | None:
value = resp.headers.get(name)
@ -226,6 +228,9 @@ def unwrap[R: BaseModel](result: Result[R]) -> R:
match result:
case Success(data=data):
return data
case UnknownApiError(status_code=status, body=body, headers=headers):
raise_if_provider_unavailable(status, body, headers)
raise AssertionError(result)
case _:
raise AssertionError(result)
@ -238,6 +243,9 @@ def unwrap_status[R: BaseModel](result: Result[R], expected_status: int) -> R:
return data
case Success(status_code=status_code):
raise AssertionError(f"expected HTTP {expected_status}, got {status_code}")
case UnknownApiError(status_code=status, body=body, headers=headers):
raise_if_provider_unavailable(status, body, headers)
raise AssertionError(result)
case _:
raise AssertionError(result)
@ -250,13 +258,24 @@ def is_ok[R: BaseModel](result: Result[R]) -> bool:
return False
def require_successful_call(result: StreamingResponse) -> None:
def require_successful_call(result: StreamingResponse, *, expected_provider: Literal["bedrock"] | None = None) -> None:
"""A call that should have succeeded but didn't is a hard failure, never a skip:
if the proxy can't make a call it's expected to, the test must fail."""
if result.network_error is not None:
raise NetworkFailureError(
provider_failure(result.status_code, "", result.headers), result.network_error.message
)
raise_if_provider_unavailable(
result.status_code,
result.stream_error or result.body,
result.headers,
expected_provider=expected_provider,
stream_error_code=result.stream_error_code,
)
if result.ok:
return
pytest.fail(
f"upstream call failed (status {result.status_code}); body={result.body[:300]}"
f"upstream call failed (status {result.status_code}); headers={result.headers}; body={result.body[:1000]}"
)
@ -271,6 +290,7 @@ def assert_auth_denied(result: StreamingResponse, context: str) -> None:
f"{context}: expected 401/403, got {result.status_code}: {result.body[:300]}"
)
def wire_body(json: BaseModel) -> dict[str, object]:
if isinstance(json, PartialBody):
return json.model_dump(by_alias=True, exclude_unset=True)
@ -349,6 +369,9 @@ class ClassifiableResponse(Protocol):
@property
def content(self) -> bytes: ...
@property
def headers(self) -> Mapping[str, str]: ...
def json(self) -> object: ...
@ -358,7 +381,11 @@ def classify[R: BaseModel](resp: ClassifiableResponse, response_type: type[R]) -
if resp.status_code == 429:
return RateLimitedError(body=resp.text)
if not resp.ok:
return UnknownApiError(status_code=resp.status_code, body=resp.text)
return UnknownApiError(
status_code=resp.status_code,
body=resp.text,
headers={key.lower(): value for key, value in resp.headers.items()},
)
try:
payload: Final[object] = resp.json() if resp.content else {}
return Success(status_code=resp.status_code, data=response_type.model_validate(payload))
@ -596,6 +623,22 @@ def _is_stream_error_line(line: bytes) -> bool:
def streaming_outcome(
resp: SseResponse, stream: bool, *, sent_at: float, clock: Callable[[], float] = time.monotonic
) -> StreamingResponse:
try:
return _streaming_outcome(resp, stream, sent_at=sent_at, clock=clock)
except requests.RequestException as exc:
return StreamingResponse(
status_code=resp.status_code,
call_id=_hdr(resp, "x-litellm-call-id"),
content_type=_hdr(resp, "content-type"),
headers={name.lower(): value for name, value in resp.headers.items()},
body=str(exc),
network_error=NetworkError(message=str(exc)),
)
def _streaming_outcome(
resp: SseResponse, stream: bool, *, sent_at: float, clock: Callable[[], float]
) -> StreamingResponse:
call_id: Final = _hdr(resp, "x-litellm-call-id")
response_cost: Final = _parse_response_cost(resp)
@ -610,6 +653,23 @@ def streaming_outcome(
headers=headers,
body=resp.text,
)
if "application/vnd.amazon.eventstream" in (content_type or "").lower():
bedrock_events: Final = tuple(
(event, clock() - sent_at) for event in decode_bedrock_stream(resp.iter_content(chunk_size=None))
)
return StreamingResponse(
status_code=resp.status_code,
call_id=call_id,
response_cost=response_cost,
content_type=content_type,
headers=headers,
body="<streamed>",
chunks=len(bedrock_events),
stream_events=[event.payload for event, _ in bedrock_events],
stream_event_arrivals=[arrived for _, arrived in bedrock_events],
stream_error=next((event.error for event, _ in bedrock_events if event.error is not None), None),
stream_error_code=next((event.error_code for event, _ in bedrock_events if event.error is not None), None),
)
stamped: Final = tuple((line, clock() - sent_at) for line in resp.iter_lines() if line)
payloads: Final = tuple(
(line.removeprefix(_SSE_DATA_PREFIX).decode(errors="replace"), arrived)
@ -661,7 +721,7 @@ def send(
)
)
except requests.RequestException as exc:
return StreamingResponse(status_code=-1, body=str(exc))
return StreamingResponse(status_code=-1, body=str(exc), network_error=NetworkError(message=str(exc)))
return streaming_outcome(resp, stream, sent_at=sent_at)

View file

@ -1,3 +1,10 @@
general_settings:
master_key: os.environ/LITELLM_MASTER_KEY
store_model_in_db: true
litellm_settings:
cache: true
cache_params:
type: redis
host: 127.0.0.1
port: 6379

View file

@ -0,0 +1,111 @@
from typing import Final
from e2e_http import StreamingResponse, require_successful_call
from pydantic import BaseModel
class _TextDelta(BaseModel):
text: str = ""
stop_reason: str | None = None
class _ContentBlockDelta(BaseModel):
delta: _TextDelta
class _MessageStart(BaseModel):
role: str
class _MessageStop(BaseModel):
stopReason: str
class _ConverseUsage(BaseModel):
inputTokens: int
outputTokens: int
totalTokens: int
class _Metadata(BaseModel):
usage: _ConverseUsage
class _ConverseEvent(BaseModel):
messageStart: _MessageStart | None = None
contentBlockDelta: _ContentBlockDelta | None = None
messageStop: _MessageStop | None = None
metadata: _Metadata | None = None
class _InvokeUsage(BaseModel):
input_tokens: int = 0
output_tokens: int = 0
class _InvokeMessage(BaseModel):
role: str
usage: _InvokeUsage
class _InvokeEvent(BaseModel):
type: str
message: _InvokeMessage | None = None
delta: _TextDelta | None = None
usage: _InvokeUsage | None = None
def _require_native_stream(result: StreamingResponse) -> None:
require_successful_call(result, expected_provider="bedrock")
assert "application/vnd.amazon.eventstream" in (result.content_type or "").lower(), result.content_type
assert result.stream_error is None, result.stream_error
assert result.stream_events, "Bedrock stream returned no decoded events"
def assert_converse_stream(result: StreamingResponse) -> None:
_require_native_stream(result)
events: Final = tuple(_ConverseEvent.model_validate_json(event) for event in result.stream_events)
starts: Final = tuple(i for i, event in enumerate(events) if event.messageStart is not None)
deltas: Final = tuple(i for i, event in enumerate(events) if event.contentBlockDelta is not None)
stops: Final = tuple(i for i, event in enumerate(events) if event.messageStop is not None)
metadata: Final = tuple(i for i, event in enumerate(events) if event.metadata is not None)
assert len(starts) == len(stops) == len(metadata) == 1, "Converse requires messageStart, messageStop and metadata"
assert deltas, "Converse stream returned no content deltas"
assert starts[0] < deltas[0] <= deltas[-1] < stops[0] < metadata[0] == len(events) - 1, "Converse event order"
start: Final = events[starts[0]].messageStart
stop: Final = events[stops[0]].messageStop
tail: Final = events[metadata[0]].metadata
assert start is not None and start.role == "assistant", "Converse stream has no assistant"
text: Final = "".join(event.contentBlockDelta.delta.text for event in events if event.contentBlockDelta is not None)
assert text.strip(), "Converse stream returned empty text"
assert stop is not None and stop.stopReason in {"end_turn", "max_tokens"}, (
"Converse stream has no valid stop reason"
)
assert tail is not None and tail.usage.inputTokens > 0 and tail.usage.outputTokens > 0, "Converse usage is missing"
assert tail.usage.totalTokens == tail.usage.inputTokens + tail.usage.outputTokens, "Converse usage does not add up"
def assert_invoke_stream(result: StreamingResponse) -> None:
_require_native_stream(result)
events: Final = tuple(_InvokeEvent.model_validate_json(event) for event in result.stream_events)
starts: Final = tuple(i for i, event in enumerate(events) if event.type == "message_start")
deltas: Final = tuple(i for i, event in enumerate(events) if event.type == "content_block_delta")
stops: Final = tuple(i for i, event in enumerate(events) if event.type == "message_stop")
finishes: Final = tuple(
i
for i, event in enumerate(events)
if event.type == "message_delta" and event.delta is not None and event.delta.stop_reason is not None
)
assert len(starts) == len(stops) == len(finishes) == 1, "Invoke requires message_start, finish and message_stop"
assert deltas, "Invoke stream returned no content deltas"
assert starts[0] < deltas[0] <= deltas[-1] < finishes[0] < stops[0] == len(events) - 1, "Invoke event order"
message: Final = events[starts[0]].message
assert message is not None and message.role == "assistant", "Invoke stream has no assistant"
assert message.usage.input_tokens > 0, "Invoke input usage is missing"
text: Final = "".join(
event.delta.text for event in events if event.type == "content_block_delta" and event.delta is not None
)
assert text.strip(), "Invoke stream returned empty text"
finish: Final = events[finishes[0]]
assert finish.delta is not None and finish.delta.stop_reason in {"end_turn", "max_tokens"}, "Invoke stop reason"
assert finish.usage is not None and finish.usage.output_tokens > 0, "Invoke output usage is missing"

View file

@ -74,6 +74,7 @@ class ResponsesRequest(BaseModel):
instructions: str | None = None
stream: bool = False
tools: list[ResponsesFunctionTool] | None = None
tool_choice: str | None = None
guardrails: list[str] | None = None
cache: dict[str, bool] | None = {"no-cache": True}
@ -351,7 +352,7 @@ class EndpointsClient:
)
def responses_with_tools(
self, key: str, model: str, text: str, tools: list[ResponsesFunctionTool]
self, key: str, model: str, text: str, tools: list[ResponsesFunctionTool], *, tool_choice: str | None = None
) -> StreamingResponse:
return self._send(
"/v1/responses",
@ -361,6 +362,7 @@ class EndpointsClient:
input=text,
instructions="You are a helpful assistant",
tools=tools,
tool_choice=tool_choice,
),
)

View file

@ -7,19 +7,18 @@ missing messages and invalid model handling without crashing the proxy.
from __future__ import annotations
import pytest
from bedrock_config import bedrock_params
from bedrock_stream import assert_converse_stream, assert_invoke_stream
from e2e_config import unique_marker
from e2e_http import (
assert_client_error,
StreamingResponse,
require_successful_call,
)
from lifecycle import ResourceManager
from models import LiteLLMParamsBody
from proxy_client import ProxyClient
from pydantic import BaseModel
pytestmark = pytest.mark.e2e
BEDROCK_BACKEND = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"
pytestmark = [pytest.mark.e2e, pytest.mark.replayable]
class ConverseContent(BaseModel):
@ -71,17 +70,17 @@ def _register(proxy: ProxyClient, resources: ResourceManager) -> tuple[str, str]
model = f"e2e-bedrock-native-{unique_marker()}"
model_id = proxy.create_model(
model,
LiteLLMParamsBody(
model=BEDROCK_BACKEND,
aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID",
aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY",
aws_region_name="os.environ/AWS_REGION",
),
bedrock_params(),
)
resources.defer(lambda: proxy.delete_model(model_id))
return model, resources.key()
def _assert_input_error(result: StreamingResponse, expected_detail: str) -> None:
assert result.status_code == 400, f"expected Bedrock validation error, got {result.status_code}: {result.body[:500]}"
assert expected_detail in result.body.lower(), f"validation error did not name {expected_detail}: {result.body[:500]}"
def _default_converse() -> ConverseBody:
return ConverseBody(
messages=[ConverseMessage(role="user", content=[ConverseContent(text="Hello")])],
@ -107,7 +106,7 @@ class TestBedrockNative:
headers=proxy.transport.bearer(key),
json=_default_converse(),
)
require_successful_call(result)
require_successful_call(result, expected_provider="bedrock")
response = ConverseResponse.model_validate_json(result.body)
assert response.output.message.role == "assistant"
assert any(part.text.strip() for part in response.output.message.content)
@ -121,9 +120,7 @@ class TestBedrockNative:
json=_default_converse(),
stream=True,
)
require_successful_call(result)
assert result.stream_error is None, result.stream_error
assert result.chunks > 0, "converse-stream returned no events"
assert_converse_stream(result)
@pytest.mark.covers("llm.bedrock_native.bedrock_invoke.basic.nonstream.works")
def test_invoke_returns_message(self, proxy: ProxyClient, resources: ResourceManager) -> None:
@ -133,7 +130,7 @@ class TestBedrockNative:
headers=proxy.transport.bearer(key),
json=_default_invoke(),
)
require_successful_call(result)
require_successful_call(result, expected_provider="bedrock")
response = InvokeResponse.model_validate_json(result.body)
assert any(part.text.strip() for part in response.content)
@ -146,9 +143,7 @@ class TestBedrockNative:
json=_default_invoke(),
stream=True,
)
require_successful_call(result)
assert result.stream_error is None, result.stream_error
assert result.chunks > 0, "invoke stream returned no events"
assert_invoke_stream(result)
@pytest.mark.covers("llm.bedrock_native.bedrock_converse.input_validation.nonstream.works")
def test_converse_missing_messages_returns_error(self, proxy: ProxyClient, resources: ResourceManager) -> None:
@ -158,7 +153,7 @@ class TestBedrockNative:
headers=proxy.transport.bearer(key),
json=ConverseBody(inferenceConfig=ConverseInferenceConfig()),
)
assert_client_error(result, "converse missing messages")
_assert_input_error(result, "messages")
@pytest.mark.covers("llm.bedrock_native.bedrock_converse.input_validation.nonstream.works")
def test_converse_empty_messages_returns_client_error(self, proxy: ProxyClient, resources: ResourceManager) -> None:
@ -168,7 +163,7 @@ class TestBedrockNative:
headers=proxy.transport.bearer(key),
json=ConverseBody(messages=[]),
)
assert_client_error(result, "converse empty messages")
_assert_input_error(result, "user message")
@pytest.mark.covers("llm.bedrock_native.bedrock_converse.input_validation.nonstream.works")
def test_converse_invalid_model_returns_error(self, proxy: ProxyClient, resources: ResourceManager) -> None:
@ -190,7 +185,7 @@ class TestBedrockNative:
headers=proxy.transport.bearer(key),
json=InvokeBody(anthropic_version="bedrock-2023-05-31", max_tokens=50),
)
assert_client_error(result, "invoke missing messages")
_assert_input_error(result, "messages")
@pytest.mark.covers("llm.bedrock_native.bedrock_invoke.input_validation.nonstream.works")
def test_invoke_missing_max_tokens_returns_error(self, proxy: ProxyClient, resources: ResourceManager) -> None:
@ -203,7 +198,7 @@ class TestBedrockNative:
messages=[InvokeMessage(role="user", content="Hello")],
),
)
assert_client_error(result, "invoke missing max_tokens")
_assert_input_error(result, "max_tokens")
@pytest.mark.covers("llm.bedrock_native.bedrock_invoke.input_validation.nonstream.works")
def test_invoke_invalid_temperature_returns_client_error(
@ -220,4 +215,4 @@ class TestBedrockNative:
temperature=5.0,
),
)
assert_client_error(result, "invoke invalid temperature")
_assert_input_error(result, "temperature")

View file

@ -22,10 +22,11 @@ from pathlib import Path
from typing import Final
import pytest
from bedrock_config import bedrock_params
from pydantic import BaseModel
from e2e_config import unique_marker
from e2e_http import StreamingResponse, unwrap
from e2e_http import StreamingResponse, require_successful_call, unwrap
from lifecycle import ResourceManager
from models import (
ChatBody,
@ -47,7 +48,6 @@ COHERE_BACKEND = "cohere/command-r-08-2024"
GEMINI_BACKEND = "gemini/gemini-2.5-flash"
OPENAI_BACKEND = "openai/gpt-5.6"
ANTHROPIC_BACKEND = "anthropic/claude-haiku-4-5-20251001"
BEDROCK_CONVERSE_BACKEND = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"
class _StreamToolCallFunction(BaseModel):
@ -66,6 +66,7 @@ class _StreamDelta(BaseModel):
class _StreamChoice(BaseModel):
delta: _StreamDelta = _StreamDelta()
finish_reason: str | None = None
class _StreamChunk(BaseModel):
@ -124,21 +125,16 @@ def _streamed_text(events: list[str]) -> str:
def _assert_streamed_completion(result: StreamingResponse) -> None:
"""A streamed /chat/completions must deliver real content, not a clean-but-empty
stream (the #28991 class on the streaming path)."""
assert result.ok and result.is_streaming, f"stream was not established: {result}"
require_successful_call(result)
assert result.is_streaming, f"stream was not established: {result}"
assert result.stream_error is None, f"stream carried an error event: {result.stream_error}"
assert len(result.stream_events) > 1, f"stream did not deliver multiple data events: {result}"
assert _streamed_text(result.stream_events).strip(), (
f"stream completed with no content deltas: {result.stream_events[:3]}"
)
def _bedrock_params() -> LiteLLMParamsBody:
return LiteLLMParamsBody(
model=BEDROCK_CONVERSE_BACKEND,
aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID",
aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY",
aws_region_name="os.environ/AWS_REGION",
)
assert result.stream_done, "chat stream ended without [DONE]"
chunks: Final = tuple(_StreamChunk.model_validate_json(event) for event in result.stream_events)
assert any(choice.finish_reason for chunk in chunks for choice in chunk.choices), "chat stream has no finish reason"
class _WeatherArgs(BaseModel):
@ -621,6 +617,7 @@ class TestOpenAIChatCompletions:
assert args.location.strip(), f"streamed tool call arguments missing location: {arguments!r}"
@pytest.mark.replayable
class TestBedrockConverseChatCompletions:
"""Bedrock Converse via /chat/completions, the customer's AWS stack. A non-OpenAI
provider must return real content on both the non-streamed and streamed paths.
@ -628,7 +625,7 @@ class TestBedrockConverseChatCompletions:
def _register(self, client: PassthroughClient, resources: ResourceManager, prefix: str) -> str:
model = f"{prefix}-{unique_marker()}"
model_id = client.proxy.create_model(model, _bedrock_params())
model_id = client.proxy.create_model(model, bedrock_params())
resources.defer(lambda: client.proxy.delete_model(model_id))
return model

View file

@ -11,6 +11,7 @@ import json
from typing import cast
import pytest
from bedrock_config import bedrock_params
from e2e_config import unique_marker
from e2e_http import (
assert_client_error,
@ -38,8 +39,6 @@ class _OptionalResponsesBody(BaseModel):
max_output_tokens: int | None = None
BEDROCK_CONVERSE_BACKEND = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"
WEATHER_TOOL = ResponsesFunctionTool(
name="get_weather",
description="Get the weather for a location",
@ -50,15 +49,6 @@ WEATHER_TOOL = ResponsesFunctionTool(
)
def _bedrock_params() -> LiteLLMParamsBody:
return LiteLLMParamsBody(
model=BEDROCK_CONVERSE_BACKEND,
aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID",
aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY",
aws_region_name="os.environ/AWS_REGION",
)
class WeatherArguments(BaseModel):
location: str
@ -260,12 +250,13 @@ class TestResponses:
arguments = WeatherArguments.model_validate(raw_arguments)
assert arguments.location, f"function call arguments missing location: {function_call.arguments}"
@pytest.mark.replayable
@pytest.mark.covers("llm.responses.bedrock_converse.basic.nonstream.works")
def test_responses_bedrock_returns_completion(
self, endpoints_client: EndpointsClient, resources: ResourceManager
) -> None:
model = f"e2e-responses-{unique_marker()}"
model_id = endpoints_client.create_model(model, _bedrock_params())
model_id = endpoints_client.create_model(model, bedrock_params())
resources.defer(lambda: endpoints_client.delete_model(model_id))
key = resources.key()
@ -274,17 +265,21 @@ class TestResponses:
parsed = ResponsesResult.model_validate_json(result.body)
assert parsed.text.strip(), f"/responses over bedrock returned no output text: {result.body[:300]}"
@pytest.mark.replayable
@pytest.mark.covers("llm.responses.bedrock_converse.tool_use.nonstream.works")
def test_responses_bedrock_returns_function_call(
self, endpoints_client: EndpointsClient, resources: ResourceManager
) -> None:
model = f"e2e-responses-{unique_marker()}"
model_id = endpoints_client.create_model(model, _bedrock_params())
model_id = endpoints_client.create_model(model, bedrock_params())
resources.defer(lambda: endpoints_client.delete_model(model_id))
key = resources.key()
result = endpoints_client.responses_with_tools(
key, model, "What is the weather in San Francisco? Use the get_weather tool.", [WEATHER_TOOL]
key,
model,
"What is the weather in San Francisco? Use the get_weather tool.",
[WEATHER_TOOL],
)
require_successful_call(result)
parsed = ResponsesResult.model_validate_json(result.body)
@ -295,6 +290,32 @@ class TestResponses:
arguments = WeatherArguments.model_validate(raw_arguments)
assert arguments.location, f"function call arguments missing location: {function_call.arguments}"
@pytest.mark.replayable
@pytest.mark.covers("llm.responses.bedrock_converse.tool_use.nonstream.works")
def test_responses_bedrock_required_tool_returns_function_call(
self, endpoints_client: EndpointsClient, resources: ResourceManager
) -> None:
model = f"e2e-responses-required-{unique_marker()}"
model_id = endpoints_client.create_model(model, bedrock_params())
resources.defer(lambda: endpoints_client.delete_model(model_id))
key = resources.key()
result = endpoints_client.responses_with_tools(
key,
model,
"What is the weather in San Francisco? Use the get_weather tool.",
[WEATHER_TOOL],
tool_choice="required",
)
require_successful_call(result)
parsed = ResponsesResult.model_validate_json(result.body)
function_call = next((call for call in parsed.function_calls if call.name == "get_weather"), None)
assert function_call is not None, f"no required get_weather function call over bedrock: {result.body[:500]}"
assert function_call.arguments is not None
raw_arguments = cast(object, json.loads(function_call.arguments))
arguments = WeatherArguments.model_validate(raw_arguments)
assert arguments.location, f"function call arguments missing location: {function_call.arguments}"
@pytest.mark.skip(reason="stage red: product gap, /v1/responses 500s (aresponses TypeError) on missing input instead of 400")
@pytest.mark.covers("llm.responses.openai.input_validation.nonstream.works")
def test_missing_input_returns_error(
@ -344,6 +365,7 @@ class TestResponses:
)
assert_client_error(result, "responses empty input")
def _parse_stream_event(
event: str,
) -> ResponsesOutputTextDeltaEvent | None:

View file

@ -0,0 +1,135 @@
from collections.abc import Mapping
from typing import Final, Literal
from pydantic import BaseModel, ValidationError
class ProviderFailure(BaseModel):
status_code: int
provider: Literal["bedrock"] | None = None
error_code: str | None = None
request_id: str | None = None
call_id: str | None = None
evidence: Literal["headers", "body", "stream_event"] | None = None
def junit_properties(self) -> tuple[tuple[str, str], ...]:
fields: Final = (
("status_code", self.status_code),
("provider", self.provider),
("error_code", self.error_code),
("request_id", self.request_id),
("call_id", self.call_id),
("evidence", self.evidence),
)
return tuple((f"upstream_{key}", str(value)) for key, value in fields if value is not None)
class ProviderUnavailableError(AssertionError):
def __init__(self, failure: ProviderFailure, body: str) -> None:
self.failure: Final = failure
super().__init__(f"{failure.model_dump_json(exclude_none=True)}; body={body[:1000]}")
class NetworkFailureError(AssertionError):
def __init__(self, failure: ProviderFailure, message: str) -> None:
self.failure: Final = failure
super().__init__(f"HTTP transfer failed; {failure.model_dump_json(exclude_none=True)}; body={message[:1000]}")
class _ErrorMessage(BaseModel):
message: str = ""
class _NativeError(BaseModel):
error: str = ""
class _ErrorEnvelope(BaseModel):
error: _ErrorMessage | None = None
detail: _NativeError | None = None
_BEDROCK_UNAVAILABLE: Final = "Bedrock is unable to process your request."
def _bedrock_unavailable_body(body: str, expected_provider: Literal["bedrock"] | None) -> bool:
try:
envelope: Final = _ErrorEnvelope.model_validate_json(body)
if envelope.error is not None:
message: Final = envelope.error.message
return message.startswith("litellm.ServiceUnavailableError: BedrockException - ") and (
_BEDROCK_UNAVAILABLE in message
)
if expected_provider == "bedrock" and envelope.detail is not None:
return _ErrorMessage.model_validate_json(envelope.detail.error).message == _BEDROCK_UNAVAILABLE
except ValidationError:
return False
return False
def provider_failure(
status_code: int,
body: str,
headers: Mapping[str, str],
*,
expected_provider: Literal["bedrock"] | None = None,
stream_error_code: str | None = None,
) -> ProviderFailure:
normalized: Final = {key.lower(): value for key, value in headers.items()}
request_id: Final = normalized.get("llm_provider-x-amzn-requestid") or normalized.get("x-amzn-requestid")
error_code: Final = normalized.get("llm_provider-x-amzn-errortype") or normalized.get("x-amzn-errortype")
call_id: Final = normalized.get("x-litellm-call-id")
if (
200 <= status_code < 300
and expected_provider == "bedrock"
and stream_error_code == "serviceUnavailableException"
):
return ProviderFailure(
status_code=status_code,
provider="bedrock",
error_code="ServiceUnavailableException",
request_id=request_id,
call_id=call_id,
evidence="stream_event",
)
if status_code == 503:
bedrock_body: Final = _bedrock_unavailable_body(body, expected_provider)
if (
request_id
and error_code
and error_code.split(":", 1)[0] == "ServiceUnavailableException"
and (expected_provider == "bedrock" or bedrock_body)
):
return ProviderFailure(
status_code=status_code,
provider="bedrock",
error_code="ServiceUnavailableException",
request_id=request_id,
call_id=call_id,
evidence="headers",
)
if bedrock_body:
return ProviderFailure(
status_code=status_code,
provider="bedrock",
error_code="ServiceUnavailableException",
request_id=request_id,
call_id=call_id,
evidence="body",
)
return ProviderFailure(status_code=status_code, request_id=request_id, call_id=call_id, error_code=error_code)
def raise_if_provider_unavailable(
status_code: int,
body: str,
headers: Mapping[str, str],
*,
expected_provider: Literal["bedrock"] | None = None,
stream_error_code: str | None = None,
) -> None:
failure: Final = provider_failure(
status_code, body, headers, expected_provider=expected_provider, stream_error_code=stream_error_code
)
if failure.provider is not None:
raise ProviderUnavailableError(failure, body)

View file

@ -55,6 +55,13 @@ from types import MappingProxyType
from typing import Final, Literal, assert_never
from urllib.parse import parse_qsl, urlsplit
from bedrock_edge import (
bedrock_region,
bedrock_upstream,
recording_credentials,
sign_bedrock_forward,
valid_bedrock_signature,
)
from e2e_http import (
NetworkError,
StreamChunk,
@ -631,7 +638,8 @@ def _is_streamed(headers: Mapping[str, str]) -> bool:
move nearly every recording to the streamed shape for no gain. The content type
is the header that says "consume this as it arrives", and it is already how the
harness defines streaming everywhere else."""
return "text/event-stream" in _header_value(headers, "content-type").lower()
content_type: Final = _header_value(headers, "content-type").lower()
return any(kind in content_type for kind in ("text/event-stream", "application/vnd.amazon.eventstream"))
def _upstream_url(upstream_base: str, upstream_path: str, query: str) -> str:
@ -783,26 +791,29 @@ def handle_edge_request(
Socket-free so unit tests exercise every branch without a server."""
split: Final = urlsplit(raw_path)
mount, _, upstream_path = split.path.lstrip("/").partition("/")
upstream_base: Final = mounts.get(mount)
upstream_base: Final = mounts.get(mount) or bedrock_upstream(mount)
if upstream_base is None:
return _text_reply(
404, f"unknown provider mount {mount!r}; known mounts: {', '.join(sorted(mounts))}"
)
request: Final = edge_request(
method, split.path, split.query, body, _header_value(headers, "content-type")
return _text_reply(404, f"unknown provider mount {mount!r}; known mounts: {', '.join(sorted(mounts))}")
request: Final = edge_request(method, split.path, split.query, body, _header_value(headers, "content-type"))
region: Final = bedrock_region(mount)
if region is not None and not valid_bedrock_signature(method, raw_path, headers, body, region):
return _text_reply(403, "Bedrock provider edge rejected the test SigV4 signature")
url: Final = _upstream_url(upstream_base, upstream_path, split.query)
forwarded_headers: Final = (
sign_bedrock_forward(method, url, headers, body, region, recording_credentials())
if region is not None and not isinstance(backend, ReplayEdge)
else headers
)
match backend:
case LiveEdge():
return _handle_live(
method, _upstream_url(upstream_base, upstream_path, split.query), headers, body, timeout
)
return _handle_live(method, url, forwarded_headers, body, timeout)
case RecordEdge():
return _handle_record(
backend,
request,
method=method,
url=_upstream_url(upstream_base, upstream_path, split.query),
headers=headers,
url=url,
headers=forwarded_headers,
body=body,
timeout=timeout,
)
@ -1020,10 +1031,8 @@ def provider_edge_api_base(
case "live":
return None
case "record" | "replay":
if mount not in EDGE_MOUNTS:
raise ValueError(
f"unknown provider mount {mount!r}; known mounts: {', '.join(sorted(EDGE_MOUNTS))}"
)
if mount not in EDGE_MOUNTS and bedrock_upstream(mount) is None:
raise ValueError(f"unknown provider mount {mount!r}; known mounts: {', '.join(sorted(EDGE_MOUNTS))}")
return _shared_edge(mode, bundle_dir, bind_host, advertise_host, forward_timeout).api_base(mount)
case _:
assert_never(mode)

View file

@ -2,7 +2,7 @@
# Config when any e2e suite under tests/e2e/ is run directly, e.g.
# uv run pytest tests/e2e/quota_management/spend_tracking/ -v
# The e2e marker is also registered in conftest.py for runs rooted elsewhere.
addopts = --strict-markers --strict-config --reruns 1 --only-rerun "kind='network'" --only-rerun "status_code=5[0-9][0-9]"
addopts = --strict-markers --strict-config --reruns 1 --only-rerun "^AssertionError: kind='network' message=" --rerun-except "ProviderUnavailableError" --rerun-except "NetworkFailureError"
markers =
e2e: live test that requires a running proxy and real provider keys
replayable: edge-wired test whose provider traffic replays from a fixture bundle, so it makes zero provider calls in replay mode; the record/replay CI lane selects it with -m replayable

View file

@ -0,0 +1,177 @@
import base64
import threading
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Final
import pytest
from bedrock_edge import (
TEST_ACCESS_KEY,
TEST_SECRET_KEY,
bedrock_upstream,
recording_credentials_from,
sign_bedrock_forward,
valid_bedrock_signature,
)
from botocore.auth import SigV4Auth
from botocore.awsrequest import AWSRequest
from botocore.client import BaseClient
from botocore.credentials import Credentials
from botocore.session import get_session
from botocore.stub import Stubber
from fixture_bundle import BundleRecorder, LoadedBundle, RecordedHttpResponse, load_bundle, prepare_bundle
from fixture_mode import current_test_key
from provider_edge import EdgeReply, RecordEdge, ReplayEdge, ReplaySource, edge_request, handle_edge_request
_PATH: Final = "/bedrock-us-east-1/model/us.anthropic.claude-haiku-4-5-20251001-v1:0/converse"
_BODY: Final = b'{"messages":[{"role":"user","content":[{"text":"Hello"}]}]}'
def _signed_headers() -> dict[str, str]:
request: Final = AWSRequest(
method="POST",
url=f"http://127.0.0.1:1234{_PATH}",
data=_BODY,
headers={"content-type": "application/json", "host": "127.0.0.1:1234"},
)
SigV4Auth(Credentials(TEST_ACCESS_KEY, TEST_SECRET_KEY), "bedrock", "us-east-1").add_auth(request)
return {key.lower(): value for key, value in request.headers.items()}
class TestBedrockSigningBoundary:
def test_recording_resolves_the_real_role_once_without_using_proxy_test_credentials(self) -> None:
client: Final = get_session().create_client(
"sts",
region_name="us-east-1",
aws_access_key_id="recording-access",
aws_secret_access_key="recording-secret",
)
def client_creator(service: str, **credentials: str | None) -> BaseClient:
assert service == "sts"
assert credentials == {
"aws_access_key_id": "recording-access",
"aws_secret_access_key": "recording-secret",
"aws_session_token": "source-session",
}
return client
with Stubber(client) as stub:
stub.add_response(
"assume_role",
{
"Credentials": {
"AccessKeyId": "ASIA1234567890123456",
"SecretAccessKey": "role-secret",
"SessionToken": "role-session",
"Expiration": datetime.now(timezone.utc) + timedelta(hours=1),
},
},
{
"RoleArn": "arn:aws:iam::123456789012:role/e2e-test",
"RoleSessionName": "test-recording",
"ExternalId": "external",
},
)
result: Final = recording_credentials_from(
{
"AWS_ACCESS_KEY_ID": "recording-access",
"AWS_SECRET_ACCESS_KEY": "recording-secret",
"AWS_SESSION_TOKEN": "source-session",
"AWS_ROLE_NAME": "arn:aws:iam::123456789012:role/e2e-test",
"AWS_SESSION_NAME": "test-recording",
"AWS_EXTERNAL_ID": "external",
},
client_creator,
)
first: Final = result.get_frozen_credentials()
assert first.access_key == "ASIA1234567890123456"
assert first.token == "role-session"
assert result.get_frozen_credentials() == first
stub.assert_no_pending_responses()
def test_accepts_a_valid_signature_from_the_proxy(self) -> None:
assert valid_bedrock_signature("POST", _PATH, _signed_headers(), _BODY, "us-east-1")
@pytest.mark.parametrize("change", ["body", "path", "region", "host", "authorization"])
def test_rejects_changes_to_signed_request_inputs(self, change: str) -> None:
headers: Final = _signed_headers()
assert not valid_bedrock_signature(
"POST",
_PATH + "/tampered" if change == "path" else _PATH,
{**headers, change: "tampered"} if change in {"host", "authorization"} else headers,
b"{}" if change == "body" else _BODY,
"us-west-2" if change == "region" else "us-east-1",
)
def test_forward_is_resigned_for_aws_and_preserves_the_body_and_feature_headers(self) -> None:
url: Final = "https://bedrock-runtime.us-east-1.amazonaws.com/model/example/converse"
credentials: Final = Credentials("recording-access", "recording-secret", "recording-session")
incoming: Final = {**_signed_headers(), "anthropic-beta": "feature-flag"}
forwarded: Final = sign_bedrock_forward("POST", url, incoming, _BODY, "us-east-1", credentials)
assert "Credential=recording-access/" in forwarded["Authorization"]
assert TEST_ACCESS_KEY not in forwarded["Authorization"]
assert forwarded["X-Amz-Security-Token"] == "recording-session"
assert forwarded["anthropic-beta"] == "feature-flag"
assert "127.0.0.1" not in str(forwarded)
@pytest.mark.parametrize(
"mount", ["bedrock-localhost", "bedrock-us-east-1.evil.test", "bedrock-us-east-1/anything"]
)
def test_mount_cannot_select_an_arbitrary_upstream_host(self, mount: str) -> None:
assert bedrock_upstream(mount) is None
def test_bad_signature_cannot_reach_aws_or_write_a_recording(self, tmp_path: Path) -> None:
recorder: Final = prepare_bundle(tmp_path / "bundle")
assert isinstance(recorder, BundleRecorder)
result: Final = handle_edge_request(
RecordEdge(recorder=recorder, lock=threading.Lock()),
{},
"POST",
_PATH,
_signed_headers(),
b"tampered",
timeout=0.1,
)
assert isinstance(result, EdgeReply) and result.status_code == 403
bundle: Final = load_bundle(tmp_path / "bundle")
assert isinstance(bundle, LoadedBundle) and not bundle.interactions
def test_replay_validates_signature_before_consuming_the_recording(self, tmp_path: Path) -> None:
root: Final = tmp_path / "bundle"
recorder: Final = prepare_bundle(root)
assert isinstance(recorder, BundleRecorder)
recorder.record(
test_key=current_test_key(),
request=edge_request("POST", _PATH, "", _BODY, "application/json"),
response=RecordedHttpResponse(
status_code=200,
headers={"content-type": "application/json"},
body_b64=base64.b64encode(b'{"output":{"message":{"role":"assistant"}}}').decode(),
),
)
loaded: Final = load_bundle(root)
assert isinstance(loaded, LoadedBundle)
source: Final = ReplaySource(loaded)
rejected: Final = handle_edge_request(
ReplayEdge(source),
{},
"POST",
_PATH,
{},
_BODY,
timeout=0.1,
)
assert isinstance(rejected, EdgeReply) and rejected.status_code == 403
result: Final = handle_edge_request(
ReplayEdge(source),
{},
"POST",
_PATH,
_signed_headers(),
_BODY,
timeout=0.1,
)
assert isinstance(result, EdgeReply) and result.status_code == 200
assert result.body == b'{"output":{"message":{"role":"assistant"}}}'
assert source.leftover_error(current_test_key()) is None

View file

@ -0,0 +1,212 @@
import base64
import json
import struct
import zlib
from collections.abc import Iterator
from dataclasses import dataclass
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from threading import Thread
from types import MappingProxyType
from typing import Final
import pytest
from bedrock_eventstream import decode_bedrock_stream
from botocore.eventstream import ChecksumMismatch
from e2e_http import URL, NoBody, StreamingResponse, require_successful_call, send, streaming_outcome
from llm_translation.bedrock_stream import assert_converse_stream, assert_invoke_stream
from provider_diagnostics import NetworkFailureError, ProviderUnavailableError
from pydantic import JsonValue, TypeAdapter
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
def _frame(payload: str, event_type: str, message_type: str = "event") -> bytes:
headers: Final = {
":message-type": message_type,
":event-type" if message_type == "event" else ":exception-type": event_type,
":content-type": "application/json",
}
encoded_headers: Final = b"".join(
bytes([len(key)]) + key.encode() + b"\x07" + struct.pack("!H", len(value)) + value.encode()
for key, value in headers.items()
)
prelude: Final = struct.pack("!II", 16 + len(encoded_headers) + len(payload.encode()), len(encoded_headers))
message: Final = prelude + struct.pack("!I", zlib.crc32(prelude)) + encoded_headers + payload.encode()
return message + struct.pack("!I", zlib.crc32(message))
_CONVERSE: Final = (
'{"messageStart":{"role":"assistant"}}',
'{"contentBlockDelta":{"delta":{"text":"Hello"},"contentBlockIndex":0}}',
'{"contentBlockStop":{"contentBlockIndex":0}}',
'{"messageStop":{"stopReason":"end_turn"}}',
'{"metadata":{"usage":{"inputTokens":10,"outputTokens":1,"totalTokens":11}}}',
)
_INVOKE: Final = (
'{"type":"message_start","message":{"role":"assistant","usage":{"input_tokens":10,"output_tokens":0}}}',
'{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hello"}}',
'{"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":1}}',
'{"type":"message_stop"}',
)
@dataclass(frozen=True, slots=True)
class _BinaryResponse:
chunks: tuple[bytes, ...]
status_code: int = 200
text: str = ""
headers = MappingProxyType(
{
"content-type": "application/vnd.amazon.eventstream",
"x-amzn-requestid": "stream-request",
"x-litellm-call-id": "stream-call",
}
)
def iter_content(self, chunk_size: int | None = 1) -> Iterator[bytes]:
return iter(self.chunks)
def iter_lines(self) -> Iterator[bytes]:
raise AssertionError("binary Bedrock streams must not be consumed as text lines")
def _stream(events: tuple[str, ...]) -> StreamingResponse:
return StreamingResponse(
status_code=200,
content_type="application/vnd.amazon.eventstream",
body="<streamed>",
stream_events=list(events),
chunks=len(events),
)
class TestBedrockEventStream:
@pytest.mark.parametrize("size", [1, 7, 4096])
def test_converse_binary_frames_survive_arbitrary_transport_splits(self, size: int) -> None:
wire: Final = b"".join(
_frame(json.dumps(value), key)
for payload in _CONVERSE
for key, value in _JSON_OBJECT.validate_json(payload).items()
)
chunks: Final = tuple(wire[i : i + size] for i in range(0, len(wire), size))
result: Final = streaming_outcome(_BinaryResponse(chunks), True, sent_at=0.0, clock=lambda: 1.0)
assert result.chunks == len(_CONVERSE)
assert_converse_stream(result)
def test_invoke_payload_is_decoded_from_its_base64_envelope(self) -> None:
wire: Final = b"".join(
_frame(json.dumps({"bytes": base64.b64encode(event.encode()).decode()}), "chunk") for event in _INVOKE
)
result: Final = streaming_outcome(_BinaryResponse((wire,)), True, sent_at=0.0)
assert result.stream_events == list(_INVOKE)
assert_invoke_stream(result)
def test_binary_provider_exception_is_not_counted_as_a_successful_stream(self) -> None:
wire: Final = _frame('{"message":"unavailable"}', "serviceUnavailableException", "exception")
result: Final = streaming_outcome(_BinaryResponse((wire,)), True, sent_at=0.0)
assert result.stream_error == 'serviceUnavailableException: {"message":"unavailable"}'
with pytest.raises(ProviderUnavailableError) as caught:
assert_converse_stream(result)
assert caught.value.failure.status_code == 200
assert caught.value.failure.evidence == "stream_event"
assert caught.value.failure.request_id == "stream-request"
assert caught.value.failure.call_id == "stream-call"
@pytest.mark.parametrize("code", ["validationException", "throttlingException", "internalServerException"])
def test_other_stream_exceptions_do_not_qualify_for_the_availability_retry(self, code: str) -> None:
wire: Final = _frame('{"message":"provider rejected the request"}', code, "exception")
result: Final = streaming_outcome(_BinaryResponse((wire,)), True, sent_at=0.0)
with pytest.raises(AssertionError, match=code) as caught:
assert_converse_stream(result)
assert not isinstance(caught.value, ProviderUnavailableError)
def test_corrupt_binary_frame_is_rejected(self) -> None:
wire: Final = _frame('{"role":"assistant"}', "messageStart")
with pytest.raises(ChecksumMismatch):
tuple(decode_bedrock_stream((wire[:-1] + bytes([wire[-1] ^ 1]),)))
@pytest.mark.parametrize("invoke", [False, True])
@pytest.mark.parametrize("cut", [1, 7, 11, 12, 30, -1])
def test_incomplete_frame_after_valid_completion_is_rejected(self, invoke: bool, cut: int) -> None:
wire: Final = (
b"".join(
_frame(json.dumps({"bytes": base64.b64encode(event.encode()).decode()}), "chunk")
for event in _INVOKE
)
if invoke
else b"".join(
_frame(json.dumps(value), key)
for payload in _CONVERSE
for key, value in _JSON_OBJECT.validate_json(payload).items()
)
)
assertion: Final = assert_invoke_stream if invoke else assert_converse_stream
assertion(streaming_outcome(_BinaryResponse((wire,)), True, sent_at=0.0))
extra: Final = _frame('{"message":"unavailable"}', "serviceUnavailableException", "exception")[:cut]
with pytest.raises(AssertionError, match="incomplete Bedrock frame"):
assertion(streaming_outcome(_BinaryResponse((wire, extra)), True, sent_at=0.0))
class TestNativeStreamAssertions:
@pytest.mark.parametrize("missing", [0, 1, 3, 4])
def test_converse_rejects_missing_content_or_completion(self, missing: int) -> None:
with pytest.raises(AssertionError):
assert_converse_stream(_stream(tuple(event for i, event in enumerate(_CONVERSE) if i != missing)))
@pytest.mark.parametrize("missing", [0, 1, 2, 3])
def test_invoke_rejects_missing_content_or_completion(self, missing: int) -> None:
with pytest.raises(AssertionError):
assert_invoke_stream(_stream(tuple(event for i, event in enumerate(_INVOKE) if i != missing)))
@pytest.mark.parametrize("events", [_CONVERSE, _INVOKE])
def test_empty_text_does_not_pass_because_events_arrived(self, events: tuple[str, ...]) -> None:
assertion: Final = assert_converse_stream if events == _CONVERSE else assert_invoke_stream
with pytest.raises(AssertionError, match="empty text"):
assertion(_stream(tuple(event.replace("Hello", " ") for event in events)))
def test_converse_rejects_content_after_the_stop_event(self) -> None:
with pytest.raises(AssertionError, match="event order"):
assert_converse_stream(_stream((_CONVERSE[0], _CONVERSE[3], _CONVERSE[1], _CONVERSE[4])))
class _InterruptedStream(BaseHTTPRequestHandler):
def do_POST(self) -> None:
_ = self.rfile.read(int(self.headers.get("content-length", "0")))
binary: Final = self.path == "/binary"
data: Final = (
_frame('{"role":"assistant"}', "messageStart") if binary else b'data: {"text":"Hello"}\n\n'
)
self.send_response(200)
self.send_header("content-type", "application/vnd.amazon.eventstream" if binary else "text/event-stream")
self.send_header("transfer-encoding", "chunked")
self.send_header("x-amzn-requestid", "interrupted-request")
self.send_header("x-litellm-call-id", "interrupted-call")
self.end_headers()
_ = self.wfile.write(f"{len(data):x}\r\n".encode() + data + b"\r\n")
self.wfile.flush()
self.close_connection = True
def log_message(self, format: str, *args: object) -> None:
pass
@pytest.mark.parametrize("path", ["binary", "sse"])
def test_interrupted_http_stream_preserves_network_classification_and_ids(path: str) -> None:
server: Final = ThreadingHTTPServer(("127.0.0.1", 0), _InterruptedStream)
worker: Final = Thread(target=server.serve_forever, daemon=True)
worker.start()
try:
result: Final = send(
URL(f"http://127.0.0.1:{server.server_port}/{path}"), headers=NoBody(), json=NoBody(), stream=True, timeout=5
)
assert result.status_code == 200 and not result.ok
assert result.network_error is not None and result.network_error.kind == "network"
with pytest.raises(NetworkFailureError, match="HTTP transfer failed") as caught:
require_successful_call(result)
assert caught.value.failure.provider is None
assert caught.value.failure.request_id == "interrupted-request"
assert caught.value.failure.call_id == "interrupted-call"
finally:
server.shutdown()
server.server_close()
worker.join(timeout=5)

View file

@ -105,6 +105,9 @@ class FakeSseResponse:
def iter_lines(self) -> Iterator[bytes]:
return iter(self.lines)
def iter_content(self, chunk_size: int | None = 1) -> Iterator[bytes]:
return iter(self.lines)
def _ticking_clock(start: float, step: float) -> Callable[[], float]:
ticks: Final = iter(range(10_000))
@ -182,6 +185,7 @@ class FakeJsonResponse:
status_code: int
content: bytes
headers: Mapping[str, str] = MappingProxyType({})
@property
def ok(self) -> bool:

View file

@ -0,0 +1,88 @@
import json
from types import MappingProxyType
from typing import Final
import pytest
from e2e_http import StreamingResponse, UnknownApiError, require_successful_call, unwrap, unwrap_status
from provider_diagnostics import ProviderUnavailableError, provider_failure
_BODY: Final = json.dumps(
{
"error": {
"message": (
'litellm.ServiceUnavailableError: BedrockException - {"message":"Bedrock is unable to process your request."}'
)
}
}
)
_HEADERS: Final = MappingProxyType(
{
"LLM_Provider-X-Amzn-RequestId": "aws-request-1",
"LLM_Provider-X-Amzn-ErrorType": "ServiceUnavailableException:http://internal.amazon.com/",
"X-Litellm-Call-Id": "proxy-call-1",
}
)
def _assert_success(helper: str) -> None:
match helper:
case "unwrap":
unwrap(UnknownApiError(status_code=503, body=_BODY, headers=dict(_HEADERS)))
case "unwrap_status":
unwrap_status(UnknownApiError(status_code=503, body=_BODY, headers=dict(_HEADERS)), 200)
case _:
require_successful_call(StreamingResponse(status_code=503, body=_BODY, headers=dict(_HEADERS)))
class TestProviderFailure:
def test_aws_headers_preserve_attribution_and_both_request_ids(self) -> None:
result: Final = provider_failure(503, "unavailable", _HEADERS, expected_provider="bedrock")
assert result.provider == "bedrock"
assert result.error_code == "ServiceUnavailableException"
assert result.request_id == "aws-request-1"
assert result.call_id == "proxy-call-1"
assert result.evidence == "headers"
def test_another_aws_service_is_not_mislabeled_as_bedrock(self) -> None:
result: Final = provider_failure(503, "service unavailable", _HEADERS)
assert result.provider is None
assert result.request_id == "aws-request-1"
def test_proxy_bedrock_error_body_remains_identifiable_without_headers(self) -> None:
result: Final = provider_failure(503, _BODY, {})
assert result.provider == "bedrock" and result.evidence == "body"
assert result.request_id is None
def test_native_error_requires_explicit_bedrock_route_context(self) -> None:
body: Final = json.dumps(
{"detail": {"error": json.dumps({"message": "Bedrock is unable to process your request."})}}
)
assert provider_failure(503, body, {}).provider is None
result: Final = StreamingResponse(status_code=503, body=body)
with pytest.raises(ProviderUnavailableError) as caught:
require_successful_call(result, expected_provider="bedrock")
assert caught.value.failure.evidence == "body"
@pytest.mark.parametrize(
"status,body",
[
(503, '{"error":{"message":"proxy overloaded"}}'),
(503, "Bedrock is unable to process your request."),
(503, '{"error":{"message":"Bedrock is unable to process your request."}}'),
(503, '{"detail":{"error":"not JSON"}}'),
(429, _BODY),
(403, _BODY),
(500, _BODY),
],
)
def test_ambiguous_and_non_availability_errors_are_not_retryable(self, status: int, body: str) -> None:
assert provider_failure(status, body, {}, expected_provider="bedrock").provider is None
with pytest.raises(AssertionError) as caught:
unwrap(UnknownApiError(status_code=status, body=body))
assert not isinstance(caught.value, ProviderUnavailableError)
@pytest.mark.parametrize("helper", ["unwrap", "unwrap_status", "raw"])
def test_all_assertion_paths_emit_the_same_provider_failure(self, helper: str) -> None:
with pytest.raises(ProviderUnavailableError) as caught:
_assert_success(helper)
assert caught.value.failure == provider_failure(503, _BODY, _HEADERS)

View file

@ -996,6 +996,19 @@ class TestStreamingFidelity:
provider actually sent, not as one coalesced body. The unit of fidelity is the
HTTP transfer chunk, so every assertion here is made at the transfer layer."""
def test_binary_eventstream_record_and_replay_preserve_transfer_chunks(self, tmp_path: Path) -> None:
root: Final = tmp_path / "bundle"
binary_chunks: Final = (b"\x00\x00\x01\xff:message-", b"type\x00\xfe\r\n", b"\x00\xff\x00\x80")
with chunked_provider(chunks=binary_chunks, content_type="application/vnd.amazon.eventstream") as provider:
with running_edge(record_backend(root), {"openai": provider_url(provider)}) as edge:
recorded_head, recorded_chunks, recorded_ending = raw_stream_post(edge.port, STREAM_PATH, STREAM_BODY)
replayed_head, replayed_chunks, replayed_ending = replay_stream(root)
assert response_header(recorded_head, "content-type") == "application/vnd.amazon.eventstream"
assert response_header(replayed_head, "content-type") == "application/vnd.amazon.eventstream"
assert recorded_chunks == replayed_chunks == list(binary_chunks)
assert stream_chunks(recorded_stream(root)) == list(binary_chunks)
assert recorded_ending == replayed_ending == "terminated"
def test_a_streamed_response_records_its_chunk_boundaries(self, tmp_path: Path) -> None:
root = tmp_path / "bundle"
record_stream(root)

View file

@ -0,0 +1,220 @@
import os
import subprocess
import sys
import xml.etree.ElementTree as ET
from pathlib import Path
from typing import Final
def test_retry_policy_does_not_turn_new_diagnostics_into_recovered_passes(tmp_path: Path) -> None:
suite: Final = Path(__file__).resolve().parent
scenario: Final = tmp_path / "test_scenarios.py"
scenario.write_text(
"""
from pathlib import Path
import json
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from threading import Thread
import pytest
from e2e_http import URL, NoBody, NetworkError, StreamingResponse, UnknownApiError, require_successful_call, send, unwrap
from test_bedrock_eventstream import _CONVERSE, _frame
from llm_translation.bedrock_stream import assert_converse_stream
def attempt(name):
path = Path(__file__).with_name(name + ".attempts")
count = int(path.read_text()) + 1 if path.exists() else 1
path.write_text(str(count))
return count
def unavailable():
body = json.dumps({"error":{"message": "litellm.ServiceUnavailableError: BedrockException - Bedrock is unable to process your request."}})
unwrap(UnknownApiError(status_code=503, body=body, headers={
"x-amzn-requestid": "aws-first-attempt", "x-amzn-errortype": "ServiceUnavailableException",
"x-litellm-call-id": "proxy-first-attempt",
}))
def test_recovers():
if attempt("recovers") == 1:
unavailable()
def test_persistent():
attempt("persistent")
unavailable()
def test_proxy_503():
attempt("proxy_503")
unwrap(UnknownApiError(status_code=503, body="proxy overloaded"))
def test_auth():
attempt("auth")
unwrap(UnknownApiError(status_code=403, body="invalid credentials"))
def test_network():
attempt("network")
unwrap(NetworkError(message="connection reset"))
def test_native():
attempt("native")
require_successful_call(StreamingResponse(status_code=503, body="unavailable", headers={
"x-amzn-requestid": "native-request", "x-amzn-errortype": "ServiceUnavailableException",
}), expected_provider="bedrock")
def streamed_unavailable():
require_successful_call(StreamingResponse(
status_code=200, body="<streamed>", stream_error="serviceUnavailableException: unavailable",
stream_error_code="serviceUnavailableException", headers={
"x-amzn-requestid": "stream-request", "x-litellm-call-id": "stream-call",
},
), expected_provider="bedrock")
def test_stream_recovers():
if attempt("stream_recovers") == 1:
streamed_unavailable()
def test_stream_persistent():
attempt("stream_persistent")
streamed_unavailable()
def test_stream_transport():
if attempt("stream_transport") == 1:
require_successful_call(StreamingResponse(
status_code=200, body="connection interrupted", network_error=NetworkError(message="kind='network'"),
headers={"x-amzn-requestid": "transport-request", "x-litellm-call-id": "transport-call"},
))
def test_connection_recovers():
if attempt("connection_recovers") == 1:
require_successful_call(StreamingResponse(
status_code=-1, body="connection refused", network_error=NetworkError(message="kind='network'"),
))
def test_connection_direct_recovers():
if attempt("connection_direct_recovers") == 1:
result = StreamingResponse(status_code=-1, body="connection refused", network_error=NetworkError(message="connection refused"))
assert result.ok, f"call failed: {result}"
def test_header_marker_recovers():
if attempt("header_marker_recovers") == 1:
require_successful_call(StreamingResponse(status_code=403, body="invalid credentials", headers={"x-request-id": "kind='network'"}))
def test_behavior_recovers():
assert attempt("behavior_recovers") > 1, "completion returned no content"
def test_healthy_control():
require_successful_call(StreamingResponse(status_code=200, body="{}"))
class InterruptedOnce(BaseHTTPRequestHandler):
def do_POST(self):
self.rfile.read(int(self.headers.get("content-length", "0")))
binary = self.path.endswith("-binary")
wire = (
b"".join(_frame(json.dumps(value), key) for payload in _CONVERSE for key, value in json.loads(payload).items())
if binary else b'data: {"text":"Hello"}\\n\\ndata: [DONE]\\n\\n'
)
interrupted = attempt("http_" + self.path.lstrip("/")) == 1 and not self.path.startswith("/healthy-")
data = wire[:30] if interrupted else wire
self.send_response(200)
self.send_header("content-type", "application/vnd.amazon.eventstream" if binary else "text/event-stream")
self.send_header("transfer-encoding", "chunked")
self.send_header("x-litellm-call-id", "controlled-disconnect")
self.end_headers()
self.wfile.write(f"{len(data):x}\\r\\n".encode() + data + b"\\r\\n")
if not interrupted:
self.wfile.write(b"0\\r\\n\\r\\n")
self.wfile.flush()
self.close_connection = True
def log_message(self, *args):
pass
@pytest.fixture(scope="session")
def endpoint():
server = ThreadingHTTPServer(("127.0.0.1", 0), InterruptedOnce)
worker = Thread(target=server.serve_forever, daemon=True)
worker.start()
yield f"http://127.0.0.1:{server.server_port}"
server.shutdown()
server.server_close()
worker.join(timeout=5)
@pytest.mark.parametrize("path", ["binary", "sse"])
@pytest.mark.parametrize("mode", ["helper", "direct", "healthy"])
def test_http_stream_would_recover_on_a_second_request(endpoint, path, mode):
result = send(URL(f"{endpoint}/{mode}-{path}"), headers=NoBody(), json=NoBody(), stream=True, timeout=5)
if mode == "direct":
assert result.ok, f"stream was not established: {result}"
else:
require_successful_call(result)
if path == "binary":
assert_converse_stream(result)
else:
assert result.stream_done
""",
encoding="utf-8",
)
result: Final = subprocess.run(
[
sys.executable,
"-m",
"pytest",
str(scenario),
"-c",
str(suite / "pytest.ini"),
"--rootdir",
str(tmp_path),
"--confcutdir",
str(tmp_path),
"-p",
"conftest",
"--reruns-delay",
"0",
"--junitxml",
str(tmp_path / "results.xml"),
"-q",
],
cwd=tmp_path,
env={**os.environ, "PYTHONPATH": str(suite), "PYTEST_ADDOPTS": "", "E2E_FIXTURE_MODE": "live"},
capture_output=True,
text=True,
timeout=60,
check=False,
)
assert result.returncode == 1, result.stdout + result.stderr
assert "17 failed, 3 passed, 1 rerun" in result.stdout, result.stdout
assert {path.stem: int(path.read_text()) for path in tmp_path.glob("*.attempts")} == {
"recovers": 1,
"persistent": 1,
"proxy_503": 1,
"auth": 1,
"network": 2,
"native": 1,
"stream_recovers": 1,
"stream_persistent": 1,
"stream_transport": 1,
"connection_recovers": 1,
"behavior_recovers": 1,
"connection_direct_recovers": 1,
"header_marker_recovers": 1,
"http_helper-binary": 1,
"http_helper-sse": 1,
"http_direct-binary": 1,
"http_direct-sse": 1,
"http_healthy-binary": 1,
"http_healthy-sse": 1,
}
cases: Final = ET.parse(tmp_path / "results.xml").getroot()
recovered: Final = cases.find(".//testcase[@name='test_recovers']")
assert recovered is not None
assert recovered.find("failure") is not None
assert recovered.find("properties/property[@name='upstream_request_id'][@value='aws-first-attempt']") is not None
assert recovered.find("properties/property[@name='upstream_call_id'][@value='proxy-first-attempt']") is not None
streamed: Final = cases.find(".//testcase[@name='test_stream_recovers']")
assert streamed is not None and streamed.find("failure") is not None
assert streamed.find("properties/property[@name='upstream_status_code'][@value='200']") is not None
assert streamed.find("properties/property[@name='upstream_evidence'][@value='stream_event']") is not None
assert streamed.find("properties/property[@name='upstream_request_id'][@value='stream-request']") is not None
assert streamed.find("properties/property[@name='upstream_call_id'][@value='stream-call']") is not None
transport: Final = cases.find(".//testcase[@name='test_stream_transport']")
assert transport is not None and transport.find("failure") is not None
assert transport.find("properties/property[@name='upstream_request_id'][@value='transport-request']") is not None
assert transport.find("properties/property[@name='upstream_call_id'][@value='transport-call']") is not None