test(e2e): tolerate provider-side flakes on five full-suite cells

Mistral OCR retries a provider-relayed 429 with backoff, the Vertex vision
probe turns reasoning off so its 32 tokens go to the answer, the Vertex
cache cell spaces eight never-seen prefixes 15s apart around Google's
nondeterministic minimum-token rejection and prices the cached tokens
instead of prompt_tokens, and the Azure content-policy cell resends the
jailbreak prompt while Azure skips its filter
This commit is contained in:
mateo-berri 2026-09-22 17:36:37 -07:00
parent 7688f56256
commit 0d3be46456
8 changed files with 290 additions and 37 deletions

View file

@ -345,6 +345,40 @@ def request_with_retry[T: RetryableResponse](
return issue()
PROVIDER_RATE_LIMIT_MARKER: Final = "litellm.RateLimitError"
PROVIDER_RATE_LIMIT_ATTEMPTS: Final = 4
PROVIDER_RATE_LIMIT_BACKOFF_SECONDS: Final = 5.0
def tolerate_provider_rate_limit[R: BaseModel](
issue: Callable[[], Result[R]],
*,
attempts: int = PROVIDER_RATE_LIMIT_ATTEMPTS,
sleep: Callable[[float], None] = time.sleep,
) -> Result[R]:
"""Bounded, opt-in retry of one call the PROVIDER throttled, for a deployment
with no fallback group whose provider key is shared with the suites that run
alongside this one. The transport never retries a 429 (see request_with_retry),
and this helper only absorbs the 429s the proxy relayed from the provider, which
litellm's exception mapping prefixes with its exception class; the proxy's own
limiter and budget 429s never carry that prefix and come back untouched, so
every assertion on those still sees them. The wait honors Retry-After when the
provider sent one and backs off geometrically otherwise, and every retry prints
so the throttling stays visible in the run log instead of vanishing into green."""
for attempt in range(1, attempts):
match issue():
case RateLimitedError(body=body, retry_after_seconds=retry_after) if PROVIDER_RATE_LIMIT_MARKER in body:
delay = retry_after or PROVIDER_RATE_LIMIT_BACKOFF_SECONDS * (1 << (attempt - 1))
print(
f"e2e-http: provider rate limit relayed by the proxy; retry {attempt}/{attempts - 1} in {delay}s",
flush=True,
)
sleep(delay)
case result:
return result
return issue()
class ClassifiableResponse(Protocol):
"""What classifying an outcome reads off a response. requests.Response satisfies
it, and so does a fake, so the classification rules are testable on their own."""

View file

@ -13,7 +13,13 @@ Each case asserts the feature actually happened, not just a 200. Coverage matrix
call, so a never-seen prefix must come back cached on its very first call
(Gemini's implicit caching cannot hit a cold prefix), the cached count must
cover the marked block, and the spend row must be billed below the uncached
price of the prompt.
price of the cached tokens. Vertex's cache create is nondeterministic:
identical bodies come back 200 or with the minimum-token 400 ("The cached
content is of 1 tokens"), in failure bursts of 45 seconds and more, so up to
eight never-seen prefixes are tried with a pause after each rejection. The
billing check prices the cached tokens rather than prompt_tokens, which
Vertex reports inclusive of the cached prefix on some calls and exclusive
of it on others.
- Anthropic (claude-haiku-4-5, direct): the same ``cache_control`` prefix over
the OpenAI-compatible route; the second call must report cache-read tokens > 0.
- OpenAI (gpt-5.6): automatic prompt caching needs no request marker, so the
@ -50,7 +56,8 @@ VERTEX_MODEL = "vertex_ai/gemini-2.5-flash"
ANTHROPIC_MODEL = "anthropic/claude-haiku-4-5-20251001"
OPENAI_MODEL = "openai/gpt-5.6"
VERTEX_CACHE_TTL: Final = "300s"
VERTEX_COLD_CALL_ATTEMPTS: Final = 3
VERTEX_COLD_CALL_ATTEMPTS: Final = 8
VERTEX_COLD_CALL_PAUSE_SECONDS: Final = 15.0
VERTEX_MINIMUM_CACHED_TOKENS: Final = 1024
CACHED_SHARE_OF_PROMPT: Final = 0.9
VERTEX_CACHE_REJECTION_MARKER: Final = "minimum token count to start explicit caching"
@ -156,26 +163,36 @@ def _cold_cache_call(send: Callable[[str], Result[ChatResponse]]) -> ChatRespons
return unwrap(result)
def _first_engaged_cold_call(send: Callable[[str], Result[ChatResponse]]) -> ChatResponse | None:
for attempt in range(1, VERTEX_COLD_CALL_ATTEMPTS + 1):
candidate = _cold_cache_call(send)
if candidate is not None and _cached_read_tokens(candidate.usage) >= VERTEX_MINIMUM_CACHED_TOKENS:
return candidate
if attempt < VERTEX_COLD_CALL_ATTEMPTS:
print(
f"cache_control: vertex did not engage the cache on cold attempt {attempt}/{VERTEX_COLD_CALL_ATTEMPTS}; "
f"pausing {VERTEX_COLD_CALL_PAUSE_SECONDS}s before the next never-seen prefix",
flush=True,
)
time.sleep(VERTEX_COLD_CALL_PAUSE_SECONDS)
return None
def _first_cold_call_reads_cache(model: str, send: Callable[[str], Result[ChatResponse]]) -> ChatResponse:
completion: Final = next(
(
candidate
for candidate in (_cold_cache_call(send) for _ in range(VERTEX_COLD_CALL_ATTEMPTS))
if candidate is not None and _cached_read_tokens(candidate.usage) >= VERTEX_MINIMUM_CACHED_TOKENS
),
None,
)
completion: Final = _first_engaged_cold_call(send)
assert completion is not None, (
f"{model}: {VERTEX_COLD_CALL_ATTEMPTS} never-seen prompts marked with cache_control were each either "
f"rejected by Vertex's minimum-token check or served with fewer than {VERTEX_MINIMUM_CACHED_TOKENS} "
"cached tokens on their first call; explicit context caching did not engage"
f"{model}: {VERTEX_COLD_CALL_ATTEMPTS} never-seen prompts marked with cache_control, spread over "
f"{VERTEX_COLD_CALL_PAUSE_SECONDS * (VERTEX_COLD_CALL_ATTEMPTS - 1):.0f}s, were each either rejected by "
f"Vertex's minimum-token check or served with fewer than {VERTEX_MINIMUM_CACHED_TOKENS} cached tokens on "
"their first call; explicit context caching did not engage"
)
assert completion.choices, f"{model}: cached call returned no choices: {completion}"
usage: Final = completion.usage
cached: Final = _cached_read_tokens(usage)
assert usage and usage.prompt_tokens and cached >= CACHED_SHARE_OF_PROMPT * usage.prompt_tokens, (
f"{model}: only {cached} of {usage.prompt_tokens if usage else None} prompt tokens were served from the "
"cache; the cache_control block was not cached whole"
assert usage and usage.prompt_tokens, f"{model}: cached completion carried no prompt_tokens: {usage}"
assert cached >= CACHED_SHARE_OF_PROMPT * usage.prompt_tokens, (
f"{model}: only {cached} of {usage.prompt_tokens} prompt tokens were served from the cache; the "
"cache_control block was not cached whole"
)
return completion
@ -196,10 +213,11 @@ def _assert_billed_below_uncached_prompt(client: PassthroughClient, model: str,
assert row.prompt_tokens == usage.prompt_tokens, (
f"{model}: spend row prompt_tokens {row.prompt_tokens} != response prompt_tokens {usage.prompt_tokens}"
)
uncached_prompt_cost: Final = usage.prompt_tokens * _input_rate(client, model)
assert row.spend is not None and row.spend < uncached_prompt_cost, (
f"{model}: spend {row.spend} is not below the uncached price of the prompt alone ({uncached_prompt_cost} for "
f"{usage.prompt_tokens} tokens); cache-read pricing was not applied"
cached: Final = _cached_read_tokens(usage)
uncached_read_cost: Final = cached * _input_rate(client, model)
assert row.spend is not None and row.spend < uncached_read_cost, (
f"{model}: spend {row.spend} is not below the uncached price of the {cached} tokens read from the cache "
f"({uncached_read_cost}); cache-read pricing was not applied"
)

View file

@ -110,10 +110,11 @@ def _vision_messages() -> list[ChatMessage]:
def _assert_describes_cat(response: ChatResponse) -> None:
assert response.choices, f"vision returned no choices: {response}"
message = response.choices[0].message
content = (message.content if message else None) or ""
choice = response.choices[0]
content = (choice.message.content if choice.message else None) or ""
assert any(term in content.lower() for term in ("cat", "feline", "kitten", "kitty")), (
f"vision response did not describe the image: {content[:200]}"
f"vision response did not describe the image: {content[:200]!r} "
f"(finish_reason={choice.finish_reason!r}, usage={response.usage})"
)
@ -421,7 +422,11 @@ class TestVertexChatCompletions:
model = self._register(client, resources, "e2e-vertex-vision")
key = resources.key()
response = unwrap(client.proxy.chat(key, ChatBody(model=model, messages=_vision_messages(), max_tokens=32)))
response = unwrap(
client.proxy.chat(
key, ChatBody(model=model, messages=_vision_messages(), max_tokens=32, reasoning_effort="none")
)
)
_assert_describes_cat(response)
@pytest.mark.covers(

View file

@ -11,6 +11,11 @@ well-formed OCR document comes back. Per the e2e hard-fail contract, a case
fails when no proxy answers and also fails once a request reaches it: the proxy
fetches each provider's referenced secrets, so a
missing credential surfaces as a live provider error rather than silent green.
The Mistral key is shared with the suites that run alongside this one and the
deployment has no fallback group, so the OCR call tolerates a bounded number of
provider 429s the proxy relays (tolerate_provider_rate_limit); a 429 the proxy
issues itself still fails the case at once.
"""
from __future__ import annotations
@ -20,7 +25,7 @@ from typing import Protocol
import pytest
from e2e_config import unique_marker
from e2e_http import assert_client_error, unwrap
from e2e_http import assert_client_error, tolerate_provider_rate_limit, unwrap
from lifecycle import ResourceManager
from models import LiteLLMParamsBody, OcrBody, OcrDocument, OcrResponse
from proxy_client import ProxyClient
@ -156,7 +161,9 @@ class TestRustOcrGateway:
resources.defer(lambda: proxy.delete_model(model_id))
key = resources.key()
response = unwrap(proxy.ocr(key, OcrBody(model=model, document=case.document)))
response = unwrap(
tolerate_provider_rate_limit(lambda: proxy.ocr(key, OcrBody(model=model, document=case.document)))
)
_assert_ocr_document(response)
@pytest.mark.skip(reason="stage red: product gap, /v1/ocr 500s (aocr TypeError) on missing document instead of 400")

View file

@ -15,9 +15,10 @@ reliability behavior.
from __future__ import annotations
from collections.abc import Sequence
from collections.abc import Mapping, Sequence
from typing import Final
from pydantic import ValidationError
from pydantic import BaseModel, ValidationError
from proxy_client import ProxyClient
from e2e_config import CHEAP_OPENAI_MODEL, PROXY_BASE_URL, unique_marker
@ -367,6 +368,31 @@ def model_id_of(resp: StreamingResponse) -> str | None:
return resp.headers.get("x-litellm-model-id")
class _AzurePromptFilterResult(BaseModel):
content_filter_results: Mapping[str, object] | None = None
class _AzureAnnotatedChatBody(BaseModel):
prompt_filter_results: Sequence[_AzurePromptFilterResult] | None = None
def azure_prompt_filter_skipped(resp: StreamingResponse) -> bool:
"""True when Azure answered without running its prompt content filter: the body
carries ``prompt_filter_results`` whose ``content_filter_results`` are all empty,
where a filter that ran records a verdict per category (jailbreak included).
Azure does this intermittently and it says nothing about the prompt, so a test
that needs the filter's verdict sends the prompt again instead of reading a
skipped filter as a pass. A body without the field (an OpenAI completion served
by a fallback, or an error body) is not a skipped filter."""
try:
annotated: Final = _AzureAnnotatedChatBody.model_validate_json(resp.body)
except ValidationError:
return False
if not annotated.prompt_filter_results:
return False
return all(not entry.content_filter_results for entry in annotated.prompt_filter_results)
def _parsed(resp: StreamingResponse) -> ChatResponse | None:
try:
return ChatResponse.model_validate_json(resp.body)

View file

@ -16,10 +16,16 @@ failure: the provider refuses the prompt itself, on length or on policy, and
reroute those, not `fallbacks`. The policy refusal is a real one, from an Azure
OpenAI content filter rejecting a jailbreak prompt, and a control call first
proves the refusal reaches the customer as a 400 when no reroute is configured.
Azure intermittently answers without running its prompt filter at all (the
body's prompt_filter_results carry no verdict), which is not a pass, so both
calls send the prompt again, a bounded number of times, until the filter ran.
"""
from __future__ import annotations
from collections.abc import Callable
from typing import Final
import pytest
from complexity_router_client import ComplexityRouterClient
@ -29,6 +35,7 @@ from lifecycle import ResourceManager
from models import RouterSettingsOverride
from reliability_support import (
CONTENT_POLICY_PROMPT,
azure_prompt_filter_skipped,
chat_override,
completion_tokens_of,
content_of,
@ -64,6 +71,28 @@ def _assert_served_by_fallback(resp: StreamingResponse) -> None:
assert int(attempted) >= 1, f"x-litellm-attempted-fallbacks should be >= 1, got {attempted!r}"
AZURE_FILTER_ATTEMPTS: Final = 3
def _chat_once_azure_runs_its_filter(send: Callable[[], StreamingResponse]) -> StreamingResponse:
for attempt in range(1, AZURE_FILTER_ATTEMPTS):
resp = send()
if not azure_prompt_filter_skipped(resp):
return resp
print(
"e2e: azure answered without running its prompt filter; sending the jailbreak prompt again "
f"({attempt}/{AZURE_FILTER_ATTEMPTS - 1})",
flush=True,
)
return send()
def _filter_verdict(resp: StreamingResponse) -> str:
if azure_prompt_filter_skipped(resp):
return "azure skipped its prompt filter on every attempt"
return "the filter ran and let the prompt through"
class TestReliabilityFallbacks:
@pytest.mark.covers("reliability.fallback.5xx.routes_to_fallback")
def test_5xx_routes_to_fallback(
@ -124,17 +153,21 @@ class TestReliabilityFallbacks:
model_id = create_content_filtered_deployment(client.proxy, primary)
resources.defer(lambda: client.proxy.delete_model(model_id))
refused = chat_override(client.proxy, scoped_key, primary, f"{CONTENT_POLICY_PROMPT} {unique_marker()}")
refused = _chat_once_azure_runs_its_filter(
lambda: chat_override(client.proxy, scoped_key, primary, f"{CONTENT_POLICY_PROMPT} {unique_marker()}")
)
assert refused.status_code == 400, (
f"the content filter should have refused the jailbreak prompt with a 400, got {refused.status_code}: "
f"{refused.body[:300]}"
f"the content filter should have refused the jailbreak prompt with a 400, got {refused.status_code} "
f"({_filter_verdict(refused)}): {refused.body[:300]}"
)
resp = chat_override(
client.proxy,
scoped_key,
primary,
f"{CONTENT_POLICY_PROMPT} {unique_marker()}",
override=RouterSettingsOverride(content_policy_fallbacks=[{primary: ["gpt-5.5"]}]),
resp = _chat_once_azure_runs_its_filter(
lambda: chat_override(
client.proxy,
scoped_key,
primary,
f"{CONTENT_POLICY_PROMPT} {unique_marker()}",
override=RouterSettingsOverride(content_policy_fallbacks=[{primary: ["gpt-5.5"]}]),
)
)
_assert_served_by_fallback(resp)

View file

@ -0,0 +1,55 @@
"""Unit tests for the reliability helpers that read provider bodies, driven with
the verbatim shapes the proxy relays; nothing here needs a proxy or a provider."""
from __future__ import annotations
from typing import Final
import pytest
from e2e_http import StreamingResponse
from reliability_support import azure_prompt_filter_skipped
_SKIPPED_FILTER_BODY: Final = (
'{"id":"chatcmpl-1","object":"chat.completion","model":"reliability-policyfail-1","choices":[{"finish_reason":'
'"stop","index":0,"message":{"role":"assistant","content":"I can\'t comply."}}],"usage":{"prompt_tokens":64,'
'"completion_tokens":53,"total_tokens":117},"prompt_filter_results":[{"prompt_index":0,"content_filter_results":{}}]}'
)
_RAN_FILTER_BODY: Final = (
'{"id":"chatcmpl-2","object":"chat.completion","model":"reliability-policyfail-1","choices":[{"finish_reason":'
'"stop","index":0,"message":{"role":"assistant","content":"Sure."}}],"usage":{"prompt_tokens":64,'
'"completion_tokens":2,"total_tokens":66},"prompt_filter_results":[{"prompt_index":0,"content_filter_results":'
'{"jailbreak":{"filtered":false,"detected":false},"hate":{"filtered":false,"severity":"safe"}}}]}'
)
_FALLBACK_BODY: Final = (
'{"id":"chatcmpl-3","object":"chat.completion","model":"gpt-5.5","choices":[{"finish_reason":"stop","index":0,'
'"message":{"role":"assistant","content":"Hello."}}],"usage":{"prompt_tokens":64,"completion_tokens":2,'
'"total_tokens":66}}'
)
_REFUSAL_BODY: Final = (
'{"error":{"message":"litellm.ContentPolicyViolationError: AzureException - The response was filtered",'
'"type":"invalid_request_error","param":"prompt","code":"content_filter"}}'
)
def _response(status_code: int, body: str) -> StreamingResponse:
return StreamingResponse(status_code=status_code, body=body)
def test_a_200_whose_prompt_filter_recorded_no_verdict_is_a_skipped_filter() -> None:
assert azure_prompt_filter_skipped(_response(200, _SKIPPED_FILTER_BODY))
@pytest.mark.parametrize(
("status_code", "body"),
(
(200, _RAN_FILTER_BODY),
(200, _FALLBACK_BODY),
(400, _REFUSAL_BODY),
(200, "not json"),
(200, '{"prompt_filter_results":[]}'),
),
ids=("filter-ran", "fallback-body", "refusal", "not-json", "empty-list"),
)
def test_every_other_body_is_not_a_skipped_filter(status_code: int, body: str) -> None:
assert not azure_prompt_filter_skipped(_response(status_code, body))

View file

@ -19,15 +19,21 @@ from typing import Final
import pytest
from e2e_http import (
PROVIDER_RATE_LIMIT_ATTEMPTS,
PROVIDER_RATE_LIMIT_BACKOFF_SECONDS,
RETRY_ATTEMPTS,
TRANSIENT_STATUSES,
NoBody,
PartialBody,
RateLimitedError,
Result,
Success,
UnknownApiError,
ValidationError,
classify,
request_with_retry,
streaming_outcome,
tolerate_provider_rate_limit,
wire_body,
without_retries,
)
@ -105,6 +111,75 @@ class TestTransientRetryPolicy:
assert [r.close_calls for r in responses] == [1, 1, 0, 0]
class _Ocr(BaseModel):
pages: int
PROVIDER_429_BODY: Final = (
'{"error":{"message":"litellm.RateLimitError: RateLimitError: MistralException - (429, '
'\'{"message":"Rate limit exceeded","type":"rate_limited"}\') LiteLLM Retried: 3 times",'
'"type":"throttling_error","param":null,"code":"429"}}'
)
PROXY_429_BODY: Final = (
'{"error":{"message":"Max parallel request limit reached for key","type":"throttling_error","code":"429"}}'
)
def _results_from(results: Sequence[Result[_Ocr]]) -> Callable[[], Result[_Ocr]]:
it: Final = iter(results)
return lambda: next(it)
class TestProviderRateLimitTolerance:
def test_relayed_provider_429_is_retried_with_backoff_then_returns_the_success(self) -> None:
results: Final = (
RateLimitedError(body=PROVIDER_429_BODY),
RateLimitedError(body=PROVIDER_429_BODY),
Success(status_code=200, data=_Ocr(pages=1)),
)
sleep: Final = SleepRecorder()
assert tolerate_provider_rate_limit(_results_from(results), sleep=sleep) is results[2]
assert sleep.delays == (PROVIDER_RATE_LIMIT_BACKOFF_SECONDS, 2 * PROVIDER_RATE_LIMIT_BACKOFF_SECONDS)
def test_retry_after_from_the_provider_replaces_the_backoff(self) -> None:
results: Final = (
RateLimitedError(body=PROVIDER_429_BODY, retry_after_seconds=7),
Success(status_code=200, data=_Ocr(pages=1)),
)
sleep: Final = SleepRecorder()
assert tolerate_provider_rate_limit(_results_from(results), sleep=sleep) is results[1]
assert sleep.delays == (7,)
def test_the_proxys_own_429_comes_back_untouched(self) -> None:
results: Final = (RateLimitedError(body=PROXY_429_BODY), Success(status_code=200, data=_Ocr(pages=1)))
sleep: Final = SleepRecorder()
assert tolerate_provider_rate_limit(_results_from(results), sleep=sleep) is results[0]
assert sleep.delays == ()
@pytest.mark.parametrize(
"first",
(
Success(status_code=200, data=_Ocr(pages=1)),
UnknownApiError(status_code=500, body="boom"),
ValidationError(message="bad"),
),
ids=("success", "unknown", "validation"),
)
def test_any_other_outcome_returns_at_once(self, first: Result[_Ocr]) -> None:
sleep: Final = SleepRecorder()
assert tolerate_provider_rate_limit(_results_from((first,)), sleep=sleep) is first
assert sleep.delays == ()
def test_a_persistent_provider_429_is_bounded_and_returns_the_last_one(self) -> None:
results: Final = tuple(
RateLimitedError(body=PROVIDER_429_BODY) for _ in range(PROVIDER_RATE_LIMIT_ATTEMPTS + 1)
)
sleep: Final = SleepRecorder()
last: Final = tolerate_provider_rate_limit(_results_from(results), sleep=sleep)
assert last is results[PROVIDER_RATE_LIMIT_ATTEMPTS - 1]
assert sleep.delays == (5.0, 10.0, 20.0)
@dataclass(frozen=True, slots=True)
class FakeSseResponse:
lines: Sequence[bytes]