diff --git a/tests/e2e/e2e_http.py b/tests/e2e/e2e_http.py index 1b62a1dbd8c..237960e4fc3 100644 --- a/tests/e2e/e2e_http.py +++ b/tests/e2e/e2e_http.py @@ -345,6 +345,40 @@ def request_with_retry[T: RetryableResponse]( return issue() +PROVIDER_RATE_LIMIT_MARKER: Final = "litellm.RateLimitError" +PROVIDER_RATE_LIMIT_ATTEMPTS: Final = 4 +PROVIDER_RATE_LIMIT_BACKOFF_SECONDS: Final = 5.0 + + +def tolerate_provider_rate_limit[R: BaseModel]( + issue: Callable[[], Result[R]], + *, + attempts: int = PROVIDER_RATE_LIMIT_ATTEMPTS, + sleep: Callable[[float], None] = time.sleep, +) -> Result[R]: + """Bounded, opt-in retry of one call the PROVIDER throttled, for a deployment + with no fallback group whose provider key is shared with the suites that run + alongside this one. The transport never retries a 429 (see request_with_retry), + and this helper only absorbs the 429s the proxy relayed from the provider, which + litellm's exception mapping prefixes with its exception class; the proxy's own + limiter and budget 429s never carry that prefix and come back untouched, so + every assertion on those still sees them. The wait honors Retry-After when the + provider sent one and backs off geometrically otherwise, and every retry prints + so the throttling stays visible in the run log instead of vanishing into green.""" + for attempt in range(1, attempts): + match issue(): + case RateLimitedError(body=body, retry_after_seconds=retry_after) if PROVIDER_RATE_LIMIT_MARKER in body: + delay = retry_after or PROVIDER_RATE_LIMIT_BACKOFF_SECONDS * (1 << (attempt - 1)) + print( + f"e2e-http: provider rate limit relayed by the proxy; retry {attempt}/{attempts - 1} in {delay}s", + flush=True, + ) + sleep(delay) + case result: + return result + return issue() + + class ClassifiableResponse(Protocol): """What classifying an outcome reads off a response. requests.Response satisfies it, and so does a fake, so the classification rules are testable on their own.""" diff --git a/tests/e2e/llm_translation/test_cache_control.py b/tests/e2e/llm_translation/test_cache_control.py index ceb3620183a..bec65144c9f 100644 --- a/tests/e2e/llm_translation/test_cache_control.py +++ b/tests/e2e/llm_translation/test_cache_control.py @@ -13,7 +13,13 @@ Each case asserts the feature actually happened, not just a 200. Coverage matrix call, so a never-seen prefix must come back cached on its very first call (Gemini's implicit caching cannot hit a cold prefix), the cached count must cover the marked block, and the spend row must be billed below the uncached - price of the prompt. + price of the cached tokens. Vertex's cache create is nondeterministic: + identical bodies come back 200 or with the minimum-token 400 ("The cached + content is of 1 tokens"), in failure bursts of 45 seconds and more, so up to + eight never-seen prefixes are tried with a pause after each rejection. The + billing check prices the cached tokens rather than prompt_tokens, which + Vertex reports inclusive of the cached prefix on some calls and exclusive + of it on others. - Anthropic (claude-haiku-4-5, direct): the same ``cache_control`` prefix over the OpenAI-compatible route; the second call must report cache-read tokens > 0. - OpenAI (gpt-5.6): automatic prompt caching needs no request marker, so the @@ -50,7 +56,8 @@ VERTEX_MODEL = "vertex_ai/gemini-2.5-flash" ANTHROPIC_MODEL = "anthropic/claude-haiku-4-5-20251001" OPENAI_MODEL = "openai/gpt-5.6" VERTEX_CACHE_TTL: Final = "300s" -VERTEX_COLD_CALL_ATTEMPTS: Final = 3 +VERTEX_COLD_CALL_ATTEMPTS: Final = 8 +VERTEX_COLD_CALL_PAUSE_SECONDS: Final = 15.0 VERTEX_MINIMUM_CACHED_TOKENS: Final = 1024 CACHED_SHARE_OF_PROMPT: Final = 0.9 VERTEX_CACHE_REJECTION_MARKER: Final = "minimum token count to start explicit caching" @@ -156,26 +163,36 @@ def _cold_cache_call(send: Callable[[str], Result[ChatResponse]]) -> ChatRespons return unwrap(result) +def _first_engaged_cold_call(send: Callable[[str], Result[ChatResponse]]) -> ChatResponse | None: + for attempt in range(1, VERTEX_COLD_CALL_ATTEMPTS + 1): + candidate = _cold_cache_call(send) + if candidate is not None and _cached_read_tokens(candidate.usage) >= VERTEX_MINIMUM_CACHED_TOKENS: + return candidate + if attempt < VERTEX_COLD_CALL_ATTEMPTS: + print( + f"cache_control: vertex did not engage the cache on cold attempt {attempt}/{VERTEX_COLD_CALL_ATTEMPTS}; " + f"pausing {VERTEX_COLD_CALL_PAUSE_SECONDS}s before the next never-seen prefix", + flush=True, + ) + time.sleep(VERTEX_COLD_CALL_PAUSE_SECONDS) + return None + + def _first_cold_call_reads_cache(model: str, send: Callable[[str], Result[ChatResponse]]) -> ChatResponse: - completion: Final = next( - ( - candidate - for candidate in (_cold_cache_call(send) for _ in range(VERTEX_COLD_CALL_ATTEMPTS)) - if candidate is not None and _cached_read_tokens(candidate.usage) >= VERTEX_MINIMUM_CACHED_TOKENS - ), - None, - ) + completion: Final = _first_engaged_cold_call(send) assert completion is not None, ( - f"{model}: {VERTEX_COLD_CALL_ATTEMPTS} never-seen prompts marked with cache_control were each either " - f"rejected by Vertex's minimum-token check or served with fewer than {VERTEX_MINIMUM_CACHED_TOKENS} " - "cached tokens on their first call; explicit context caching did not engage" + f"{model}: {VERTEX_COLD_CALL_ATTEMPTS} never-seen prompts marked with cache_control, spread over " + f"{VERTEX_COLD_CALL_PAUSE_SECONDS * (VERTEX_COLD_CALL_ATTEMPTS - 1):.0f}s, were each either rejected by " + f"Vertex's minimum-token check or served with fewer than {VERTEX_MINIMUM_CACHED_TOKENS} cached tokens on " + "their first call; explicit context caching did not engage" ) assert completion.choices, f"{model}: cached call returned no choices: {completion}" usage: Final = completion.usage cached: Final = _cached_read_tokens(usage) - assert usage and usage.prompt_tokens and cached >= CACHED_SHARE_OF_PROMPT * usage.prompt_tokens, ( - f"{model}: only {cached} of {usage.prompt_tokens if usage else None} prompt tokens were served from the " - "cache; the cache_control block was not cached whole" + assert usage and usage.prompt_tokens, f"{model}: cached completion carried no prompt_tokens: {usage}" + assert cached >= CACHED_SHARE_OF_PROMPT * usage.prompt_tokens, ( + f"{model}: only {cached} of {usage.prompt_tokens} prompt tokens were served from the cache; the " + "cache_control block was not cached whole" ) return completion @@ -196,10 +213,11 @@ def _assert_billed_below_uncached_prompt(client: PassthroughClient, model: str, assert row.prompt_tokens == usage.prompt_tokens, ( f"{model}: spend row prompt_tokens {row.prompt_tokens} != response prompt_tokens {usage.prompt_tokens}" ) - uncached_prompt_cost: Final = usage.prompt_tokens * _input_rate(client, model) - assert row.spend is not None and row.spend < uncached_prompt_cost, ( - f"{model}: spend {row.spend} is not below the uncached price of the prompt alone ({uncached_prompt_cost} for " - f"{usage.prompt_tokens} tokens); cache-read pricing was not applied" + cached: Final = _cached_read_tokens(usage) + uncached_read_cost: Final = cached * _input_rate(client, model) + assert row.spend is not None and row.spend < uncached_read_cost, ( + f"{model}: spend {row.spend} is not below the uncached price of the {cached} tokens read from the cache " + f"({uncached_read_cost}); cache-read pricing was not applied" ) diff --git a/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py b/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py index 363b2a7e02e..235856d7692 100644 --- a/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py +++ b/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py @@ -110,10 +110,11 @@ def _vision_messages() -> list[ChatMessage]: def _assert_describes_cat(response: ChatResponse) -> None: assert response.choices, f"vision returned no choices: {response}" - message = response.choices[0].message - content = (message.content if message else None) or "" + choice = response.choices[0] + content = (choice.message.content if choice.message else None) or "" assert any(term in content.lower() for term in ("cat", "feline", "kitten", "kitty")), ( - f"vision response did not describe the image: {content[:200]}" + f"vision response did not describe the image: {content[:200]!r} " + f"(finish_reason={choice.finish_reason!r}, usage={response.usage})" ) @@ -421,7 +422,11 @@ class TestVertexChatCompletions: model = self._register(client, resources, "e2e-vertex-vision") key = resources.key() - response = unwrap(client.proxy.chat(key, ChatBody(model=model, messages=_vision_messages(), max_tokens=32))) + response = unwrap( + client.proxy.chat( + key, ChatBody(model=model, messages=_vision_messages(), max_tokens=32, reasoning_effort="none") + ) + ) _assert_describes_cat(response) @pytest.mark.covers( diff --git a/tests/e2e/llm_translation/test_ocr_rust_e2e.py b/tests/e2e/llm_translation/test_ocr_rust_e2e.py index c2560b199af..24c98170609 100644 --- a/tests/e2e/llm_translation/test_ocr_rust_e2e.py +++ b/tests/e2e/llm_translation/test_ocr_rust_e2e.py @@ -11,6 +11,11 @@ well-formed OCR document comes back. Per the e2e hard-fail contract, a case fails when no proxy answers and also fails once a request reaches it: the proxy fetches each provider's referenced secrets, so a missing credential surfaces as a live provider error rather than silent green. + +The Mistral key is shared with the suites that run alongside this one and the +deployment has no fallback group, so the OCR call tolerates a bounded number of +provider 429s the proxy relays (tolerate_provider_rate_limit); a 429 the proxy +issues itself still fails the case at once. """ from __future__ import annotations @@ -20,7 +25,7 @@ from typing import Protocol import pytest from e2e_config import unique_marker -from e2e_http import assert_client_error, unwrap +from e2e_http import assert_client_error, tolerate_provider_rate_limit, unwrap from lifecycle import ResourceManager from models import LiteLLMParamsBody, OcrBody, OcrDocument, OcrResponse from proxy_client import ProxyClient @@ -156,7 +161,9 @@ class TestRustOcrGateway: resources.defer(lambda: proxy.delete_model(model_id)) key = resources.key() - response = unwrap(proxy.ocr(key, OcrBody(model=model, document=case.document))) + response = unwrap( + tolerate_provider_rate_limit(lambda: proxy.ocr(key, OcrBody(model=model, document=case.document))) + ) _assert_ocr_document(response) @pytest.mark.skip(reason="stage red: product gap, /v1/ocr 500s (aocr TypeError) on missing document instead of 400") diff --git a/tests/e2e/router/reliability_support.py b/tests/e2e/router/reliability_support.py index 4388e7f11bc..503d7684cc0 100644 --- a/tests/e2e/router/reliability_support.py +++ b/tests/e2e/router/reliability_support.py @@ -15,9 +15,10 @@ reliability behavior. from __future__ import annotations -from collections.abc import Sequence +from collections.abc import Mapping, Sequence +from typing import Final -from pydantic import ValidationError +from pydantic import BaseModel, ValidationError from proxy_client import ProxyClient from e2e_config import CHEAP_OPENAI_MODEL, PROXY_BASE_URL, unique_marker @@ -367,6 +368,31 @@ def model_id_of(resp: StreamingResponse) -> str | None: return resp.headers.get("x-litellm-model-id") +class _AzurePromptFilterResult(BaseModel): + content_filter_results: Mapping[str, object] | None = None + + +class _AzureAnnotatedChatBody(BaseModel): + prompt_filter_results: Sequence[_AzurePromptFilterResult] | None = None + + +def azure_prompt_filter_skipped(resp: StreamingResponse) -> bool: + """True when Azure answered without running its prompt content filter: the body + carries ``prompt_filter_results`` whose ``content_filter_results`` are all empty, + where a filter that ran records a verdict per category (jailbreak included). + Azure does this intermittently and it says nothing about the prompt, so a test + that needs the filter's verdict sends the prompt again instead of reading a + skipped filter as a pass. A body without the field (an OpenAI completion served + by a fallback, or an error body) is not a skipped filter.""" + try: + annotated: Final = _AzureAnnotatedChatBody.model_validate_json(resp.body) + except ValidationError: + return False + if not annotated.prompt_filter_results: + return False + return all(not entry.content_filter_results for entry in annotated.prompt_filter_results) + + def _parsed(resp: StreamingResponse) -> ChatResponse | None: try: return ChatResponse.model_validate_json(resp.body) diff --git a/tests/e2e/router/test_reliability_fallbacks_e2e.py b/tests/e2e/router/test_reliability_fallbacks_e2e.py index 8bc60f2829b..54c11b163d0 100644 --- a/tests/e2e/router/test_reliability_fallbacks_e2e.py +++ b/tests/e2e/router/test_reliability_fallbacks_e2e.py @@ -16,10 +16,16 @@ failure: the provider refuses the prompt itself, on length or on policy, and reroute those, not `fallbacks`. The policy refusal is a real one, from an Azure OpenAI content filter rejecting a jailbreak prompt, and a control call first proves the refusal reaches the customer as a 400 when no reroute is configured. +Azure intermittently answers without running its prompt filter at all (the +body's prompt_filter_results carry no verdict), which is not a pass, so both +calls send the prompt again, a bounded number of times, until the filter ran. """ from __future__ import annotations +from collections.abc import Callable +from typing import Final + import pytest from complexity_router_client import ComplexityRouterClient @@ -29,6 +35,7 @@ from lifecycle import ResourceManager from models import RouterSettingsOverride from reliability_support import ( CONTENT_POLICY_PROMPT, + azure_prompt_filter_skipped, chat_override, completion_tokens_of, content_of, @@ -64,6 +71,28 @@ def _assert_served_by_fallback(resp: StreamingResponse) -> None: assert int(attempted) >= 1, f"x-litellm-attempted-fallbacks should be >= 1, got {attempted!r}" +AZURE_FILTER_ATTEMPTS: Final = 3 + + +def _chat_once_azure_runs_its_filter(send: Callable[[], StreamingResponse]) -> StreamingResponse: + for attempt in range(1, AZURE_FILTER_ATTEMPTS): + resp = send() + if not azure_prompt_filter_skipped(resp): + return resp + print( + "e2e: azure answered without running its prompt filter; sending the jailbreak prompt again " + f"({attempt}/{AZURE_FILTER_ATTEMPTS - 1})", + flush=True, + ) + return send() + + +def _filter_verdict(resp: StreamingResponse) -> str: + if azure_prompt_filter_skipped(resp): + return "azure skipped its prompt filter on every attempt" + return "the filter ran and let the prompt through" + + class TestReliabilityFallbacks: @pytest.mark.covers("reliability.fallback.5xx.routes_to_fallback") def test_5xx_routes_to_fallback( @@ -124,17 +153,21 @@ class TestReliabilityFallbacks: model_id = create_content_filtered_deployment(client.proxy, primary) resources.defer(lambda: client.proxy.delete_model(model_id)) - refused = chat_override(client.proxy, scoped_key, primary, f"{CONTENT_POLICY_PROMPT} {unique_marker()}") + refused = _chat_once_azure_runs_its_filter( + lambda: chat_override(client.proxy, scoped_key, primary, f"{CONTENT_POLICY_PROMPT} {unique_marker()}") + ) assert refused.status_code == 400, ( - f"the content filter should have refused the jailbreak prompt with a 400, got {refused.status_code}: " - f"{refused.body[:300]}" + f"the content filter should have refused the jailbreak prompt with a 400, got {refused.status_code} " + f"({_filter_verdict(refused)}): {refused.body[:300]}" ) - resp = chat_override( - client.proxy, - scoped_key, - primary, - f"{CONTENT_POLICY_PROMPT} {unique_marker()}", - override=RouterSettingsOverride(content_policy_fallbacks=[{primary: ["gpt-5.5"]}]), + resp = _chat_once_azure_runs_its_filter( + lambda: chat_override( + client.proxy, + scoped_key, + primary, + f"{CONTENT_POLICY_PROMPT} {unique_marker()}", + override=RouterSettingsOverride(content_policy_fallbacks=[{primary: ["gpt-5.5"]}]), + ) ) _assert_served_by_fallback(resp) diff --git a/tests/e2e/router/test_reliability_support.py b/tests/e2e/router/test_reliability_support.py new file mode 100644 index 00000000000..3399c3fab7e --- /dev/null +++ b/tests/e2e/router/test_reliability_support.py @@ -0,0 +1,55 @@ +"""Unit tests for the reliability helpers that read provider bodies, driven with +the verbatim shapes the proxy relays; nothing here needs a proxy or a provider.""" + +from __future__ import annotations + +from typing import Final + +import pytest + +from e2e_http import StreamingResponse +from reliability_support import azure_prompt_filter_skipped + +_SKIPPED_FILTER_BODY: Final = ( + '{"id":"chatcmpl-1","object":"chat.completion","model":"reliability-policyfail-1","choices":[{"finish_reason":' + '"stop","index":0,"message":{"role":"assistant","content":"I can\'t comply."}}],"usage":{"prompt_tokens":64,' + '"completion_tokens":53,"total_tokens":117},"prompt_filter_results":[{"prompt_index":0,"content_filter_results":{}}]}' +) +_RAN_FILTER_BODY: Final = ( + '{"id":"chatcmpl-2","object":"chat.completion","model":"reliability-policyfail-1","choices":[{"finish_reason":' + '"stop","index":0,"message":{"role":"assistant","content":"Sure."}}],"usage":{"prompt_tokens":64,' + '"completion_tokens":2,"total_tokens":66},"prompt_filter_results":[{"prompt_index":0,"content_filter_results":' + '{"jailbreak":{"filtered":false,"detected":false},"hate":{"filtered":false,"severity":"safe"}}}]}' +) +_FALLBACK_BODY: Final = ( + '{"id":"chatcmpl-3","object":"chat.completion","model":"gpt-5.5","choices":[{"finish_reason":"stop","index":0,' + '"message":{"role":"assistant","content":"Hello."}}],"usage":{"prompt_tokens":64,"completion_tokens":2,' + '"total_tokens":66}}' +) +_REFUSAL_BODY: Final = ( + '{"error":{"message":"litellm.ContentPolicyViolationError: AzureException - The response was filtered",' + '"type":"invalid_request_error","param":"prompt","code":"content_filter"}}' +) + + +def _response(status_code: int, body: str) -> StreamingResponse: + return StreamingResponse(status_code=status_code, body=body) + + +def test_a_200_whose_prompt_filter_recorded_no_verdict_is_a_skipped_filter() -> None: + assert azure_prompt_filter_skipped(_response(200, _SKIPPED_FILTER_BODY)) + + +@pytest.mark.parametrize( + ("status_code", "body"), + ( + (200, _RAN_FILTER_BODY), + (200, _FALLBACK_BODY), + (400, _REFUSAL_BODY), + (200, "not json"), + (200, '{"prompt_filter_results":[]}'), + ), + ids=("filter-ran", "fallback-body", "refusal", "not-json", "empty-list"), +) +def test_every_other_body_is_not_a_skipped_filter(status_code: int, body: str) -> None: + assert not azure_prompt_filter_skipped(_response(status_code, body)) diff --git a/tests/e2e/test_e2e_http.py b/tests/e2e/test_e2e_http.py index 7201da84924..dc06d9557b7 100644 --- a/tests/e2e/test_e2e_http.py +++ b/tests/e2e/test_e2e_http.py @@ -19,15 +19,21 @@ from typing import Final import pytest from e2e_http import ( + PROVIDER_RATE_LIMIT_ATTEMPTS, + PROVIDER_RATE_LIMIT_BACKOFF_SECONDS, RETRY_ATTEMPTS, TRANSIENT_STATUSES, NoBody, PartialBody, + RateLimitedError, + Result, Success, + UnknownApiError, ValidationError, classify, request_with_retry, streaming_outcome, + tolerate_provider_rate_limit, wire_body, without_retries, ) @@ -105,6 +111,75 @@ class TestTransientRetryPolicy: assert [r.close_calls for r in responses] == [1, 1, 0, 0] +class _Ocr(BaseModel): + pages: int + + +PROVIDER_429_BODY: Final = ( + '{"error":{"message":"litellm.RateLimitError: RateLimitError: MistralException - (429, ' + '\'{"message":"Rate limit exceeded","type":"rate_limited"}\') LiteLLM Retried: 3 times",' + '"type":"throttling_error","param":null,"code":"429"}}' +) +PROXY_429_BODY: Final = ( + '{"error":{"message":"Max parallel request limit reached for key","type":"throttling_error","code":"429"}}' +) + + +def _results_from(results: Sequence[Result[_Ocr]]) -> Callable[[], Result[_Ocr]]: + it: Final = iter(results) + return lambda: next(it) + + +class TestProviderRateLimitTolerance: + def test_relayed_provider_429_is_retried_with_backoff_then_returns_the_success(self) -> None: + results: Final = ( + RateLimitedError(body=PROVIDER_429_BODY), + RateLimitedError(body=PROVIDER_429_BODY), + Success(status_code=200, data=_Ocr(pages=1)), + ) + sleep: Final = SleepRecorder() + assert tolerate_provider_rate_limit(_results_from(results), sleep=sleep) is results[2] + assert sleep.delays == (PROVIDER_RATE_LIMIT_BACKOFF_SECONDS, 2 * PROVIDER_RATE_LIMIT_BACKOFF_SECONDS) + + def test_retry_after_from_the_provider_replaces_the_backoff(self) -> None: + results: Final = ( + RateLimitedError(body=PROVIDER_429_BODY, retry_after_seconds=7), + Success(status_code=200, data=_Ocr(pages=1)), + ) + sleep: Final = SleepRecorder() + assert tolerate_provider_rate_limit(_results_from(results), sleep=sleep) is results[1] + assert sleep.delays == (7,) + + def test_the_proxys_own_429_comes_back_untouched(self) -> None: + results: Final = (RateLimitedError(body=PROXY_429_BODY), Success(status_code=200, data=_Ocr(pages=1))) + sleep: Final = SleepRecorder() + assert tolerate_provider_rate_limit(_results_from(results), sleep=sleep) is results[0] + assert sleep.delays == () + + @pytest.mark.parametrize( + "first", + ( + Success(status_code=200, data=_Ocr(pages=1)), + UnknownApiError(status_code=500, body="boom"), + ValidationError(message="bad"), + ), + ids=("success", "unknown", "validation"), + ) + def test_any_other_outcome_returns_at_once(self, first: Result[_Ocr]) -> None: + sleep: Final = SleepRecorder() + assert tolerate_provider_rate_limit(_results_from((first,)), sleep=sleep) is first + assert sleep.delays == () + + def test_a_persistent_provider_429_is_bounded_and_returns_the_last_one(self) -> None: + results: Final = tuple( + RateLimitedError(body=PROVIDER_429_BODY) for _ in range(PROVIDER_RATE_LIMIT_ATTEMPTS + 1) + ) + sleep: Final = SleepRecorder() + last: Final = tolerate_provider_rate_limit(_results_from(results), sleep=sleep) + assert last is results[PROVIDER_RATE_LIMIT_ATTEMPTS - 1] + assert sleep.delays == (5.0, 10.0, 20.0) + + @dataclass(frozen=True, slots=True) class FakeSseResponse: lines: Sequence[bytes]