diff --git a/tests/e2e/e2e_http.py b/tests/e2e/e2e_http.py index 74ed596beb6..993b3557b08 100644 --- a/tests/e2e/e2e_http.py +++ b/tests/e2e/e2e_http.py @@ -372,6 +372,23 @@ def tolerate_provider_rate_limit[R: BaseModel]( return issue() +class ProxyErrorDetail(BaseModel): + message: str + type: str + code: str + + +class _ProxyErrorBody(BaseModel): + error: ProxyErrorDetail + + +def relayed_provider_rate_limit(outcome: RateLimitedError) -> ProxyErrorDetail | None: + """The provider's own 429 as the proxy relayed it, or None when the 429 is the proxy's own.""" + if PROVIDER_RATE_LIMIT_MARKER not in outcome.body: + return None + return _ProxyErrorBody.model_validate_json(outcome.body).error + + class ClassifiableResponse(Protocol): """What classifying an outcome reads off a response. requests.Response satisfies it, and so does a fake, so the classification rules are testable on their own.""" @@ -934,7 +951,10 @@ class PreparedForward: def prepare_forward( - method: str, url: str, headers: dict[str, str], body: bytes | None, + method: str, + url: str, + headers: dict[str, str], + body: bytes | None, ) -> PreparedForward | NetworkError: try: with requests.Session() as session: @@ -953,7 +973,8 @@ def forward_prepared_stream(prepared: PreparedForward, timeout: float) -> Stream except requests.RequestException as exc: return NetworkError(message=str(exc)) return StreamHead( - resp.status_code, {name.lower(): value for name, value in resp.headers.items()}, + resp.status_code, + {name.lower(): value for name, value in resp.headers.items()}, primed_steps(_stream_steps(resp)), ) diff --git a/tests/e2e/llm_translation/test_ocr_rust_e2e.py b/tests/e2e/llm_translation/test_ocr_rust_e2e.py index 2456f64d057..2f7fc74e650 100644 --- a/tests/e2e/llm_translation/test_ocr_rust_e2e.py +++ b/tests/e2e/llm_translation/test_ocr_rust_e2e.py @@ -11,16 +11,28 @@ well-formed OCR document comes back. Per the e2e hard-fail contract, a case fails when no proxy answers and also fails once a request reaches it: the proxy fetches each provider's referenced secrets, so a missing credential surfaces as a live provider error rather than silent green. + +The provider keys are shared with other pipelines, so a provider's rate limit can +hold across the bounded retries. The case then accepts the gateway's faithful relay +of that 429 (throttling_error, code 429) as its second expected outcome; any other +non-success still fails at once. """ from __future__ import annotations from dataclasses import dataclass -from typing import Protocol +from typing import Final, Protocol import pytest from e2e_config import unique_marker -from e2e_http import assert_client_error, tolerate_provider_rate_limit, unwrap +from e2e_http import ( + PROVIDER_RATE_LIMIT_ATTEMPTS, + RateLimitedError, + Success, + assert_client_error, + relayed_provider_rate_limit, + tolerate_provider_rate_limit, +) from lifecycle import ResourceManager from models import LiteLLMParamsBody, OcrBody, OcrDocument, OcrResponse from proxy_client import ProxyClient @@ -146,26 +158,36 @@ def _assert_ocr_document(response: OcrResponse) -> None: assert response.pages[0].markdown is not None, "first page has no markdown" +def _assert_provider_rate_limit_relayed(model: str, outcome: RateLimitedError) -> None: + detail: Final = relayed_provider_rate_limit(outcome) + assert detail is not None, f"{model}: the 429 is the gateway's own, not the provider's: {outcome.body}" + assert (detail.type, detail.code) == ("throttling_error", "429"), f"{model}: provider 429 relayed as {detail!r}" + print( + f"{model}: the provider's rate limit held across {PROVIDER_RATE_LIMIT_ATTEMPTS} attempts; " + f"the gateway relayed it as {detail.type} {detail.code}", + flush=True, + ) + + class TestRustOcrGateway: @pytest.mark.parametrize("case", RUST_OCR_CASES, ids=_CASE_IDS) - def test_rust_ocr_response( - self, proxy: ProxyClient, resources: ResourceManager, case: _OcrCase - ) -> None: + def test_rust_ocr_response(self, proxy: ProxyClient, resources: ResourceManager, case: _OcrCase) -> None: model = f"rust-ocr-{case.suffix}-{unique_marker()}" model_id = proxy.create_model(model, case.provider.litellm_params()) resources.defer(lambda: proxy.delete_model(model_id)) key = resources.key() - response = unwrap( - tolerate_provider_rate_limit(lambda: proxy.ocr(key, OcrBody(model=model, document=case.document))) - ) - _assert_ocr_document(response) + match tolerate_provider_rate_limit(lambda: proxy.ocr(key, OcrBody(model=model, document=case.document))): + case Success(data=response): + _assert_ocr_document(response) + case RateLimitedError() as outcome: + _assert_provider_rate_limit_relayed(model, outcome) + case outcome: + pytest.fail(f"{model}: {outcome!r}") @pytest.mark.skip(reason="stage red: product gap, /v1/ocr 500s (aocr TypeError) on missing document instead of 400") @pytest.mark.covers("llm.ocr.openai.input_validation.nonstream.works") - def test_missing_document_returns_error( - self, proxy: ProxyClient, resources: ResourceManager - ) -> None: + def test_missing_document_returns_error(self, proxy: ProxyClient, resources: ResourceManager) -> None: model = f"rust-ocr-val-{unique_marker()}" model_id = proxy.create_model(model, MistralOcr().litellm_params()) resources.defer(lambda: proxy.delete_model(model_id)) @@ -176,4 +198,3 @@ class TestRustOcrGateway: json=_OptionalOcrBody(model=model), ) assert_client_error(result, "ocr missing document") - diff --git a/tests/e2e/test_e2e_http.py b/tests/e2e/test_e2e_http.py index dc06d9557b7..35b227942d6 100644 --- a/tests/e2e/test_e2e_http.py +++ b/tests/e2e/test_e2e_http.py @@ -12,6 +12,7 @@ monkeypatches anything. from __future__ import annotations +import json from collections.abc import Callable, Iterator, Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType @@ -21,6 +22,7 @@ import pytest from e2e_http import ( PROVIDER_RATE_LIMIT_ATTEMPTS, PROVIDER_RATE_LIMIT_BACKOFF_SECONDS, + PROVIDER_RATE_LIMIT_MARKER, RETRY_ATTEMPTS, TRANSIENT_STATUSES, NoBody, @@ -33,6 +35,7 @@ from e2e_http import ( classify, request_with_retry, streaming_outcome, + relayed_provider_rate_limit, tolerate_provider_rate_limit, wire_body, without_retries, @@ -115,13 +118,21 @@ class _Ocr(BaseModel): pages: int -PROVIDER_429_BODY: Final = ( - '{"error":{"message":"litellm.RateLimitError: RateLimitError: MistralException - (429, ' - '\'{"message":"Rate limit exceeded","type":"rate_limited"}\') LiteLLM Retried: 3 times",' - '"type":"throttling_error","param":null,"code":"429"}}' +PROVIDER_429_BODY: Final = json.dumps( + { + "error": { + "message": ( + "litellm.RateLimitError: RateLimitError: MistralException - " + '(429, \'{"message":"Rate limit exceeded","type":"rate_limited"}\') LiteLLM Retried: 3 times' + ), + "type": "throttling_error", + "param": None, + "code": "429", + } + } ) -PROXY_429_BODY: Final = ( - '{"error":{"message":"Max parallel request limit reached for key","type":"throttling_error","code":"429"}}' +PROXY_429_BODY: Final = json.dumps( + {"error": {"message": "Max parallel request limit reached for key", "type": "throttling_error", "code": "429"}} ) @@ -170,6 +181,15 @@ class TestProviderRateLimitTolerance: assert tolerate_provider_rate_limit(_results_from((first,)), sleep=sleep) is first assert sleep.delays == () + def test_the_providers_429_is_parsed_out_of_the_relayed_envelope(self) -> None: + detail: Final = relayed_provider_rate_limit(RateLimitedError(body=PROVIDER_429_BODY)) + assert detail is not None + assert (detail.type, detail.code) == ("throttling_error", "429") + assert detail.message.startswith(PROVIDER_RATE_LIMIT_MARKER) + + def test_the_proxys_own_429_is_not_a_relayed_provider_error(self) -> None: + assert relayed_provider_rate_limit(RateLimitedError(body=PROXY_429_BODY)) is None + def test_a_persistent_provider_429_is_bounded_and_returns_the_last_one(self) -> None: results: Final = tuple( RateLimitedError(body=PROVIDER_429_BODY) for _ in range(PROVIDER_RATE_LIMIT_ATTEMPTS + 1)