test(e2e): accept a relayed provider 429 on the rust OCR cells

The gateway already retries a provider 429 three times per call and the
Mistral key is shared across pipelines, so a throttle can hold across all
four attempts of the OCR cell. After the bounded retries the cell now
accepts the gateway's faithful relay of the provider's 429 (throttling_error,
code 429) as its second expected outcome; the gateway's own 429 and any
other error still fail the cell at once.
This commit is contained in:
mateo-berri 2026-09-22 18:44:17 -07:00
parent e18fb87f69
commit 14db6dbcd6
3 changed files with 83 additions and 21 deletions

View file

@ -372,6 +372,23 @@ def tolerate_provider_rate_limit[R: BaseModel](
return issue()
class ProxyErrorDetail(BaseModel):
message: str
type: str
code: str
class _ProxyErrorBody(BaseModel):
error: ProxyErrorDetail
def relayed_provider_rate_limit(outcome: RateLimitedError) -> ProxyErrorDetail | None:
"""The provider's own 429 as the proxy relayed it, or None when the 429 is the proxy's own."""
if PROVIDER_RATE_LIMIT_MARKER not in outcome.body:
return None
return _ProxyErrorBody.model_validate_json(outcome.body).error
class ClassifiableResponse(Protocol):
"""What classifying an outcome reads off a response. requests.Response satisfies
it, and so does a fake, so the classification rules are testable on their own."""
@ -934,7 +951,10 @@ class PreparedForward:
def prepare_forward(
method: str, url: str, headers: dict[str, str], body: bytes | None,
method: str,
url: str,
headers: dict[str, str],
body: bytes | None,
) -> PreparedForward | NetworkError:
try:
with requests.Session() as session:
@ -953,7 +973,8 @@ def forward_prepared_stream(prepared: PreparedForward, timeout: float) -> Stream
except requests.RequestException as exc:
return NetworkError(message=str(exc))
return StreamHead(
resp.status_code, {name.lower(): value for name, value in resp.headers.items()},
resp.status_code,
{name.lower(): value for name, value in resp.headers.items()},
primed_steps(_stream_steps(resp)),
)

View file

@ -11,16 +11,28 @@ well-formed OCR document comes back. Per the e2e hard-fail contract, a case
fails when no proxy answers and also fails once a request reaches it: the proxy
fetches each provider's referenced secrets, so a
missing credential surfaces as a live provider error rather than silent green.
The provider keys are shared with other pipelines, so a provider's rate limit can
hold across the bounded retries. The case then accepts the gateway's faithful relay
of that 429 (throttling_error, code 429) as its second expected outcome; any other
non-success still fails at once.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import Protocol
from typing import Final, Protocol
import pytest
from e2e_config import unique_marker
from e2e_http import assert_client_error, tolerate_provider_rate_limit, unwrap
from e2e_http import (
PROVIDER_RATE_LIMIT_ATTEMPTS,
RateLimitedError,
Success,
assert_client_error,
relayed_provider_rate_limit,
tolerate_provider_rate_limit,
)
from lifecycle import ResourceManager
from models import LiteLLMParamsBody, OcrBody, OcrDocument, OcrResponse
from proxy_client import ProxyClient
@ -146,26 +158,36 @@ def _assert_ocr_document(response: OcrResponse) -> None:
assert response.pages[0].markdown is not None, "first page has no markdown"
def _assert_provider_rate_limit_relayed(model: str, outcome: RateLimitedError) -> None:
detail: Final = relayed_provider_rate_limit(outcome)
assert detail is not None, f"{model}: the 429 is the gateway's own, not the provider's: {outcome.body}"
assert (detail.type, detail.code) == ("throttling_error", "429"), f"{model}: provider 429 relayed as {detail!r}"
print(
f"{model}: the provider's rate limit held across {PROVIDER_RATE_LIMIT_ATTEMPTS} attempts; "
f"the gateway relayed it as {detail.type} {detail.code}",
flush=True,
)
class TestRustOcrGateway:
@pytest.mark.parametrize("case", RUST_OCR_CASES, ids=_CASE_IDS)
def test_rust_ocr_response(
self, proxy: ProxyClient, resources: ResourceManager, case: _OcrCase
) -> None:
def test_rust_ocr_response(self, proxy: ProxyClient, resources: ResourceManager, case: _OcrCase) -> None:
model = f"rust-ocr-{case.suffix}-{unique_marker()}"
model_id = proxy.create_model(model, case.provider.litellm_params())
resources.defer(lambda: proxy.delete_model(model_id))
key = resources.key()
response = unwrap(
tolerate_provider_rate_limit(lambda: proxy.ocr(key, OcrBody(model=model, document=case.document)))
)
_assert_ocr_document(response)
match tolerate_provider_rate_limit(lambda: proxy.ocr(key, OcrBody(model=model, document=case.document))):
case Success(data=response):
_assert_ocr_document(response)
case RateLimitedError() as outcome:
_assert_provider_rate_limit_relayed(model, outcome)
case outcome:
pytest.fail(f"{model}: {outcome!r}")
@pytest.mark.skip(reason="stage red: product gap, /v1/ocr 500s (aocr TypeError) on missing document instead of 400")
@pytest.mark.covers("llm.ocr.openai.input_validation.nonstream.works")
def test_missing_document_returns_error(
self, proxy: ProxyClient, resources: ResourceManager
) -> None:
def test_missing_document_returns_error(self, proxy: ProxyClient, resources: ResourceManager) -> None:
model = f"rust-ocr-val-{unique_marker()}"
model_id = proxy.create_model(model, MistralOcr().litellm_params())
resources.defer(lambda: proxy.delete_model(model_id))
@ -176,4 +198,3 @@ class TestRustOcrGateway:
json=_OptionalOcrBody(model=model),
)
assert_client_error(result, "ocr missing document")

View file

@ -12,6 +12,7 @@ monkeypatches anything.
from __future__ import annotations
import json
from collections.abc import Callable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from types import MappingProxyType
@ -21,6 +22,7 @@ import pytest
from e2e_http import (
PROVIDER_RATE_LIMIT_ATTEMPTS,
PROVIDER_RATE_LIMIT_BACKOFF_SECONDS,
PROVIDER_RATE_LIMIT_MARKER,
RETRY_ATTEMPTS,
TRANSIENT_STATUSES,
NoBody,
@ -33,6 +35,7 @@ from e2e_http import (
classify,
request_with_retry,
streaming_outcome,
relayed_provider_rate_limit,
tolerate_provider_rate_limit,
wire_body,
without_retries,
@ -115,13 +118,21 @@ class _Ocr(BaseModel):
pages: int
PROVIDER_429_BODY: Final = (
'{"error":{"message":"litellm.RateLimitError: RateLimitError: MistralException - (429, '
'\'{"message":"Rate limit exceeded","type":"rate_limited"}\') LiteLLM Retried: 3 times",'
'"type":"throttling_error","param":null,"code":"429"}}'
PROVIDER_429_BODY: Final = json.dumps(
{
"error": {
"message": (
"litellm.RateLimitError: RateLimitError: MistralException - "
'(429, \'{"message":"Rate limit exceeded","type":"rate_limited"}\') LiteLLM Retried: 3 times'
),
"type": "throttling_error",
"param": None,
"code": "429",
}
}
)
PROXY_429_BODY: Final = (
'{"error":{"message":"Max parallel request limit reached for key","type":"throttling_error","code":"429"}}'
PROXY_429_BODY: Final = json.dumps(
{"error": {"message": "Max parallel request limit reached for key", "type": "throttling_error", "code": "429"}}
)
@ -170,6 +181,15 @@ class TestProviderRateLimitTolerance:
assert tolerate_provider_rate_limit(_results_from((first,)), sleep=sleep) is first
assert sleep.delays == ()
def test_the_providers_429_is_parsed_out_of_the_relayed_envelope(self) -> None:
detail: Final = relayed_provider_rate_limit(RateLimitedError(body=PROVIDER_429_BODY))
assert detail is not None
assert (detail.type, detail.code) == ("throttling_error", "429")
assert detail.message.startswith(PROVIDER_RATE_LIMIT_MARKER)
def test_the_proxys_own_429_is_not_a_relayed_provider_error(self) -> None:
assert relayed_provider_rate_limit(RateLimitedError(body=PROXY_429_BODY)) is None
def test_a_persistent_provider_429_is_bounded_and_returns_the_last_one(self) -> None:
results: Final = tuple(
RateLimitedError(body=PROVIDER_429_BODY) for _ in range(PROVIDER_RATE_LIMIT_ATTEMPTS + 1)