mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
test(e2e): accept a relayed provider 429 on the rust OCR cells
The gateway already retries a provider 429 three times per call and the Mistral key is shared across pipelines, so a throttle can hold across all four attempts of the OCR cell. After the bounded retries the cell now accepts the gateway's faithful relay of the provider's 429 (throttling_error, code 429) as its second expected outcome; the gateway's own 429 and any other error still fail the cell at once.
This commit is contained in:
parent
e18fb87f69
commit
14db6dbcd6
3 changed files with 83 additions and 21 deletions
|
|
@ -372,6 +372,23 @@ def tolerate_provider_rate_limit[R: BaseModel](
|
|||
return issue()
|
||||
|
||||
|
||||
class ProxyErrorDetail(BaseModel):
|
||||
message: str
|
||||
type: str
|
||||
code: str
|
||||
|
||||
|
||||
class _ProxyErrorBody(BaseModel):
|
||||
error: ProxyErrorDetail
|
||||
|
||||
|
||||
def relayed_provider_rate_limit(outcome: RateLimitedError) -> ProxyErrorDetail | None:
|
||||
"""The provider's own 429 as the proxy relayed it, or None when the 429 is the proxy's own."""
|
||||
if PROVIDER_RATE_LIMIT_MARKER not in outcome.body:
|
||||
return None
|
||||
return _ProxyErrorBody.model_validate_json(outcome.body).error
|
||||
|
||||
|
||||
class ClassifiableResponse(Protocol):
|
||||
"""What classifying an outcome reads off a response. requests.Response satisfies
|
||||
it, and so does a fake, so the classification rules are testable on their own."""
|
||||
|
|
@ -934,7 +951,10 @@ class PreparedForward:
|
|||
|
||||
|
||||
def prepare_forward(
|
||||
method: str, url: str, headers: dict[str, str], body: bytes | None,
|
||||
method: str,
|
||||
url: str,
|
||||
headers: dict[str, str],
|
||||
body: bytes | None,
|
||||
) -> PreparedForward | NetworkError:
|
||||
try:
|
||||
with requests.Session() as session:
|
||||
|
|
@ -953,7 +973,8 @@ def forward_prepared_stream(prepared: PreparedForward, timeout: float) -> Stream
|
|||
except requests.RequestException as exc:
|
||||
return NetworkError(message=str(exc))
|
||||
return StreamHead(
|
||||
resp.status_code, {name.lower(): value for name, value in resp.headers.items()},
|
||||
resp.status_code,
|
||||
{name.lower(): value for name, value in resp.headers.items()},
|
||||
primed_steps(_stream_steps(resp)),
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -11,16 +11,28 @@ well-formed OCR document comes back. Per the e2e hard-fail contract, a case
|
|||
fails when no proxy answers and also fails once a request reaches it: the proxy
|
||||
fetches each provider's referenced secrets, so a
|
||||
missing credential surfaces as a live provider error rather than silent green.
|
||||
|
||||
The provider keys are shared with other pipelines, so a provider's rate limit can
|
||||
hold across the bounded retries. The case then accepts the gateway's faithful relay
|
||||
of that 429 (throttling_error, code 429) as its second expected outcome; any other
|
||||
non-success still fails at once.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Protocol
|
||||
from typing import Final, Protocol
|
||||
|
||||
import pytest
|
||||
from e2e_config import unique_marker
|
||||
from e2e_http import assert_client_error, tolerate_provider_rate_limit, unwrap
|
||||
from e2e_http import (
|
||||
PROVIDER_RATE_LIMIT_ATTEMPTS,
|
||||
RateLimitedError,
|
||||
Success,
|
||||
assert_client_error,
|
||||
relayed_provider_rate_limit,
|
||||
tolerate_provider_rate_limit,
|
||||
)
|
||||
from lifecycle import ResourceManager
|
||||
from models import LiteLLMParamsBody, OcrBody, OcrDocument, OcrResponse
|
||||
from proxy_client import ProxyClient
|
||||
|
|
@ -146,26 +158,36 @@ def _assert_ocr_document(response: OcrResponse) -> None:
|
|||
assert response.pages[0].markdown is not None, "first page has no markdown"
|
||||
|
||||
|
||||
def _assert_provider_rate_limit_relayed(model: str, outcome: RateLimitedError) -> None:
|
||||
detail: Final = relayed_provider_rate_limit(outcome)
|
||||
assert detail is not None, f"{model}: the 429 is the gateway's own, not the provider's: {outcome.body}"
|
||||
assert (detail.type, detail.code) == ("throttling_error", "429"), f"{model}: provider 429 relayed as {detail!r}"
|
||||
print(
|
||||
f"{model}: the provider's rate limit held across {PROVIDER_RATE_LIMIT_ATTEMPTS} attempts; "
|
||||
f"the gateway relayed it as {detail.type} {detail.code}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
|
||||
class TestRustOcrGateway:
|
||||
@pytest.mark.parametrize("case", RUST_OCR_CASES, ids=_CASE_IDS)
|
||||
def test_rust_ocr_response(
|
||||
self, proxy: ProxyClient, resources: ResourceManager, case: _OcrCase
|
||||
) -> None:
|
||||
def test_rust_ocr_response(self, proxy: ProxyClient, resources: ResourceManager, case: _OcrCase) -> None:
|
||||
model = f"rust-ocr-{case.suffix}-{unique_marker()}"
|
||||
model_id = proxy.create_model(model, case.provider.litellm_params())
|
||||
resources.defer(lambda: proxy.delete_model(model_id))
|
||||
key = resources.key()
|
||||
|
||||
response = unwrap(
|
||||
tolerate_provider_rate_limit(lambda: proxy.ocr(key, OcrBody(model=model, document=case.document)))
|
||||
)
|
||||
_assert_ocr_document(response)
|
||||
match tolerate_provider_rate_limit(lambda: proxy.ocr(key, OcrBody(model=model, document=case.document))):
|
||||
case Success(data=response):
|
||||
_assert_ocr_document(response)
|
||||
case RateLimitedError() as outcome:
|
||||
_assert_provider_rate_limit_relayed(model, outcome)
|
||||
case outcome:
|
||||
pytest.fail(f"{model}: {outcome!r}")
|
||||
|
||||
@pytest.mark.skip(reason="stage red: product gap, /v1/ocr 500s (aocr TypeError) on missing document instead of 400")
|
||||
@pytest.mark.covers("llm.ocr.openai.input_validation.nonstream.works")
|
||||
def test_missing_document_returns_error(
|
||||
self, proxy: ProxyClient, resources: ResourceManager
|
||||
) -> None:
|
||||
def test_missing_document_returns_error(self, proxy: ProxyClient, resources: ResourceManager) -> None:
|
||||
model = f"rust-ocr-val-{unique_marker()}"
|
||||
model_id = proxy.create_model(model, MistralOcr().litellm_params())
|
||||
resources.defer(lambda: proxy.delete_model(model_id))
|
||||
|
|
@ -176,4 +198,3 @@ class TestRustOcrGateway:
|
|||
json=_OptionalOcrBody(model=model),
|
||||
)
|
||||
assert_client_error(result, "ocr missing document")
|
||||
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ monkeypatches anything.
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Callable, Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
|
|
@ -21,6 +22,7 @@ import pytest
|
|||
from e2e_http import (
|
||||
PROVIDER_RATE_LIMIT_ATTEMPTS,
|
||||
PROVIDER_RATE_LIMIT_BACKOFF_SECONDS,
|
||||
PROVIDER_RATE_LIMIT_MARKER,
|
||||
RETRY_ATTEMPTS,
|
||||
TRANSIENT_STATUSES,
|
||||
NoBody,
|
||||
|
|
@ -33,6 +35,7 @@ from e2e_http import (
|
|||
classify,
|
||||
request_with_retry,
|
||||
streaming_outcome,
|
||||
relayed_provider_rate_limit,
|
||||
tolerate_provider_rate_limit,
|
||||
wire_body,
|
||||
without_retries,
|
||||
|
|
@ -115,13 +118,21 @@ class _Ocr(BaseModel):
|
|||
pages: int
|
||||
|
||||
|
||||
PROVIDER_429_BODY: Final = (
|
||||
'{"error":{"message":"litellm.RateLimitError: RateLimitError: MistralException - (429, '
|
||||
'\'{"message":"Rate limit exceeded","type":"rate_limited"}\') LiteLLM Retried: 3 times",'
|
||||
'"type":"throttling_error","param":null,"code":"429"}}'
|
||||
PROVIDER_429_BODY: Final = json.dumps(
|
||||
{
|
||||
"error": {
|
||||
"message": (
|
||||
"litellm.RateLimitError: RateLimitError: MistralException - "
|
||||
'(429, \'{"message":"Rate limit exceeded","type":"rate_limited"}\') LiteLLM Retried: 3 times'
|
||||
),
|
||||
"type": "throttling_error",
|
||||
"param": None,
|
||||
"code": "429",
|
||||
}
|
||||
}
|
||||
)
|
||||
PROXY_429_BODY: Final = (
|
||||
'{"error":{"message":"Max parallel request limit reached for key","type":"throttling_error","code":"429"}}'
|
||||
PROXY_429_BODY: Final = json.dumps(
|
||||
{"error": {"message": "Max parallel request limit reached for key", "type": "throttling_error", "code": "429"}}
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -170,6 +181,15 @@ class TestProviderRateLimitTolerance:
|
|||
assert tolerate_provider_rate_limit(_results_from((first,)), sleep=sleep) is first
|
||||
assert sleep.delays == ()
|
||||
|
||||
def test_the_providers_429_is_parsed_out_of_the_relayed_envelope(self) -> None:
|
||||
detail: Final = relayed_provider_rate_limit(RateLimitedError(body=PROVIDER_429_BODY))
|
||||
assert detail is not None
|
||||
assert (detail.type, detail.code) == ("throttling_error", "429")
|
||||
assert detail.message.startswith(PROVIDER_RATE_LIMIT_MARKER)
|
||||
|
||||
def test_the_proxys_own_429_is_not_a_relayed_provider_error(self) -> None:
|
||||
assert relayed_provider_rate_limit(RateLimitedError(body=PROXY_429_BODY)) is None
|
||||
|
||||
def test_a_persistent_provider_429_is_bounded_and_returns_the_last_one(self) -> None:
|
||||
results: Final = tuple(
|
||||
RateLimitedError(body=PROVIDER_429_BODY) for _ in range(PROVIDER_RATE_LIMIT_ATTEMPTS + 1)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue