mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
test(e2e): drop the harness unit tests, the live cells cover the helpers
This commit is contained in:
parent
14db6dbcd6
commit
fc88955bdc
2 changed files with 0 additions and 149 deletions
|
|
@ -1,54 +0,0 @@
|
|||
"""Contract tests for the helpers that read provider bodies; no proxy or provider needed."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from e2e_http import StreamingResponse
|
||||
from reliability_support import azure_prompt_filter_skipped
|
||||
|
||||
_SKIPPED_FILTER_BODY: Final = (
|
||||
'{"id":"chatcmpl-1","object":"chat.completion","model":"reliability-policyfail-1","choices":[{"finish_reason":'
|
||||
'"stop","index":0,"message":{"role":"assistant","content":"I can\'t comply."}}],"usage":{"prompt_tokens":64,'
|
||||
'"completion_tokens":53,"total_tokens":117},"prompt_filter_results":[{"prompt_index":0,"content_filter_results":{}}]}'
|
||||
)
|
||||
_RAN_FILTER_BODY: Final = (
|
||||
'{"id":"chatcmpl-2","object":"chat.completion","model":"reliability-policyfail-1","choices":[{"finish_reason":'
|
||||
'"stop","index":0,"message":{"role":"assistant","content":"Sure."}}],"usage":{"prompt_tokens":64,'
|
||||
'"completion_tokens":2,"total_tokens":66},"prompt_filter_results":[{"prompt_index":0,"content_filter_results":'
|
||||
'{"jailbreak":{"filtered":false,"detected":false},"hate":{"filtered":false,"severity":"safe"}}}]}'
|
||||
)
|
||||
_FALLBACK_BODY: Final = (
|
||||
'{"id":"chatcmpl-3","object":"chat.completion","model":"gpt-5.5","choices":[{"finish_reason":"stop","index":0,'
|
||||
'"message":{"role":"assistant","content":"Hello."}}],"usage":{"prompt_tokens":64,"completion_tokens":2,'
|
||||
'"total_tokens":66}}'
|
||||
)
|
||||
_REFUSAL_BODY: Final = (
|
||||
'{"error":{"message":"litellm.ContentPolicyViolationError: AzureException - The response was filtered",'
|
||||
'"type":"invalid_request_error","param":"prompt","code":"content_filter"}}'
|
||||
)
|
||||
|
||||
|
||||
def _response(status_code: int, body: str) -> StreamingResponse:
|
||||
return StreamingResponse(status_code=status_code, body=body)
|
||||
|
||||
|
||||
def test_a_200_whose_prompt_filter_recorded_no_verdict_is_a_skipped_filter() -> None:
|
||||
assert azure_prompt_filter_skipped(_response(200, _SKIPPED_FILTER_BODY))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("status_code", "body"),
|
||||
(
|
||||
(200, _RAN_FILTER_BODY),
|
||||
(200, _FALLBACK_BODY),
|
||||
(400, _REFUSAL_BODY),
|
||||
(200, "not json"),
|
||||
(200, '{"prompt_filter_results":[]}'),
|
||||
),
|
||||
ids=("filter-ran", "fallback-body", "refusal", "not-json", "empty-list"),
|
||||
)
|
||||
def test_every_other_body_is_not_a_skipped_filter(status_code: int, body: str) -> None:
|
||||
assert not azure_prompt_filter_skipped(_response(status_code, body))
|
||||
|
|
@ -12,7 +12,6 @@ monkeypatches anything.
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Callable, Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
|
|
@ -20,23 +19,15 @@ from typing import Final
|
|||
|
||||
import pytest
|
||||
from e2e_http import (
|
||||
PROVIDER_RATE_LIMIT_ATTEMPTS,
|
||||
PROVIDER_RATE_LIMIT_BACKOFF_SECONDS,
|
||||
PROVIDER_RATE_LIMIT_MARKER,
|
||||
RETRY_ATTEMPTS,
|
||||
TRANSIENT_STATUSES,
|
||||
NoBody,
|
||||
PartialBody,
|
||||
RateLimitedError,
|
||||
Result,
|
||||
Success,
|
||||
UnknownApiError,
|
||||
ValidationError,
|
||||
classify,
|
||||
request_with_retry,
|
||||
streaming_outcome,
|
||||
relayed_provider_rate_limit,
|
||||
tolerate_provider_rate_limit,
|
||||
wire_body,
|
||||
without_retries,
|
||||
)
|
||||
|
|
@ -114,92 +105,6 @@ class TestTransientRetryPolicy:
|
|||
assert [r.close_calls for r in responses] == [1, 1, 0, 0]
|
||||
|
||||
|
||||
class _Ocr(BaseModel):
|
||||
pages: int
|
||||
|
||||
|
||||
PROVIDER_429_BODY: Final = json.dumps(
|
||||
{
|
||||
"error": {
|
||||
"message": (
|
||||
"litellm.RateLimitError: RateLimitError: MistralException - "
|
||||
'(429, \'{"message":"Rate limit exceeded","type":"rate_limited"}\') LiteLLM Retried: 3 times'
|
||||
),
|
||||
"type": "throttling_error",
|
||||
"param": None,
|
||||
"code": "429",
|
||||
}
|
||||
}
|
||||
)
|
||||
PROXY_429_BODY: Final = json.dumps(
|
||||
{"error": {"message": "Max parallel request limit reached for key", "type": "throttling_error", "code": "429"}}
|
||||
)
|
||||
|
||||
|
||||
def _results_from(results: Sequence[Result[_Ocr]]) -> Callable[[], Result[_Ocr]]:
|
||||
it: Final = iter(results)
|
||||
return lambda: next(it)
|
||||
|
||||
|
||||
class TestProviderRateLimitTolerance:
|
||||
def test_relayed_provider_429_is_retried_with_backoff_then_returns_the_success(self) -> None:
|
||||
results: Final = (
|
||||
RateLimitedError(body=PROVIDER_429_BODY),
|
||||
RateLimitedError(body=PROVIDER_429_BODY),
|
||||
Success(status_code=200, data=_Ocr(pages=1)),
|
||||
)
|
||||
sleep: Final = SleepRecorder()
|
||||
assert tolerate_provider_rate_limit(_results_from(results), sleep=sleep) is results[2]
|
||||
assert sleep.delays == (PROVIDER_RATE_LIMIT_BACKOFF_SECONDS, 2 * PROVIDER_RATE_LIMIT_BACKOFF_SECONDS)
|
||||
|
||||
def test_retry_after_from_the_provider_replaces_the_backoff(self) -> None:
|
||||
results: Final = (
|
||||
RateLimitedError(body=PROVIDER_429_BODY, retry_after_seconds=7),
|
||||
Success(status_code=200, data=_Ocr(pages=1)),
|
||||
)
|
||||
sleep: Final = SleepRecorder()
|
||||
assert tolerate_provider_rate_limit(_results_from(results), sleep=sleep) is results[1]
|
||||
assert sleep.delays == (7,)
|
||||
|
||||
def test_the_proxys_own_429_comes_back_untouched(self) -> None:
|
||||
results: Final = (RateLimitedError(body=PROXY_429_BODY), Success(status_code=200, data=_Ocr(pages=1)))
|
||||
sleep: Final = SleepRecorder()
|
||||
assert tolerate_provider_rate_limit(_results_from(results), sleep=sleep) is results[0]
|
||||
assert sleep.delays == ()
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"first",
|
||||
(
|
||||
Success(status_code=200, data=_Ocr(pages=1)),
|
||||
UnknownApiError(status_code=500, body="boom"),
|
||||
ValidationError(message="bad"),
|
||||
),
|
||||
ids=("success", "unknown", "validation"),
|
||||
)
|
||||
def test_any_other_outcome_returns_at_once(self, first: Result[_Ocr]) -> None:
|
||||
sleep: Final = SleepRecorder()
|
||||
assert tolerate_provider_rate_limit(_results_from((first,)), sleep=sleep) is first
|
||||
assert sleep.delays == ()
|
||||
|
||||
def test_the_providers_429_is_parsed_out_of_the_relayed_envelope(self) -> None:
|
||||
detail: Final = relayed_provider_rate_limit(RateLimitedError(body=PROVIDER_429_BODY))
|
||||
assert detail is not None
|
||||
assert (detail.type, detail.code) == ("throttling_error", "429")
|
||||
assert detail.message.startswith(PROVIDER_RATE_LIMIT_MARKER)
|
||||
|
||||
def test_the_proxys_own_429_is_not_a_relayed_provider_error(self) -> None:
|
||||
assert relayed_provider_rate_limit(RateLimitedError(body=PROXY_429_BODY)) is None
|
||||
|
||||
def test_a_persistent_provider_429_is_bounded_and_returns_the_last_one(self) -> None:
|
||||
results: Final = tuple(
|
||||
RateLimitedError(body=PROVIDER_429_BODY) for _ in range(PROVIDER_RATE_LIMIT_ATTEMPTS + 1)
|
||||
)
|
||||
sleep: Final = SleepRecorder()
|
||||
last: Final = tolerate_provider_rate_limit(_results_from(results), sleep=sleep)
|
||||
assert last is results[PROVIDER_RATE_LIMIT_ATTEMPTS - 1]
|
||||
assert sleep.delays == (5.0, 10.0, 20.0)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class FakeSseResponse:
|
||||
lines: Sequence[bytes]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue