test(e2e): drop the harness unit tests, the live cells cover the helpers

This commit is contained in:
mateo-berri 2026-09-22 19:04:03 -07:00
parent 14db6dbcd6
commit fc88955bdc
2 changed files with 0 additions and 149 deletions

View file

@ -1,54 +0,0 @@
"""Contract tests for the helpers that read provider bodies; no proxy or provider needed."""
from __future__ import annotations
from typing import Final
import pytest
from e2e_http import StreamingResponse
from reliability_support import azure_prompt_filter_skipped
_SKIPPED_FILTER_BODY: Final = (
'{"id":"chatcmpl-1","object":"chat.completion","model":"reliability-policyfail-1","choices":[{"finish_reason":'
'"stop","index":0,"message":{"role":"assistant","content":"I can\'t comply."}}],"usage":{"prompt_tokens":64,'
'"completion_tokens":53,"total_tokens":117},"prompt_filter_results":[{"prompt_index":0,"content_filter_results":{}}]}'
)
_RAN_FILTER_BODY: Final = (
'{"id":"chatcmpl-2","object":"chat.completion","model":"reliability-policyfail-1","choices":[{"finish_reason":'
'"stop","index":0,"message":{"role":"assistant","content":"Sure."}}],"usage":{"prompt_tokens":64,'
'"completion_tokens":2,"total_tokens":66},"prompt_filter_results":[{"prompt_index":0,"content_filter_results":'
'{"jailbreak":{"filtered":false,"detected":false},"hate":{"filtered":false,"severity":"safe"}}}]}'
)
_FALLBACK_BODY: Final = (
'{"id":"chatcmpl-3","object":"chat.completion","model":"gpt-5.5","choices":[{"finish_reason":"stop","index":0,'
'"message":{"role":"assistant","content":"Hello."}}],"usage":{"prompt_tokens":64,"completion_tokens":2,'
'"total_tokens":66}}'
)
_REFUSAL_BODY: Final = (
'{"error":{"message":"litellm.ContentPolicyViolationError: AzureException - The response was filtered",'
'"type":"invalid_request_error","param":"prompt","code":"content_filter"}}'
)
def _response(status_code: int, body: str) -> StreamingResponse:
return StreamingResponse(status_code=status_code, body=body)
def test_a_200_whose_prompt_filter_recorded_no_verdict_is_a_skipped_filter() -> None:
assert azure_prompt_filter_skipped(_response(200, _SKIPPED_FILTER_BODY))
@pytest.mark.parametrize(
("status_code", "body"),
(
(200, _RAN_FILTER_BODY),
(200, _FALLBACK_BODY),
(400, _REFUSAL_BODY),
(200, "not json"),
(200, '{"prompt_filter_results":[]}'),
),
ids=("filter-ran", "fallback-body", "refusal", "not-json", "empty-list"),
)
def test_every_other_body_is_not_a_skipped_filter(status_code: int, body: str) -> None:
assert not azure_prompt_filter_skipped(_response(status_code, body))

View file

@ -12,7 +12,6 @@ monkeypatches anything.
from __future__ import annotations
import json
from collections.abc import Callable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from types import MappingProxyType
@ -20,23 +19,15 @@ from typing import Final
import pytest
from e2e_http import (
PROVIDER_RATE_LIMIT_ATTEMPTS,
PROVIDER_RATE_LIMIT_BACKOFF_SECONDS,
PROVIDER_RATE_LIMIT_MARKER,
RETRY_ATTEMPTS,
TRANSIENT_STATUSES,
NoBody,
PartialBody,
RateLimitedError,
Result,
Success,
UnknownApiError,
ValidationError,
classify,
request_with_retry,
streaming_outcome,
relayed_provider_rate_limit,
tolerate_provider_rate_limit,
wire_body,
without_retries,
)
@ -114,92 +105,6 @@ class TestTransientRetryPolicy:
assert [r.close_calls for r in responses] == [1, 1, 0, 0]
class _Ocr(BaseModel):
pages: int
PROVIDER_429_BODY: Final = json.dumps(
{
"error": {
"message": (
"litellm.RateLimitError: RateLimitError: MistralException - "
'(429, \'{"message":"Rate limit exceeded","type":"rate_limited"}\') LiteLLM Retried: 3 times'
),
"type": "throttling_error",
"param": None,
"code": "429",
}
}
)
PROXY_429_BODY: Final = json.dumps(
{"error": {"message": "Max parallel request limit reached for key", "type": "throttling_error", "code": "429"}}
)
def _results_from(results: Sequence[Result[_Ocr]]) -> Callable[[], Result[_Ocr]]:
it: Final = iter(results)
return lambda: next(it)
class TestProviderRateLimitTolerance:
def test_relayed_provider_429_is_retried_with_backoff_then_returns_the_success(self) -> None:
results: Final = (
RateLimitedError(body=PROVIDER_429_BODY),
RateLimitedError(body=PROVIDER_429_BODY),
Success(status_code=200, data=_Ocr(pages=1)),
)
sleep: Final = SleepRecorder()
assert tolerate_provider_rate_limit(_results_from(results), sleep=sleep) is results[2]
assert sleep.delays == (PROVIDER_RATE_LIMIT_BACKOFF_SECONDS, 2 * PROVIDER_RATE_LIMIT_BACKOFF_SECONDS)
def test_retry_after_from_the_provider_replaces_the_backoff(self) -> None:
results: Final = (
RateLimitedError(body=PROVIDER_429_BODY, retry_after_seconds=7),
Success(status_code=200, data=_Ocr(pages=1)),
)
sleep: Final = SleepRecorder()
assert tolerate_provider_rate_limit(_results_from(results), sleep=sleep) is results[1]
assert sleep.delays == (7,)
def test_the_proxys_own_429_comes_back_untouched(self) -> None:
results: Final = (RateLimitedError(body=PROXY_429_BODY), Success(status_code=200, data=_Ocr(pages=1)))
sleep: Final = SleepRecorder()
assert tolerate_provider_rate_limit(_results_from(results), sleep=sleep) is results[0]
assert sleep.delays == ()
@pytest.mark.parametrize(
"first",
(
Success(status_code=200, data=_Ocr(pages=1)),
UnknownApiError(status_code=500, body="boom"),
ValidationError(message="bad"),
),
ids=("success", "unknown", "validation"),
)
def test_any_other_outcome_returns_at_once(self, first: Result[_Ocr]) -> None:
sleep: Final = SleepRecorder()
assert tolerate_provider_rate_limit(_results_from((first,)), sleep=sleep) is first
assert sleep.delays == ()
def test_the_providers_429_is_parsed_out_of_the_relayed_envelope(self) -> None:
detail: Final = relayed_provider_rate_limit(RateLimitedError(body=PROVIDER_429_BODY))
assert detail is not None
assert (detail.type, detail.code) == ("throttling_error", "429")
assert detail.message.startswith(PROVIDER_RATE_LIMIT_MARKER)
def test_the_proxys_own_429_is_not_a_relayed_provider_error(self) -> None:
assert relayed_provider_rate_limit(RateLimitedError(body=PROXY_429_BODY)) is None
def test_a_persistent_provider_429_is_bounded_and_returns_the_last_one(self) -> None:
results: Final = tuple(
RateLimitedError(body=PROVIDER_429_BODY) for _ in range(PROVIDER_RATE_LIMIT_ATTEMPTS + 1)
)
sleep: Final = SleepRecorder()
last: Final = tolerate_provider_rate_limit(_results_from(results), sleep=sleep)
assert last is results[PROVIDER_RATE_LIMIT_ATTEMPTS - 1]
assert sleep.delays == (5.0, 10.0, 20.0)
@dataclass(frozen=True, slots=True)
class FakeSseResponse:
lines: Sequence[bytes]