mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-05 08:07:05 +00:00
7aef79b774
* test(e2e): send no-cache on every cacheable request body, opt in only where a hit is the assertion
The e2e proxy runs with the response cache on, so any test that re-sends an
identical chat, messages, responses, completions, embeddings or rerank body
reads back a redis copy of an earlier call instead of reaching the provider.
Five tests in the last week failed that way. Default cache: {"no-cache": true}
on those request models and pass cache=None only in the two tests whose
assertion is the cache hit itself.
* test(e2e): give image edits and OCR a 180s client timeout
Both routes wait on providers that can legitimately take longer than the
60s transport-wide request timeout (gpt-image edits, Azure Document
Intelligence), and a client-side read timeout there fails a green request.
post/upload now accept a per-call timeout like get already does; only those
two call sites use it.
* test(e2e): rerun once on network errors and upstream 5xx only
Assertion failures still fail on the first attempt; only an outcome whose
error string carries the e2e_http network kind or a 5xx status gets one
more try. Test Engine records every attempt, so the flake rate stays
visible while a single provider blip no longer reds the rc run.
* test(e2e): let the reseed burst survive one upstream failure and print why
The burst is the precondition, not the property: one 5xx among six
concurrent calls still leaves five workers racing the cold counter, which
is what the reseed assertion measures. Two or more failures still abort,
and the failing bodies are now in the message instead of only the status
codes.
* test(e2e): keep polling Jaeger through a transient query failure
poll_traces_for_call already waits up to POLL_TIMEOUT for spans to land,
but a single refused connection to the query API failed the test on the
spot. Jaeger restarted twice during today's gate runs (19:05 and 19:41
UTC, each under a minute) and took ten and three otel tests with it while
the same tests passed on the rc build minutes later. A network failure
now counts as not-yet inside the same deadline; if Jaeger is still
unreachable when the deadline passes the test fails with that error, and
any non-network failure still fails immediately.
79 lines
2.8 KiB
Python
79 lines
2.8 KiB
Python
"""Shared helpers for the reliability e2e tests (fallbacks, timeouts, cache).
|
|
|
|
These are plain functions over the router suite's shared ProxyClient, not a
|
|
fixture/client class: the tests reuse the router `client` fixture and pass
|
|
`client.proxy`. Fallbacks and timeouts are driven by REAL deployments that all
|
|
point at the real `openai/gpt-5.5`; a bad base URL yields a real connection
|
|
error and a 1ms deadline yields a real timeout, and each test wires the
|
|
reroute per request through a `router_settings_override` in the /chat/completions
|
|
body, so a single long-lived proxy serves every reliability behavior.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from pydantic import ValidationError
|
|
|
|
from proxy_client import ProxyClient
|
|
from e2e_http import StreamingResponse
|
|
from models import (
|
|
ChatMessage,
|
|
ChatResponse,
|
|
LiteLLMParamsBody,
|
|
ReliabilityChatBody,
|
|
RouterSettingsOverride,
|
|
)
|
|
|
|
REAL_MODEL = "openai/gpt-5.5"
|
|
REAL_KEY = "os.environ/OPENAI_API_KEY"
|
|
|
|
|
|
def create_bad_base_deployment(proxy: ProxyClient, name: str) -> str:
|
|
"""Register a deployment pointing at an unreachable base, so every call to it
|
|
fails with a real connection error the fallback can reroute around."""
|
|
return proxy.create_model(
|
|
name, LiteLLMParamsBody(model=REAL_MODEL, api_key=REAL_KEY, api_base="http://127.0.0.1:9/v1")
|
|
)
|
|
|
|
|
|
def create_timeout_deployment(proxy: ProxyClient, name: str) -> str:
|
|
"""Register a deployment with a 1ms deadline the real backend always exceeds."""
|
|
return proxy.create_model(name, LiteLLMParamsBody(model=REAL_MODEL, api_key=REAL_KEY, timeout=0.001))
|
|
|
|
|
|
def chat_override(
|
|
proxy: ProxyClient,
|
|
key: str,
|
|
model: str,
|
|
content: str,
|
|
override: RouterSettingsOverride | None = None,
|
|
stream: bool = False,
|
|
cache: dict[str, bool] | None = {"no-cache": True},
|
|
) -> StreamingResponse:
|
|
"""POST /chat/completions with an optional per-request router_settings_override,
|
|
returning the raw outcome so tests read status, body, and reliability headers."""
|
|
return proxy.transport.send(
|
|
"/chat/completions",
|
|
headers=proxy.transport.bearer(key),
|
|
json=ReliabilityChatBody(
|
|
model=model,
|
|
messages=[ChatMessage(role="user", content=content)],
|
|
max_tokens=64,
|
|
stream=stream,
|
|
router_settings_override=override,
|
|
cache=cache,
|
|
),
|
|
stream=stream,
|
|
)
|
|
|
|
|
|
def content_of(resp: StreamingResponse) -> str | None:
|
|
"""The assistant message content of a successful chat response, or None when the
|
|
body is not a success shape (an error body, or an elided streamed body)."""
|
|
try:
|
|
parsed = ChatResponse.model_validate_json(resp.body)
|
|
except ValidationError:
|
|
return None
|
|
if not parsed.choices:
|
|
return None
|
|
message = parsed.choices[0].message
|
|
return message.content if message is not None else None
|