mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
test(e2e): refresh retired provider fixtures
This commit is contained in:
parent
7472f43421
commit
9d64fa4201
10 changed files with 22 additions and 27 deletions
|
|
@ -38,7 +38,7 @@ from models import (
|
|||
from proxy_client import ProxyClient
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
GUARDRAIL_BACKEND: Final = "gemini/gemini-2.5-flash"
|
||||
GUARDRAIL_BACKEND: Final = "gemini/gemini-3.8-flash"
|
||||
|
||||
GuardrailMode = Literal["pre_call", "post_call", "during_call", "logging_only"]
|
||||
BlockedWordAction = Literal["BLOCK", "MASK"]
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ from passthrough_client import PassthroughClient
|
|||
pytestmark = pytest.mark.e2e
|
||||
|
||||
COHERE_BACKEND = "cohere/command-r-08-2024"
|
||||
GEMINI_BACKEND = "gemini/gemini-2.5-flash"
|
||||
GEMINI_BACKEND = "gemini/gemini-3.8-flash"
|
||||
VERTEX_BACKEND: Final = "vertex_ai/gemini-2.5-flash"
|
||||
AZURE_OPENAI_BACKEND: Final = "azure/gpt-5.4-nano"
|
||||
AZURE_OPENAI_API_VERSION: Final = "v1"
|
||||
|
|
@ -243,7 +243,7 @@ _PERSON_SCHEMA: dict[str, object] = {
|
|||
|
||||
OPENAI_CHAT_MODEL: Final = "gpt-5.5"
|
||||
ANTHROPIC_CHAT_MODEL: Final = "claude-haiku-4-5"
|
||||
GEMINI_FLASH_MODEL: Final = "gemini-2.5-flash"
|
||||
GEMINI_FLASH_MODEL: Final = "gemini-3.8-flash"
|
||||
|
||||
CHAT_MODELS: Final = (
|
||||
pytest.param(
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ teardown) instead of relying on a statically configured model, so the check is
|
|||
self-contained and never inherits pricing another suite or a stale config left on
|
||||
the shared proxy. custom-priced-flash sets input/output rates deliberately far
|
||||
above the canonical gemini price; the isolation sibling shares the same
|
||||
gemini/gemini-2.5-flash backend but sets no override. Three behaviors are checked
|
||||
gemini/gemini-3.8-flash backend but sets no override. Three behaviors are checked
|
||||
independently:
|
||||
|
||||
- billing: a real call's logged cost breakdown charges input and output tokens at
|
||||
|
|
@ -37,10 +37,8 @@ from models import (
|
|||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
BACKEND_MODEL = "gemini/gemini-2.5-flash"
|
||||
BACKEND_MODEL = "gemini/gemini-3.8-flash"
|
||||
GEMINI_API_KEY = "os.environ/GEMINI_API_KEY"
|
||||
# Deliberately ~100x above canonical gemini-2.5-flash (input 3e-7 / output 2.5e-6)
|
||||
# so an override that is ignored or under-applied bills at the base rate and fails.
|
||||
CUSTOM_INPUT_RATE = 5e-05
|
||||
CUSTOM_OUTPUT_RATE = 1e-04
|
||||
|
||||
|
|
@ -78,7 +76,7 @@ def _provision(
|
|||
input_cost_per_token: float | None,
|
||||
output_cost_per_token: float | None,
|
||||
) -> str:
|
||||
"""Register a fresh gemini/gemini-2.5-flash deployment (deleted on teardown) and
|
||||
"""Register a fresh gemini/gemini-3.8-flash deployment (deleted on teardown) and
|
||||
return its model name. With the cost fields set the deployment carries a custom
|
||||
pricing override; with them None it is a plain sibling on the same backend. The
|
||||
marker keeps the name unique so concurrent runs on the shared proxy never
|
||||
|
|
@ -251,7 +249,7 @@ class TestCustomPricing:
|
|||
assert sibling_entry is not None, f"{sibling} absent from /model/info"
|
||||
|
||||
# custom-priced-flash overrides pricing; the sibling shares the same
|
||||
# gemini/gemini-2.5-flash backend but sets no override, so it must keep its
|
||||
# gemini/gemini-3.8-flash backend but sets no override, so it must keep its
|
||||
# own price. Equal rates mean the override leaked into the shared cost map.
|
||||
assert (
|
||||
sibling_entry.model_info.input_cost_per_token
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ from pydantic import BaseModel
|
|||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
UPSTREAM_MODEL = "gemini/gemini-2.5-flash"
|
||||
UPSTREAM_MODEL = "gemini/gemini-3.8-flash"
|
||||
|
||||
|
||||
class _GenerateContentPart(BaseModel):
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ from sdk_clients import SdkClients
|
|||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
IMAGE_EDIT_BACKEND: Final = "openai/gpt-image-1"
|
||||
IMAGE_EDIT_BACKEND: Final = "openai/gpt-image-2.5-sunburst"
|
||||
|
||||
_TEST_PNG = base64.b64decode(
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAS0lEQVR42u3PMQ0AAAwDoPo3"
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ and verifies the proxy still logged a costed SpendLogs row
|
|||
(call_type="pass_through_endpoint"), correlated by the id the caller was served:
|
||||
the x-litellm-call-id header on gemini, the `msg_...` message id on anthropic.
|
||||
|
||||
Covered: gemini ("gemini-2.5-flash") + anthropic ("claude-haiku-4-5"), streaming +
|
||||
Covered: gemini ("gemini-3.8-flash") + anthropic ("claude-haiku-4-5"), streaming +
|
||||
non-streaming, plus native tool calls. See LLM_TRANSLATION_COVERAGE_MATRIX.md.
|
||||
|
||||
A passthrough call returning non-2xx fails hard (never a skip); once it returns
|
||||
|
|
@ -32,7 +32,7 @@ from passthrough_client import (
|
|||
completed_responses_object,
|
||||
)
|
||||
|
||||
GEMINI_MODEL: Final = "gemini-2.5-flash"
|
||||
GEMINI_MODEL: Final = "gemini-3.8-flash"
|
||||
ANTHROPIC_PASSTHROUGH_MODEL: Final = "claude-haiku-4-5"
|
||||
EMBEDDING_MODEL = "text-embedding-3-small"
|
||||
REALTIME_MODEL = "gpt-realtime-2"
|
||||
|
|
@ -291,7 +291,7 @@ class TestPassthroughModelAllowlist:
|
|||
|
||||
result = client.anthropic_message(key, ANTHROPIC_PASSTHROUGH_MODEL, f"say hi {unique_marker()}")
|
||||
assert result.status_code == 403, (
|
||||
"a key restricted to gemini-2.5-flash must be denied a claude passthrough call, "
|
||||
"a key restricted to gemini-3.8-flash must be denied a claude passthrough call, "
|
||||
f"got {result.status_code}: {result.body[:300]}"
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -57,7 +57,7 @@ OPENAI_VISION_BACKEND: Final = "openai/gpt-4o"
|
|||
ANTHROPIC_BACKEND: Final = "anthropic/claude-haiku-4-5"
|
||||
BEDROCK_CONVERSE_BACKEND: Final = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"
|
||||
VERTEX_BACKEND: Final = "vertex_ai/gemini-2.5-flash"
|
||||
GEMINI_BACKEND: Final = "gemini/gemini-2.5-flash"
|
||||
GEMINI_BACKEND: Final = "gemini/gemini-3.8-flash"
|
||||
OPENAI_RESPONSES_BACKEND: Final = "openai/gpt-5.5"
|
||||
INSTRUCTIONS = "You are a helpful assistant"
|
||||
CAT_IMAGE_URL = "https://upload.wikimedia.org/wikipedia/commons/3/3a/Cat03.jpg"
|
||||
|
|
|
|||
|
|
@ -55,7 +55,7 @@ from spend_e2e_client import (
|
|||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
CHAT_MODEL: Final = "gemini-2.5-flash"
|
||||
CHAT_MODEL: Final = "gemini-3.8-flash"
|
||||
MESSAGES_MODEL: Final = "claude-haiku-4-5"
|
||||
RESPONSES_MODEL: Final = "openai-responses-codex"
|
||||
EMBED_MODEL: Final = "openai-text-embedding-3-small"
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
Run against a proxy started with the gateway config. Coverage rationale:
|
||||
SPEND_TRACKING_COVERAGE_MATRIX.md.
|
||||
|
||||
Model names are literals from that config: chat tests hit "gemini-2.5-flash",
|
||||
Model names are literals from that config: chat tests hit "gemini-3.8-flash",
|
||||
embedding tests hit "openai-text-embedding-3-small".
|
||||
|
||||
Every test: fresh scoped key (isolation) -> real provider call -> unwrap (hard
|
||||
|
|
@ -37,7 +37,7 @@ from spend_reconciliation import BACKEND as TRAFFIC_BACKEND
|
|||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
GEMINI_MODEL = "gemini-2.5-flash"
|
||||
GEMINI_MODEL = "gemini-3.8-flash"
|
||||
CLAUDE_MODEL = "claude-haiku-4-5"
|
||||
CODEX_MODEL = "openai-responses-codex"
|
||||
EMBEDDING_MODEL = "openai-text-embedding-3-small"
|
||||
|
|
@ -780,7 +780,7 @@ def test_spend_calculate_returns_nonzero_cost(client: SpendClient) -> None:
|
|||
GEMINI_MODEL, "estimate the cost of this request"
|
||||
)
|
||||
assert cost > 0, (
|
||||
"/spend/calculate returned 0 for gemini-2.5-flash; "
|
||||
"/spend/calculate returned 0 for gemini-3.8-flash; "
|
||||
"cost map may be missing this model"
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -57,11 +57,9 @@ CONTENT_POLICY_PROMPT = (
|
|||
COOLDOWN_SECONDS = 30.0
|
||||
REPLICA_PROPAGATION_SECONDS = 15.0
|
||||
|
||||
# The smallest-context chat model OpenAI still serves (16385 tokens). A prompt
|
||||
# past that limit comes back as a real `context_length_exceeded` 400, which is
|
||||
# what litellm maps to ContextWindowExceededError.
|
||||
SMALL_CONTEXT_MODEL = "openai/gpt-3.5-turbo"
|
||||
SMALL_CONTEXT_LIMIT_TOKENS = 16385
|
||||
SMALL_CONTEXT_MODEL: Final = "openai/gpt-5.4-nano"
|
||||
# model_prices_and_context_window.json, gpt-5.4-nano, verified 2026-11-02
|
||||
SMALL_CONTEXT_LIMIT_TOKENS: Final = 272000
|
||||
|
||||
|
||||
def oversized_prompt(marker: str) -> str:
|
||||
|
|
@ -107,7 +105,7 @@ def create_timeout_deployment(proxy: ProxyClient, name: str) -> str:
|
|||
|
||||
@step(f"Add a deployment named {{name}} that calls the small-context model {SMALL_CONTEXT_MODEL}")
|
||||
def create_small_context_deployment(proxy: ProxyClient, name: str) -> str:
|
||||
"""Register a deployment on the smallest-context model OpenAI still serves, so an
|
||||
"""Register a deployment with a smaller context window than the backup, so an
|
||||
oversized prompt earns a real context-window refusal from the provider."""
|
||||
return proxy.create_model(name, LiteLLMParamsBody(model=SMALL_CONTEXT_MODEL, api_key=REAL_KEY))
|
||||
|
||||
|
|
@ -263,8 +261,7 @@ def spend_only_request_of(proxy: ProxyClient, spent_key: str) -> None:
|
|||
|
||||
@step(f"Add a deployment named {{name}} that calls {SMALL_CONTEXT_MODEL} and takes all of its group's traffic")
|
||||
def create_always_picked_small_context_deployment(proxy: ProxyClient, name: str) -> str:
|
||||
"""The always-picked half of a retry pair on the smallest-context model OpenAI
|
||||
still serves: it holds all of the model group's shuffle weight, so an oversized
|
||||
"""The always-picked half of a retry pair on the smaller-context model: it holds all of the model group's shuffle weight, so an oversized
|
||||
prompt opens on it and earns a real context-window refusal, which never benches
|
||||
a deployment, so only the retry itself can steer the request off it."""
|
||||
return proxy.register_model(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue