diff --git a/.github/workflows/test-code-quality.yml b/.github/workflows/test-code-quality.yml index 872a1799d98..1ce77a23810 100644 --- a/.github/workflows/test-code-quality.yml +++ b/.github/workflows/test-code-quality.yml @@ -115,6 +115,12 @@ jobs: - name: check_fastuuid_usage run: uv run --no-sync python ./tests/code_coverage_tests/check_fastuuid_usage.py + - name: check_e2e_model_freshness_unit_tests + run: uv run --no-sync python -m pytest tests/code_coverage_tests/test_check_e2e_model_freshness.py -q + + - name: check_e2e_model_freshness + run: uv run --no-sync python ./tests/code_coverage_tests/check_e2e_model_freshness.py + - name: memory_test run: uv run --no-sync python ./tests/code_coverage_tests/memory_test.py diff --git a/tests/code_coverage_tests/check_e2e_model_freshness.py b/tests/code_coverage_tests/check_e2e_model_freshness.py new file mode 100644 index 00000000000..e4c59a89324 --- /dev/null +++ b/tests/code_coverage_tests/check_e2e_model_freshness.py @@ -0,0 +1,199 @@ +"""Fail CI when the e2e model matrix (tests/e2e/model_matrix.py) rots. + +Checks, in order: +- every pin resolves to an entry in model_prices_and_context_window.json +- no pin is within DEPRECATION_WINDOW_DAYS of its deprecation_date +- the gateway config inside tests/e2e/docker-compose.yml matches GATEWAY_MODELS + exactly, and every fallback references a configured alias +- no test under tests/e2e/ hardcodes a versioned model literal instead of + importing a pin from model_matrix.py (triple-quoted strings are exempt as + docstrings/prose; the scan is token-based so it survives syntax newer than + the interpreter running it) +""" + +import datetime +import importlib.util +import io +import json +import re +import sys +import tokenize +from pathlib import Path +from types import ModuleType + +import yaml +from pydantic import BaseModel, ConfigDict + +REPO_ROOT = Path(__file__).resolve().parents[2] +E2E_DIR = REPO_ROOT / "tests" / "e2e" +MATRIX_PATH = E2E_DIR / "model_matrix.py" +COMPOSE_PATH = E2E_DIR / "docker-compose.yml" +PRICING_PATH = REPO_ROOT / "model_prices_and_context_window.json" +DEPRECATION_WINDOW_DAYS = 30 +MODEL_LITERAL_RE = re.compile( + r"(?:gpt|gemini|claude|haiku|sonnet|opus|text-embedding)-\d|rerank-v\d" +) + + +class PricingEntry(BaseModel): + model_config = ConfigDict(extra="ignore") + deprecation_date: str | None = None + + +class GatewayLiteLLMParams(BaseModel): + model_config = ConfigDict(extra="ignore") + model: str + + +class GatewayModelEntry(BaseModel): + model_config = ConfigDict(extra="ignore") + model_name: str + litellm_params: GatewayLiteLLMParams + + +class GatewayRouterSettings(BaseModel): + model_config = ConfigDict(extra="ignore") + fallbacks: list[dict[str, list[str]]] = [] + + +class GatewayConfig(BaseModel): + model_config = ConfigDict(extra="ignore") + model_list: list[GatewayModelEntry] + router_settings: GatewayRouterSettings = GatewayRouterSettings() + + +def load_matrix() -> ModuleType: + spec = importlib.util.spec_from_file_location("e2e_model_matrix", MATRIX_PATH) + assert spec is not None and spec.loader is not None, f"cannot load {MATRIX_PATH}" + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def load_pricing() -> dict[str, PricingEntry]: + raw = json.loads(PRICING_PATH.read_text(encoding="utf-8")) + return { + key: PricingEntry.model_validate(value) + for key, value in raw.items() + if key != "sample_spec" + } + + +def load_gateway_config() -> GatewayConfig: + compose = yaml.safe_load(COMPOSE_PATH.read_text(encoding="utf-8")) + content = compose["configs"]["litellm_config"]["content"] + return GatewayConfig.model_validate(yaml.safe_load(content)) + + +def resolve_pricing_entry( + pricing: dict[str, PricingEntry], canonical: str, model_id: str +) -> PricingEntry | None: + return pricing.get(canonical) or pricing.get(model_id) + + +def existence_violations(matrix: ModuleType, pricing: dict[str, PricingEntry]) -> tuple[str, ...]: + return tuple( + f"{pin.backend}: neither '{pin.canonical}' nor '{pin.model_id}' is in " + f"model_prices_and_context_window.json; the model was removed or the pin is a typo" + for pin in matrix.ALL_PINS + if resolve_pricing_entry(pricing, pin.canonical, pin.model_id) is None + ) + + +def deprecation_violations( + matrix: ModuleType, pricing: dict[str, PricingEntry], today: datetime.date +) -> tuple[str, ...]: + def violation(pin) -> str | None: + entry = resolve_pricing_entry(pricing, pin.canonical, pin.model_id) + if entry is None or entry.deprecation_date is None: + return None + deprecation = datetime.date.fromisoformat(entry.deprecation_date) + if (deprecation - today).days > DEPRECATION_WINDOW_DAYS: + return None + return ( + f"{pin.backend}: deprecation_date {entry.deprecation_date} is within " + f"{DEPRECATION_WINDOW_DAYS} days; bump this pin in tests/e2e/model_matrix.py" + ) + + return tuple(v for v in (violation(pin) for pin in matrix.ALL_PINS) if v is not None) + + +def gateway_sync_violations(matrix: ModuleType, config: GatewayConfig) -> tuple[str, ...]: + expected = frozenset((pin.alias, pin.backend) for pin in matrix.GATEWAY_MODELS) + actual = frozenset( + (entry.model_name, entry.litellm_params.model) for entry in config.model_list + ) + aliases = frozenset(entry.model_name for entry in config.model_list) + missing = tuple( + f"docker-compose.yml gateway config is missing model_name '{alias}' -> '{backend}' " + f"from GATEWAY_MODELS" + for alias, backend in sorted(expected - actual) + ) + extra = tuple( + f"docker-compose.yml gateway config has model_name '{alias}' -> '{backend}' " + f"that is not in GATEWAY_MODELS; add a pin to tests/e2e/model_matrix.py" + for alias, backend in sorted(actual - expected) + ) + dangling_fallbacks = tuple( + f"docker-compose.yml fallback references '{name}', which is not a configured model_name" + for mapping in config.router_settings.fallbacks + for source, targets in mapping.items() + for name in (source, *targets) + if name not in aliases + ) + return missing + extra + dangling_fallbacks + + +def is_flagged_token(token: tokenize.TokenInfo) -> bool: + fstring_middle = getattr(tokenize, "FSTRING_MIDDLE", None) + if token.type == fstring_middle: + return bool(MODEL_LITERAL_RE.search(token.string)) + if token.type != tokenize.STRING: + return False + if token.string.lstrip("rbufRBUF").startswith(('"""', "'''")): + return False + return bool(MODEL_LITERAL_RE.search(token.string)) + + +def literal_violations_in_source(source: str, relative_path: str) -> tuple[str, ...]: + tokens = tokenize.generate_tokens(io.StringIO(source).readline) + return tuple( + f"{relative_path}:{token.start[0]}: hardcoded model literal in {token.string.strip()!r}; " + f"import a pin from tests/e2e/model_matrix.py instead" + for token in tokens + if is_flagged_token(token) + ) + + +def hardcoded_literal_violations(e2e_dir: Path) -> tuple[str, ...]: + return tuple( + violation + for path in sorted(e2e_dir.rglob("*.py")) + if path != MATRIX_PATH + for violation in literal_violations_in_source( + path.read_text(encoding="utf-8"), str(path.relative_to(REPO_ROOT)) + ) + ) + + +def main() -> int: + matrix = load_matrix() + pricing = load_pricing() + config = load_gateway_config() + violations = ( + existence_violations(matrix, pricing) + + deprecation_violations(matrix, pricing, datetime.date.today()) + + gateway_sync_violations(matrix, config) + + hardcoded_literal_violations(E2E_DIR) + ) + if violations: + print("e2e model freshness check failed:") + for violation in violations: + print(f" - {violation}") + return 1 + print(f"e2e model freshness check passed for {len(matrix.ALL_PINS)} pins") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/code_coverage_tests/test_check_e2e_model_freshness.py b/tests/code_coverage_tests/test_check_e2e_model_freshness.py new file mode 100644 index 00000000000..7c1f9861edb --- /dev/null +++ b/tests/code_coverage_tests/test_check_e2e_model_freshness.py @@ -0,0 +1,158 @@ +"""Unit tests for check_e2e_model_freshness.py. + +Each test feeds the check synthetic inputs and asserts it flags exactly the rot +it exists to catch: a pin vanishing from the pricing map, a pin nearing its +deprecation_date, the compose gateway config drifting from GATEWAY_MODELS, and +a hardcoded model literal sneaking into an e2e test. +""" + +import datetime +import importlib.util +from pathlib import Path +from types import SimpleNamespace + +CHECK_PATH = Path(__file__).resolve().parent / "check_e2e_model_freshness.py" + + +def _load_check(): + spec = importlib.util.spec_from_file_location("check_e2e_model_freshness", CHECK_PATH) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +check = _load_check() +matrix = check.load_matrix() +ModelPin = matrix.ModelPin +TODAY = datetime.date(2026, 7, 8) + + +def _matrix_of(*pins, gateway=()): + return SimpleNamespace(ALL_PINS=tuple(pins), GATEWAY_MODELS=tuple(gateway)) + + +def _pricing(**entries): + return { + key: check.PricingEntry(deprecation_date=deprecation) + for key, deprecation in entries.items() + } + + +class TestExistence: + def test_pin_missing_from_pricing_map_is_flagged(self): + pin = ModelPin("gemini", "gemini-99-flash") + violations = check.existence_violations(_matrix_of(pin), _pricing()) + assert len(violations) == 1 + assert "gemini/gemini-99-flash" in violations[0] + + def test_pin_resolving_via_prefixed_key_passes(self): + pin = ModelPin("gemini", "gemini-99-flash") + pricing = _pricing(**{"gemini/gemini-99-flash": None}) + assert check.existence_violations(_matrix_of(pin), pricing) == () + + def test_pin_resolving_via_bare_model_id_passes(self): + pin = ModelPin("openai", "gpt-99") + pricing = _pricing(**{"gpt-99": None}) + assert check.existence_violations(_matrix_of(pin), pricing) == () + + def test_pricing_key_override_is_used(self): + pin = ModelPin("azure", "my-deployment-name", pricing_key="azure/gpt-99") + pricing = _pricing(**{"azure/gpt-99": None}) + assert check.existence_violations(_matrix_of(pin), pricing) == () + + +class TestDeprecation: + def test_deprecation_within_window_is_flagged(self): + pin = ModelPin("openai", "gpt-99") + pricing = _pricing(**{"gpt-99": "2026-07-20"}) + violations = check.deprecation_violations(_matrix_of(pin), pricing, TODAY) + assert len(violations) == 1 + assert "2026-07-20" in violations[0] + + def test_already_deprecated_is_flagged(self): + pin = ModelPin("openai", "gpt-99") + pricing = _pricing(**{"gpt-99": "2026-01-01"}) + assert len(check.deprecation_violations(_matrix_of(pin), pricing, TODAY)) == 1 + + def test_deprecation_beyond_window_passes(self): + pin = ModelPin("openai", "gpt-99") + pricing = _pricing(**{"gpt-99": "2027-01-01"}) + assert check.deprecation_violations(_matrix_of(pin), pricing, TODAY) == () + + def test_no_deprecation_date_passes(self): + pin = ModelPin("openai", "gpt-99") + pricing = _pricing(**{"gpt-99": None}) + assert check.deprecation_violations(_matrix_of(pin), pricing, TODAY) == () + + +class TestGatewaySync: + def _config(self, models, fallbacks=()): + return check.GatewayConfig( + model_list=[ + check.GatewayModelEntry( + model_name=name, + litellm_params=check.GatewayLiteLLMParams(model=backend), + ) + for name, backend in models + ], + router_settings=check.GatewayRouterSettings(fallbacks=list(fallbacks)), + ) + + def test_matching_config_passes(self): + pin = ModelPin("gemini", "gemini-99-flash") + config = self._config([(pin.alias, pin.backend)]) + assert check.gateway_sync_violations(_matrix_of(gateway=(pin,)), config) == () + + def test_stale_compose_model_is_flagged_both_ways(self): + pin = ModelPin("gemini", "gemini-99-flash") + config = self._config([("gemini-98-flash", "gemini/gemini-98-flash")]) + violations = check.gateway_sync_violations(_matrix_of(gateway=(pin,)), config) + assert any("missing model_name 'gemini-99-flash'" in v for v in violations) + assert any("has model_name 'gemini-98-flash'" in v for v in violations) + + def test_fallback_referencing_unknown_alias_is_flagged(self): + pin = ModelPin("gemini", "gemini-99-flash") + config = self._config( + [(pin.alias, pin.backend)], + fallbacks=[{pin.alias: ["gpt-99"]}], + ) + violations = check.gateway_sync_violations(_matrix_of(gateway=(pin,)), config) + assert len(violations) == 1 + assert "gpt-99" in violations[0] + + +class TestLiteralScan: + def test_hardcoded_model_literal_is_flagged(self): + source = 'MODEL = "gemini-2.5-flash"\n' + violations = check.literal_violations_in_source(source, "suite/test_x.py") + assert len(violations) == 1 + assert "suite/test_x.py:1" in violations[0] + + def test_pin_usage_is_not_flagged(self): + source = ( + "from model_matrix import GEMINI_CHAT\n" + "MODEL = GEMINI_CHAT.alias\n" + 'MESSAGE = f"model {GEMINI_CHAT.alias} denied"\n' + ) + assert check.literal_violations_in_source(source, "suite/test_x.py") == () + + def test_docstrings_and_comments_are_exempt(self): + source = ( + '"""Drives gemini-2.5-flash against the proxy."""\n' + "MODEL = None # was gemini-2.5-flash\n" + ) + assert check.literal_violations_in_source(source, "suite/test_x.py") == () + + def test_rerank_literal_is_flagged(self): + source = 'MODEL = "cohere/rerank-v3.5"\n' + assert len(check.literal_violations_in_source(source, "suite/test_x.py")) == 1 + + def test_syntax_newer_than_interpreter_is_still_scanned(self): + source = ( + "def unwrap[R](result: Result[R]) -> R: ...\n" + 'MODEL = "gpt-4o-mini"\n' + ) + violations = check.literal_violations_in_source(source, "suite/test_x.py") + assert len(violations) == 1 + assert "suite/test_x.py:2" in violations[0] diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index a4c507ca5ea..9d6cb72834b 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -32,13 +32,17 @@ class TestPromptCompression: def test_prompt_compression_accumulate_spend(self, key_id, user_id): for _ in range(10): - response = self.resources.gateway.post("gemini-2.5-flash", key_id, user_id) + response = self.resources.gateway.post(GEMINI_CHAT.alias, key_id, user_id) compressed_value = ... assert response.cost == compressed_value # the cost was actually reduced ``` That snippet only conveys intent. What you actually write uses the real harness: the `client` fixture for your suite, the `scoped_key` fixture for an auto-deleted key, typed pydantic bodies from `models.py`, and `unwrap(...)` on the tagged-union result. `tests/e2e/llm_translation/test_custom_pricing_e2e.py` is the reference to copy from; it creates a scoped key, drives a real gemini call, polls `/spend/logs` to a deadline for the cost-breakdown row, then asserts the input and output costs match the configured custom rates and that a sibling deployment kept its own price. Read it before writing yours +## Models come from model_matrix.py, never hardcoded + +Every model a test drives is a pin imported from `model_matrix.py` (`GEMINI_CHAT.alias`, `OPENAI_CHAT_MINI.backend`, ...), named for the role it plays, never its version. Bumping a model version is then a one-file change (plus `docker-compose.yml`, which cannot import Python). `tests/code_coverage_tests/check_e2e_model_freshness.py` enforces this in CI: it fails on hardcoded versioned model literals in any `.py` file here, on pins missing from `model_prices_and_context_window.json` or near their `deprecation_date`, and on drift between `GATEWAY_MODELS` and the compose gateway config + ## Use the shared transport; never touch requests directly Every HTTP call goes through the shared transport, never through `requests.*` in a test. `e2e_http.py` is the only module permitted to call `requests.*`, and that is enforced in CI by `tests/code_coverage_tests/check_e2e_no_raw_requests.py`. A test that imports requests will fail the check diff --git a/tests/e2e/CONTRIBUTING.md b/tests/e2e/CONTRIBUTING.md index 4395d299c5b..784a8b4b4b3 100644 --- a/tests/e2e/CONTRIBUTING.md +++ b/tests/e2e/CONTRIBUTING.md @@ -9,7 +9,7 @@ When contributing to this directory, please first discuss the change you wish to ## Setup -The suites run against a live proxy, so bring one up first. `docker-compose.yml` here starts that proxy with a throwaway Postgres and Redis; `docker compose down -v` resets everything, so no state leaks between runs. The proxy config is inlined in the compose file under `configs`, prewired with example models (`gpt-5.5`, `claude-haiku-4-5`, `gemini-2.5-flash`, `openai-text-embedding-3-small`) whose keys come from your `.env`. If your test needs another model, a pricing override, or a guardrail declared up front, add it to that inline config and read it back in the test rather than hardcoding values +The suites run against a live proxy, so bring one up first. `docker-compose.yml` here starts that proxy with a throwaway Postgres and Redis; `docker compose down -v` resets everything, so no state leaks between runs. The proxy config is inlined in the compose file under `configs`, prewired with the models pinned in `model_matrix.py` (kept in sync by `tests/code_coverage_tests/check_e2e_model_freshness.py`) whose keys come from your `.env`. If your test needs another model, a pricing override, or a guardrail declared up front, add it to that inline config and read it back in the test rather than hardcoding values ## Running the tests locally @@ -113,7 +113,7 @@ class TestPromptCompression: def test_prompt_compression_accumulate_spend(self, key_id, user_id): for _ in range(10): - response = self.resources.gateway.post("gemini-2.5-flash", key_id, user_id) + response = self.resources.gateway.post(GEMINI_CHAT.alias, key_id, user_id) compressed_value = ... assert response.cost == compressed_value # the cost was actually reduced ``` diff --git a/tests/e2e/access_control/access_control_client.py b/tests/e2e/access_control/access_control_client.py index d7bc9c280aa..7f2ead9ff79 100644 --- a/tests/e2e/access_control/access_control_client.py +++ b/tests/e2e/access_control/access_control_client.py @@ -6,6 +6,7 @@ from dataclasses import dataclass from e2e_gateway import Gateway, build_gateway from e2e_http import StreamingResponse +from model_matrix import OPENAI_CHAT_MINI from models import ( ChatBody, ChatMessage, @@ -46,7 +47,7 @@ class AccessControlClient: headers=self.gateway.transport.bearer(key), json=ModelNewBody( model_name=model_name, - litellm_params=LiteLLMParamsBody(model="openai/gpt-4o-mini"), + litellm_params=LiteLLMParamsBody(model=OPENAI_CHAT_MINI.backend), model_info=ModelInfoBody(id=model_name), ), ) diff --git a/tests/e2e/access_control/test_access_control_e2e.py b/tests/e2e/access_control/test_access_control_e2e.py index ce649fa2400..68846493e21 100644 --- a/tests/e2e/access_control/test_access_control_e2e.py +++ b/tests/e2e/access_control/test_access_control_e2e.py @@ -24,11 +24,12 @@ from access_control_client import ( ) from e2e_config import unique_marker from lifecycle import ResourceManager +from model_matrix import GEMINI_CHAT, OPENAI_CHAT pytestmark = pytest.mark.e2e -ALLOWED_MODEL = "gemini-2.5-flash" -DISALLOWED_MODEL = "gpt-5.5" +ALLOWED_MODEL = GEMINI_CHAT.alias +DISALLOWED_MODEL = OPENAI_CHAT.alias def _is_json(body: str) -> bool: diff --git a/tests/e2e/batches/capabilities.py b/tests/e2e/batches/capabilities.py index 522e3162e24..2d45b5cc8bf 100644 --- a/tests/e2e/batches/capabilities.py +++ b/tests/e2e/batches/capabilities.py @@ -13,6 +13,7 @@ import base64 from dataclasses import dataclass from typing import Literal +from model_matrix import AZURE_BATCH, BEDROCK_ANTHROPIC_CHAT, GEMINI_CHAT, OPENAI_CHAT_MINI, VERTEX_CHAT from models import LiteLLMParamsBody Scenario = Literal["encoded", "unified", "model_param", "provider_fallback"] @@ -39,26 +40,26 @@ class Provider: match self.name: case "openai": return LiteLLMParamsBody( - model="openai/gpt-4o-mini", + model=OPENAI_CHAT_MINI.backend, api_key="os.environ/OPENAI_API_KEY", ) case "azure": return LiteLLMParamsBody( - model="azure/gpt-4.1-mini-batch", + model=AZURE_BATCH.backend, api_base="os.environ/AZURE_API_BASE", api_key="os.environ/AZURE_API_KEY", api_version="2024-07-01-preview", ) case "vertex_ai": return LiteLLMParamsBody( - model="vertex_ai/gemini-2.5-flash", + model=VERTEX_CHAT.backend, vertex_project="os.environ/VERTEXAI_PROJECT", vertex_location="us-central1", vertex_credentials="os.environ/VERTEXAI_CREDENTIALS", ) case "bedrock": return LiteLLMParamsBody( - model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + model=BEDROCK_ANTHROPIC_CHAT.backend, s3_access_key_id="os.environ/AWS_ACCESS_KEY_ID", s3_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY", s3_region_name="os.environ/AWS_REGION", @@ -96,15 +97,15 @@ class Capability: PROVIDERS: tuple[Provider, ...] = ( - Provider("openai", "openai-batch", "gpt-4o-mini", can_cancel=True, can_list=True), - Provider("azure", "azure-batch", "gpt-4.1-mini-batch", can_cancel=True, can_list=True), + Provider("openai", "openai-batch", OPENAI_CHAT_MINI.alias, can_cancel=True, can_list=True), + Provider("azure", "azure-batch", AZURE_BATCH.alias, can_cancel=True, can_list=True), Provider( - "vertex_ai", "vertex-batch", "gemini-2.5-flash", can_cancel=True, can_list=True + "vertex_ai", "vertex-batch", GEMINI_CHAT.alias, can_cancel=True, can_list=True ), Provider( "bedrock", "bedrock-batch", - "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + BEDROCK_ANTHROPIC_CHAT.backend, can_cancel=False, can_list=False, ), diff --git a/tests/e2e/batches/test_batches_e2e.py b/tests/e2e/batches/test_batches_e2e.py index a998f962c04..a89fe13288b 100644 --- a/tests/e2e/batches/test_batches_e2e.py +++ b/tests/e2e/batches/test_batches_e2e.py @@ -49,6 +49,7 @@ from e2e_http import ( unwrap, ) from lifecycle import ResourceManager +from model_matrix import OPENAI_CHAT_MINI from models import KeyGenerateBody, SpendLogRow, SpendLogsParams pytestmark = pytest.mark.e2e @@ -339,7 +340,7 @@ def test_rate_limited_batch_create_leaves_no_unattributed_spend_row( file = unwrap( client.upload_file( - content=render_jsonl("gpt-4o-mini"), + content=render_jsonl(OPENAI_CHAT_MINI.alias), form=FileUploadForm(purpose="batch"), model="openai-batch", key=key, diff --git a/tests/e2e/budgets/test_budget_enforcement_e2e.py b/tests/e2e/budgets/test_budget_enforcement_e2e.py index d03288b637a..6f40191312a 100644 --- a/tests/e2e/budgets/test_budget_enforcement_e2e.py +++ b/tests/e2e/budgets/test_budget_enforcement_e2e.py @@ -20,6 +20,7 @@ from budget_client import BudgetClient, is_budget_block from e2e_config import unique_marker from e2e_http import require_successful_call from lifecycle import run_case +from model_matrix import ANTHROPIC_CHAT pytestmark = pytest.mark.e2e @@ -31,7 +32,7 @@ def _assert_budget_blocks(client: BudgetClient, key: str, *, user: str = "") -> for _ in range(40): result = client.chat( key, - "claude-haiku-4-5", + ANTHROPIC_CHAT.alias, f"spend {unique_marker()}", max_tokens=16, user=user or None, @@ -88,7 +89,7 @@ class EndUserBudgetCase(_BudgetCase): customer = f"e2e-budget-cust-{unique_marker()}" self.client.create_customer(customer, max_budget=3e-6) self._undo.append(lambda: self.client.delete_customers([customer])) - self.key = self.client.generate_key(models=["claude-haiku-4-5"]) + self.key = self.client.generate_key(models=[ANTHROPIC_CHAT.alias]) self._undo.append(lambda: self.client.delete_key(self.key)) self._customer = customer diff --git a/tests/e2e/budgets/test_budget_fallback_e2e.py b/tests/e2e/budgets/test_budget_fallback_e2e.py index 6114bfe6fc9..c8a993c8e44 100644 --- a/tests/e2e/budgets/test_budget_fallback_e2e.py +++ b/tests/e2e/budgets/test_budget_fallback_e2e.py @@ -12,11 +12,12 @@ import pytest from budget_client import BudgetClient, model_budget from e2e_config import unique_marker from lifecycle import ResourceManager +from model_matrix import ANTHROPIC_CHAT, OPENAI_CHAT pytestmark = pytest.mark.e2e -PRIMARY_MODEL = "claude-haiku-4-5" -FALLBACK_MODEL = "gpt-5.5" +PRIMARY_MODEL = ANTHROPIC_CHAT.alias +FALLBACK_MODEL = OPENAI_CHAT.alias def test_budget_fallback_reroutes_anthropic_messages_to_openai( diff --git a/tests/e2e/budgets/test_budget_reset_e2e.py b/tests/e2e/budgets/test_budget_reset_e2e.py index dcf776db9a2..d970a0a7199 100644 --- a/tests/e2e/budgets/test_budget_reset_e2e.py +++ b/tests/e2e/budgets/test_budget_reset_e2e.py @@ -16,13 +16,14 @@ from budget_client import BudgetClient, is_budget_block from e2e_config import unique_marker from e2e_http import require_successful_call from lifecycle import ResourceManager +from model_matrix import ANTHROPIC_CHAT pytestmark = pytest.mark.e2e def _call(client: BudgetClient, key: str): return client.chat( - key, "claude-haiku-4-5", f"reset {unique_marker()}", max_tokens=16 + key, ANTHROPIC_CHAT.alias, f"reset {unique_marker()}", max_tokens=16 ) diff --git a/tests/e2e/budgets/test_model_max_budget_e2e.py b/tests/e2e/budgets/test_model_max_budget_e2e.py index 44e6a333ef0..6e2a09e371f 100644 --- a/tests/e2e/budgets/test_model_max_budget_e2e.py +++ b/tests/e2e/budgets/test_model_max_budget_e2e.py @@ -14,11 +14,12 @@ from budget_client import BudgetClient, is_budget_block, model_budget from e2e_config import unique_marker from e2e_http import require_successful_call from lifecycle import ResourceManager +from model_matrix import ANTHROPIC_CHAT, GEMINI_CHAT pytestmark = pytest.mark.e2e -CAPPED_MODEL = "claude-haiku-4-5" -FREE_MODEL = "gemini-2.5-flash" +CAPPED_MODEL = ANTHROPIC_CHAT.alias +FREE_MODEL = GEMINI_CHAT.alias def _call(client: BudgetClient, key: str, model: str): diff --git a/tests/e2e/budgets/test_multi_window_budget_e2e.py b/tests/e2e/budgets/test_multi_window_budget_e2e.py index 553ad1ce701..6ea894afb69 100644 --- a/tests/e2e/budgets/test_multi_window_budget_e2e.py +++ b/tests/e2e/budgets/test_multi_window_budget_e2e.py @@ -16,6 +16,7 @@ from budget_client import BudgetClient, is_budget_block from e2e_config import unique_marker from e2e_http import require_successful_call from lifecycle import ResourceManager +from model_matrix import ANTHROPIC_CHAT from models import BudgetWindow pytestmark = pytest.mark.e2e @@ -25,7 +26,7 @@ WINDOW_SECONDS = 30 # the tight window; calls succeed again only after it elaps def _call(client: BudgetClient, key: str): return client.chat( - key, "claude-haiku-4-5", f"window {unique_marker()}", max_tokens=16 + key, ANTHROPIC_CHAT.alias, f"window {unique_marker()}", max_tokens=16 ) diff --git a/tests/e2e/budgets/test_soft_budget_e2e.py b/tests/e2e/budgets/test_soft_budget_e2e.py index 407de7ae467..da4212e5058 100644 --- a/tests/e2e/budgets/test_soft_budget_e2e.py +++ b/tests/e2e/budgets/test_soft_budget_e2e.py @@ -13,6 +13,7 @@ from budget_client import BudgetClient, is_budget_block from e2e_config import unique_marker from e2e_http import require_successful_call from lifecycle import ResourceManager +from model_matrix import ANTHROPIC_CHAT pytestmark = pytest.mark.e2e @@ -26,7 +27,7 @@ def test_soft_budget_does_not_block( for _ in range(3): result = client.chat( - key, "claude-haiku-4-5", f"hi {unique_marker()}", max_tokens=16 + key, ANTHROPIC_CHAT.alias, f"hi {unique_marker()}", max_tokens=16 ) assert not is_budget_block(result), ( "soft_budget blocked a request; it must alert only, not block " diff --git a/tests/e2e/budgets/test_spend_counter_reseed_e2e.py b/tests/e2e/budgets/test_spend_counter_reseed_e2e.py index a6860aeef43..c8b9987babf 100644 --- a/tests/e2e/budgets/test_spend_counter_reseed_e2e.py +++ b/tests/e2e/budgets/test_spend_counter_reseed_e2e.py @@ -28,6 +28,7 @@ from budget_client import BudgetClient from e2e_config import unique_marker from e2e_http import StreamingResponse from lifecycle import ResourceManager +from model_matrix import ANTHROPIC_CHAT if TYPE_CHECKING: import redis @@ -35,7 +36,7 @@ if TYPE_CHECKING: pytestmark = pytest.mark.e2e -MODEL = "claude-haiku-4-5" +MODEL = ANTHROPIC_CHAT.alias ACCUMULATE_CALLS = 24 BURST = 6 # proxy_batch_write_at (60s) flushes the spend to the DB and default_redis_ttl (20s) diff --git a/tests/e2e/budgets/test_tag_budget_e2e.py b/tests/e2e/budgets/test_tag_budget_e2e.py index 7cec5bc96c1..5536308a450 100644 --- a/tests/e2e/budgets/test_tag_budget_e2e.py +++ b/tests/e2e/budgets/test_tag_budget_e2e.py @@ -14,6 +14,7 @@ from budget_client import BudgetClient, is_budget_block from e2e_config import unique_marker from e2e_http import require_successful_call from lifecycle import ResourceManager +from model_matrix import ANTHROPIC_CHAT pytestmark = pytest.mark.e2e @@ -23,7 +24,7 @@ TINY_BUDGET = 1e-6 def _tagged_call(client: BudgetClient, key: str, tag: str): result = client.chat( key, - "claude-haiku-4-5", + ANTHROPIC_CHAT.alias, f"hi {unique_marker()}", tags=[tag], max_tokens=16, diff --git a/tests/e2e/budgets/test_team_member_budget_e2e.py b/tests/e2e/budgets/test_team_member_budget_e2e.py index 301617bfdca..680966dede3 100644 --- a/tests/e2e/budgets/test_team_member_budget_e2e.py +++ b/tests/e2e/budgets/test_team_member_budget_e2e.py @@ -22,11 +22,12 @@ from budget_client import BudgetClient, is_budget_block from e2e_config import unique_marker from e2e_http import Success, require_successful_call from lifecycle import ResourceManager +from model_matrix import ANTHROPIC_CHAT from models import ChatBody, ChatMessage pytestmark = pytest.mark.e2e -MODEL = "claude-haiku-4-5" +MODEL = ANTHROPIC_CHAT.alias TEAM_BUDGET = 100.0 MEMBER_BUDGET = 3e-6 BURST = 6 diff --git a/tests/e2e/budgets/test_team_member_budget_reset_e2e.py b/tests/e2e/budgets/test_team_member_budget_reset_e2e.py index 2749f16a26e..d09a7818d62 100644 --- a/tests/e2e/budgets/test_team_member_budget_reset_e2e.py +++ b/tests/e2e/budgets/test_team_member_budget_reset_e2e.py @@ -7,6 +7,7 @@ from budget_client import BudgetClient from e2e_config import unique_marker from e2e_http import require_successful_call from lifecycle import ResourceManager +from model_matrix import ANTHROPIC_CHAT pytestmark = pytest.mark.e2e @@ -33,7 +34,7 @@ def test_team_member_budget_reset_keeps_advancing(client: BudgetClient, resource # the member can spend within the team while the window is live key = client.generate_key(team_id=team_id, user_id=user_id) resources.defer(lambda: client.delete_key(key)) - require_successful_call(client.chat(key, "claude-haiku-4-5", f"reset {unique_marker()}", max_tokens=16)) + require_successful_call(client.chat(key, ANTHROPIC_CHAT.alias, f"reset {unique_marker()}", max_tokens=16)) # once the window elapses the reset job must move budget_reset_at forward; a job # that skips the member's budget row (the #25109 regression) leaves it pinned at diff --git a/tests/e2e/budgets/test_team_multi_window_budget_e2e.py b/tests/e2e/budgets/test_team_multi_window_budget_e2e.py index c58e74db965..6c7d62223b3 100644 --- a/tests/e2e/budgets/test_team_multi_window_budget_e2e.py +++ b/tests/e2e/budgets/test_team_multi_window_budget_e2e.py @@ -21,6 +21,7 @@ from budget_client import BudgetClient, is_budget_block from e2e_config import unique_marker from e2e_http import require_successful_call from lifecycle import ResourceManager +from model_matrix import ANTHROPIC_CHAT from models import BudgetWindow pytestmark = pytest.mark.e2e @@ -29,7 +30,7 @@ WINDOW_SECONDS = 30 def _call(client: BudgetClient, key: str): - return client.chat(key, "claude-haiku-4-5", f"team-window {unique_marker()}", max_tokens=16) + return client.chat(key, ANTHROPIC_CHAT.alias, f"team-window {unique_marker()}", max_tokens=16) def test_team_short_window_blocks_then_resets(client: BudgetClient, resources: ResourceManager) -> None: diff --git a/tests/e2e/docker-compose.yml b/tests/e2e/docker-compose.yml index cdf5d6cbf6f..9b4e758c7fb 100644 --- a/tests/e2e/docker-compose.yml +++ b/tests/e2e/docker-compose.yml @@ -23,7 +23,7 @@ configs: allowed_fails: 5 cooldown_time: 30 fallbacks: - - gemini-2.5-flash: ["gpt-5.5", "claude-haiku-4-5"] + - gemini-3.5-flash: ["gpt-5.5", "claude-haiku-4-5"] model_list: - model_name: gpt-5.5 @@ -36,9 +36,9 @@ configs: model: anthropic/claude-haiku-4-5 api_key: os.environ/ANTHROPIC_API_KEY - - model_name: gemini-2.5-flash + - model_name: gemini-3.5-flash litellm_params: - model: gemini/gemini-2.5-flash + model: gemini/gemini-3.5-flash api_key: os.environ/GEMINI_API_KEY - model_name: openai-text-embedding-3-small diff --git a/tests/e2e/llm_translation/test_audio_speech_e2e.py b/tests/e2e/llm_translation/test_audio_speech_e2e.py index f7a04d94cb3..8187d1c368b 100644 --- a/tests/e2e/llm_translation/test_audio_speech_e2e.py +++ b/tests/e2e/llm_translation/test_audio_speech_e2e.py @@ -13,6 +13,7 @@ from e2e_config import unique_marker from e2e_http import require_successful_call from endpoints_client import EndpointsClient from lifecycle import ResourceManager +from model_matrix import OPENAI_TTS from models import LiteLLMParamsBody pytestmark = pytest.mark.e2e @@ -26,7 +27,7 @@ class TestAudioSpeech: model_id = endpoints_client.create_model( model, LiteLLMParamsBody( - model="openai/gpt-4o-mini-tts", api_key="os.environ/OPENAI_API_KEY" + model=OPENAI_TTS.backend, api_key="os.environ/OPENAI_API_KEY" ), ) resources.defer(lambda: endpoints_client.delete_model(model_id)) diff --git a/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py b/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py index 5cc4ff308fa..efa57ccd2c5 100644 --- a/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py +++ b/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py @@ -15,15 +15,16 @@ import pytest from e2e_config import unique_marker from e2e_http import unwrap +from model_matrix import ANTHROPIC_CHAT, GEMINI_CHAT, OPENAI_CHAT from models import ChatBody, ChatMessage from passthrough_client import PassthroughClient pytestmark = pytest.mark.e2e CHAT_MODELS: tuple[tuple[str, str], ...] = ( - ("gpt-5.5", "openai"), - ("claude-haiku-4-5", "anthropic"), - ("gemini-2.5-flash", "gemini"), + (OPENAI_CHAT.alias, "openai"), + (ANTHROPIC_CHAT.alias, "anthropic"), + (GEMINI_CHAT.alias, "gemini"), ) diff --git a/tests/e2e/llm_translation/test_custom_pricing_e2e.py b/tests/e2e/llm_translation/test_custom_pricing_e2e.py index 7894b447be9..15b55bcb4f4 100644 --- a/tests/e2e/llm_translation/test_custom_pricing_e2e.py +++ b/tests/e2e/llm_translation/test_custom_pricing_e2e.py @@ -5,12 +5,13 @@ teardown) instead of relying on a statically configured model, so the check is self-contained and never inherits pricing another suite or a stale config left on the shared proxy. custom-priced-flash sets input/output rates deliberately far above the canonical gemini price; the isolation sibling shares the same -gemini/gemini-2.5-flash backend but sets no override. Three behaviors are checked -independently: +gemini flash backend (GEMINI_CHAT in model_matrix.py) but sets no override. +Three behaviors are checked independently: - billing: a real call's logged cost breakdown charges input and output tokens at - the custom rates, each component checked separately (a base-rate bill lands - ~100x lower; a swapped input/output rate passes a total-only check but not this) + the custom rates, each component checked separately (a base-rate bill lands an + order of magnitude lower; a swapped input/output rate passes a total-only check + but not this) - reporting: /model/info surfaces those rates for the deployment - isolation: the sibling keeps its own price; an override that leaks into the shared backend cost map (LIT-3897) misprices it, making the sibling's rate match @@ -27,6 +28,7 @@ from e2e_gateway import Gateway from e2e_http import Success, unwrap from endpoints_client import EndpointsClient from lifecycle import ResourceManager +from model_matrix import GEMINI_CHAT from models import ( ChatBody, ChatMessage, @@ -37,9 +39,9 @@ from models import ( pytestmark = pytest.mark.e2e -BACKEND_MODEL = "gemini/gemini-2.5-flash" +BACKEND_MODEL = GEMINI_CHAT.backend GEMINI_API_KEY = "os.environ/GEMINI_API_KEY" -# Deliberately ~100x above canonical gemini-2.5-flash (input 3e-7 / output 2.5e-6) +# Deliberately an order of magnitude above the canonical gemini flash rates # so an override that is ignored or under-applied bills at the base rate and fails. CUSTOM_INPUT_RATE = 5e-05 CUSTOM_OUTPUT_RATE = 1e-04 @@ -78,7 +80,7 @@ def _provision( input_cost_per_token: float | None, output_cost_per_token: float | None, ) -> str: - """Register a fresh gemini/gemini-2.5-flash deployment (deleted on teardown) and + """Register a fresh gemini flash deployment (deleted on teardown) and return its model name. With the cost fields set the deployment carries a custom pricing override; with them None it is a plain sibling on the same backend. The marker keeps the name unique so concurrent runs on the shared proxy never @@ -230,7 +232,7 @@ class TestCustomPricing: assert sibling_entry is not None, f"{sibling} absent from /model/info" # custom-priced-flash overrides pricing; the sibling shares the same - # gemini/gemini-2.5-flash backend but sets no override, so it must keep its + # gemini flash backend but sets no override, so it must keep its # own price. Equal rates mean the override leaked into the shared cost map. assert ( sibling_entry.model_info.input_cost_per_token diff --git a/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py b/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py index 56f2de8bd4f..4e990868a98 100644 --- a/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py +++ b/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py @@ -14,6 +14,7 @@ from e2e_config import unique_marker from e2e_http import require_successful_call from endpoints_client import EmbeddingsResult, EndpointsClient from lifecycle import ResourceManager +from model_matrix import OPENAI_EMBEDDING from models import LiteLLMParamsBody pytestmark = pytest.mark.e2e @@ -27,7 +28,7 @@ class TestEmbeddingsEndpoint: model_id = endpoints_client.create_model( model, LiteLLMParamsBody( - model="openai/text-embedding-3-small", api_key="os.environ/OPENAI_API_KEY" + model=OPENAI_EMBEDDING.backend, api_key="os.environ/OPENAI_API_KEY" ), ) resources.defer(lambda: endpoints_client.delete_model(model_id)) diff --git a/tests/e2e/llm_translation/test_messages_e2e.py b/tests/e2e/llm_translation/test_messages_e2e.py index b0a48f22118..65999f76a32 100644 --- a/tests/e2e/llm_translation/test_messages_e2e.py +++ b/tests/e2e/llm_translation/test_messages_e2e.py @@ -13,6 +13,7 @@ from e2e_config import unique_marker from e2e_http import require_successful_call from endpoints_client import EndpointsClient, MessagesResult from lifecycle import ResourceManager +from model_matrix import ANTHROPIC_CHAT from models import LiteLLMParamsBody pytestmark = pytest.mark.e2e @@ -26,7 +27,7 @@ class TestAnthropicMessages: model_id = endpoints_client.create_model( model, LiteLLMParamsBody( - model="anthropic/claude-haiku-4-5", api_key="os.environ/ANTHROPIC_API_KEY" + model=ANTHROPIC_CHAT.backend, api_key="os.environ/ANTHROPIC_API_KEY" ), ) resources.defer(lambda: endpoints_client.delete_model(model_id)) diff --git a/tests/e2e/llm_translation/test_passthrough_e2e.py b/tests/e2e/llm_translation/test_passthrough_e2e.py index 37d55c665b3..a4787d6903f 100644 --- a/tests/e2e/llm_translation/test_passthrough_e2e.py +++ b/tests/e2e/llm_translation/test_passthrough_e2e.py @@ -4,7 +4,7 @@ Each test sends a NATIVE provider request through the proxy's passthrough route and verifies the proxy still logged a costed SpendLogs row (call_type="pass_through_endpoint"), correlated by the x-litellm-call-id header. -Covered: gemini ("gemini-2.5-flash") + anthropic ("claude-haiku-4-5"), streaming + +Covered: gemini (GEMINI_CHAT.alias) + anthropic (ANTHROPIC_CHAT.alias), streaming + non-streaming, plus native tool calls. See LLM_TRANSLATION_COVERAGE_MATRIX.md. A passthrough call returning non-2xx fails hard (never a skip); once it returns @@ -15,6 +15,7 @@ import pytest from e2e_config import unique_marker from e2e_http import StreamingResponse, require_successful_call +from model_matrix import ANTHROPIC_CHAT, GEMINI_CHAT from models import SpendLogRow from passthrough_client import ( AnthropicTool, @@ -55,7 +56,7 @@ def test_gemini_passthrough_nonstreaming_logs_cost( ) -> None: tag = f"e2e-passthrough-{unique_marker()}" result = client.gemini_generate( - scoped_key, "gemini-2.5-flash", "Say hello in one word", tags=[tag, "gemini"] + scoped_key, GEMINI_CHAT.alias, "Say hello in one word", tags=[tag, "gemini"] ) require_successful_call(result) @@ -68,7 +69,7 @@ def test_gemini_passthrough_nonstreaming_logs_cost( def test_gemini_passthrough_streaming_logs_cost( client: PassthroughClient, scoped_key: str ) -> None: - result = client.gemini_stream(scoped_key, "gemini-2.5-flash", "Count to five") + result = client.gemini_stream(scoped_key, GEMINI_CHAT.alias, "Count to five") require_successful_call(result) assert result.chunks > 0, "streaming passthrough produced no events" @@ -81,7 +82,7 @@ def test_gemini_passthrough_tool_call_logs_cost( ) -> None: result = client.gemini_generate( scoped_key, - "gemini-2.5-flash", + GEMINI_CHAT.alias, "What is the weather in Paris? Use the get_weather tool.", tools=[ GeminiTool( @@ -112,7 +113,7 @@ def test_gemini_passthrough_tool_call_logs_cost( def test_anthropic_passthrough_nonstreaming_logs_cost( client: PassthroughClient, scoped_key: str ) -> None: - result = client.anthropic_message(scoped_key, "claude-haiku-4-5", "Say hello") + result = client.anthropic_message(scoped_key, ANTHROPIC_CHAT.alias, "Say hello") require_successful_call(result) row = _fetch_cost_breakdown(client, result) @@ -124,7 +125,7 @@ def test_anthropic_passthrough_streaming_logs_cost( client: PassthroughClient, scoped_key: str ) -> None: result = client.anthropic_message( - scoped_key, "claude-haiku-4-5", "Count to five", stream=True + scoped_key, ANTHROPIC_CHAT.alias, "Count to five", stream=True ) require_successful_call(result) assert result.chunks > 0, "streaming passthrough produced no events" @@ -138,7 +139,7 @@ def test_anthropic_passthrough_tool_call_logs_cost( ) -> None: result = client.anthropic_message( scoped_key, - "claude-haiku-4-5", + ANTHROPIC_CHAT.alias, "What is the weather in Paris? Use the get_weather tool.", tools=[ AnthropicTool( diff --git a/tests/e2e/llm_translation/test_provider_features_e2e.py b/tests/e2e/llm_translation/test_provider_features_e2e.py index cf05a4306b4..64df2abdadf 100644 --- a/tests/e2e/llm_translation/test_provider_features_e2e.py +++ b/tests/e2e/llm_translation/test_provider_features_e2e.py @@ -27,6 +27,7 @@ from pydantic import BaseModel from e2e_config import unique_marker from e2e_http import unwrap from lifecycle import ResourceManager +from model_matrix import BEDROCK_ANTHROPIC_CHAT, OPENAI_CHAT from models import ChatBody, ChatMessage, ChatResponse, LiteLLMParamsBody from passthrough_client import PassthroughClient @@ -86,7 +87,7 @@ class TestServiceTier: model_id = client.gateway.create_model( model, LiteLLMParamsBody( - model="openai/gpt-5.5", api_key="os.environ/OPENAI_API_KEY" + model=OPENAI_CHAT.backend, api_key="os.environ/OPENAI_API_KEY" ), ) resources.defer(lambda: client.gateway.delete_model(model_id)) @@ -121,7 +122,7 @@ class TestPromptCaching: model_id = client.gateway.create_model( model, LiteLLMParamsBody( - model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + model=BEDROCK_ANTHROPIC_CHAT.backend, aws_region_name="us-east-1", ), ) diff --git a/tests/e2e/llm_translation/test_rerank_e2e.py b/tests/e2e/llm_translation/test_rerank_e2e.py index 4b30ac1ea5c..5866cc7d031 100644 --- a/tests/e2e/llm_translation/test_rerank_e2e.py +++ b/tests/e2e/llm_translation/test_rerank_e2e.py @@ -13,6 +13,7 @@ from e2e_config import unique_marker from e2e_http import require_successful_call from endpoints_client import EndpointsClient, RerankResult from lifecycle import ResourceManager +from model_matrix import COHERE_RERANK from models import LiteLLMParamsBody pytestmark = pytest.mark.e2e @@ -32,7 +33,7 @@ class TestRerank: model = f"e2e-rerank-{unique_marker()}" model_id = endpoints_client.create_model( model, - LiteLLMParamsBody(model="cohere/rerank-v3.5", api_key="os.environ/COHERE_API_KEY"), + LiteLLMParamsBody(model=COHERE_RERANK.backend, api_key="os.environ/COHERE_API_KEY"), ) resources.defer(lambda: endpoints_client.delete_model(model_id)) key = resources.key() diff --git a/tests/e2e/llm_translation/test_responses_e2e.py b/tests/e2e/llm_translation/test_responses_e2e.py index 743de79880f..350d843fe90 100644 --- a/tests/e2e/llm_translation/test_responses_e2e.py +++ b/tests/e2e/llm_translation/test_responses_e2e.py @@ -13,6 +13,7 @@ from e2e_config import unique_marker from e2e_http import require_successful_call from endpoints_client import EndpointsClient, ResponsesResult from lifecycle import ResourceManager +from model_matrix import OPENAI_CHAT_MINI from models import LiteLLMParamsBody pytestmark = pytest.mark.e2e @@ -25,7 +26,7 @@ class TestResponses: model = f"e2e-responses-{unique_marker()}" model_id = endpoints_client.create_model( model, - LiteLLMParamsBody(model="openai/gpt-4o-mini", api_key="os.environ/OPENAI_API_KEY"), + LiteLLMParamsBody(model=OPENAI_CHAT_MINI.backend, api_key="os.environ/OPENAI_API_KEY"), ) resources.defer(lambda: endpoints_client.delete_model(model_id)) key = resources.key() diff --git a/tests/e2e/llm_translation/test_vertex_passthrough_e2e.py b/tests/e2e/llm_translation/test_vertex_passthrough_e2e.py index 78d2bb358d2..5bcab1a2cfa 100644 --- a/tests/e2e/llm_translation/test_vertex_passthrough_e2e.py +++ b/tests/e2e/llm_translation/test_vertex_passthrough_e2e.py @@ -30,12 +30,13 @@ from pydantic import BaseModel from e2e_config import unique_marker from e2e_http import NoBody, require_successful_call, unwrap from lifecycle import ResourceManager +from model_matrix import GEMINI_CHAT from models import SpendLogRow from passthrough_client import PassthroughClient pytestmark = pytest.mark.e2e -VERTEX_MODEL = "gemini-2.5-flash" +VERTEX_MODEL = GEMINI_CHAT.alias # The added deployment's region and the passthrough URL's region are the same constant, # so they always agree; the proxy registers passthrough credentials per project+region. VERTEX_LOCATION = os.environ.get("VERTEXAI_LOCATION", "us-central1") diff --git a/tests/e2e/logging/test_prometheus_cardinality_e2e.py b/tests/e2e/logging/test_prometheus_cardinality_e2e.py index 163293a3009..d1d5599fc08 100644 --- a/tests/e2e/logging/test_prometheus_cardinality_e2e.py +++ b/tests/e2e/logging/test_prometheus_cardinality_e2e.py @@ -23,10 +23,11 @@ from prometheus_client.parser import text_string_to_metric_families from e2e_config import unique_marker from lifecycle import ResourceManager from logging_client import LoggingClient +from model_matrix import GEMINI_CHAT pytestmark = pytest.mark.e2e -DRIVER_MODEL = "gemini-2.5-flash" +DRIVER_MODEL = GEMINI_CHAT.alias REQUESTS_METRIC = "litellm_requests_metric_total" ALIAS_LABEL = "api_key_alias" DISTINCT_KEYS = 3 diff --git a/tests/e2e/management/test_key_models_dropdown_e2e.py b/tests/e2e/management/test_key_models_dropdown_e2e.py index f0ba21699e0..8234e453499 100644 --- a/tests/e2e/management/test_key_models_dropdown_e2e.py +++ b/tests/e2e/management/test_key_models_dropdown_e2e.py @@ -17,6 +17,7 @@ import pytest from e2e_config import PROXY_BASE_URL, unique_marker from lifecycle import ResourceManager from management_client import ManagementClient +from model_matrix import OPENAI_CHAT from models import KeyGenerateBody, TeamNewBody pytest.importorskip("playwright.sync_api", reason="playwright not installed") @@ -77,7 +78,7 @@ def _open_key_edit_form(page: Page, key_alias: str) -> None: def _provision_team(client: ManagementClient, resources: ResourceManager, alias: str) -> str: - team_id = client.create_team(TeamNewBody(team_alias=alias, models=["all-proxy-models", "gpt-5.5"])) + team_id = client.create_team(TeamNewBody(team_alias=alias, models=["all-proxy-models", OPENAI_CHAT.alias])) resources.defer(lambda: client.delete_team(team_id)) return team_id @@ -85,7 +86,7 @@ def _provision_team(client: ManagementClient, resources: ResourceManager, alias: def _provision_key( client: ManagementClient, resources: ResourceManager, alias: str, team_id: str | None = None ) -> str: - key = client.gateway.generate_key(KeyGenerateBody(key_alias=alias, models=["gpt-5.5"], team_id=team_id)) + key = client.gateway.generate_key(KeyGenerateBody(key_alias=alias, models=[OPENAI_CHAT.alias], team_id=team_id)) resources.defer(lambda: client.gateway.delete_key(key)) return key @@ -98,7 +99,7 @@ class TestKeyModelsDropdownUI: ) -> None: _open_create_key_modal(ui_page) - options = _models_dropdown_texts(ui_page, must_contain="gpt-5.5") + options = _models_dropdown_texts(ui_page, must_contain=OPENAI_CHAT.alias) assert "All Proxy Models" in options, f"teamless create lost 'All Proxy Models': {options}" assert "All Team Models" not in options, f"teamless create offered 'All Team Models': {options}" @@ -120,7 +121,7 @@ class TestKeyModelsDropdownUI: _select_team(ui_page, team_alias) options = _models_dropdown_texts(ui_page, must_contain="All Team Models") - assert "gpt-5.5" in options, f"team key create lost the team's own model: {options}" + assert OPENAI_CHAT.alias in options, f"team key create lost the team's own model: {options}" assert "All Proxy Models" not in options, f"team key create offered 'All Proxy Models': {options}" assert "all-proxy-models" not in options, f"team key create offered the raw sentinel: {options}" @@ -140,7 +141,7 @@ class TestKeyModelsDropdownUI: _open_key_edit_form(ui_page, key_alias) - options = _models_dropdown_texts(ui_page, must_contain="gpt-5.5") + options = _models_dropdown_texts(ui_page, must_contain=OPENAI_CHAT.alias) assert "All Proxy Models" in options, f"teamless edit lost 'All Proxy Models': {options}" assert "All Team Models" not in options, f"teamless edit offered 'All Team Models': {options}" @@ -156,6 +157,6 @@ class TestKeyModelsDropdownUI: _open_key_edit_form(ui_page, key_alias) options = _models_dropdown_texts(ui_page, must_contain="All Team Models") - assert "gpt-5.5" in options, f"team key edit lost the team's own model: {options}" + assert OPENAI_CHAT.alias in options, f"team key edit lost the team's own model: {options}" assert "All Proxy Models" not in options, f"team key edit offered 'All Proxy Models': {options}" assert "all-proxy-models" not in options, f"team key edit offered the raw sentinel: {options}" diff --git a/tests/e2e/management/test_management_e2e.py b/tests/e2e/management/test_management_e2e.py index cbd5db0d59f..3af5e4ded8f 100644 --- a/tests/e2e/management/test_management_e2e.py +++ b/tests/e2e/management/test_management_e2e.py @@ -22,6 +22,7 @@ from management_client import ( ROUTE_NOT_ALLOWED_MARKER, ManagementClient, ) +from model_matrix import GEMINI_CHAT, OPENAI_CHAT from models import KeyGenerateBody, OrgNewBody, TeamNewBody, UserNewBody pytestmark = pytest.mark.e2e @@ -111,42 +112,42 @@ class TestKeyRoutes: key = _generate_key( client, resources, - KeyGenerateBody(models=["gemini-2.5-flash"], key_alias=alias, tpm_limit=424242), + KeyGenerateBody(models=[GEMINI_CHAT.alias], key_alias=alias, tpm_limit=424242), ) info = client.gateway.key_info(key) assert info.key_alias == alias, f"/key/info reports key_alias {info.key_alias!r}, configured {alias!r}" - assert info.models == ["gemini-2.5-flash"], ( - f"/key/info reports models {info.models}, configured ['gemini-2.5-flash']" + assert info.models == [GEMINI_CHAT.alias], ( + f"/key/info reports models {info.models}, configured [{GEMINI_CHAT.alias!r}]" ) assert info.tpm_limit == 424242, ( f"/key/info reports tpm_limit {info.tpm_limit}, configured 424242" ) - _poll_chat_ok(client, key, "gemini-2.5-flash") + _poll_chat_ok(client, key, GEMINI_CHAT.alias) _assert_model_denied( - client.chat_status(key, "gpt-5.5", f"say hi {unique_marker()}"), "gpt-5.5" + client.chat_status(key, OPENAI_CHAT.alias, f"say hi {unique_marker()}"), OPENAI_CHAT.alias ) @pytest.mark.covers("mgmt.key.update.persists") def test_update_models_persists_and_flips_enforcement( self, client: ManagementClient, resources: ResourceManager ) -> None: - key = _generate_key(client, resources, KeyGenerateBody(models=["gemini-2.5-flash"])) - _poll_chat_ok(client, key, "gemini-2.5-flash") + key = _generate_key(client, resources, KeyGenerateBody(models=[GEMINI_CHAT.alias])) + _poll_chat_ok(client, key, GEMINI_CHAT.alias) _assert_model_denied( - client.chat_status(key, "gpt-5.5", f"say hi {unique_marker()}"), "gpt-5.5" + client.chat_status(key, OPENAI_CHAT.alias, f"say hi {unique_marker()}"), OPENAI_CHAT.alias ) - client.update_key_models(key, ["gpt-5.5"]) + client.update_key_models(key, [OPENAI_CHAT.alias]) info = client.gateway.key_info(key) - assert info.models == ["gpt-5.5"], ( - f"/key/info reports models {info.models} after /key/update to ['gpt-5.5']" + assert info.models == [OPENAI_CHAT.alias], ( + f"/key/info reports models {info.models} after /key/update to [{OPENAI_CHAT.alias!r}]" ) - _poll_model_access_granted(client, key, "gpt-5.5") - _poll_chat_denied(client, key, "gemini-2.5-flash") + _poll_model_access_granted(client, key, OPENAI_CHAT.alias) + _poll_chat_denied(client, key, GEMINI_CHAT.alias) @pytest.mark.covers("mgmt.key.delete.persists") def test_delete_revokes_the_key_on_chat(self, client: ManagementClient, resources: ResourceManager) -> None: @@ -154,13 +155,13 @@ class TestKeyRoutes: design: the deferred cleanup must survive this test failing before the in-body delete, and a repeat /key/delete is a cheap no-op the warn-only teardown absorbs.""" - key = _generate_key(client, resources, KeyGenerateBody(models=["gemini-2.5-flash"])) - _poll_chat_ok(client, key, "gemini-2.5-flash") + key = _generate_key(client, resources, KeyGenerateBody(models=[GEMINI_CHAT.alias])) + _poll_chat_ok(client, key, GEMINI_CHAT.alias) client.delete_key_strict(key) def rejected() -> bool | None: - outcome = client.chat_status(key, "gemini-2.5-flash", f"say hi {unique_marker()}") + outcome = client.chat_status(key, GEMINI_CHAT.alias, f"say hi {unique_marker()}") return True if outcome.status_code == 401 else None _ = _poll(client, rejected, "deleted key was still accepted on chat (never rejected 401) at the deadline") @@ -172,12 +173,12 @@ class TestTeamRoutes: self, client: ManagementClient, resources: ResourceManager ) -> None: alias = f"e2e-mgmt-team-{unique_marker()}" - team_id = _create_team(client, resources, alias, ["gemini-2.5-flash"]) + team_id = _create_team(client, resources, alias, [GEMINI_CHAT.alias]) info = client.team_info(team_id) assert info.team_alias == alias, f"/team/info reports team_alias {info.team_alias!r}, configured {alias!r}" - assert info.models == ["gemini-2.5-flash"], ( - f"/team/info reports models {info.models}, configured ['gemini-2.5-flash']" + assert info.models == [GEMINI_CHAT.alias], ( + f"/team/info reports models {info.models}, configured [{GEMINI_CHAT.alias!r}]" ) key = _generate_key(client, resources, KeyGenerateBody(team_id=team_id)) @@ -195,7 +196,7 @@ class TestTeamRoutes: resources, UserNewBody(user_email=f"e2e-mgmt-{unique_marker()}@example.com", user_role="internal_user"), ) - team_id = _create_team(client, resources, f"e2e-mgmt-team-{unique_marker()}", ["gemini-2.5-flash"]) + team_id = _create_team(client, resources, f"e2e-mgmt-team-{unique_marker()}", [GEMINI_CHAT.alias]) client.add_team_member(team_id, user_id) member = next( @@ -230,15 +231,15 @@ class TestOrganizationRoutes: self, client: ManagementClient, resources: ResourceManager ) -> None: alias = f"e2e-mgmt-org-{unique_marker()}" - org_id = client.create_org(OrgNewBody(organization_alias=alias, models=["gemini-2.5-flash"])) + org_id = client.create_org(OrgNewBody(organization_alias=alias, models=[GEMINI_CHAT.alias])) resources.defer(lambda: client.delete_org(org_id)) info = client.org_info(org_id) assert info.organization_alias == alias, ( f"/organization/info reports alias {info.organization_alias!r}, configured {alias!r}" ) - assert info.models == ["gemini-2.5-flash"], ( - f"/organization/info reports models {info.models}, configured ['gemini-2.5-flash']" + assert info.models == [GEMINI_CHAT.alias], ( + f"/organization/info reports models {info.models}, configured [{GEMINI_CHAT.alias!r}]" ) diff --git a/tests/e2e/model_matrix.py b/tests/e2e/model_matrix.py new file mode 100644 index 00000000000..1c59b63ac09 --- /dev/null +++ b/tests/e2e/model_matrix.py @@ -0,0 +1,66 @@ +"""Single source of truth for every model the e2e suite drives. + +Bump a model version here instead of editing individual tests. Constants are +named for the role a model plays, never its version, so a bump touches this +file (plus docker-compose.yml, which cannot import Python) and nothing else. +tests/code_coverage_tests/check_e2e_model_freshness.py fails CI when a pin +disappears from model_prices_and_context_window.json, approaches its +deprecation_date, drifts from the docker-compose gateway config, or when a +test hardcodes a model literal instead of importing a pin. +""" + +from dataclasses import dataclass + + +@dataclass(frozen=True, slots=True) +class ModelPin: + provider: str + model_id: str + gateway_alias: str = "" + pricing_key: str = "" + + @property + def backend(self) -> str: + return f"{self.provider}/{self.model_id}" + + @property + def alias(self) -> str: + return self.gateway_alias or self.model_id + + @property + def canonical(self) -> str: + return self.pricing_key or self.backend + + +GEMINI_CHAT = ModelPin("gemini", "gemini-3.5-flash") +OPENAI_CHAT = ModelPin("openai", "gpt-5.5") +OPENAI_CHAT_MINI = ModelPin("openai", "gpt-5.4-mini") +ANTHROPIC_CHAT = ModelPin("anthropic", "claude-haiku-4-5") +OPENAI_EMBEDDING = ModelPin( + "openai", "text-embedding-3-small", gateway_alias="openai-text-embedding-3-small" +) +OPENAI_TTS = ModelPin("openai", "gpt-4o-mini-tts") +VERTEX_CHAT = ModelPin("vertex_ai", "gemini-3.5-flash") +AZURE_BATCH = ModelPin("azure", "gpt-4.1-mini-batch", pricing_key="azure/gpt-4.1-mini") +BEDROCK_ANTHROPIC_CHAT = ModelPin("bedrock", "us.anthropic.claude-haiku-4-5-20251001-v1:0") +COHERE_RERANK = ModelPin("cohere", "rerank-v3.5") + +GATEWAY_MODELS: tuple[ModelPin, ...] = ( + OPENAI_CHAT, + ANTHROPIC_CHAT, + GEMINI_CHAT, + OPENAI_EMBEDDING, +) + +ALL_PINS: tuple[ModelPin, ...] = ( + GEMINI_CHAT, + OPENAI_CHAT, + OPENAI_CHAT_MINI, + ANTHROPIC_CHAT, + OPENAI_EMBEDDING, + OPENAI_TTS, + VERTEX_CHAT, + AZURE_BATCH, + BEDROCK_ANTHROPIC_CHAT, + COHERE_RERANK, +) diff --git a/tests/e2e/spend_tracking/conftest.py b/tests/e2e/spend_tracking/conftest.py index 0e80764236b..a03c72b640f 100644 --- a/tests/e2e/spend_tracking/conftest.py +++ b/tests/e2e/spend_tracking/conftest.py @@ -19,6 +19,7 @@ from typing import Iterator import pytest +from model_matrix import ANTHROPIC_CHAT, GEMINI_CHAT, OPENAI_EMBEDDING from models import LiteLLMParamsBody from spend_e2e_client import SpendClient, build_client @@ -31,9 +32,9 @@ def _driver_params(provider_model: str, env_var: str) -> LiteLLMParamsBody: DRIVER_MODELS: tuple[tuple[str, str, str], ...] = ( - ("gemini-2.5-flash", "gemini/gemini-2.5-flash", "GEMINI_API_KEY"), - ("claude-haiku-4-5", "anthropic/claude-haiku-4-5", "ANTHROPIC_API_KEY"), - ("openai-text-embedding-3-small", "openai/text-embedding-3-small", "OPENAI_API_KEY"), + (GEMINI_CHAT.alias, GEMINI_CHAT.backend, "GEMINI_API_KEY"), + (ANTHROPIC_CHAT.alias, ANTHROPIC_CHAT.backend, "ANTHROPIC_API_KEY"), + (OPENAI_EMBEDDING.alias, OPENAI_EMBEDDING.backend, "OPENAI_API_KEY"), ) diff --git a/tests/e2e/spend_tracking/test_spend_tracking_e2e.py b/tests/e2e/spend_tracking/test_spend_tracking_e2e.py index 3495011eab6..75eceebb39c 100644 --- a/tests/e2e/spend_tracking/test_spend_tracking_e2e.py +++ b/tests/e2e/spend_tracking/test_spend_tracking_e2e.py @@ -3,8 +3,8 @@ Run against a proxy started with the gateway config. Coverage rationale: SPEND_TRACKING_COVERAGE_MATRIX.md. -Model names are literals from that config: chat tests hit "gemini-2.5-flash", -embedding tests hit "openai-text-embedding-3-small". +Model names come from model_matrix.py pins baked into that config: chat tests +hit GEMINI_CHAT, embedding tests hit OPENAI_EMBEDDING. Every test: fresh scoped key (isolation) -> real provider call -> unwrap (hard fail if the proxy couldn't make a call it should) -> poll /spend/logs to a @@ -23,6 +23,7 @@ import pytest from e2e_http import Result, Success from lifecycle import ResourceManager +from model_matrix import ANTHROPIC_CHAT, GEMINI_CHAT, OPENAI_EMBEDDING from models import ChatResponse, SpendLogs, SpendLogsParams from spend_e2e_client import SpendClient, SpendLogRow, is_ok, unique_marker, unwrap @@ -65,7 +66,7 @@ def test_chat_completion_writes_nonzero_spend_row( chat = unwrap( client.chat( scoped_key, - "gemini-2.5-flash", + GEMINI_CHAT.alias, f"reply with one word {unique_marker()}", max_tokens=16, ) @@ -79,7 +80,7 @@ def test_chat_completion_writes_nonzero_spend_row( assert (row.spend or 0) > 0, f"chat row should cost > 0: {_summarize(rows)}" assert row.status == "success" assert row.cache_hit != "True", "fresh call must not be a cache hit" - assert "gemini-2.5-flash" in (row.model or "") + assert GEMINI_CHAT.alias in (row.model or "") prompt = row.prompt_tokens or 0 completion = row.completion_tokens or 0 @@ -98,7 +99,7 @@ def test_streaming_chat_completion_tracks_spend( ) -> None: result = client.chat_stream( scoped_key, - "gemini-2.5-flash", + GEMINI_CHAT.alias, f"count to three {unique_marker()}", max_tokens=64, ) @@ -126,7 +127,7 @@ def test_embedding_writes_nonzero_spend_row( _ = unwrap( client.embed( scoped_key, - "openai-text-embedding-3-small", + OPENAI_EMBEDDING.alias, f"vectorize this sentence {unique_marker()}", ) ) @@ -139,7 +140,7 @@ def test_embedding_writes_nonzero_spend_row( ) assert (row.prompt_tokens or 0) > 0 assert (row.completion_tokens or 0) == 0, "embeddings have no completion tokens" - assert "text-embedding-3-small" in (row.model or "") + assert OPENAI_EMBEDDING.model_id in (row.model or "") def test_cache_hit_is_zero_cost_and_suffixed( @@ -150,8 +151,8 @@ def test_cache_hit_is_zero_cost_and_suffixed( # populated. The marker keeps each run isolated - a fixed prompt would persist # in the shared response cache across runs and make both calls hit (flaky). prompt = f"What is the capital of France? Answer in one word. {unique_marker()}" - _ = unwrap(client.chat(scoped_key, "gemini-2.5-flash", prompt, max_tokens=16)) - _ = unwrap(client.chat(scoped_key, "gemini-2.5-flash", prompt, max_tokens=16)) + _ = unwrap(client.chat(scoped_key, GEMINI_CHAT.alias, prompt, max_tokens=16)) + _ = unwrap(client.chat(scoped_key, GEMINI_CHAT.alias, prompt, max_tokens=16)) rows = client.poll_logs_for_key( scoped_key, predicate=lambda rs: any(r.cache_hit == "True" for r in rs) @@ -182,7 +183,7 @@ def test_key_spend_equals_sum_of_logs(client: SpendClient, scoped_key: str) -> N _ = unwrap( client.chat( scoped_key, - "gemini-2.5-flash", + GEMINI_CHAT.alias, f"say hi {unique_marker()}", max_tokens=16, ) @@ -216,7 +217,7 @@ def test_burst_of_concurrent_calls_loses_no_spend( def call(idx: int) -> Result[ChatResponse]: return client.chat( scoped_key, - "gemini-2.5-flash", + GEMINI_CHAT.alias, f"burst call {idx} {unique_marker()}", max_tokens=16, ) @@ -265,7 +266,7 @@ def test_spend_logs_v2_pagination_caps_pages_and_keeps_total( _ = unwrap( client.chat( scoped_key, - "gemini-2.5-flash", + GEMINI_CHAT.alias, f"page fodder {unique_marker()}", max_tokens=16, ) @@ -305,7 +306,7 @@ def test_request_tags_round_trip(client: SpendClient, scoped_key: str) -> None: tag = f"e2e-spend-{unique_marker()}" _ = unwrap( client.chat( - scoped_key, "gemini-2.5-flash", "tagged request", tags=[tag], max_tokens=16 + scoped_key, GEMINI_CHAT.alias, "tagged request", tags=[tag], max_tokens=16 ) ) @@ -327,7 +328,7 @@ def test_tag_spend_matches_sum_of_tagged_logs( _ = unwrap( client.chat( scoped_key, - "gemini-2.5-flash", + GEMINI_CHAT.alias, f"hi {unique_marker()}", tags=[tag], max_tokens=16, @@ -359,7 +360,7 @@ def test_end_user_spend_attributed_on_row( ) -> None: customer = resources.customer(f"e2e-cust-{unique_marker()}") _ = unwrap( - client.chat(scoped_key, "gemini-2.5-flash", "hi", user=customer, max_tokens=16) + client.chat(scoped_key, GEMINI_CHAT.alias, "hi", user=customer, max_tokens=16) ) rows = client.poll_logs_for_key( @@ -381,27 +382,27 @@ def test_each_model_on_a_shared_key_gets_its_own_row( sibling deployment, or collapses both calls onto one request_id fails here.""" gemini = unwrap( client.chat( - scoped_key, "gemini-2.5-flash", f"one word {unique_marker()}", max_tokens=16 + scoped_key, GEMINI_CHAT.alias, f"one word {unique_marker()}", max_tokens=16 ) ) claude = unwrap( client.chat( - scoped_key, "claude-haiku-4-5", f"one word {unique_marker()}", max_tokens=16 + scoped_key, ANTHROPIC_CHAT.alias, f"one word {unique_marker()}", max_tokens=16 ) ) def both_models_costed(rows: list[SpendLogRow]) -> bool: costed = [r.model or "" for r in rows if (r.spend or 0) > 0] - return any("gemini-2.5-flash" in m for m in costed) and any( - "claude-haiku-4-5" in m for m in costed + return any(GEMINI_CHAT.alias in m for m in costed) and any( + ANTHROPIC_CHAT.alias in m for m in costed ) rows = client.poll_logs_for_key(scoped_key, min_rows=2, predicate=both_models_costed) gemini_row = _require_row( - rows, lambda r: "gemini-2.5-flash" in (r.model or ""), "for the gemini call" + rows, lambda r: GEMINI_CHAT.alias in (r.model or ""), "for the gemini call" ) claude_row = _require_row( - rows, lambda r: "claude-haiku-4-5" in (r.model or ""), "for the claude call" + rows, lambda r: ANTHROPIC_CHAT.alias in (r.model or ""), "for the claude call" ) assert (gemini_row.spend or 0) > 0, f"gemini row should cost > 0: {_summarize(rows)}" @@ -422,7 +423,7 @@ def test_each_model_on_a_shared_key_gets_its_own_row( def test_failure_call_writes_failure_status_row( client: SpendClient, scoped_key: str ) -> None: - result = client.chat(scoped_key, "gemini-2.5-flash", "", max_tokens=1) + result = client.chat(scoped_key, GEMINI_CHAT.alias, "", max_tokens=1) if is_ok(result): pytest.skip("call unexpectedly succeeded; could not induce a failure row") @@ -440,11 +441,11 @@ def test_failure_call_writes_failure_status_row( def test_spend_calculate_returns_nonzero_cost(client: SpendClient) -> None: cost = client.calculate_spend( - "gemini-2.5-flash", "estimate the cost of this request" + GEMINI_CHAT.alias, "estimate the cost of this request" ) assert cost > 0, ( - "/spend/calculate returned 0 for gemini-2.5-flash; " - "cost map may be missing this model" + f"/spend/calculate returned 0 for {GEMINI_CHAT.alias}; " + f"cost map may be missing this model" ) @@ -458,7 +459,7 @@ def test_spend_logs_endpoint_returns_spend( call's nonzero spend must surface before the deadline.""" unwrap( client.chat( - scoped_key, "gemini-2.5-flash", f"spend logs {unique_marker()}", max_tokens=16 + scoped_key, GEMINI_CHAT.alias, f"spend logs {unique_marker()}", max_tokens=16 ) ) diff --git a/tests/e2e/test_e2e_gateway.py b/tests/e2e/test_e2e_gateway.py index a6dcc6112d6..ceb64121aec 100644 --- a/tests/e2e/test_e2e_gateway.py +++ b/tests/e2e/test_e2e_gateway.py @@ -22,6 +22,7 @@ from e2e_http import ( StreamingResponse, Success, ) +from model_matrix import OPENAI_CHAT_MINI from models import ( LiteLLMParamsBody, ModelDeleteBody, @@ -109,7 +110,7 @@ def test_gateway_create_model_registers_deployment_and_returns_model_id() -> Non gateway = Gateway(transport=transport) model_id = gateway.create_model( - "e2e-test-model", LiteLLMParamsBody(model="openai/gpt-4o-mini") + "e2e-test-model", LiteLLMParamsBody(model=OPENAI_CHAT_MINI.backend) ) assert model_id == "registered-id" @@ -126,7 +127,7 @@ def test_batch_client_create_model_registers_a_batch_mode_deployment() -> None: client = BatchClient(gateway=Gateway(transport=transport)) model_id = client.create_model( - "e2e-batch-model", LiteLLMParamsBody(model="openai/gpt-4o-mini") + "e2e-batch-model", LiteLLMParamsBody(model=OPENAI_CHAT_MINI.backend) ) assert model_id == "registered-id"