mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-13 23:11:40 +00:00
09f2a9528a
Model IDs were hardcoded across the e2e suite, so every provider deprecation meant a scatter-shot edit and stale models (gemini-2.5-flash, gpt-4o-mini) kept running in CI. Tests now import role-named pins from tests/e2e/model_matrix.py, and a bump is a one-file change plus the compose gateway config. check_e2e_model_freshness.py (wired into test-code-quality) fails when a pin is missing from model_prices_and_context_window.json, is within 30 days of its deprecation_date, drifts from the docker-compose gateway config, or when a test hardcodes a versioned model literal again. Bumps: gemini-2.5-flash -> gemini-3.5-flash, gpt-4o-mini -> gpt-5.4-mini
56 lines
2.1 KiB
Python
56 lines
2.1 KiB
Python
"""Spend-tracking suite's `client` fixture and driver-model registration.
|
|
|
|
The shared lifecycle (resources/scoped_key), proxy liveness skip, and e2e marker
|
|
live in the parent tests/e2e/conftest.py. SpendClient exposes the shared Gateway
|
|
(GatewayProvider), so the `resources` fixture cleans up keys and customers this
|
|
suite creates.
|
|
|
|
The suite drives real calls through three deployments. On the stage gateway they
|
|
are baked into the proxy config; on a local dev proxy they usually are not, so
|
|
`driver_models` registers whichever are missing via /model/new and deletes only
|
|
the ones it created, never a config-baked deployment. Each registration carries
|
|
the provider key from the test runner's env when set (so a local proxy whose
|
|
container env lacks the key still works); otherwise it falls back to an
|
|
os.environ reference resolved from the proxy's own env, the stage convention.
|
|
"""
|
|
|
|
import os
|
|
from typing import Iterator
|
|
|
|
import pytest
|
|
|
|
from model_matrix import ANTHROPIC_CHAT, GEMINI_CHAT, OPENAI_EMBEDDING
|
|
from models import LiteLLMParamsBody
|
|
from spend_e2e_client import SpendClient, build_client
|
|
|
|
|
|
def _driver_params(provider_model: str, env_var: str) -> LiteLLMParamsBody:
|
|
return LiteLLMParamsBody(
|
|
model=provider_model,
|
|
api_key=os.environ.get(env_var) or f"os.environ/{env_var}",
|
|
)
|
|
|
|
|
|
DRIVER_MODELS: tuple[tuple[str, str, str], ...] = (
|
|
(GEMINI_CHAT.alias, GEMINI_CHAT.backend, "GEMINI_API_KEY"),
|
|
(ANTHROPIC_CHAT.alias, ANTHROPIC_CHAT.backend, "ANTHROPIC_API_KEY"),
|
|
(OPENAI_EMBEDDING.alias, OPENAI_EMBEDDING.backend, "OPENAI_API_KEY"),
|
|
)
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def client() -> SpendClient:
|
|
return build_client()
|
|
|
|
|
|
@pytest.fixture(scope="session", autouse=True)
|
|
def driver_models(client: SpendClient) -> Iterator[None]:
|
|
existing = frozenset(entry.model_name for entry in client.gateway.model_info())
|
|
created = tuple(
|
|
client.gateway.create_model(name, _driver_params(provider_model, env_var))
|
|
for name, provider_model, env_var in DRIVER_MODELS
|
|
if name not in existing
|
|
)
|
|
yield
|
|
for model_id in created:
|
|
client.gateway.delete_model(model_id)
|