diff --git a/tests/e2e/claude_code/_compat_models.py b/tests/e2e/claude_code/_compat_models.py new file mode 100644 index 00000000000..89239ca9716 --- /dev/null +++ b/tests/e2e/claude_code/_compat_models.py @@ -0,0 +1,67 @@ +"""Load Claude Code matrix deployments from test_config.yaml for /model/new.""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import cast + +import yaml + +from models import LiteLLMParamsBody + +CONFIG_PATH = Path(__file__).resolve().parent / "test_config.yaml" + +_YAML_TO_PYDANTIC_ALIASES = { + "vertex_ai_project": "vertex_project", + "vertex_ai_location": "vertex_location", + "vertex_ai_credentials": "vertex_credentials", +} + + +@dataclass(frozen=True, slots=True) +class CompatDeployment: + model_name: str + litellm_params: LiteLLMParamsBody + + +def _normalize_params(raw: Mapping[str, object]) -> dict[str, object]: + return {_YAML_TO_PYDANTIC_ALIASES.get(key, key): value for key, value in raw.items()} + + +ConfigReader = Callable[[Path], str] + + +def _default_reader(path: Path) -> str: + return path.read_text() + + +def load_all_deployments( + config_path: Path = CONFIG_PATH, + reader: ConfigReader = _default_reader, +) -> tuple[CompatDeployment, ...]: + doc = yaml.safe_load(reader(config_path)) + if not isinstance(doc, dict): + return () + model_list = doc.get("model_list") or [] + if not isinstance(model_list, list): + return () + return tuple( + CompatDeployment( + model_name=str(entry["model_name"]), + litellm_params=LiteLLMParamsBody( + **_normalize_params(cast(Mapping[str, object], entry["litellm_params"])) + ), + ) + for entry in model_list + if isinstance(entry, dict) and "model_name" in entry and "litellm_params" in entry + ) + + +def all_expected_model_names( + *, + config_path: Path = CONFIG_PATH, + reader: ConfigReader = _default_reader, +) -> frozenset[str]: + return frozenset(d.model_name for d in load_all_deployments(config_path, reader)) diff --git a/tests/e2e/claude_code/_pr_gate_unit_tests/test_compat_models.py b/tests/e2e/claude_code/_pr_gate_unit_tests/test_compat_models.py new file mode 100644 index 00000000000..2da981782fe --- /dev/null +++ b/tests/e2e/claude_code/_pr_gate_unit_tests/test_compat_models.py @@ -0,0 +1,75 @@ +"""Unit tests for the compat matrix deployment loader and yaml drift checks.""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest + +from claude_code._compat_models import ( + all_expected_model_names, + load_all_deployments, +) + +CLAUDE_CODE_DIR = Path(__file__).resolve().parents[1] + + +def _cell_declared_model_names() -> frozenset[str]: + pattern = re.compile(r'"(claude-[a-zA-Z0-9._-]+)"') + found: set[str] = set() + for path in CLAUDE_CODE_DIR.glob("*/test_*.py"): + if path.parent.name.startswith("_"): + continue + for match in pattern.finditer(path.read_text()): + name = match.group(1) + if "/" in name or "@" in name: + continue + found.add(name) + return frozenset(found) + + +def test_yaml_covers_every_cell_declared_model_name() -> None: + yaml_names = all_expected_model_names() + cell_names = _cell_declared_model_names() + missing = cell_names - yaml_names + assert not missing, ( + f"compat cells reference model names not declared in " + f"test_config.yaml: {sorted(missing)}. Add a matching " + f"model_list entry so the session fixture can register them." + ) + + +def test_yaml_has_no_unused_declarations() -> None: + yaml_names = all_expected_model_names() + cell_names = _cell_declared_model_names() + unused = yaml_names - cell_names + assert not unused, ( + f"test_config.yaml declares model names no cell references: " + f"{sorted(unused)}. Delete them or add the cell." + ) + + +def test_load_returns_fifteen_deployments() -> None: + assert len(load_all_deployments()) == 15 + + +def test_deployments_are_hashable_and_frozen() -> None: + d = load_all_deployments()[0] + with pytest.raises((AttributeError, TypeError)): + d.model_name = "mutated" # type: ignore[misc] + + +def test_vertex_yaml_keys_populate_pydantic_body() -> None: + vertex = tuple(d for d in load_all_deployments() if d.model_name.endswith("-vertex")) + assert vertex, "no vertex deployments found in yaml" + for d in vertex: + assert d.litellm_params.vertex_project, ( + f"{d.model_name} lost its vertex_project after normalization" + ) + assert d.litellm_params.vertex_location, ( + f"{d.model_name} lost its vertex_location after normalization" + ) + assert d.litellm_params.use_in_pass_through is True, ( + f"{d.model_name} must set use_in_pass_through for the passthrough row" + ) diff --git a/tests/e2e/claude_code/conftest.py b/tests/e2e/claude_code/conftest.py index d2bfa1a54bf..7454cba8871 100644 --- a/tests/e2e/claude_code/conftest.py +++ b/tests/e2e/claude_code/conftest.py @@ -43,6 +43,8 @@ from typing import Any, Dict, FrozenSet, List, Optional, Tuple import pytest import yaml +from claude_code._compat_models import CompatDeployment, load_all_deployments + VALID_STATUSES = {"pass", "fail", "not_applicable", "not_tested"} RESULTS_ARTIFACT_ENV = "COMPAT_RESULTS_PATH" DEFAULT_ARTIFACT_PATH = "compat-results.json" @@ -548,3 +550,69 @@ def pytest_sessionfinish(session, exitstatus): ) summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True)) _print_rate_limit_summary(summary) + + +def _proxy_env_configured() -> bool: + has_url = bool(os.environ.get("LITELLM_PROXY_URL") or os.environ.get("LITELLM_PROXY_BASE_URL")) + has_key = bool(os.environ.get("LITELLM_MASTER_KEY") or os.environ.get("LITELLM_PROXY_API_KEY")) + return has_url and has_key + + +def _build_control_gateway(): + from e2e_gateway import build_gateway + + return build_gateway() + + +def _register_deployment(gateway: Any, deployment: CompatDeployment) -> str: + return gateway.create_model(deployment.model_name, deployment.litellm_params) + + +def _try_register( + gateway: Any, deployment: CompatDeployment +) -> tuple[str | None, tuple[str, str] | None]: + from requests import RequestException + + try: + return _register_deployment(gateway, deployment), None + except (AssertionError, RequestException) as exc: + return None, (deployment.model_name, str(exc)) + + +@pytest.fixture(scope="session", autouse=True) +def _compat_models_registered() -> Any: + """POST /model/new for every deployment in test_config.yaml; delete on exit. + + No-ops when proxy env is unset so unit-test trees stay hermetic. + Per-deployment register failures are printed and skipped: cells that need + that alias fail with Invalid model name, which is the right signal when + the proxy lacks that provider's credentials. + """ + if not _proxy_env_configured(): + yield + return + + from requests import RequestException + + gateway = _build_control_gateway() + outcomes = tuple(_try_register(gateway, d) for d in load_all_deployments()) + registered_ids = tuple(model_id for model_id, err in outcomes if model_id is not None) + failures = tuple(err for _model_id, err in outcomes if err is not None) + if failures: + summary = "\n".join(f" - {name}: {reason}" for name, reason in failures) + print( + f"[compat fixture] {len(failures)} of " + f"{len(failures) + len(registered_ids)} deployments " + f"failed to register (proxy likely missing that provider's " + f"credentials); cells that target them will fail loudly:\n" + f"{summary}", + flush=True, + ) + try: + yield + finally: + for model_id in registered_ids: + try: + gateway.delete_model(model_id) + except (AssertionError, RequestException): + pass diff --git a/tests/e2e/claude_code/test_config.yaml b/tests/e2e/claude_code/test_config.yaml index eec68d11dcf..5594c2884c8 100644 --- a/tests/e2e/claude_code/test_config.yaml +++ b/tests/e2e/claude_code/test_config.yaml @@ -59,21 +59,26 @@ model_list: aws_region_name: us-east-1 # ---- Vertex AI ---- + # use_in_pass_through registers project/location with the /vertex_ai + # passthrough router for the passthrough matrix row. - model_name: claude-haiku-4-5-vertex litellm_params: model: vertex_ai/claude-haiku-4-5 vertex_ai_project: os.environ/VERTEXAI_PROJECT vertex_ai_location: os.environ/VERTEXAI_LOCATION + use_in_pass_through: true - model_name: claude-sonnet-4-6-vertex litellm_params: model: vertex_ai/claude-sonnet-4-6 vertex_ai_project: os.environ/VERTEXAI_PROJECT vertex_ai_location: os.environ/VERTEXAI_LOCATION + use_in_pass_through: true - model_name: claude-opus-4-7-vertex litellm_params: model: vertex_ai/claude-opus-4-7 vertex_ai_project: os.environ/VERTEXAI_PROJECT vertex_ai_location: os.environ/VERTEXAI_LOCATION + use_in_pass_through: true # ---- Microsoft Foundry (Anthropic deployments on Azure) ---- - model_name: claude-haiku-4-5-azure diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 4140967f3e0..5c3bc3bdd1d 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -431,6 +431,7 @@ class LiteLLMParamsBody(BaseModel): vertex_project: str | None = None vertex_location: str | None = None vertex_credentials: str | None = None + use_in_pass_through: bool | None = None gcs_bucket_name: str | None = None bucket_name: str | None = None s3_bucket_name: str | None = None