mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
fa992071f3
Merge main and move the Codex compatibility work onto main's tests/e2e/codex suite, replacing the parallel tests/e2e/client_apps suite and its harness tests. A makereport hook in codex/compat_results.py appends one JSON line per decided cell when CODEX_COMPAT_RESULTS_PATH is set, and codex/matrix_builder.py builds a table in the Claude Code matrix shape and exits 3 when a cell that passed no longer does. The row id comes from the test file name and the label from the class name, so a new row needs no other edit. The suite now pins @openai/codex 0.166.0 in its own package.json and lockfile, adds Gemini on Vertex AI as a setup, and adds prompt caching, reasoning effort, vision, structured output and web search rows. Web search carries strict xfails where Codex or LiteLLM leaves the search tool out, and auto-compaction now also expects Vertex AI to fail, for the same bridge gap Claude on Bedrock has
43 lines
1.4 KiB
Python
43 lines
1.4 KiB
Python
from __future__ import annotations
|
|
|
|
from collections.abc import Iterator
|
|
from dataclasses import dataclass
|
|
from typing import Final
|
|
|
|
import pytest
|
|
from cli_harness.rate_limiter import get_default_limiter
|
|
from codex.compat_results import pytest_runtest_makereport as pytest_runtest_makereport
|
|
from codex.harness import CodexHarness, codex_harness
|
|
from codex.setups import Deployment, ModelSetup
|
|
from e2e_config import PROXY_BASE_URL, unique_marker
|
|
from lifecycle import ResourceManager
|
|
from proxy_client import ProxyClient
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class _Client:
|
|
proxy: ProxyClient
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def client(proxy: ProxyClient) -> _Client:
|
|
return _Client(proxy)
|
|
|
|
|
|
@pytest.fixture
|
|
def setup(deployment: Deployment, proxy: ProxyClient, resources: ResourceManager) -> ModelSetup:
|
|
model: Final = f"e2e-codex-{unique_marker()}"
|
|
model_id: Final = proxy.create_model(model, deployment.litellm_params, provider_live=True)
|
|
resources.defer(lambda: proxy.delete_model(model_id))
|
|
return ModelSetup(model=model, rate_bucket=deployment.rate_bucket)
|
|
|
|
|
|
@pytest.fixture
|
|
def codex(setup: ModelSetup, resources: ResourceManager) -> Iterator[CodexHarness]:
|
|
with codex_harness(
|
|
gateway=PROXY_BASE_URL,
|
|
api_key=resources.key(models=[setup.model], user_id=None),
|
|
rate_bucket=setup.rate_bucket,
|
|
limiter=get_default_limiter(),
|
|
) as harness:
|
|
yield harness
|