mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-04 07:27:51 +00:00
50af27eaa6
* test(e2e): add the Codex file edit and long session rows The file edit row has Codex change a random value into config.txt with the patch tool and checks the file on disk. The long session row has it follow a chain of random file names one cat at a time. new_workspace now returns the resolved path, because Codex reports patched files by their real path (/private/var on macOS). * test(e2e): make the Codex long session prove every earlier command output reached the model * test(e2e): Codex auto-compaction through LiteLLM (#49389) * test(e2e): add the Codex auto-compaction row Codex reads a random value, compacts through the gateway with its remote compaction forced on, and is asked for the value after the file holding it is deleted. The row reads the compaction from the thread's rollout file and marks Claude on Bedrock as a strict expected failure, since the Responses-to-chat bridge ignores the compaction_trigger item * refactor(e2e): validate Codex rollout lines with the models directly
96 lines
3.1 KiB
Python
96 lines
3.1 KiB
Python
"""The credentials are `os.environ/` references the gateway resolves, the same names the Claude Code matrix's
|
|
`test_config.yaml` uses, and Bedrock rides the gateway's default AWS identity."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
from typing import Final, Literal
|
|
|
|
import pytest
|
|
from _pytest.mark.structures import ParameterSet
|
|
from cli_harness.rate_limiter import RateBucket
|
|
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
|
|
from models import LiteLLMParamsBody
|
|
|
|
GPT_ON_BEDROCK_MANTLE: Final = "bedrock_mantle/openai.gpt-6.1-sol"
|
|
GPT_ON_BEDROCK_RUNTIME: Final = "bedrock/us.openai.gpt-6.1-sol"
|
|
GPT_ON_AZURE_OPENAI: Final = "azure/gpt-6.1-sol"
|
|
GPT_ON_OPENAI: Final = "openai/gpt-6.1-sol"
|
|
CLAUDE_ON_BEDROCK: Final = "bedrock/us.anthropic.claude-sonnet-5-5"
|
|
_BEDROCK_REGION: Final = "us-east-1"
|
|
|
|
type SetupId = Literal["bedrock_mantle", "bedrock_runtime", "azure_openai", "openai", "claude_bedrock"]
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class Deployment:
|
|
litellm_params: LiteLLMParamsBody
|
|
rate_bucket: RateBucket
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class ModelSetup:
|
|
model: str
|
|
rate_bucket: RateBucket
|
|
|
|
|
|
def _drives(provider: Provider, model: str) -> pytest.MarkDecorator:
|
|
return meta(
|
|
Subject(
|
|
domain=Domain.LLM_TRANSLATION,
|
|
route=Route.RESPONSES,
|
|
providers=(provider,),
|
|
models=(model,),
|
|
mode=Mode.STREAM,
|
|
)
|
|
)
|
|
|
|
|
|
SETUPS: Final = (
|
|
pytest.param(
|
|
Deployment(
|
|
LiteLLMParamsBody(model=GPT_ON_BEDROCK_MANTLE, aws_region_name=_BEDROCK_REGION), RateBucket.BEDROCK_MANTLE
|
|
),
|
|
id="bedrock_mantle",
|
|
marks=_drives(Provider.BEDROCK_MANTLE, GPT_ON_BEDROCK_MANTLE),
|
|
),
|
|
pytest.param(
|
|
Deployment(
|
|
LiteLLMParamsBody(model=GPT_ON_BEDROCK_RUNTIME, aws_region_name=_BEDROCK_REGION),
|
|
RateBucket.BEDROCK_CONVERSE,
|
|
),
|
|
id="bedrock_runtime",
|
|
marks=_drives(Provider.BEDROCK, GPT_ON_BEDROCK_RUNTIME),
|
|
),
|
|
pytest.param(
|
|
Deployment(
|
|
LiteLLMParamsBody(
|
|
model=GPT_ON_AZURE_OPENAI, api_base="os.environ/AZURE_API_BASE", api_key="os.environ/AZURE_API_KEY"
|
|
),
|
|
RateBucket.AZURE_OPENAI,
|
|
),
|
|
id="azure_openai",
|
|
marks=_drives(Provider.AZURE, GPT_ON_AZURE_OPENAI),
|
|
),
|
|
pytest.param(
|
|
Deployment(LiteLLMParamsBody(model=GPT_ON_OPENAI, api_key="os.environ/OPENAI_API_KEY"), RateBucket.OPENAI),
|
|
id="openai",
|
|
marks=_drives(Provider.OPENAI, GPT_ON_OPENAI),
|
|
),
|
|
pytest.param(
|
|
Deployment(
|
|
LiteLLMParamsBody(model=CLAUDE_ON_BEDROCK, aws_region_name=_BEDROCK_REGION), RateBucket.BEDROCK_CONVERSE
|
|
),
|
|
id="claude_bedrock",
|
|
marks=_drives(Provider.BEDROCK, CLAUDE_ON_BEDROCK),
|
|
),
|
|
)
|
|
|
|
|
|
def with_expected_failure(setup_id: SetupId, *, reason: str) -> tuple[ParameterSet, ...]:
|
|
return tuple(
|
|
pytest.param(*setup.values, id=setup.id, marks=(*setup.marks, pytest.mark.xfail(strict=True, reason=reason)))
|
|
if setup.id == setup_id
|
|
else setup
|
|
for setup in SETUPS
|
|
)
|