test(e2e): sync main suite into rc 1.109.0

Copy tests/e2e from main at f91774e2f9 so RC qualification uses the canonical suite. Application code is unchanged.
This commit is contained in:
Yuneng Jiang 2026-11-02 11:14:26 -08:00
parent ff3983c1f6
commit ac2c9883aa
No known key found for this signature in database

View file

@ -15,48 +15,85 @@ pytestmark = pytest.mark.e2e
_BETA_CONFIG: Final = files("litellm").joinpath("anthropic_beta_headers_config.json")
_ANTHROPIC_MODEL: Final = "anthropic/claude-sonnet-5-5"
_VERTEX_MODEL: Final = "vertex_ai/claude-sonnet-5-5"
_VERTEX_LOCATION: Final = "global"
_AZURE_AI_MODEL: Final = "azure_ai/claude-fable-5-1"
_BEDROCK_MODEL_ID: Final = "us.anthropic.claude-fable-5-1"
_BETA_CONFIG_FILE: Final = TypeAdapter(dict[str, JsonValue])
_BETA_MAPPING: Final = TypeAdapter(dict[str, str | None])
_CODE_EXECUTION: Final[ToolUnionParam] = {"type": "code_execution_20250825", "name": "code_execution"}
# A live Vertex probe on 2026-10-31: each Claude 4.5 and 4.6 model takes one of these and Claude 5.5 takes neither
_VERTEX_PER_MODEL_BETAS: Final = frozenset({"computer-use-2025-01-24", "computer-use-2025-11-24"})
_NO_PER_MODEL_BETAS: Final[frozenset[str]] = frozenset()
def _mapped_beta_headers(provider: str) -> str:
mapping: Final = _BETA_MAPPING.validate_python(_BETA_CONFIG_FILE.validate_json(_BETA_CONFIG.read_text())[provider])
return ",".join(name for name, value in mapping.items() if value is not None)
def _mapped_beta_headers(provider: str, per_model: frozenset[str]) -> str:
config: Final = _BETA_CONFIG_FILE.validate_json(_BETA_CONFIG.read_text())
mapping: Final = _BETA_MAPPING.validate_python(config[provider])
header_only: Final = _BETA_MAPPING.validate_python(config.get(f"{provider}_header_only", {}))
forwarded: Final = {name for name, value in {**mapping, **header_only}.items() if value is not None}
assert per_model <= forwarded, per_model - forwarded
return ",".join(sorted(forwarded - per_model))
def _deployment(backend: str) -> LiteLLMParamsBody:
if backend.startswith("anthropic/"):
return LiteLLMParamsBody(model=backend, api_key="os.environ/ANTHROPIC_API_KEY")
return LiteLLMParamsBody(
model=backend,
aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID",
aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY",
aws_region_name="us-east-1",
)
match backend.split("/", 1)[0]:
case "anthropic":
return LiteLLMParamsBody(model=backend, api_key="os.environ/ANTHROPIC_API_KEY")
case "vertex_ai":
return LiteLLMParamsBody(
model=backend,
vertex_project="os.environ/VERTEXAI_PROJECT",
vertex_location=_VERTEX_LOCATION,
vertex_credentials="os.environ/VERTEXAI_CREDENTIALS",
)
case "azure_ai":
return LiteLLMParamsBody(
model=backend,
api_base="os.environ/AZURE_AI_API_BASE",
api_key="os.environ/AZURE_AI_API_KEY",
)
case _:
return LiteLLMParamsBody(
model=backend,
aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID",
aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY",
aws_region_name="us-east-1",
)
class TestAnthropicBetaHeaders:
@pytest.mark.parametrize(
("backend", "beta_provider", "tools"),
("backend", "beta_provider", "tools", "per_model"),
[
pytest.param(_ANTHROPIC_MODEL, "anthropic", [_CODE_EXECUTION], id="anthropic"),
pytest.param(f"bedrock/invoke/{_BEDROCK_MODEL_ID}", "bedrock", [], id="bedrock-invoke"),
pytest.param(f"bedrock/converse/{_BEDROCK_MODEL_ID}", "bedrock_converse", [], id="bedrock-converse"),
pytest.param(_ANTHROPIC_MODEL, "anthropic", [_CODE_EXECUTION], _NO_PER_MODEL_BETAS, id="anthropic"),
pytest.param(_VERTEX_MODEL, "vertex_ai", [], _VERTEX_PER_MODEL_BETAS, id="vertex-ai"),
pytest.param(_AZURE_AI_MODEL, "azure_ai", [], _NO_PER_MODEL_BETAS, id="azure-ai"),
pytest.param(
f"bedrock/invoke/{_BEDROCK_MODEL_ID}", "bedrock", [], _NO_PER_MODEL_BETAS, id="bedrock-invoke"
),
pytest.param(
f"bedrock/converse/{_BEDROCK_MODEL_ID}",
"bedrock_converse",
[],
_NO_PER_MODEL_BETAS,
id="bedrock-converse",
),
],
)
@pytest.mark.covers(
"llm.messages.anthropic.basic.nonstream.works",
"llm.messages.bedrock_invoke.basic.nonstream.works",
"llm.messages.bedrock_converse.basic.nonstream.works",
"llm.messages.vertex.basic.nonstream.works",
"llm.messages.azure_foundry.basic.nonstream.works",
)
@meta(
Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.MESSAGES,
providers=(Provider.ANTHROPIC, Provider.BEDROCK),
models=(_ANTHROPIC_MODEL, _BEDROCK_MODEL_ID),
providers=(Provider.ANTHROPIC, Provider.BEDROCK, Provider.VERTEX_AI, Provider.AZURE_AI),
models=(_ANTHROPIC_MODEL, _BEDROCK_MODEL_ID, _VERTEX_MODEL, _AZURE_AI_MODEL),
mode=Mode.NONSTREAM,
)
)
@ -68,6 +105,7 @@ class TestAnthropicBetaHeaders:
backend: str,
beta_provider: str,
tools: list[ToolUnionParam],
per_model: frozenset[str],
) -> None:
model: Final = f"e2e-messages-beta-{unique_marker()}"
model_id: Final = proxy.create_model(model, _deployment(backend))
@ -78,7 +116,7 @@ class TestAnthropicBetaHeaders:
max_tokens=64,
messages=[{"role": "user", "content": "Say 'hello' and nothing else"}],
tools=tools,
extra_headers={"anthropic-beta": _mapped_beta_headers(beta_provider)},
extra_headers={"anthropic-beta": _mapped_beta_headers(beta_provider, per_model)},
extra_body=NO_PROXY_CACHE,
)