mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-21 00:21:49 +00:00
test(integration): drive scripted wires and provider wiring from data
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
77f6166c39
commit
6ccba7fdb5
7 changed files with 317 additions and 158 deletions
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
These tests exercise a running gateway, PostgreSQL and Redis with an owned local upstream. CircleCI owns this suite. Tests are grouped by behavior, with no automatic test retries or fallback to paid provider calls
|
||||
|
||||
The `cost` group runs the scripted-wire cost matrix through the shared integration upstream. The upstream serves the test-owned cost map over loopback through `LITELLM_MODEL_COST_MAP_URL`; cost goldens are checked into the integration suite and must not be copied into the E2E coverage registry
|
||||
The `cost` group runs the scripted-wire cost matrix through the shared integration upstream. A provider speaking an existing response shape is a `wires.json` row, a `cases.json` `providers` row and cost-map entries; a new response shape needs a renderer in `scripted_wires.py`
|
||||
|
||||
Use `tests/integration/run.py management`, `accounting`, `database`, `providers`, `extensions`, `sdk` or `cost` to run a selected group. Set `INTEGRATION_PROXY_URL`, `INTEGRATION_UPSTREAM_URL`, `INTEGRATION_MASTER_KEY` and `DATABASE_URL` to an isolated test deployment. The runner selects the new domain directories explicitly; the legacy OCI and sandbox selections remain separate
|
||||
|
||||
|
|
|
|||
|
|
@ -34,100 +34,26 @@ import time
|
|||
import zlib
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal, TypeAlias
|
||||
from typing import Final, Literal, TypeAlias, assert_never
|
||||
from urllib.parse import unquote, urlsplit
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter, model_validator
|
||||
|
||||
Wire: TypeAlias = Literal[
|
||||
Wire: TypeAlias = str
|
||||
Shape: TypeAlias = Literal[
|
||||
"openai_chat",
|
||||
"openai_responses",
|
||||
"anthropic_messages",
|
||||
"gemini_generate",
|
||||
"together_chat",
|
||||
"fireworks_chat",
|
||||
"azure_chat",
|
||||
"bedrock_converse",
|
||||
"vertex_generate",
|
||||
]
|
||||
|
||||
WIRE_MOUNTS: Final[Mapping[str, str]] = MappingProxyType(
|
||||
{
|
||||
"openai_chat": "openai",
|
||||
"openai_responses": "openai",
|
||||
"anthropic_messages": "anthropic",
|
||||
"gemini_generate": "gemini",
|
||||
"together_chat": "together",
|
||||
"fireworks_chat": "fireworks",
|
||||
"azure_chat": "azure",
|
||||
"bedrock_converse": "bedrock",
|
||||
"vertex_generate": "vertex",
|
||||
}
|
||||
)
|
||||
|
||||
StreamUsage: TypeAlias = Literal["final_chunk", "absent"]
|
||||
ServiceTier: TypeAlias = Literal["flex", "priority"]
|
||||
TerminalKind: TypeAlias = Literal["completed", "incomplete", "unvalidated", "prompt_blocked"]
|
||||
|
||||
# Which terminal variant each wire can represent.
|
||||
_TERMINAL_CAPS: Final[Mapping[str, frozenset[str]]] = MappingProxyType(
|
||||
{
|
||||
"openai_responses": frozenset({"incomplete", "unvalidated"}),
|
||||
"gemini_generate": frozenset({"prompt_blocked"}),
|
||||
"vertex_generate": frozenset({"prompt_blocked"}),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
_BASE_USAGE_FIELDS: Final = frozenset({"fresh_input_tokens", "output_tokens"})
|
||||
_OPENAI_FAMILY_USAGE: Final = frozenset(
|
||||
{
|
||||
"cache_read_tokens",
|
||||
"reasoning_tokens",
|
||||
"audio_input_tokens",
|
||||
"audio_output_tokens",
|
||||
"web_search_calls",
|
||||
}
|
||||
)
|
||||
_CACHE_WRITE_USAGE: Final = frozenset({"cache_write_5m_tokens", "cache_write_1h_tokens"})
|
||||
_GEMINI_USAGE: Final = frozenset(
|
||||
{
|
||||
"cache_read_tokens",
|
||||
"reasoning_tokens",
|
||||
"audio_input_tokens",
|
||||
"audio_output_tokens",
|
||||
"image_input_tokens",
|
||||
"video_input_tokens",
|
||||
"web_search_calls",
|
||||
"google_maps_calls",
|
||||
}
|
||||
)
|
||||
|
||||
_USAGE_CAPS: Final[Mapping[str, frozenset[str]]] = MappingProxyType(
|
||||
{
|
||||
wire: usage
|
||||
for wire, usage in (
|
||||
("openai_chat", _OPENAI_FAMILY_USAGE),
|
||||
("azure_chat", _OPENAI_FAMILY_USAGE),
|
||||
("together_chat", _OPENAI_FAMILY_USAGE),
|
||||
("fireworks_chat", _OPENAI_FAMILY_USAGE),
|
||||
(
|
||||
"openai_responses",
|
||||
frozenset(
|
||||
{"cache_read_tokens", "reasoning_tokens", "web_search_calls", "file_search_calls"}
|
||||
),
|
||||
),
|
||||
(
|
||||
"anthropic_messages",
|
||||
frozenset({"cache_read_tokens", "web_search_calls"}) | _CACHE_WRITE_USAGE,
|
||||
),
|
||||
("bedrock_converse", frozenset({"cache_read_tokens"}) | _CACHE_WRITE_USAGE),
|
||||
("gemini_generate", _GEMINI_USAGE),
|
||||
("vertex_generate", _GEMINI_USAGE),
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class ScriptedToolCall(BaseModel):
|
||||
|
|
@ -166,6 +92,32 @@ class ScriptedUsage(BaseModel):
|
|||
file_search_calls: int = 0
|
||||
|
||||
|
||||
class WireSpec(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
shape: Shape
|
||||
mount: str
|
||||
usage: frozenset[str]
|
||||
terminals: frozenset[TerminalKind]
|
||||
|
||||
|
||||
def _load_wires() -> Mapping[str, WireSpec]:
|
||||
adapter: Final = TypeAdapter(dict[str, WireSpec])
|
||||
loaded: Final = adapter.validate_json((Path(__file__).resolve().with_name("wires.json")).read_bytes())
|
||||
known_usage_fields: Final = frozenset(ScriptedUsage.model_fields) - _BASE_USAGE_FIELDS
|
||||
unknown: Final = {
|
||||
wire: sorted(spec.usage - known_usage_fields)
|
||||
for wire, spec in loaded.items()
|
||||
if spec.usage - known_usage_fields
|
||||
}
|
||||
if unknown:
|
||||
raise ValueError(f"wires.json has unknown usage fields: {unknown}")
|
||||
return MappingProxyType(loaded)
|
||||
|
||||
|
||||
WIRES: Final[Mapping[str, WireSpec]] = _load_wires()
|
||||
|
||||
|
||||
class ScriptedOutput(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
|
|
@ -205,9 +157,14 @@ class Scenario(BaseModel):
|
|||
|
||||
@model_validator(mode="after")
|
||||
def _check_terminal_supported(self) -> Scenario:
|
||||
spec: Final = WIRES.get(self.wire)
|
||||
if spec is None:
|
||||
raise ValueError(
|
||||
f"unknown wire {self.wire}; known wires: {', '.join(sorted(WIRES))}"
|
||||
)
|
||||
if (
|
||||
self.output.terminal != "completed"
|
||||
and self.output.terminal not in _TERMINAL_CAPS.get(self.wire, frozenset())
|
||||
and self.output.terminal not in spec.terminals
|
||||
):
|
||||
raise ValueError(
|
||||
f"wire {self.wire} cannot emit terminal={self.output.terminal}"
|
||||
|
|
@ -216,7 +173,7 @@ class Scenario(BaseModel):
|
|||
field
|
||||
for field in self.usage.model_fields_set
|
||||
if getattr(self.usage, field)
|
||||
and field not in (_USAGE_CAPS.get(self.wire, frozenset()) | _BASE_USAGE_FIELDS)
|
||||
and field not in (spec.usage | _BASE_USAGE_FIELDS)
|
||||
)
|
||||
if unsupported:
|
||||
raise ValueError(
|
||||
|
|
@ -230,7 +187,7 @@ class Scenario(BaseModel):
|
|||
|
||||
@property
|
||||
def mount(self) -> str:
|
||||
return WIRE_MOUNTS[self.wire]
|
||||
return WIRES[self.wire].mount
|
||||
|
||||
|
||||
class ScenarioRegistered(BaseModel):
|
||||
|
|
@ -1266,33 +1223,32 @@ def _render(
|
|||
return RenderedResponse(
|
||||
200, "application/json", _json_bytes(_responses_body(scenario, requested_model))
|
||||
)
|
||||
if scenario.wire == "bedrock_converse":
|
||||
if stream:
|
||||
return RenderedResponse(
|
||||
200, "application/vnd.amazon.eventstream", _bedrock_eventstream(scenario)
|
||||
)
|
||||
return RenderedResponse(200, "application/json", _json_bytes(_bedrock_body(scenario)))
|
||||
if scenario.wire == "vertex_generate":
|
||||
if stream:
|
||||
return RenderedResponse(200, "text/event-stream", _gemini_sse(scenario, requested_model))
|
||||
return RenderedResponse(200, "application/json", _json_bytes(_gemini_body(scenario, requested_model)))
|
||||
if scenario.wire == "anthropic_messages":
|
||||
if stream:
|
||||
return RenderedResponse(200, "text/event-stream", _anthropic_sse(scenario, requested_model))
|
||||
return RenderedResponse(200, "application/json", _json_bytes(_anthropic_body(scenario, requested_model)))
|
||||
if scenario.wire == "gemini_generate":
|
||||
if stream:
|
||||
return RenderedResponse(200, "text/event-stream", _gemini_sse(scenario, requested_model))
|
||||
return RenderedResponse(200, "application/json", _json_bytes(_gemini_body(scenario, requested_model)))
|
||||
if scenario.wire == "openai_responses":
|
||||
if stream:
|
||||
return RenderedResponse(200, "text/event-stream", _responses_sse(scenario, requested_model))
|
||||
return RenderedResponse(200, "application/json", _json_bytes(_responses_body(scenario, requested_model)))
|
||||
# openai_chat, together_chat, fireworks_chat and azure_chat share the
|
||||
# OpenAI chat shape.
|
||||
if stream:
|
||||
return RenderedResponse(200, "text/event-stream", _openai_chat_sse(scenario, requested_model))
|
||||
return RenderedResponse(200, "application/json", _json_bytes(_openai_chat_body(scenario, requested_model)))
|
||||
shape: Final = WIRES[scenario.wire].shape
|
||||
match shape:
|
||||
case "bedrock_converse":
|
||||
if stream:
|
||||
return RenderedResponse(
|
||||
200, "application/vnd.amazon.eventstream", _bedrock_eventstream(scenario)
|
||||
)
|
||||
return RenderedResponse(200, "application/json", _json_bytes(_bedrock_body(scenario)))
|
||||
case "gemini_generate":
|
||||
if stream:
|
||||
return RenderedResponse(200, "text/event-stream", _gemini_sse(scenario, requested_model))
|
||||
return RenderedResponse(200, "application/json", _json_bytes(_gemini_body(scenario, requested_model)))
|
||||
case "anthropic_messages":
|
||||
if stream:
|
||||
return RenderedResponse(200, "text/event-stream", _anthropic_sse(scenario, requested_model))
|
||||
return RenderedResponse(200, "application/json", _json_bytes(_anthropic_body(scenario, requested_model)))
|
||||
case "openai_responses":
|
||||
if stream:
|
||||
return RenderedResponse(200, "text/event-stream", _responses_sse(scenario, requested_model))
|
||||
return RenderedResponse(200, "application/json", _json_bytes(_responses_body(scenario, requested_model)))
|
||||
case "openai_chat":
|
||||
if stream:
|
||||
return RenderedResponse(200, "text/event-stream", _openai_chat_sse(scenario, requested_model))
|
||||
return RenderedResponse(200, "application/json", _json_bytes(_openai_chat_body(scenario, requested_model)))
|
||||
case _:
|
||||
assert_never(shape)
|
||||
|
||||
|
||||
# ---------- registry + request routing ----------
|
||||
|
|
|
|||
|
|
@ -19,12 +19,12 @@ from starlette.routing import Route
|
|||
|
||||
from _fake_openai_endpoint_server import chat_completions, completions, embeddings, health, moderations
|
||||
from integration._support.scripted_wires import (
|
||||
WIRE_MOUNTS,
|
||||
RenderedResponse,
|
||||
Scenario,
|
||||
ScenarioDeleted,
|
||||
ScenarioRegistered,
|
||||
ScenarioStore,
|
||||
WIRES,
|
||||
Wire,
|
||||
render,
|
||||
)
|
||||
|
|
@ -218,7 +218,7 @@ class ScenarioHandle:
|
|||
return f"{self.control_url}/{self.scenario_id}/{self._mount()}"
|
||||
|
||||
def _mount(self) -> str:
|
||||
return WIRE_MOUNTS[self.wire]
|
||||
return WIRES[self.wire].mount
|
||||
|
||||
|
||||
def register_scenario(scenario: Scenario) -> ScenarioHandle:
|
||||
|
|
|
|||
119
tests/integration/_support/wires.json
Normal file
119
tests/integration/_support/wires.json
Normal file
|
|
@ -0,0 +1,119 @@
|
|||
{
|
||||
"openai_chat": {
|
||||
"shape": "openai_chat",
|
||||
"mount": "openai",
|
||||
"usage": [
|
||||
"cache_read_tokens",
|
||||
"reasoning_tokens",
|
||||
"audio_input_tokens",
|
||||
"audio_output_tokens",
|
||||
"web_search_calls"
|
||||
],
|
||||
"terminals": []
|
||||
},
|
||||
"openai_responses": {
|
||||
"shape": "openai_responses",
|
||||
"mount": "openai",
|
||||
"usage": [
|
||||
"cache_read_tokens",
|
||||
"reasoning_tokens",
|
||||
"web_search_calls",
|
||||
"file_search_calls"
|
||||
],
|
||||
"terminals": [
|
||||
"incomplete",
|
||||
"unvalidated"
|
||||
]
|
||||
},
|
||||
"anthropic_messages": {
|
||||
"shape": "anthropic_messages",
|
||||
"mount": "anthropic",
|
||||
"usage": [
|
||||
"cache_read_tokens",
|
||||
"web_search_calls",
|
||||
"cache_write_5m_tokens",
|
||||
"cache_write_1h_tokens"
|
||||
],
|
||||
"terminals": []
|
||||
},
|
||||
"gemini_generate": {
|
||||
"shape": "gemini_generate",
|
||||
"mount": "gemini",
|
||||
"usage": [
|
||||
"cache_read_tokens",
|
||||
"reasoning_tokens",
|
||||
"audio_input_tokens",
|
||||
"audio_output_tokens",
|
||||
"image_input_tokens",
|
||||
"video_input_tokens",
|
||||
"web_search_calls",
|
||||
"google_maps_calls"
|
||||
],
|
||||
"terminals": [
|
||||
"prompt_blocked"
|
||||
]
|
||||
},
|
||||
"together_chat": {
|
||||
"shape": "openai_chat",
|
||||
"mount": "together",
|
||||
"usage": [
|
||||
"cache_read_tokens",
|
||||
"reasoning_tokens",
|
||||
"audio_input_tokens",
|
||||
"audio_output_tokens",
|
||||
"web_search_calls"
|
||||
],
|
||||
"terminals": []
|
||||
},
|
||||
"fireworks_chat": {
|
||||
"shape": "openai_chat",
|
||||
"mount": "fireworks",
|
||||
"usage": [
|
||||
"cache_read_tokens",
|
||||
"reasoning_tokens",
|
||||
"audio_input_tokens",
|
||||
"audio_output_tokens",
|
||||
"web_search_calls"
|
||||
],
|
||||
"terminals": []
|
||||
},
|
||||
"azure_chat": {
|
||||
"shape": "openai_chat",
|
||||
"mount": "azure",
|
||||
"usage": [
|
||||
"cache_read_tokens",
|
||||
"reasoning_tokens",
|
||||
"audio_input_tokens",
|
||||
"audio_output_tokens",
|
||||
"web_search_calls"
|
||||
],
|
||||
"terminals": []
|
||||
},
|
||||
"bedrock_converse": {
|
||||
"shape": "bedrock_converse",
|
||||
"mount": "bedrock",
|
||||
"usage": [
|
||||
"cache_read_tokens",
|
||||
"cache_write_5m_tokens",
|
||||
"cache_write_1h_tokens"
|
||||
],
|
||||
"terminals": []
|
||||
},
|
||||
"vertex_generate": {
|
||||
"shape": "gemini_generate",
|
||||
"mount": "vertex",
|
||||
"usage": [
|
||||
"cache_read_tokens",
|
||||
"reasoning_tokens",
|
||||
"audio_input_tokens",
|
||||
"audio_output_tokens",
|
||||
"image_input_tokens",
|
||||
"video_input_tokens",
|
||||
"web_search_calls",
|
||||
"google_maps_calls"
|
||||
],
|
||||
"terminals": [
|
||||
"prompt_blocked"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
@ -1,4 +1,78 @@
|
|||
{
|
||||
"providers": [
|
||||
{
|
||||
"litellm_provider": "openai",
|
||||
"mode": "chat",
|
||||
"wire": "openai_chat",
|
||||
"model_prefix": "openai",
|
||||
"litellm_params": {}
|
||||
},
|
||||
{
|
||||
"litellm_provider": "openai",
|
||||
"mode": "responses",
|
||||
"wire": "openai_responses",
|
||||
"model_prefix": "openai/responses",
|
||||
"litellm_params": {}
|
||||
},
|
||||
{
|
||||
"litellm_provider": "anthropic",
|
||||
"mode": "chat",
|
||||
"wire": "anthropic_messages",
|
||||
"model_prefix": "anthropic",
|
||||
"litellm_params": {}
|
||||
},
|
||||
{
|
||||
"litellm_provider": "gemini",
|
||||
"mode": "chat",
|
||||
"wire": "gemini_generate",
|
||||
"model_prefix": null,
|
||||
"litellm_params": {}
|
||||
},
|
||||
{
|
||||
"litellm_provider": "together_ai",
|
||||
"mode": "chat",
|
||||
"wire": "together_chat",
|
||||
"model_prefix": null,
|
||||
"litellm_params": {}
|
||||
},
|
||||
{
|
||||
"litellm_provider": "fireworks_ai",
|
||||
"mode": "chat",
|
||||
"wire": "fireworks_chat",
|
||||
"model_prefix": null,
|
||||
"litellm_params": {}
|
||||
},
|
||||
{
|
||||
"litellm_provider": "azure",
|
||||
"mode": "chat",
|
||||
"wire": "azure_chat",
|
||||
"model_prefix": null,
|
||||
"litellm_params": {
|
||||
"api_version": "2025-04-01-preview"
|
||||
}
|
||||
},
|
||||
{
|
||||
"litellm_provider": "bedrock_converse",
|
||||
"mode": "chat",
|
||||
"wire": "bedrock_converse",
|
||||
"model_prefix": "bedrock/converse",
|
||||
"litellm_params": {
|
||||
"aws_access_key_id": "AKIASCRIPTEDPROVIDER",
|
||||
"aws_secret_access_key": "scripted-secret",
|
||||
"aws_region_name": "us-east-1"
|
||||
}
|
||||
},
|
||||
{
|
||||
"litellm_provider": "vertex_ai-language-models",
|
||||
"mode": "chat",
|
||||
"wire": "vertex_generate",
|
||||
"model_prefix": "vertex_ai",
|
||||
"litellm_params": {
|
||||
"vertex_project": "cc-scripted-project",
|
||||
"vertex_location": "us-central1"
|
||||
}
|
||||
}
|
||||
],
|
||||
"deployments": [
|
||||
{
|
||||
"map_key": "azure/gpt-5.4-mini",
|
||||
|
|
|
|||
|
|
@ -27,7 +27,14 @@ from types import MappingProxyType
|
|||
from typing import Final, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter
|
||||
from integration._support.scripted_wires import Scenario, ScriptedOutput, ScriptedToolCall, ScriptedUsage, Wire
|
||||
from integration._support.scripted_wires import (
|
||||
WIRES,
|
||||
Scenario,
|
||||
ScriptedOutput,
|
||||
ScriptedToolCall,
|
||||
ScriptedUsage,
|
||||
Wire,
|
||||
)
|
||||
|
||||
COST_MAP_PATH: Final = Path(__file__).resolve().parent / "cost_map.json"
|
||||
CASES_PATH: Final = Path(__file__).resolve().parent / "cases.json"
|
||||
|
|
@ -251,9 +258,20 @@ class Case(BaseModel):
|
|||
)
|
||||
|
||||
|
||||
class _ProviderWiringRow(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
litellm_provider: str
|
||||
mode: str
|
||||
wire: str
|
||||
model_prefix: str | None
|
||||
litellm_params: Mapping[str, str]
|
||||
|
||||
|
||||
class _CasesFile(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
providers: tuple[_ProviderWiringRow, ...] = ()
|
||||
deployments: tuple[DeploymentSpec, ...] = ()
|
||||
cases: tuple[Case, ...] = ()
|
||||
|
||||
|
|
@ -267,7 +285,7 @@ _DEPLOYMENTS: Final[Mapping[str, DeploymentSpec]] = MappingProxyType(
|
|||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _ProviderWiring:
|
||||
"""How a (litellm_provider, mode) pair maps to a sidecar wire, the provider
|
||||
"""How a (litellm_provider, mode) pair maps to a provider wire, the provider
|
||||
prefix on the registered litellm model string, and extra litellm_params."""
|
||||
|
||||
wire: Wire
|
||||
|
|
@ -275,42 +293,26 @@ class _ProviderWiring:
|
|||
litellm_params: Mapping[str, str]
|
||||
|
||||
|
||||
_AZURE_PARAMS: Final[Mapping[str, str]] = MappingProxyType({"api_version": "2025-04-01-preview"})
|
||||
_BEDROCK_PARAMS: Final[Mapping[str, str]] = MappingProxyType(
|
||||
{
|
||||
"aws_access_key_id": "AKIASCRIPTEDPROVIDER",
|
||||
"aws_secret_access_key": "scripted-secret",
|
||||
"aws_region_name": "us-east-1",
|
||||
}
|
||||
)
|
||||
_VERTEX_PARAMS: Final[Mapping[str, str]] = MappingProxyType(
|
||||
{
|
||||
"vertex_project": "cc-scripted-project",
|
||||
"vertex_location": "us-central1",
|
||||
}
|
||||
)
|
||||
def _provider_wiring(rows: tuple[_ProviderWiringRow, ...]) -> Mapping[tuple[str, str], _ProviderWiring]:
|
||||
unknown_wires: Final = sorted({row.wire for row in rows if row.wire not in WIRES})
|
||||
if unknown_wires:
|
||||
raise ValueError(
|
||||
f"cases.json providers has unknown wires: {unknown_wires}; "
|
||||
f"known wires are {sorted(WIRES)}"
|
||||
)
|
||||
return MappingProxyType(
|
||||
{
|
||||
(row.litellm_provider, row.mode): _ProviderWiring(
|
||||
row.wire,
|
||||
row.model_prefix,
|
||||
MappingProxyType(dict(row.litellm_params)),
|
||||
)
|
||||
for row in rows
|
||||
}
|
||||
)
|
||||
|
||||
_PROVIDER_WIRING: Final[Mapping[tuple[str, str], _ProviderWiring]] = MappingProxyType(
|
||||
{
|
||||
("openai", "chat"): _ProviderWiring("openai_chat", "openai", MappingProxyType({})),
|
||||
("openai", "responses"): _ProviderWiring(
|
||||
"openai_responses", "openai/responses", MappingProxyType({})
|
||||
),
|
||||
("anthropic", "chat"): _ProviderWiring(
|
||||
"anthropic_messages", "anthropic", MappingProxyType({})
|
||||
),
|
||||
("gemini", "chat"): _ProviderWiring("gemini_generate", None, MappingProxyType({})),
|
||||
("together_ai", "chat"): _ProviderWiring("together_chat", None, MappingProxyType({})),
|
||||
("fireworks_ai", "chat"): _ProviderWiring("fireworks_chat", None, MappingProxyType({})),
|
||||
("azure", "chat"): _ProviderWiring("azure_chat", None, _AZURE_PARAMS),
|
||||
("bedrock_converse", "chat"): _ProviderWiring(
|
||||
"bedrock_converse", "bedrock/converse", _BEDROCK_PARAMS
|
||||
),
|
||||
("vertex_ai-language-models", "chat"): _ProviderWiring(
|
||||
"vertex_generate", "vertex_ai", _VERTEX_PARAMS
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
_PROVIDER_WIRING: Final[Mapping[tuple[str, str], _ProviderWiring]] = _provider_wiring(CASES_FILE.providers)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -390,11 +392,7 @@ def _frontier() -> tuple[FrontierModel, ...]:
|
|||
pair = (entry.litellm_provider, entry.mode)
|
||||
wiring = _PROVIDER_WIRING.get(pair)
|
||||
if wiring is None:
|
||||
raise ValueError(
|
||||
f"cost_map entry {map_key} has no wiring for "
|
||||
f"(litellm_provider={pair[0]}, mode={pair[1]}); add a "
|
||||
f"_ProviderWiring row in cost_matrix.py"
|
||||
)
|
||||
continue
|
||||
siblings = groups[pair]
|
||||
override_key = (
|
||||
siblings[(siblings.index(map_key) + 1) % len(siblings)] if len(siblings) > 1 else None
|
||||
|
|
@ -567,6 +565,13 @@ def matrix_data_errors() -> tuple[str, ...]:
|
|||
for case in CASES
|
||||
if (case.family == "transport") != (not case.owns and not case.fallback_for)
|
||||
)
|
||||
missing_provider_rows: Final = sorted(
|
||||
f"cost_map entry {map_key} has no providers row for "
|
||||
f"(litellm_provider={entry.litellm_provider}, mode={entry.mode}); "
|
||||
f"add a providers row in cases.json"
|
||||
for map_key, entry in COST_MAP.items()
|
||||
if (entry.litellm_provider, entry.mode) not in _PROVIDER_WIRING
|
||||
)
|
||||
input_rates: Final = tuple(entry.input_cost_per_token for entry in COST_MAP.values())
|
||||
findings: Final = (
|
||||
(
|
||||
|
|
@ -615,5 +620,10 @@ def matrix_data_errors() -> tuple[str, ...]:
|
|||
if family_violations
|
||||
else None
|
||||
),
|
||||
(
|
||||
f"cost_map entries without providers rows: {missing_provider_rows}"
|
||||
if missing_provider_rows
|
||||
else None
|
||||
),
|
||||
)
|
||||
return tuple(finding for finding in findings if finding is not None)
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import pytest
|
|||
from pydantic import JsonValue
|
||||
|
||||
from integration._support.client import JSON_OBJECT, Gateway
|
||||
from integration._support.scripted_wires import ScriptedUsage, Wire
|
||||
from integration._support.scripted_wires import WIRES, ScriptedUsage, Wire
|
||||
from integration.cost_calculation.conftest import (
|
||||
approx_equal,
|
||||
assert_total_is_sum_of_components,
|
||||
|
|
@ -50,12 +50,12 @@ _MATRIX: Final = tuple(
|
|||
for model in FRONTIER_MODELS
|
||||
for case in cases_for(model)
|
||||
)
|
||||
_CACHE_WIRES: Final = frozenset({"anthropic_messages", "bedrock_converse"})
|
||||
_WEB_SEARCH_OPTION_WIRES: Final = frozenset({"openai_chat", "azure_chat", "openai_responses"})
|
||||
_CACHE_SHAPES: Final = frozenset({"anthropic_messages", "bedrock_converse"})
|
||||
_WEB_SEARCH_OPTION_SHAPES: Final = frozenset({"openai_chat", "openai_responses"})
|
||||
|
||||
|
||||
def _cache_control(usage: ScriptedUsage, wire: Wire) -> dict[str, JsonValue] | None:
|
||||
if wire not in _CACHE_WIRES:
|
||||
if WIRES[wire].shape not in _CACHE_SHAPES:
|
||||
return None
|
||||
if not (usage.cache_read_tokens or usage.cache_write_5m_tokens or usage.cache_write_1h_tokens):
|
||||
return None
|
||||
|
|
@ -148,7 +148,7 @@ def _chat_body(model: FrontierModel, case: Case, model_name: str, marker: str) -
|
|||
**({"audio": {"voice": "alloy", "format": "pcm16"}} if case.audio_output else {}),
|
||||
**(
|
||||
{"web_search_options": {"search_context_size": case.web_search}}
|
||||
if case.web_search is not None and model.wire in _WEB_SEARCH_OPTION_WIRES
|
||||
if case.web_search is not None and WIRES[model.wire].shape in _WEB_SEARCH_OPTION_SHAPES
|
||||
else {}
|
||||
),
|
||||
**({"tools": tools} if tools else {}),
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue