From 0f3c4ccfbba894c0a66d92ed44ca18f312bf75f0 Mon Sep 17 00:00:00 2001 From: jesus Date: Fri, 11 Sep 2026 00:00:17 +0000 Subject: [PATCH 001/210] fix(auth): inherit organization_alias from the org for JWT and team-linked keys Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/auth/user_api_key_auth.py | 42 ++++++- .../proxy/auth/test_user_api_key_auth.py | 104 +++++++++++++++++- 2 files changed, 143 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 20ab9904f46..110c524ecdf 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -54,6 +54,7 @@ from litellm.proxy.auth.auth_checks import ( get_end_user_object, get_jwt_key_mapping_object, get_object_permission, + get_org_object, get_project_object, get_team_object, get_user_object, @@ -2398,6 +2399,37 @@ def _token_can_vouch_for_team(valid_token: UserAPIKeyAuth, lookup_error: BaseExc return PrismaDBExceptionHandler.should_allow_request_on_db_unavailable() +async def _inherit_org_identity( + user_api_key_auth_obj: UserAPIKeyAuth, + team_object: LiteLLM_TeamTableCachedObj | None, + prisma_client: PrismaClient | None, + user_api_key_cache: UserApiKeyCache, + parent_otel_span: Span | None, + proxy_logging_obj: ProxyLogging | None, +) -> None: + if user_api_key_auth_obj.org_id is None and team_object is not None and team_object.organization_id is not None: + user_api_key_auth_obj.org_id = team_object.organization_id + if ( + user_api_key_auth_obj.org_id is None + or user_api_key_auth_obj.organization_alias is not None + or prisma_client is None + ): + return + try: + org_object: Final = await get_org_object( + org_id=user_api_key_auth_obj.org_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + ) + except Exception: + verbose_proxy_logger.debug("org alias lookup failed for org_id=%s", user_api_key_auth_obj.org_id, exc_info=True) + return + if org_object is not None: + user_api_key_auth_obj.organization_alias = org_object.organization_alias + + @tracer.wrap() async def _run_centralized_common_checks( user_api_key_auth_obj: UserAPIKeyAuth, @@ -2622,8 +2654,14 @@ async def _run_centralized_common_checks( ) global_proxy_spend: float | None = None if isinstance(global_spend_result, BaseException) else global_spend_result - if user_api_key_auth_obj.org_id is None and team_object is not None and team_object.organization_id is not None: - user_api_key_auth_obj.org_id = team_object.organization_id + await _inherit_org_identity( + user_api_key_auth_obj=user_api_key_auth_obj, + team_object=cast(LiteLLM_TeamTableCachedObj | None, team_object), + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + ) # common_checks identifies admin via user_object, not the token # (non_proxy_admin_allowed_routes_check). JWT admin shortcut and diff --git a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py index 6cce6d0316b..03efbfa7185 100644 --- a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py +++ b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py @@ -23,6 +23,7 @@ from litellm.proxy._types import ( LiteLLM_JWTAuth, LiteLLM_BudgetTable, LiteLLM_EndUserTable, + LiteLLM_OrganizationTable, LiteLLM_UserTable, LitellmUserRoles, ProxyErrorTypes, @@ -31,7 +32,7 @@ from litellm.proxy._types import ( JWTRoutingOverride, ) from litellm.proxy.auth.handle_jwt import JWTHandler -from litellm.proxy.auth.auth_checks import get_key_object, _cache_key_object +from litellm.proxy.auth.auth_checks import OrganizationNotFoundError, get_key_object, _cache_key_object from litellm.proxy.auth.route_checks import RouteChecks from litellm.proxy.auth.user_api_key_auth import ( _check_key_model_budget_with_fallback, @@ -5293,6 +5294,107 @@ async def test_centralized_common_checks_backfills_org_id_from_team(key_org_id, setattr(_proxy_server_mod, k, v) +@pytest.mark.asyncio +@pytest.mark.parametrize( + "key_org_id,team_id,team_org_id,existing_alias,lookup_mode,expected_org_id,expected_alias", + [ + (None, "t1", "org-from-team", None, "success", "org-from-team", "acme-org"), + ("org-jwt", None, None, None, "success", "org-jwt", "acme-org"), + ("org-pinned", None, None, "preset", "success", "org-pinned", "preset"), + ("org-missing", None, None, None, "missing", "org-missing", None), + ], +) +async def test_centralized_common_checks_inherits_org_alias( + key_org_id, + team_id, + team_org_id, + existing_alias, + lookup_mode, + expected_org_id, + expected_alias, +): + import litellm.proxy.proxy_server as _proxy_server_mod + from fastapi import Request + from starlette.datastructures import URL + + from litellm.proxy._types import LiteLLM_TeamTableCachedObj + + token = UserAPIKeyAuth( + api_key="sk-test", + user_id="u", + team_id=team_id, + org_id=key_org_id, + organization_alias=existing_alias, + ) + request = Request(scope={"type": "http"}) + request._url = URL(url="/chat/completions") + + fetched_team = ( + LiteLLM_TeamTableCachedObj(team_id="t1", organization_id=team_org_id) if team_id is not None else None + ) + organization = LiteLLM_OrganizationTable( + organization_id=expected_org_id, + organization_alias="acme-org", + budget_id="budget-id", + models=[], + created_by="test", + updated_by="test", + ) + + attrs = _proxy_attrs_for_centralized_checks(user_custom_auth=None) + attrs["prisma_client"] = MagicMock() + originals = {a: getattr(_proxy_server_mod, a, None) for a in attrs} + try: + for k, v in attrs.items(): + setattr(_proxy_server_mod, k, v) + identity_seen_by_common_checks = [] + with ( + patch( + "litellm.proxy.auth.user_api_key_auth.get_team_object", + new_callable=AsyncMock, + return_value=fetched_team, + ) as mock_get_team_object, + patch( + "litellm.proxy.auth.user_api_key_auth.get_org_object", + new_callable=AsyncMock, + return_value=organization, + ) as mock_get_org_object, + patch( + "litellm.proxy.auth.user_api_key_auth.common_checks", + new_callable=AsyncMock, + side_effect=lambda **kw: identity_seen_by_common_checks.append( + (kw["valid_token"].org_id, kw["valid_token"].organization_alias) + ), + ) as mock_checks, + ): + if lookup_mode == "missing": + mock_get_org_object.side_effect = OrganizationNotFoundError("x") + + await _run_centralized_common_checks( + user_api_key_auth_obj=token, + request=request, + request_data={"model": "gpt-4o"}, + route="/chat/completions", + ) + + mock_checks.assert_awaited_once() + assert token.org_id == expected_org_id + assert token.organization_alias == expected_alias + assert identity_seen_by_common_checks == [(expected_org_id, expected_alias)] + if team_id is None: + mock_get_team_object.assert_not_awaited() + else: + mock_get_team_object.assert_awaited_once() + if existing_alias is not None: + mock_get_org_object.assert_not_awaited() + else: + mock_get_org_object.assert_awaited_once() + assert mock_get_org_object.await_args.kwargs["org_id"] == expected_org_id + finally: + for k, v in originals.items(): + setattr(_proxy_server_mod, k, v) + + @pytest.mark.asyncio async def test_cli_session_token_org_backfilled_from_team(monkeypatch): """LIT-4688 root cause: CLI session tokens (from /sso/cli/poll) are minted From 61ac4f57394e022382459235baa598ac89ce00d1 Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 15 Sep 2026 23:03:56 +0000 Subject: [PATCH 002/210] test(e2e): add scripted-provider cost calculation suite Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/CLAUDE.md | 3 +- tests/e2e/conftest.py | 9 +- tests/e2e/cost_calculation/conftest.py | 139 ++++ tests/e2e/cost_calculation/cost_matrix.py | 458 +++++++++++++ tests/e2e/cost_calculation/scripted_client.py | 70 ++ .../e2e/cost_calculation/scripted_provider.py | 631 ++++++++++++++++++ .../test_token_pricing_e2e.py | 115 ++++ .../cost_calculation/test_wire_formats_e2e.py | 186 ++++++ tests/e2e/cost_map.json | 352 ++++++++++ .../coverage_registry/quota_management.yaml | 2 + tests/e2e/e2e_config.py | 16 + tests/e2e/pytest.ini | 1 + 12 files changed, 1980 insertions(+), 2 deletions(-) create mode 100644 tests/e2e/cost_calculation/conftest.py create mode 100644 tests/e2e/cost_calculation/cost_matrix.py create mode 100644 tests/e2e/cost_calculation/scripted_client.py create mode 100644 tests/e2e/cost_calculation/scripted_provider.py create mode 100644 tests/e2e/cost_calculation/test_token_pricing_e2e.py create mode 100644 tests/e2e/cost_calculation/test_wire_formats_e2e.py create mode 100644 tests/e2e/cost_map.json diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index 0541ce25d4b..b6c3840f626 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -21,6 +21,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments split round robin over `/chat/completions` and `/v1/messages`, one endpoint per simulated user, with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests on every endpoint, budgeting RSS and CPU-per-request as ratios against the same run's healthy phase, and holding p50/p90/p99 latency and log-bytes-per-request to flat ceilings (a ratio cannot bound those two: an open breaker skips Redis instead of waiting on it, so the chaos phase can measure cheaper than baseline while still being far slower than a user should see); needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` and `E2E_PROXY_LOG` set, marked `redis_chaos`, deselected unless `E2E_REDIS_CHAOS` is set and excluded from the per-PR selector like the rest of `load/`, driven by `.github/workflows/test-e2e-redis-chaos.yml` and by the Buildkite `e2e-redis-chaos` step in project-releaser, which runs the proxy, Postgres and Valkey co-located with pytest in one pod and sets the opt-in), and markerless harness unit tests for the locust, process-usage, and session-anomaly aggregation logic - `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate, JWT auth (access tokens issued by a real Keycloak realm, `idp.py` plus `idp_realm.json`, whose JWKS the proxy's `JWT_PUBLIC_KEY_URL` points at; see CONTRIBUTING.md for the start command and config block), and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite - `gateway/` - proxy configuration only (`litellm-config.yml`); no tests +- `cost_calculation/` - cost accounting against a dedicated proxy whose whole model cost map is the test-owned `tests/e2e/cost_map.json` (loaded via `LITELLM_MODEL_COST_MAP_URL`), with provider calls answered by the scripted-provider sidecar in `scripted_provider.py`; asserts literal rate arithmetic on scripted usage across every provider wire and pricing component, deselected unless `E2E_COST_MAP_STACK` is set - `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture; ships its own driver/builder/publisher plus `_*_unit_tests/` trees. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`); the CLI-driving path stays bespoke - `ui/` - the Admin UI browser suite: Playwright in TypeScript, driving the dashboard served by a live proxy on port 4000 (seeded postgres + mock LLM upstream; see its `run_e2e.sh`). It is a self-contained npm package with its own lockfile and does not use the Python harness, pytest markers, or the shared transport; the Python rules in this file (typed models, `Result` unions, basedpyright zero-error gate) do not apply inside it. Its only Python file, `fixtures/mock_llm_server/server.py`, is excluded from the e2e basedpyright gate via the root `pyrightconfig.json` @@ -221,7 +222,7 @@ other... ``` ## Hard Rules -- no unit tests of any kind under `tests/e2e`. a product feature is proven end to end against a live proxy, never with a unit test, and the harness itself is not unit-tested here either. no monkeypatching or mock tests. if a contributor asks you to write an end to end test, do NOT stage a unit test with it; if you find a product gap, call it out in the PR description +- no unit tests of any kind under `tests/e2e`. a product feature is proven end to end against a live proxy, never with a unit test, and the harness itself is not unit-tested here either. no monkeypatching or mock tests; the one carve-out is a scripted upstream served through a real HTTP sidecar (the cost_calculation suite's scripted provider), allowed because provider-response-shape coverage needs a controlled usage payload and every hop from the proxy's upstream call to the spend row still executes for real. if a contributor asks you to write an end to end test, do NOT stage a unit test with it; if you find a product gap, call it out in the PR description - use model management endpoints to create new models for a test. this could be in a conftest / inline for each test. ask the user what they want. diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index b1a75d5f862..7ab41b8ff68 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -22,9 +22,9 @@ from typing import Final import pytest import requests - from e2e_config import ( CONTROL_PLANE_BASE_URL, + COST_MAP_OPT_IN_ENV, FIXTURE_DIR, FIXTURE_MODE_RAW, MANAGED_FILES_OPT_IN_ENV, @@ -53,6 +53,7 @@ OPT_IN_MARKERS: Final = MappingProxyType( "managed_files": MANAGED_FILES_OPT_IN_ENV, "prompt_caching_stack": PROMPT_CACHING_OPT_IN_ENV, "redis_chaos": REDIS_CHAOS_OPT_IN_ENV, + "cost_map_stack": COST_MAP_OPT_IN_ENV, } ) @@ -120,6 +121,12 @@ def pytest_configure(config: pytest.Config) -> None: "redis_chaos: load test that pauses the proxy's Redis outright mid-run; needs a proxy booted from " "gateway/redis_chaos_ci_config.yml on the same host, and is deselected unless E2E_REDIS_CHAOS is set", ) + config.addinivalue_line( + "markers", + "cost_map_stack: needs a proxy whose whole cost map is tests/e2e/cost_map.json " + "(LITELLM_MODEL_COST_MAP_URL) plus a scripted-provider sidecar; deselected unless " + "E2E_COST_MAP_STACK is set", + ) def pytest_sessionstart(session: pytest.Session) -> None: diff --git a/tests/e2e/cost_calculation/conftest.py b/tests/e2e/cost_calculation/conftest.py new file mode 100644 index 00000000000..1bba3d50e1d --- /dev/null +++ b/tests/e2e/cost_calculation/conftest.py @@ -0,0 +1,139 @@ +"""Cost-calculation suite fixtures. + +Runs against a dedicated proxy whose whole model cost map is the test-owned +``tests/e2e/cost_map.json`` (LITELLM_MODEL_COST_MAP_URL), so every deployment +bills at rates the test asserts literal arithmetic on. Provider calls are +answered by the scripted-provider sidecar (``scripted_provider.py``), registered +per scenario over its control API. + +Deselected unless E2E_COST_MAP_STACK is set (marker `cost_map_stack`). +""" + +from __future__ import annotations + +import importlib.util +import sys +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path +from types import ModuleType +from typing import Final, Protocol, cast + +import pytest + +from cost_matrix import Case, FrontierModel +from e2e_config import COST_MAP_PROXY_URL +from lifecycle import ResourceManager +from models import LiteLLMParamsBody, ModelInfoBody, ModelNewBody +from proxy_client import ProxyClient, build_proxy_client +from scripted_client import ScenarioHandle, delete_scenario, register_scenario +from scripted_provider import Scenario + + +def _load_cost_rows() -> ModuleType: + """Load quota_management/spend_tracking/cost_rows.py by path (the e2e tree + has no package layout), the same trick the mcp suite uses for + logging/datadog_reader.py.""" + path = ( + Path(__file__).resolve().parent.parent + / "quota_management" + / "spend_tracking" + / "cost_rows.py" + ) + name = "e2e_spend_tracking_cost_rows" + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +class SpendCostBreakdown(Protocol): + input_cost: float | None + output_cost: float | None + cache_read_cost: float | None + cache_creation_cost: float | None + reasoning_cost: float | None + tool_usage_cost: float | None + total_cost: float | None + service_tier: str | None + + def model_dump(self) -> dict[str, object]: ... + + +class SpendRowMetadata(Protocol): + cost_breakdown: SpendCostBreakdown | None + + +class SpendCostRow(Protocol): + """The slice of spend_tracking.cost_rows.CostRow this suite reads.""" + + spend: float | None + prompt_tokens: int | None + completion_tokens: int | None + metadata: SpendRowMetadata | None + + @property + def breakdown(self) -> SpendCostBreakdown: ... + + +class CostRowsModule(Protocol): + """cost_rows.py loaded by path has no importable name for basedpyright, so + its surface is declared here and reached through a single cast.""" + + approx_equal: Callable[[float, float], bool] + assert_total_is_sum_of_components: Callable[[SpendCostRow], None] + poll_cost_row_where: Callable[ + [ProxyClient, str, Callable[[SpendCostRow], bool]], SpendCostRow | None + ] + + +cost_rows: Final[CostRowsModule] = cast(CostRowsModule, _load_cost_rows()) + + +@dataclass(frozen=True, slots=True) +class CostCalcClient: + """The suite's client: a ProxyClient pointed at the cost-map proxy pod.""" + + proxy: ProxyClient + + +@pytest.fixture(scope="session") +def client() -> CostCalcClient: + proxy = build_proxy_client( + base_url=COST_MAP_PROXY_URL, + control_plane_base_url=COST_MAP_PROXY_URL, + replica_urls=(COST_MAP_PROXY_URL,), + ) + return CostCalcClient(proxy=proxy) + + +def register_scenario_deployment( + client: CostCalcClient, + resources: ResourceManager, + model: FrontierModel, + case: Case, + marker: str, +) -> tuple[str, ScenarioHandle]: + """Register the case's scenario on the sidecar plus a deployment pointed at + it; both are torn down by ``resources``. Returns the callable model_name.""" + scenario: Scenario = case.scenario( + scenario_id=f"sc-{marker}", model=model, text=f"scripted answer {marker}" + ) + handle = register_scenario(scenario) + resources.defer(lambda: delete_scenario(handle)) + model_name = f"{model.model_name}-{marker}" + model_id = client.proxy.register_model( + ModelNewBody( + model_name=model_name, + litellm_params=LiteLLMParamsBody( + model=model.litellm_model, + api_key="sk-scripted-provider", + api_base=handle.api_base(), + ), + model_info=ModelInfoBody(), + ) + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + return model_name, handle diff --git a/tests/e2e/cost_calculation/cost_matrix.py b/tests/e2e/cost_calculation/cost_matrix.py new file mode 100644 index 00000000000..bc466d7d823 --- /dev/null +++ b/tests/e2e/cost_calculation/cost_matrix.py @@ -0,0 +1,458 @@ +"""The cost-calculation matrix: frontier model set, the pricing-component cases +each model runs, and the expected-cost arithmetic. + +Rates come from ``tests/e2e/cost_map.json``, which the proxy under test loads as +its ENTIRE model cost map (LITELLM_MODEL_COST_MAP_URL), so an entry's rates are +exactly what the proxy bills and nothing in the suite depends on the bundled +map. Each model's rates are a distinct multiple of a shared base set, so a +component billed at the wrong model's rate (or the wrong case's rate) can never +coincidentally match. + +Case applicability is pricing-field-gated AND wire-gated: a case runs for a +model only when the entry carries the rate the case exercises and the wire can +report the token kind that rate prices. When the wire cannot report a kind +(e.g. Anthropic has no reasoning-token field, Responses reports no cache +creation), the case is absent from the matrix rather than silently zero. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Literal + +from pydantic import BaseModel, ConfigDict, TypeAdapter + +from scripted_provider import Scenario, ScriptedOutput, ScriptedUsage, Wire + +COST_MAP_PATH: Final = Path(__file__).resolve().parent.parent / "cost_map.json" + + +class SearchContextCostPerQuery(BaseModel): + model_config = ConfigDict(frozen=True) + + search_context_size_low: float | None = None + search_context_size_medium: float | None = None + search_context_size_high: float | None = None + + +class CostMapEntry(BaseModel): + """The pricing fields of a cost-map entry the matrix reads. Shaped like a + ``model_prices_and_context_window.json`` entry; unmodelled keys are ignored.""" + + model_config = ConfigDict(frozen=True, extra="ignore") + + litellm_provider: str + mode: str + input_cost_per_token: float | None = None + output_cost_per_token: float | None = None + cache_read_input_token_cost: float | None = None + cache_creation_input_token_cost: float | None = None + cache_creation_input_token_cost_above_1hr: float | None = None + output_cost_per_reasoning_token: float | None = None + input_cost_per_audio_token: float | None = None + output_cost_per_audio_token: float | None = None + input_cost_per_token_above_200k_tokens: float | None = None + output_cost_per_token_above_200k_tokens: float | None = None + input_cost_per_token_flex: float | None = None + output_cost_per_token_flex: float | None = None + input_cost_per_token_priority: float | None = None + output_cost_per_token_priority: float | None = None + search_context_cost_per_query: SearchContextCostPerQuery | None = None + web_search_billing_unit: str | None = None + + +_COST_MAP_ADAPTER: Final = TypeAdapter(dict[str, CostMapEntry]) +_COST_MAP: Final[dict[str, CostMapEntry]] = _COST_MAP_ADAPTER.validate_python( + json.loads(COST_MAP_PATH.read_text()) +) + +TIER_THRESHOLD_TOKENS: Final = 200_000 + + +@dataclass(frozen=True, slots=True) +class FrontierModel: + """One deployment under test: the model_name the suite registers, the + provider-prefixed litellm model string, the wire the scripted upstream + speaks, its cost-map key, and the sibling map model the response_model + override case reports.""" + + model_name: str + litellm_model: str + wire: Wire + map_key: str + override_model: str + + @property + def rates(self) -> CostMapEntry: + return _COST_MAP[self.map_key] + + @property + def override_rates(self) -> CostMapEntry: + return _COST_MAP[self.override_map_key] + + @property + def override_map_key(self) -> str: + return _OVERRIDE_MAP_KEYS[self.override_model] + + @property + def provider(self) -> str: + return self.rates.litellm_provider + + @property + def api_key(self) -> str: + # The scripted upstream ignores auth; a fixed bogus key proves the suite + # spends zero real provider calls. + return "sk-scripted-provider" + + +# Response-model override targets: emit a sibling's bare provider-facing name so +# the biller's provider-prefixed lookup lands on that sibling's map key. +_OVERRIDE_MODELS: Final[dict[str, str]] = { + "gpt-5.6": "gpt-5.4-mini", + "gpt-5.5-pro": "gpt-5.3-codex", + "gpt-5.3-codex": "gpt-5.5-pro", + "gpt-5.4-mini": "gpt-5.6", + "claude-opus-5": "claude-sonnet-5", + "claude-sonnet-5": "claude-opus-5", + "claude-haiku-4-5": "claude-sonnet-5", + "gemini/gemini-3.8-flash": "gemini-3.1-pro-preview", + "gemini/gemini-3.1-pro-preview": "gemini-3.8-flash", + "together_ai/moonshotai/Kimi-K3": "zai-org/GLM-5.3", + "together_ai/zai-org/GLM-5.3": "moonshotai/Kimi-K3", + "fireworks_ai/kimi-k3": "qwen3p8-max", + "fireworks_ai/qwen3p8-max": "kimi-k3", + "fireworks_ai/deepseek-v4p1-flash": "kimi-k3", +} + +_OVERRIDE_MAP_KEYS: Final[dict[str, str]] = { + "gpt-5.4-mini": "gpt-5.4-mini", + "gpt-5.6": "gpt-5.6", + "gpt-5.3-codex": "gpt-5.3-codex", + "gpt-5.5-pro": "gpt-5.5-pro", + "claude-sonnet-5": "claude-sonnet-5", + "claude-opus-5": "claude-opus-5", + "gemini-3.1-pro-preview": "gemini/gemini-3.1-pro-preview", + "gemini-3.8-flash": "gemini/gemini-3.8-flash", + "zai-org/GLM-5.3": "together_ai/zai-org/GLM-5.3", + "moonshotai/Kimi-K3": "together_ai/moonshotai/Kimi-K3", + "qwen3p8-max": "fireworks_ai/qwen3p8-max", + "kimi-k3": "fireworks_ai/kimi-k3", +} + + +_FRONTIER_SPECS: Final[tuple[tuple[str, str, Wire], ...]] = ( + ("gpt-5.6", "openai/gpt-5.6", "openai_chat"), + ("gpt-5.5-pro", "openai/gpt-5.5-pro", "openai_responses"), + ("gpt-5.3-codex", "openai/gpt-5.3-codex", "openai_responses"), + ("gpt-5.4-mini", "openai/gpt-5.4-mini", "openai_chat"), + ("claude-opus-5", "anthropic/claude-opus-5", "anthropic_messages"), + ("claude-sonnet-5", "anthropic/claude-sonnet-5", "anthropic_messages"), + ("claude-haiku-4-5", "anthropic/claude-haiku-4-5", "anthropic_messages"), + ("gemini/gemini-3.8-flash", "gemini/gemini-3.8-flash", "gemini_generate"), + ("gemini/gemini-3.1-pro-preview", "gemini/gemini-3.1-pro-preview", "gemini_generate"), + ("together_ai/moonshotai/Kimi-K3", "together_ai/moonshotai/Kimi-K3", "together_chat"), + ("together_ai/zai-org/GLM-5.3", "together_ai/zai-org/GLM-5.3", "together_chat"), + ("fireworks_ai/kimi-k3", "fireworks_ai/kimi-k3", "fireworks_chat"), + ("fireworks_ai/qwen3p8-max", "fireworks_ai/qwen3p8-max", "fireworks_chat"), + ("fireworks_ai/deepseek-v4p1-flash", "fireworks_ai/deepseek-v4p1-flash", "fireworks_chat"), +) + + +def _frontier() -> tuple[FrontierModel, ...]: + return tuple( + FrontierModel( + model_name=f"cc-{map_key.replace('/', '-').lower()}", + litellm_model=litellm_model, + wire=wire, + map_key=map_key, + override_model=_OVERRIDE_MODELS[map_key], + ) + for map_key, litellm_model, wire in _FRONTIER_SPECS + ) + + +FRONTIER_MODELS: Final[tuple[FrontierModel, ...]] = _frontier() + +# Token kinds each wire can report, gating which pricing cases apply. +_WIRE_CAPS: Final[dict[str, frozenset[str]]] = { + "openai_chat": frozenset( + { + "cache_read", "cache_write_5m", "cache_write_1h", "reasoning", "audio", + "web_search", "response_model", "absent_usage", + } + ), + "openai_responses": frozenset({"cache_read", "reasoning", "web_search", "response_model", "absent_usage"}), + # Product gap: litellm hard-indexes message_delta["usage"] in + # anthropic/chat/handler.py, so a usage-absent anthropic stream raises + # KeyError; the real wire always carries it, so the case cannot be + # represented. + "anthropic_messages": frozenset({"cache_read", "cache_write_5m", "cache_write_1h", "web_search", "response_model"}), + # Product gap: the gemini transform sets ModelResponse.model from the + # request and drops the provider's modelVersion, so a response-model + # override can never be priced on this wire. + "gemini_generate": frozenset({"cache_read", "reasoning", "audio", "web_search", "absent_usage"}), + "together_chat": frozenset( + { + "cache_read", "cache_write_5m", "cache_write_1h", "reasoning", "audio", + "web_search", "response_model", "absent_usage", + } + ), + "fireworks_chat": frozenset( + { + "cache_read", "cache_write_5m", "cache_write_1h", "reasoning", "audio", + "web_search", "response_model", "absent_usage", + } + ), +} + +CaseName = Literal[ + "basic", + "cache_read", + "cache_write_5m", + "cache_write_1h", + "reasoning", + "audio", + "tiered", + "service_tier_flex", + "service_tier_priority", + "web_search", + "stream", + "stream_no_usage", + "response_model_override", +] + + +@dataclass(frozen=True, slots=True) +class Case: + name: CaseName + usage: ScriptedUsage + stream: bool = False + stream_usage: Literal["final_chunk", "absent"] = "final_chunk" + service_tier: Literal["flex", "priority"] | None = None + # For web_search the wire's reported call count is not always what gets + # billed: chat-completions surfaces only expose url_citation annotations, so + # the biller floors to one call; responses/messages/gemini report a real + # count. + billed_web_search_calls: int = 0 + response_model_override: bool = False + exact_spend: bool = True + # stream_usage=absent on a wire with no proxy-side token recount means the + # bill is exactly zero; asserted as such rather than skipped. + expect_zero_bill: bool = False + + def scenario(self, scenario_id: str, model: FrontierModel, text: str) -> Scenario: + return Scenario( + scenario_id=scenario_id, + wire=model.wire, + usage=self.usage, + output=ScriptedOutput( + text=text, + response_model=model.override_model if self.response_model_override else None, + ), + stream_usage=self.stream_usage, + service_tier=self.service_tier, + ) + + +_BASIC_USAGE: Final = ScriptedUsage(fresh_input_tokens=120, output_tokens=40) + + +def _web_search_case(model: FrontierModel) -> Case: + counts_exactly = model.wire in ("openai_responses", "anthropic_messages", "gemini_generate") + return Case( + name="web_search", + usage=ScriptedUsage(fresh_input_tokens=100, output_tokens=30, web_search_calls=3), + billed_web_search_calls=3 if counts_exactly else 1, + ) + + +def cases_for(model: FrontierModel) -> tuple[Case, ...]: + rates = model.rates + caps = _WIRE_CAPS[model.wire] + cases: list[Case] = [Case(name="basic", usage=_BASIC_USAGE)] + if rates.cache_read_input_token_cost is not None and "cache_read" in caps: + cases.append( + Case(name="cache_read", usage=ScriptedUsage(fresh_input_tokens=100, cache_read_tokens=50, output_tokens=30)) + ) + if rates.cache_creation_input_token_cost is not None and "cache_write_5m" in caps: + cases.append( + Case( + name="cache_write_5m", + usage=ScriptedUsage(fresh_input_tokens=90, cache_write_5m_tokens=60, output_tokens=30), + ) + ) + if ( + rates.cache_creation_input_token_cost_above_1hr is not None + and rates.cache_creation_input_token_cost is not None + and "cache_write_1h" in caps + ): + cases.append( + Case( + name="cache_write_1h", + usage=ScriptedUsage( + fresh_input_tokens=90, + cache_write_5m_tokens=20, + cache_write_1h_tokens=40, + output_tokens=30, + ), + ) + ) + if rates.output_cost_per_reasoning_token is not None and "reasoning" in caps: + cases.append( + Case( + name="reasoning", + usage=ScriptedUsage(fresh_input_tokens=100, output_tokens=30, reasoning_tokens=70), + ) + ) + if ( + rates.input_cost_per_audio_token is not None + and rates.output_cost_per_audio_token is not None + and "audio" in caps + ): + cases.append( + Case( + name="audio", + usage=ScriptedUsage( + fresh_input_tokens=100, audio_input_tokens=25, output_tokens=30, audio_output_tokens=15 + ), + ) + ) + if ( + rates.input_cost_per_token_above_200k_tokens is not None + and rates.output_cost_per_token_above_200k_tokens is not None + ): + cases.append( + Case( + name="tiered", + usage=ScriptedUsage( + fresh_input_tokens=TIER_THRESHOLD_TOKENS + 1, output_tokens=30 + ), + ) + ) + if rates.input_cost_per_token_flex is not None and rates.output_cost_per_token_flex is not None: + cases.append( + Case(name="service_tier_flex", usage=_BASIC_USAGE, service_tier="flex") + ) + if rates.input_cost_per_token_priority is not None and rates.output_cost_per_token_priority is not None: + cases.append( + Case(name="service_tier_priority", usage=_BASIC_USAGE, service_tier="priority") + ) + if rates.search_context_cost_per_query is not None and "web_search" in caps: + cases.append(_web_search_case(model)) + cases.append(Case(name="stream", usage=_BASIC_USAGE, stream=True)) + if "absent_usage" in caps: + cases.append( + Case( + name="stream_no_usage", + usage=_BASIC_USAGE, + stream=True, + stream_usage="absent", + exact_spend=False, + # The responses surface bills only provider-reported usage; + # with no usage in the stream the spend row is zero. Other + # wires recount tokens proxy-side and bill a nonzero amount. + expect_zero_bill=model.wire == "openai_responses", + ) + ) + if "response_model" in caps: + cases.append(Case(name="response_model_override", usage=_BASIC_USAGE, response_model_override=True)) + return tuple(cases) + + +@dataclass(frozen=True, slots=True) +class ExpectedCost: + """The expected bill split the way the spend row's cost_breakdown reports + it: the gross input component (cache reads/writes folded in), the output + component, and the tool-usage component.""" + + input_cost: float + output_cost: float + tool_cost: float + + @property + def total(self) -> float: + return self.input_cost + self.output_cost + self.tool_cost + + +def expected_breakdown(model: FrontierModel, case: Case) -> ExpectedCost: + """Literal arithmetic on the test-map rates over the scripted token counts. + + Input = fresh*in + read*read + 5m*create + 1h*create_1h + audio_in*audio_in; + output = text*out + reasoning*reasoning + audio_out*audio_out; plus the + billed web-search calls at the medium search-context rate. Above-threshold + swaps every input/output rate to its ``_above_200k_tokens`` variant when + total prompt tokens exceed the threshold; a service tier swaps input/output + to the tier's variants, falling back to the base rate when a variant is + unset -- mirroring _get_token_base_cost in litellm's cost calculator. + """ + rates = model.override_rates if case.response_model_override else model.rates + u = case.usage + prompt_tokens = ( + u.fresh_input_tokens + u.cache_read_tokens + u.cache_write_5m_tokens + + u.cache_write_1h_tokens + u.audio_input_tokens + ) + tiered = prompt_tokens > TIER_THRESHOLD_TOKENS + in_rate = rates.input_cost_per_token or 0.0 + out_rate = rates.output_cost_per_token or 0.0 + if case.service_tier == "flex": + in_rate = rates.input_cost_per_token_flex or in_rate + out_rate = rates.output_cost_per_token_flex or out_rate + if case.service_tier == "priority": + in_rate = rates.input_cost_per_token_priority or in_rate + out_rate = rates.output_cost_per_token_priority or out_rate + if tiered: + in_rate = rates.input_cost_per_token_above_200k_tokens or in_rate + out_rate = rates.output_cost_per_token_above_200k_tokens or out_rate + input_cost = ( + u.fresh_input_tokens * in_rate + + u.cache_read_tokens * (rates.cache_read_input_token_cost or 0.0) + + u.cache_write_5m_tokens * (rates.cache_creation_input_token_cost or 0.0) + + u.cache_write_1h_tokens * (rates.cache_creation_input_token_cost_above_1hr or 0.0) + + u.audio_input_tokens * (rates.input_cost_per_audio_token or 0.0) + ) + output_cost = ( + u.output_tokens * out_rate + + u.reasoning_tokens * (rates.output_cost_per_reasoning_token or out_rate) + + u.audio_output_tokens * (rates.output_cost_per_audio_token or out_rate) + ) + search = rates.search_context_cost_per_query + tool_cost = case.billed_web_search_calls * ( + search.search_context_size_medium if search and search.search_context_size_medium else 0.0 + ) + return ExpectedCost(input_cost=input_cost, output_cost=output_cost, tool_cost=tool_cost) + + +def expected_cost(model: FrontierModel, case: Case) -> float: + return expected_breakdown(model, case).total + + +def expected_token_columns(model: FrontierModel, case: Case) -> tuple[int, int]: + """(prompt_tokens, completion_tokens) the spend row should carry, per the + wire's normalization: Anthropic folds cache read/write into prompt_tokens, + everyone else reports the totals the wire emitted.""" + u = case.usage + if model.wire == "anthropic_messages": + return ( + u.fresh_input_tokens + u.cache_read_tokens + u.cache_write_5m_tokens + u.cache_write_1h_tokens, + u.output_tokens, + ) + if model.wire == "gemini_generate": + return ( + u.fresh_input_tokens + u.cache_read_tokens + u.audio_input_tokens, + u.output_tokens + u.reasoning_tokens + u.audio_output_tokens, + ) + if model.wire == "openai_responses": + return ( + u.fresh_input_tokens + u.cache_read_tokens, + u.output_tokens + u.reasoning_tokens, + ) + return ( + u.fresh_input_tokens + + u.cache_read_tokens + + u.cache_write_5m_tokens + + u.cache_write_1h_tokens + + u.audio_input_tokens, + u.output_tokens + u.reasoning_tokens + u.audio_output_tokens, + ) diff --git a/tests/e2e/cost_calculation/scripted_client.py b/tests/e2e/cost_calculation/scripted_client.py new file mode 100644 index 00000000000..dceec02630a --- /dev/null +++ b/tests/e2e/cost_calculation/scripted_client.py @@ -0,0 +1,70 @@ +"""Client side of the scripted-provider sidecar: register scenarios over its +control API through the shared transport helpers and get back a handle whose +``api_base`` is what a /model/new deployment should register for the proxy to +reach the scripted wire.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Final + +from e2e_config import SCRIPTED_PROVIDER_CONTROL_URL, SCRIPTED_PROVIDER_PROXY_BASE +from e2e_http import URL, NoBody, unwrap, post +from e2e_http import delete as http_delete +from scripted_provider import ( + Scenario, + ScenarioDeleted, + ScenarioRegistered, + Wire, +) + + +@dataclass(frozen=True, slots=True) +class ScenarioHandle: + scenario_id: str + wire: Wire + proxy_base: str + + def api_base(self) -> str: + return f"{self.proxy_base}/{self.scenario_id}/{self._mount()}" + + def _mount(self) -> str: + return { + "openai_chat": "openai", + "openai_responses": "openai", + "anthropic_messages": "anthropic", + "gemini_generate": "gemini", + "together_chat": "together", + "fireworks_chat": "fireworks", + }[self.wire] + + +def register_scenario(scenario: Scenario) -> ScenarioHandle: + """POST the scenario to the sidecar's control API and return its handle.""" + result = unwrap( + post( + URL(f"{SCRIPTED_PROVIDER_CONTROL_URL}/_scenarios"), + headers=NoBody(), + json=scenario, + response_type=ScenarioRegistered, + ) + ) + return ScenarioHandle( + scenario_id=result.scenario_id, + wire=scenario.wire, + proxy_base=SCRIPTED_PROVIDER_PROXY_BASE, + ) + + +def delete_scenario(handle: ScenarioHandle) -> None: + unwrap( + http_delete( + URL(f"{SCRIPTED_PROVIDER_CONTROL_URL}/_scenarios/{handle.scenario_id}"), + headers=NoBody(), + json=NoBody(), + response_type=ScenarioDeleted, + ) + ) + + +CONTROL_URL: Final = SCRIPTED_PROVIDER_CONTROL_URL diff --git a/tests/e2e/cost_calculation/scripted_provider.py b/tests/e2e/cost_calculation/scripted_provider.py new file mode 100644 index 00000000000..93a6f49ec25 --- /dev/null +++ b/tests/e2e/cost_calculation/scripted_provider.py @@ -0,0 +1,631 @@ +"""Scripted provider sidecar for the cost-calculation e2e suite. + +A standalone process (``python -m cost_calculation.scripted_provider``) that +pretends to be an LLM provider for the proxy under test. The suite registers a +Scenario over a small control API; the provider wire routes then answer the +proxy's upstream calls with the scripted usage figures, in the exact wire shape +the real provider would emit (OpenAI chat completions, OpenAI Responses, +Anthropic Messages, Gemini generateContent, or the OpenAI-compatible Together / +Fireworks surfaces). Because the usage is scripted, expected spend is literal +arithmetic on the test cost map's rates, with no dependency on what a real +provider would report. + +Layout on one port: + +- ``GET /health`` liveness +- ``POST /_scenarios`` register a Scenario JSON, returns its id +- ``DELETE /_scenarios/`` remove it +- ``POST ///`` provider wire; mount is one of + ``openai``, ``anthropic``, ``gemini``, ``together``, ``fireworks`` and the + remainder is whatever path the provider client appends (``chat/completions``, + ``responses``, ``v1/messages``, ``models/:generateContent`` ...) + +A request carrying ``"stream": true`` (or the ``:streamGenerateContent`` Gemini +verb) gets an SSE answer; ``stream_usage`` on the Scenario decides whether the +final stream chunk carries usage or the provider reports none. +""" + +from __future__ import annotations + +import json +import sys +import threading +import time +from dataclasses import dataclass +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Final, Literal +from urllib.parse import urlsplit + +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError + +Wire = Literal[ + "openai_chat", + "openai_responses", + "anthropic_messages", + "gemini_generate", + "together_chat", + "fireworks_chat", +] + +_WIRE_MOUNTS: Final[dict[str, str]] = { + "openai_chat": "openai", + "openai_responses": "openai", + "anthropic_messages": "anthropic", + "gemini_generate": "gemini", + "together_chat": "together", + "fireworks_chat": "fireworks", +} + +StreamUsage = Literal["final_chunk", "absent"] +ServiceTier = Literal["flex", "priority"] + + +class ScriptedUsage(BaseModel): + """Physical token counts the scripted response reports. ``fresh_input_tokens`` + is the uncached, never-written, non-audio input count; ``output_tokens`` is + the non-reasoning, non-audio output count. Renderers add the cached, written, + audio, and reasoning counts into the wire's total fields the way the real + provider does (inside prompt_tokens for OpenAI/Gemini, as uncached-only + input_tokens for Anthropic).""" + + model_config = ConfigDict(frozen=True) + + fresh_input_tokens: int = 0 + output_tokens: int = 0 + cache_read_tokens: int = 0 + cache_write_5m_tokens: int = 0 + cache_write_1h_tokens: int = 0 + reasoning_tokens: int = 0 + audio_input_tokens: int = 0 + audio_output_tokens: int = 0 + web_search_calls: int = 0 + + +class ScriptedOutput(BaseModel): + model_config = ConfigDict(frozen=True) + + text: str + finish_reason: str = "stop" + # When set, emitted verbatim as the response's model field, letting a test + # prove the biller prices the provider-reported model. + response_model: str | None = None + # OpenAI-compatible providers can report a provider-computed cost; emitted as + # the top-level "cost" field on the together/fireworks wire. + provider_cost: float | None = None + + +class Scenario(BaseModel): + model_config = ConfigDict(frozen=True) + + scenario_id: str + wire: Wire + usage: ScriptedUsage + output: ScriptedOutput + stream_usage: StreamUsage = "final_chunk" + service_tier: ServiceTier | None = None + + @property + def mount(self) -> str: + return _WIRE_MOUNTS[self.wire] + + +class ScenarioRegistered(BaseModel): + scenario_id: str + + +class ScenarioDeleted(BaseModel): + deleted: bool + + +class HealthStatus(BaseModel): + status: str + + +@dataclass(frozen=True, slots=True) +class RenderedResponse: + status_code: int + content_type: str + body: bytes + + +def _json_bytes(payload: dict[str, object]) -> bytes: + return json.dumps(payload).encode("utf-8") + + +def _sse(events: tuple[tuple[str | None, dict[str, object] | str], ...]) -> bytes: + frames: list[str] = [] + for event_name, data in events: + head = f"event: {event_name}\n" if event_name is not None else "" + payload = data if isinstance(data, str) else json.dumps(data) + frames.append(f"{head}data: {payload}\n\n") + return "".join(frames).encode("utf-8") + + +# ---------- per-wire usage shapes ---------- + + +def _openai_usage(u: ScriptedUsage) -> dict[str, object]: + prompt_tokens = ( + u.fresh_input_tokens + + u.cache_read_tokens + + u.cache_write_5m_tokens + + u.cache_write_1h_tokens + + u.audio_input_tokens + ) + completion_tokens = u.output_tokens + u.reasoning_tokens + u.audio_output_tokens + prompt_details: dict[str, object] = {} + if u.cache_read_tokens: + prompt_details["cached_tokens"] = u.cache_read_tokens + if u.cache_write_5m_tokens or u.cache_write_1h_tokens: + prompt_details["cache_write_tokens"] = u.cache_write_5m_tokens + u.cache_write_1h_tokens + prompt_details["cache_creation_token_details"] = { + "ephemeral_5m_input_tokens": u.cache_write_5m_tokens, + "ephemeral_1h_input_tokens": u.cache_write_1h_tokens, + } + if u.audio_input_tokens: + prompt_details["audio_tokens"] = u.audio_input_tokens + completion_details: dict[str, object] = {} + if u.reasoning_tokens: + completion_details["reasoning_tokens"] = u.reasoning_tokens + if u.audio_output_tokens: + completion_details["audio_tokens"] = u.audio_output_tokens + usage: dict[str, object] = { + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "total_tokens": prompt_tokens + completion_tokens, + } + if prompt_details: + usage["prompt_tokens_details"] = prompt_details + if completion_details: + usage["completion_tokens_details"] = completion_details + return usage + + +def _anthropic_usage(u: ScriptedUsage) -> dict[str, object]: + # Anthropic reports uncached-only input_tokens; cache reads and writes ride + # top-level fields, with the 5m/1h write split under cache_creation. + usage: dict[str, object] = { + "input_tokens": u.fresh_input_tokens, + "output_tokens": u.output_tokens, + } + if u.cache_read_tokens: + usage["cache_read_input_tokens"] = u.cache_read_tokens + if u.cache_write_5m_tokens or u.cache_write_1h_tokens: + usage["cache_creation_input_tokens"] = u.cache_write_5m_tokens + u.cache_write_1h_tokens + usage["cache_creation"] = { + "ephemeral_5m_input_tokens": u.cache_write_5m_tokens, + "ephemeral_1h_input_tokens": u.cache_write_1h_tokens, + } + if u.web_search_calls: + usage["server_tool_use"] = {"web_search_requests": u.web_search_calls} + return usage + + +def _gemini_usage(u: ScriptedUsage) -> dict[str, object]: + # promptTokenCount carries the cached count inside it; TEXT modality is the + # cached-inclusive text count so litellm's implicit-caching subtraction lands + # on the fresh figure. candidatesTokenCount includes reasoning + audio. + prompt_tokens = u.fresh_input_tokens + u.cache_read_tokens + u.audio_input_tokens + candidates = u.output_tokens + u.reasoning_tokens + u.audio_output_tokens + usage: dict[str, object] = { + "promptTokenCount": prompt_tokens, + "candidatesTokenCount": candidates, + "totalTokenCount": prompt_tokens + candidates, + } + if u.cache_read_tokens: + usage["cachedContentTokenCount"] = u.cache_read_tokens + if u.reasoning_tokens: + usage["thoughtsTokenCount"] = u.reasoning_tokens + prompt_details = [{"modality": "TEXT", "tokenCount": u.fresh_input_tokens + u.cache_read_tokens}] + if u.audio_input_tokens: + prompt_details.append({"modality": "AUDIO", "tokenCount": u.audio_input_tokens}) + usage["promptTokensDetails"] = prompt_details + if u.audio_output_tokens: + usage["candidatesTokensDetails"] = [ + {"modality": "TEXT", "tokenCount": u.output_tokens + u.reasoning_tokens}, + {"modality": "AUDIO", "tokenCount": u.audio_output_tokens}, + ] + return usage + + +def _responses_usage(u: ScriptedUsage) -> dict[str, object]: + input_tokens = u.fresh_input_tokens + u.cache_read_tokens + u.audio_input_tokens + output_tokens = u.output_tokens + u.reasoning_tokens + u.audio_output_tokens + usage: dict[str, object] = { + "input_tokens": input_tokens, + "output_tokens": output_tokens, + "total_tokens": input_tokens + output_tokens, + } + input_details: dict[str, object] = {} + if u.cache_read_tokens: + input_details["cached_tokens"] = u.cache_read_tokens + if input_details: + usage["input_tokens_details"] = input_details + if u.reasoning_tokens: + usage["output_tokens_details"] = {"reasoning_tokens": u.reasoning_tokens} + return usage + + +# ---------- per-wire responses ---------- + + +def _openai_message(scenario: Scenario) -> dict[str, object]: + message: dict[str, object] = {"role": "assistant", "content": scenario.output.text} + if scenario.usage.web_search_calls: + message["annotations"] = [ + { + "type": "url_citation", + "url_citation": { + "url": "https://scripted.example/source", + "title": "scripted source", + "start_index": 0, + "end_index": 1, + }, + } + for _ in range(scenario.usage.web_search_calls) + ] + return message + + +def _openai_chat_body(scenario: Scenario, requested_model: str) -> dict[str, object]: + body: dict[str, object] = { + "id": f"chatcmpl-{scenario.scenario_id}", + "object": "chat.completion", + "created": int(time.time()), + "model": scenario.output.response_model or requested_model, + "choices": [ + { + "index": 0, + "message": _openai_message(scenario), + "finish_reason": scenario.output.finish_reason, + } + ], + "usage": _openai_usage(scenario.usage), + } + if scenario.service_tier is not None: + body["service_tier"] = scenario.service_tier + if scenario.output.provider_cost is not None: + body["cost"] = scenario.output.provider_cost + return body + + +def _openai_chunk(scenario: Scenario, requested_model: str, **kw: object) -> dict[str, object]: + chunk: dict[str, object] = { + "id": f"chatcmpl-{scenario.scenario_id}", + "object": "chat.completion.chunk", + "created": int(time.time()), + "model": scenario.output.response_model or requested_model, + } + chunk.update(kw) + return chunk + + +def _openai_chat_sse(scenario: Scenario, requested_model: str) -> bytes: + _EMPTY_DELTA: Final[dict[str, object]] = {} + delta: dict[str, object] = {"role": "assistant", "content": scenario.output.text} + if scenario.usage.web_search_calls: + delta["annotations"] = _openai_message(scenario)["annotations"] + events: list[tuple[str | None, dict[str, object] | str]] = [ + ( + None, + _openai_chunk( + scenario, + requested_model, + choices=[{"index": 0, "delta": {"role": "assistant"}, "finish_reason": None}], + ), + ), + ( + None, + _openai_chunk( + scenario, + requested_model, + choices=[{"index": 0, "delta": delta, "finish_reason": None}], + ), + ), + ( + None, + _openai_chunk( + scenario, + requested_model, + choices=[ + { + "index": 0, + "delta": _EMPTY_DELTA, + "finish_reason": scenario.output.finish_reason, + } + ], + ), + ), + ] + if scenario.stream_usage == "final_chunk": + events.append( + (None, _openai_chunk(scenario, requested_model, choices=(), usage=_openai_usage(scenario.usage))) + ) + events.append((None, "[DONE]")) + return _sse(tuple(events)) + + +def _anthropic_body(scenario: Scenario, requested_model: str) -> dict[str, object]: + return { + "id": f"msg_{scenario.scenario_id}", + "type": "message", + "role": "assistant", + "model": scenario.output.response_model or requested_model, + "content": [{"type": "text", "text": scenario.output.text}], + "stop_reason": "end_turn" if scenario.output.finish_reason == "stop" else scenario.output.finish_reason, + "usage": _anthropic_usage(scenario.usage), + } + + +def _anthropic_sse(scenario: Scenario, requested_model: str) -> bytes: + emit_usage = scenario.stream_usage == "final_chunk" + input_usage = {k: v for k, v in _anthropic_usage(scenario.usage).items() if k != "output_tokens"} + message_start: dict[str, object] = { + "type": "message_start", + "message": { + "id": f"msg_{scenario.scenario_id}", + "type": "message", + "role": "assistant", + "model": scenario.output.response_model or requested_model, + "content": [], + "stop_reason": None, + **({"usage": input_usage} if emit_usage else {}), + }, + } + message_delta: dict[str, object] = { + "type": "message_delta", + "delta": { + "stop_reason": "end_turn" if scenario.output.finish_reason == "stop" else scenario.output.finish_reason + }, + **({"usage": {"output_tokens": scenario.usage.output_tokens}} if emit_usage else {}), + } + return _sse( + ( + ("message_start", message_start), + ( + "content_block_start", + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "text", "text": ""}, + }, + ), + ( + "content_block_delta", + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "text_delta", "text": scenario.output.text}, + }, + ), + ("content_block_stop", {"type": "content_block_stop", "index": 0}), + ("message_delta", message_delta), + ("message_stop", {"type": "message_stop"}), + ) + ) + + +def _gemini_body(scenario: Scenario, requested_model: str) -> dict[str, object]: + candidate: dict[str, object] = { + "content": {"parts": [{"text": scenario.output.text}], "role": "model"}, + "finishReason": "STOP" if scenario.output.finish_reason == "stop" else scenario.output.finish_reason.upper(), + "index": 0, + } + if scenario.usage.web_search_calls: + candidate["groundingMetadata"] = { + "webSearchQueries": [f"query {i}" for i in range(scenario.usage.web_search_calls)] + } + return { + "candidates": [candidate], + "usageMetadata": _gemini_usage(scenario.usage), + "modelVersion": scenario.output.response_model or requested_model, + } + + +def _gemini_sse(scenario: Scenario, requested_model: str) -> bytes: + first = _gemini_body(scenario, requested_model) + if scenario.stream_usage == "absent": + first = {k: v for k, v in first.items() if k != "usageMetadata"} + events: list[tuple[str | None, dict[str, object] | str]] = [(None, first)] + if scenario.stream_usage == "final_chunk": + events.append( + ( + None, + { + "candidates": [], + "usageMetadata": _gemini_usage(scenario.usage), + "modelVersion": scenario.output.response_model or requested_model, + }, + ) + ) + return _sse(tuple(events)) + + +def _responses_body(scenario: Scenario, requested_model: str) -> dict[str, object]: + output: list[dict[str, object]] = [ + {"type": "web_search_call", "id": f"ws_{i}", "status": "completed"} + for i in range(scenario.usage.web_search_calls) + ] + output.append( + { + "type": "message", + "id": f"msg_{scenario.scenario_id}", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": scenario.output.text, + "annotations": [], + } + ], + } + ) + return { + "id": f"resp_{scenario.scenario_id}", + "object": "response", + "created_at": int(time.time()), + "status": "completed", + "model": scenario.output.response_model or requested_model, + "output": output, + "usage": _responses_usage(scenario.usage), + } + + +def _responses_sse(scenario: Scenario, requested_model: str) -> bytes: + completed = _responses_body(scenario, requested_model) + if scenario.stream_usage == "absent": + completed = {k: v for k, v in completed.items() if k != "usage"} + created = {**completed, "status": "in_progress", "usage": None} + return _sse( + ( + ("response.created", {"type": "response.created", "response": created}), + ( + "response.output_text.delta", + { + "type": "response.output_text.delta", + "item_id": f"msg_{scenario.scenario_id}", + "output_index": scenario.usage.web_search_calls, + "content_index": 0, + "delta": scenario.output.text, + }, + ), + ("response.completed", {"type": "response.completed", "response": completed}), + ) + ) + + +def _render(scenario: Scenario, *, stream: bool, requested_model: str) -> RenderedResponse: + if scenario.wire == "anthropic_messages": + if stream: + return RenderedResponse(200, "text/event-stream", _anthropic_sse(scenario, requested_model)) + return RenderedResponse(200, "application/json", _json_bytes(_anthropic_body(scenario, requested_model))) + if scenario.wire == "gemini_generate": + if stream: + return RenderedResponse(200, "text/event-stream", _gemini_sse(scenario, requested_model)) + return RenderedResponse(200, "application/json", _json_bytes(_gemini_body(scenario, requested_model))) + if scenario.wire == "openai_responses": + if stream: + return RenderedResponse(200, "text/event-stream", _responses_sse(scenario, requested_model)) + return RenderedResponse(200, "application/json", _json_bytes(_responses_body(scenario, requested_model))) + # openai_chat, together_chat, fireworks_chat share the OpenAI chat shape. + if stream: + return RenderedResponse(200, "text/event-stream", _openai_chat_sse(scenario, requested_model)) + return RenderedResponse(200, "application/json", _json_bytes(_openai_chat_body(scenario, requested_model))) + + +# ---------- registry + request routing ---------- + + +class _ScenarioStore: + def __init__(self) -> None: + self._lock: Final = threading.Lock() + self._scenarios: dict[str, Scenario] = {} # mutable-ok: server state, guarded by _lock + + def put(self, scenario: Scenario) -> None: + with self._lock: + self._scenarios[scenario.scenario_id] = scenario + + def drop(self, scenario_id: str) -> bool: + with self._lock: + return self._scenarios.pop(scenario_id, None) is not None + + def get(self, scenario_id: str) -> Scenario | None: + with self._lock: + return self._scenarios.get(scenario_id) + + +_REQUEST_BODY: Final = TypeAdapter(dict[str, object]) + + +def _request_body(body: bytes) -> dict[str, object]: + try: + return _REQUEST_BODY.validate_json(body) + except ValueError: + return {} + + +def _request_wants_stream(path_tail: str, body: bytes) -> bool: + if ":streamGenerateContent" in path_tail: + return True + if not body: + return False + return _request_body(body).get("stream") is True + + +def _request_model(body: bytes) -> str: + model = _request_body(body).get("model") + return model if isinstance(model, str) else "unknown" + + +def handle_request(store: _ScenarioStore, method: str, raw_path: str, body: bytes) -> RenderedResponse: + path = urlsplit(raw_path).path + segments = [segment for segment in path.split("/") if segment] + if method == "GET" and segments == ["health"]: + return RenderedResponse(200, "application/json", _json_bytes({"status": "ok"})) + if segments and segments[0] == "_scenarios": + if method == "POST" and len(segments) == 1: + try: + scenario = Scenario.model_validate_json(body) + except ValidationError as exc: + return RenderedResponse(400, "application/json", _json_bytes({"error": str(exc)})) + store.put(scenario) + return RenderedResponse(200, "application/json", _json_bytes({"scenario_id": scenario.scenario_id})) + if method == "DELETE" and len(segments) == 2: + deleted = store.drop(segments[1]) + return RenderedResponse( + 200 if deleted else 404, "application/json", _json_bytes({"deleted": deleted}) + ) + return RenderedResponse(404, "application/json", _json_bytes({"error": "unknown control route"})) + if len(segments) < 2 or method != "POST": + return RenderedResponse(404, "application/json", _json_bytes({"error": f"no route for {method} {path}"})) + scenario_id, mount = segments[0], segments[1] + scenario = store.get(scenario_id) + if scenario is None: + return RenderedResponse(404, "application/json", _json_bytes({"error": f"unknown scenario {scenario_id}"})) + if scenario.mount != mount: + return RenderedResponse( + 400, + "application/json", + _json_bytes({"error": f"scenario {scenario_id} is wire {scenario.wire}, not mount {mount}"}), + ) + tail = "/".join(segments[2:]) + return _render(scenario, stream=_request_wants_stream(tail, body), requested_model=_request_model(body)) + + +class _ScriptedHandler(BaseHTTPRequestHandler): + store: Final[_ScenarioStore] = _ScenarioStore() + + def _dispatch(self, method: str) -> None: + length = int(self.headers.get("content-length") or 0) + body = self.rfile.read(length) if length else b"" + rendered = handle_request(self.store, method, self.path, body) + self.send_response(rendered.status_code) + self.send_header("content-type", rendered.content_type) + self.send_header("content-length", str(len(rendered.body))) + self.end_headers() + self.wfile.write(rendered.body) + + def do_GET(self) -> None: + self._dispatch("GET") + + def do_POST(self) -> None: + self._dispatch("POST") + + def do_DELETE(self) -> None: + self._dispatch("DELETE") + + + +DEFAULT_PORT: Final = 9100 + + +def serve(port: int = DEFAULT_PORT, bind_host: str = "127.0.0.1") -> None: + server = ThreadingHTTPServer((bind_host, port), _ScriptedHandler) + sys.stderr.write(f"scripted-provider listening on http://{bind_host}:{port}\n") + server.serve_forever() + + +if __name__ == "__main__": + port_arg = int(sys.argv[1]) if len(sys.argv) > 1 else DEFAULT_PORT + serve(port=port_arg) diff --git a/tests/e2e/cost_calculation/test_token_pricing_e2e.py b/tests/e2e/cost_calculation/test_token_pricing_e2e.py new file mode 100644 index 00000000000..8d7678cf9ca --- /dev/null +++ b/tests/e2e/cost_calculation/test_token_pricing_e2e.py @@ -0,0 +1,115 @@ +"""Token-pricing e2e: every (frontier model, pricing-component case) cell runs a +scripted-usage call through a deployment registered on the cost-map proxy, and +the spend row plus response-cost header must equal literal arithmetic on the +test map's rates. + +Nothing here touches a real provider or the bundled cost map: the proxy's +upstream is the scripted-provider sidecar and its entire cost map is +tests/e2e/cost_map.json. +""" + +from __future__ import annotations + +import pytest + +from conftest import CostCalcClient, cost_rows, register_scenario_deployment +from cost_matrix import ( + FRONTIER_MODELS, + Case, + FrontierModel, + cases_for, + expected_cost, + expected_token_columns, +) +from e2e_config import unique_marker +from lifecycle import ResourceManager +from models import ChatBody, ChatMessage, ChatStreamOptions + +pytestmark = [pytest.mark.e2e, pytest.mark.cost_map_stack] + +_MATRIX: list[tuple[FrontierModel, Case]] = [ + (model, case) for model in FRONTIER_MODELS for case in cases_for(model) +] + + +def _case_id(param: tuple[FrontierModel, Case]) -> str: + model, case = param + return f"{model.map_key.replace('/', '-')}-{case.name}" + + +def _chat_body(model_name: str, marker: str, case: Case) -> ChatBody: + return ChatBody( + model=model_name, + messages=[ChatMessage(role="user", content=f"{marker} scripted pricing call")], + stream=case.stream, + stream_options=ChatStreamOptions(include_usage=True) if case.stream else None, + service_tier=case.service_tier, + ) + + +class TestTokenPricing: + @pytest.mark.parametrize("model_case", _MATRIX, ids=_case_id) + @pytest.mark.covers("quota_management.spend_tracking.cost_matrix.logs_cost") + def test_scripted_usage_bills_at_map_rates( + self, + client: CostCalcClient, + resources: ResourceManager, + scoped_key: str, + model_case: tuple[FrontierModel, Case], + ) -> None: + model, case = model_case + marker = unique_marker() + model_name, _handle = register_scenario_deployment(client, resources, model, case, marker) + response = client.proxy.transport.send( + "/chat/completions", + headers=client.proxy.transport.bearer(scoped_key), + json=_chat_body(model_name, marker, case), + stream=case.stream, + ) + assert response.ok, ( + f"{model.map_key}/{case.name}: proxy returned {response.status_code}: {response.body[:400]}" + ) + assert response.stream_error is None, f"stream carried an error event: {response.stream_error}" + + expected = expected_cost(model, case) + if case.exact_spend and not case.stream: + # Streamed responses commit headers before the bill is computed, so + # the x-litellm-response-cost header is asserted only on non-stream + # calls. + assert response.response_cost is not None and cost_rows.approx_equal( + response.response_cost, expected + ), ( + f"x-litellm-response-cost {response.response_cost} != expected {expected}" + ) + + row = cost_rows.poll_cost_row_where( + client.proxy, + scoped_key, + lambda r: r.metadata is not None and r.metadata.cost_breakdown is not None, + ) + assert row is not None, f"no spend row with a cost breakdown landed for {model.map_key}/{case.name}" + + if not case.exact_spend and case.expect_zero_bill: + # The provider reported no usage and this wire has no proxy-side + # recount, so the bill is exactly zero. + assert row.spend is not None and row.spend == 0, f"no-usage stream billed {row.spend}: {row}" + return + if not case.exact_spend: + # stream_usage=absent: the provider reported no usage, so the row's + # token counts are the proxy's own recount; only assert a bill landed. + assert row.spend is not None and row.spend > 0, f"no-usage stream billed nothing: {row}" + return + + assert row.spend is not None and cost_rows.approx_equal(row.spend, expected), ( + f"{model.map_key}/{case.name}: spend {row.spend} != expected {expected} " + f"(breakdown {row.breakdown.model_dump()})" + ) + + prompt_tokens, completion_tokens = expected_token_columns(model, case) + assert row.prompt_tokens == prompt_tokens, ( + f"prompt_tokens {row.prompt_tokens} != {prompt_tokens}" + ) + assert row.completion_tokens == completion_tokens, ( + f"completion_tokens {row.completion_tokens} != {completion_tokens}" + ) + cost_rows.assert_total_is_sum_of_components(row) diff --git a/tests/e2e/cost_calculation/test_wire_formats_e2e.py b/tests/e2e/cost_calculation/test_wire_formats_e2e.py new file mode 100644 index 00000000000..b1ef675d9ef --- /dev/null +++ b/tests/e2e/cost_calculation/test_wire_formats_e2e.py @@ -0,0 +1,186 @@ +"""Wire-format e2e: one scripted upstream per provider wire, answering with a +usage payload where every token kind the wire can report is nonzero. The spend +row's gross input cost must equal fresh tokens at the input rate plus each cache +and audio component at its own rate -- proving the wire's usage shape landed the +cached tokens inside the total (OpenAI/Gemini) or as separate fields +(Anthropic), and that the biller subtracted them before billing fresh tokens. + +Also covers the Responses API wire (an openai/gpt-5.5-pro deployment bridged by +the proxy to POST /responses) and a streamed Anthropic-messages case. +""" + +from __future__ import annotations + +import pytest + +from conftest import CostCalcClient, cost_rows, register_scenario_deployment +from cost_matrix import ( + FRONTIER_MODELS, + Case, + FrontierModel, + expected_breakdown, + expected_token_columns, +) +from e2e_config import unique_marker +from lifecycle import ResourceManager +from models import ChatBody, ChatMessage, ChatStreamOptions +from scripted_provider import ScriptedUsage + +pytestmark = [pytest.mark.e2e, pytest.mark.cost_map_stack] + +_MODELS: dict[str, FrontierModel] = {model.map_key: model for model in FRONTIER_MODELS} + +# One scripted usage per wire, every reportable token kind nonzero. +_WIRE_USAGE: dict[str, tuple[str, ScriptedUsage]] = { + "openai_chat": ( + "gpt-5.6", + ScriptedUsage( + fresh_input_tokens=80, + cache_read_tokens=40, + cache_write_5m_tokens=20, + cache_write_1h_tokens=10, + output_tokens=25, + reasoning_tokens=15, + audio_input_tokens=5, + audio_output_tokens=3, + ), + ), + "openai_responses": ( + "gpt-5.5-pro", + ScriptedUsage( + fresh_input_tokens=80, cache_read_tokens=40, output_tokens=25, reasoning_tokens=15 + ), + ), + "anthropic_messages": ( + "claude-sonnet-5", + ScriptedUsage( + fresh_input_tokens=80, + cache_read_tokens=40, + cache_write_5m_tokens=20, + cache_write_1h_tokens=10, + output_tokens=25, + ), + ), + "gemini_generate": ( + "gemini/gemini-3.8-flash", + ScriptedUsage( + fresh_input_tokens=80, + cache_read_tokens=40, + output_tokens=25, + reasoning_tokens=15, + audio_input_tokens=5, + audio_output_tokens=3, + ), + ), + "together_chat": ( + "together_ai/moonshotai/Kimi-K3", + ScriptedUsage( + fresh_input_tokens=80, + cache_read_tokens=40, + cache_write_5m_tokens=20, + cache_write_1h_tokens=10, + output_tokens=25, + reasoning_tokens=15, + audio_input_tokens=5, + audio_output_tokens=3, + ), + ), + "fireworks_chat": ( + "fireworks_ai/kimi-k3", + ScriptedUsage(fresh_input_tokens=80, cache_read_tokens=40, output_tokens=25), + ), +} + + +class TestWireFormats: + @pytest.mark.parametrize("wire", tuple(_WIRE_USAGE)) + @pytest.mark.covers("quota_management.spend_tracking.scripted_wire.logs_cost") + def test_wire_usage_shape_bills_each_component( + self, + client: CostCalcClient, + resources: ResourceManager, + scoped_key: str, + wire: str, + ) -> None: + map_key, usage = _WIRE_USAGE[wire] + model = _MODELS[map_key] + case = Case(name="basic", usage=usage) + marker = unique_marker() + model_name, _handle = register_scenario_deployment(client, resources, model, case, marker) + response = client.proxy.transport.send( + "/chat/completions", + headers=client.proxy.transport.bearer(scoped_key), + json=ChatBody( + model=model_name, + messages=[ChatMessage(role="user", content=f"{marker} scripted wire call")], + ), + ) + assert response.ok, f"{wire}: proxy returned {response.status_code}: {response.body[:400]}" + + expected = expected_breakdown(model, case) + row = cost_rows.poll_cost_row_where( + client.proxy, + scoped_key, + lambda r: r.metadata is not None and r.metadata.cost_breakdown is not None, + ) + assert row is not None, f"{wire}: no spend row landed" + assert row.spend is not None and cost_rows.approx_equal(row.spend, expected.total), ( + f"{wire}: spend {row.spend} != expected {expected.total} " + f"(breakdown {row.breakdown.model_dump()})" + ) + breakdown = row.breakdown + assert breakdown.input_cost is not None and cost_rows.approx_equal( + breakdown.input_cost, expected.input_cost + ), ( + f"{wire}: gross input_cost {breakdown.input_cost} != expected {expected.input_cost}; " + "cached/written tokens billed at the input rate" + ) + assert breakdown.output_cost is not None and cost_rows.approx_equal( + breakdown.output_cost, expected.output_cost + ), f"{wire}: output_cost {breakdown.output_cost} != expected {expected.output_cost}" + + prompt_tokens, completion_tokens = expected_token_columns(model, case) + assert row.prompt_tokens == prompt_tokens, ( + f"{wire}: prompt_tokens {row.prompt_tokens} != {prompt_tokens}" + ) + assert row.completion_tokens == completion_tokens, ( + f"{wire}: completion_tokens {row.completion_tokens} != {completion_tokens}" + ) + cost_rows.assert_total_is_sum_of_components(row) + + @pytest.mark.covers("quota_management.spend_tracking.scripted_wire.logs_cost") + def test_anthropic_streamed_usage_bills_each_component( + self, client: CostCalcClient, resources: ResourceManager, scoped_key: str + ) -> None: + map_key, usage = _WIRE_USAGE["anthropic_messages"] + model = _MODELS[map_key] + case = Case(name="stream", usage=usage, stream=True) + marker = unique_marker() + model_name, _handle = register_scenario_deployment(client, resources, model, case, marker) + response = client.proxy.transport.send( + "/chat/completions", + headers=client.proxy.transport.bearer(scoped_key), + json=ChatBody( + model=model_name, + messages=[ChatMessage(role="user", content=f"{marker} scripted anthropic stream")], + stream=True, + stream_options=ChatStreamOptions(include_usage=True), + ), + stream=True, + ) + assert response.ok, f"anthropic stream: proxy returned {response.status_code}: {response.body[:400]}" + assert response.stream_done, "anthropic stream did not reach its terminal event" + assert response.stream_error is None, f"stream carried an error event: {response.stream_error}" + + expected = expected_breakdown(model, case) + row = cost_rows.poll_cost_row_where( + client.proxy, + scoped_key, + lambda r: r.metadata is not None and r.metadata.cost_breakdown is not None, + ) + assert row is not None, "anthropic stream: no spend row landed" + assert row.spend is not None and cost_rows.approx_equal(row.spend, expected.total), ( + f"anthropic stream: spend {row.spend} != expected {expected.total} " + f"(breakdown {row.breakdown.model_dump()})" + ) + cost_rows.assert_total_is_sum_of_components(row) diff --git a/tests/e2e/cost_map.json b/tests/e2e/cost_map.json new file mode 100644 index 00000000000..b761710bae3 --- /dev/null +++ b/tests/e2e/cost_map.json @@ -0,0 +1,352 @@ +{ + "claude-haiku-4-5": { + "cache_creation_input_token_cost": 0.00021, + "cache_creation_input_token_cost_above_1hr": 0.00028000000000000003, + "cache_read_input_token_cost": 7e-06, + "input_cost_per_token": 7.000000000000001e-05, + "litellm_provider": "anthropic", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 0.00014000000000000001, + "search_context_cost_per_query": { + "search_context_size_high": 0.03, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.02 + }, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_web_search": true + }, + "claude-opus-5": { + "cache_creation_input_token_cost": 0.00015000000000000001, + "cache_creation_input_token_cost_above_1hr": 0.0002, + "cache_read_input_token_cost": 4.9999999999999996e-06, + "input_cost_per_token": 5e-05, + "litellm_provider": "anthropic", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 0.0001, + "search_context_cost_per_query": { + "search_context_size_high": 0.03, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.02 + }, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_web_search": true + }, + "claude-sonnet-5": { + "cache_creation_input_token_cost": 0.00018, + "cache_creation_input_token_cost_above_1hr": 0.00024000000000000003, + "cache_read_input_token_cost": 6e-06, + "input_cost_per_token": 6.000000000000001e-05, + "litellm_provider": "anthropic", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 0.00012000000000000002, + "search_context_cost_per_query": { + "search_context_size_high": 0.03, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.02 + }, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_web_search": true + }, + "fireworks_ai/deepseek-v4p1-flash": { + "cache_read_input_token_cost": 1.4e-05, + "input_cost_per_token": 0.00014000000000000001, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 0.00028000000000000003, + "search_context_cost_per_query": { + "search_context_size_high": 0.03, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.02 + }, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_web_search": true + }, + "fireworks_ai/kimi-k3": { + "cache_read_input_token_cost": 1.2e-05, + "input_cost_per_token": 0.00012000000000000002, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 0.00024000000000000003, + "search_context_cost_per_query": { + "search_context_size_high": 0.03, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.02 + }, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_web_search": true + }, + "fireworks_ai/qwen3p8-max": { + "cache_read_input_token_cost": 1.3e-05, + "input_cost_per_token": 0.00013000000000000002, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 0.00026000000000000003, + "search_context_cost_per_query": { + "search_context_size_high": 0.03, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.02 + }, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_web_search": true + }, + "gemini/gemini-3.1-pro-preview": { + "cache_read_input_token_cost": 9e-06, + "input_cost_per_audio_token": 0.00054, + "input_cost_per_token": 9e-05, + "input_cost_per_token_above_200k_tokens": 0.00072, + "input_cost_per_token_flex": 0.000135, + "input_cost_per_token_priority": 0.000153, + "litellm_provider": "gemini", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_audio_token": 0.0006299999999999999, + "output_cost_per_reasoning_token": 0.00045000000000000004, + "output_cost_per_token": 0.00018, + "output_cost_per_token_above_200k_tokens": 0.0008100000000000001, + "output_cost_per_token_flex": 0.00022500000000000002, + "output_cost_per_token_priority": 0.000243, + "search_context_cost_per_query": { + "search_context_size_high": 0.03, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.02 + }, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_web_search": true, + "web_search_billing_unit": "per_query" + }, + "gemini/gemini-3.8-flash": { + "cache_read_input_token_cost": 8e-06, + "input_cost_per_audio_token": 0.00048, + "input_cost_per_token": 8e-05, + "input_cost_per_token_above_200k_tokens": 0.00064, + "input_cost_per_token_flex": 0.00012, + "input_cost_per_token_priority": 0.000136, + "litellm_provider": "gemini", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_audio_token": 0.00056, + "output_cost_per_reasoning_token": 0.0004, + "output_cost_per_token": 0.00016, + "output_cost_per_token_above_200k_tokens": 0.00072, + "output_cost_per_token_flex": 0.0002, + "output_cost_per_token_priority": 0.000216, + "search_context_cost_per_query": { + "search_context_size_high": 0.03, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.02 + }, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_web_search": true, + "web_search_billing_unit": "per_query" + }, + "gpt-5.3-codex": { + "cache_read_input_token_cost": 3e-06, + "input_cost_per_token": 3.0000000000000004e-05, + "input_cost_per_token_above_200k_tokens": 0.00024000000000000003, + "input_cost_per_token_flex": 4.5e-05, + "input_cost_per_token_priority": 5.1e-05, + "litellm_provider": "openai", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "output_cost_per_reasoning_token": 0.00015000000000000001, + "output_cost_per_token": 6.000000000000001e-05, + "output_cost_per_token_above_200k_tokens": 0.00027, + "output_cost_per_token_flex": 7.500000000000001e-05, + "output_cost_per_token_priority": 8.099999999999999e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.03, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.02 + }, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_web_search": true + }, + "gpt-5.4-mini": { + "cache_creation_input_token_cost": 0.00012, + "cache_creation_input_token_cost_above_1hr": 0.00016, + "cache_read_input_token_cost": 4e-06, + "input_cost_per_audio_token": 0.00024, + "input_cost_per_token": 4e-05, + "input_cost_per_token_above_200k_tokens": 0.00032, + "input_cost_per_token_flex": 6e-05, + "input_cost_per_token_priority": 6.8e-05, + "litellm_provider": "openai", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_audio_token": 0.00028, + "output_cost_per_reasoning_token": 0.0002, + "output_cost_per_token": 8e-05, + "output_cost_per_token_above_200k_tokens": 0.00036, + "output_cost_per_token_flex": 0.0001, + "output_cost_per_token_priority": 0.000108, + "search_context_cost_per_query": { + "search_context_size_high": 0.03, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.02 + }, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_web_search": true + }, + "gpt-5.5-pro": { + "cache_read_input_token_cost": 2e-06, + "input_cost_per_token": 2e-05, + "input_cost_per_token_above_200k_tokens": 0.00016, + "input_cost_per_token_flex": 3e-05, + "input_cost_per_token_priority": 3.4e-05, + "litellm_provider": "openai", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "output_cost_per_reasoning_token": 0.0001, + "output_cost_per_token": 4e-05, + "output_cost_per_token_above_200k_tokens": 0.00018, + "output_cost_per_token_flex": 5e-05, + "output_cost_per_token_priority": 5.4e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.03, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.02 + }, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_web_search": true + }, + "gpt-5.6": { + "cache_creation_input_token_cost": 3e-05, + "cache_creation_input_token_cost_above_1hr": 4e-05, + "cache_read_input_token_cost": 1e-06, + "input_cost_per_audio_token": 6e-05, + "input_cost_per_token": 1e-05, + "input_cost_per_token_above_200k_tokens": 8e-05, + "input_cost_per_token_flex": 1.5e-05, + "input_cost_per_token_priority": 1.7e-05, + "litellm_provider": "openai", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_audio_token": 7e-05, + "output_cost_per_reasoning_token": 5e-05, + "output_cost_per_token": 2e-05, + "output_cost_per_token_above_200k_tokens": 9e-05, + "output_cost_per_token_flex": 2.5e-05, + "output_cost_per_token_priority": 2.7e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.03, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.02 + }, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_web_search": true + }, + "together_ai/moonshotai/Kimi-K3": { + "cache_creation_input_token_cost": 0.00030000000000000003, + "cache_creation_input_token_cost_above_1hr": 0.0004, + "cache_read_input_token_cost": 9.999999999999999e-06, + "input_cost_per_audio_token": 0.0006000000000000001, + "input_cost_per_token": 0.0001, + "input_cost_per_token_above_200k_tokens": 0.0008, + "input_cost_per_token_flex": 0.00015000000000000001, + "input_cost_per_token_priority": 0.00017, + "litellm_provider": "together_ai", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_audio_token": 0.0006999999999999999, + "output_cost_per_reasoning_token": 0.0005, + "output_cost_per_token": 0.0002, + "output_cost_per_token_above_200k_tokens": 0.0009000000000000001, + "output_cost_per_token_flex": 0.00025, + "output_cost_per_token_priority": 0.00027, + "search_context_cost_per_query": { + "search_context_size_high": 0.03, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.02 + }, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_web_search": true + }, + "together_ai/zai-org/GLM-5.3": { + "cache_creation_input_token_cost": 0.00033, + "cache_creation_input_token_cost_above_1hr": 0.00044, + "cache_read_input_token_cost": 1.1e-05, + "input_cost_per_audio_token": 0.00066, + "input_cost_per_token": 0.00011, + "input_cost_per_token_above_200k_tokens": 0.00088, + "input_cost_per_token_flex": 0.000165, + "input_cost_per_token_priority": 0.000187, + "litellm_provider": "together_ai", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_audio_token": 0.00077, + "output_cost_per_reasoning_token": 0.00055, + "output_cost_per_token": 0.00022, + "output_cost_per_token_above_200k_tokens": 0.00099, + "output_cost_per_token_flex": 0.000275, + "output_cost_per_token_priority": 0.000297, + "search_context_cost_per_query": { + "search_context_size_high": 0.03, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.02 + }, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_web_search": true + } +} diff --git a/tests/e2e/coverage_registry/quota_management.yaml b/tests/e2e/coverage_registry/quota_management.yaml index ad0914d455b..6b40e70125c 100644 --- a/tests/e2e/coverage_registry/quota_management.yaml +++ b/tests/e2e/coverage_registry/quota_management.yaml @@ -63,3 +63,5 @@ - {id: quota_management.spend_tracking.key_attribution.health_rows_keep_service_account, module: quota_management, tier: P1, behavior: spend_tracking, variant: key_attribution, assertions: [health_rows_keep_service_account], exercised_on: [chat_completions], source: "proxy/health_check.py", rationale: "A /health probe's spend row stays keyed by the literal litellm-internal-health-check service account rather than a hash of it, so health spend never appears as an unattributed key"} - {id: quota_management.spend_tracking.key_attribution.retrieve_batch_cost_joins_retrieving_key, module: quota_management, tier: P1, behavior: spend_tracking, variant: key_attribution, assertions: [retrieve_batch_cost_joins_retrieving_key], exercised_on: [batches], source: "proxy/batches_endpoints/endpoints.py", rationale: "The retrieve that first sees a batch in a terminal state prices it inline and writes its {provider_batch_id}_batch_cost row against the retrieving key, so the batch each run creates is one OpenAI fails at validation within seconds and the test retrieves it by its raw provider id with the same key until it is failed; a raw id is never owned by the CheckBatchCost poller, and the row must carry that key's token hash and alias"} - {id: quota_management.spend_tracking.key_attribution.poller_batch_cost_joins_creating_key, module: quota_management, tier: P1, behavior: spend_tracking, variant: key_attribution, assertions: [poller_batch_cost_joins_creating_key], exercised_on: [batches], source: "enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py", rationale: "The CheckBatchCost poller bills a completed, positive-cost batch created through a unified id against the key that created it, a different writer from the inline retrieve. No test claims this cell yet: OpenAI's completion window is 24h and both e2e stacks boot a fresh Postgres per build, so a completed batch is out of one run's reach and the managed list never shows an earlier run's batch; the cell stays visible as a gap until a run can hand a completed batch to the poller"} +- {id: quota_management.spend_tracking.cost_matrix.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: cost_matrix, assertions: [logs_cost], exercised_on: [chat_completions], source: "litellm_core_utils/llm_cost_calc/utils.py", rationale: "A scripted-usage call through the cost-map proxy bills every reported token kind at the deployment's test-map rate (input, output, cache read, 5m/1h cache write, reasoning, audio, above-threshold tiers, flex/priority service tiers, web search, response-model override) and lands on the row's cost_breakdown, streamed or not"} +- {id: quota_management.spend_tracking.scripted_wire.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: scripted_wire, assertions: [logs_cost], exercised_on: [chat_completions], source: "litellm_core_utils/llm_cost_calc/utils.py", rationale: "Each provider wire shape (openai chat, responses, anthropic messages, gemini generateContent, together, fireworks) parses usage into the same spend components: the gross input cost is fresh tokens at the input rate plus each cache/audio component at its own rate, streamed anthropic included"} diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py index 896cb3e7efe..a891d9dcba2 100644 --- a/tests/e2e/e2e_config.py +++ b/tests/e2e/e2e_config.py @@ -145,6 +145,22 @@ WEEKLY_ANOMALY_OPT_IN_ENV = "E2E_WEEKLY_ANOMALY" MANAGED_FILES_OPT_IN_ENV = "E2E_MANAGED_FILES_STACK" PROMPT_CACHING_OPT_IN_ENV = "E2E_PROMPT_CACHING_STACK" REDIS_CHAOS_OPT_IN_ENV = "E2E_REDIS_CHAOS" +# The cost_calculation suite needs a proxy booted with LITELLM_MODEL_COST_MAP_URL +# pointing at tests/e2e/cost_map.json (its whole map is test-owned rates) plus a +# scripted-provider sidecar; deselected unless the opt-in env var is set. +COST_MAP_OPT_IN_ENV = "E2E_COST_MAP_STACK" +# Base URL of the proxy running the test cost map. Defaults to the shared proxy +# so a local run only has to set the opt-in and boot the proxy accordingly. +COST_MAP_PROXY_URL = os.environ.get("E2E_COST_MAP_PROXY_URL", PROXY_BASE_URL).rstrip("/") +# Where the test runner reaches the scripted-provider sidecar's control API. +SCRIPTED_PROVIDER_CONTROL_URL = os.environ.get( + "E2E_SCRIPTED_PROVIDER_CONTROL_URL", "http://127.0.0.1:9100" +).rstrip("/") +# The api_base root deployments register with: how the proxy (possibly in +# another container) reaches the sidecar's provider wire. +SCRIPTED_PROVIDER_PROXY_BASE = os.environ.get( + "E2E_SCRIPTED_PROVIDER_PROXY_BASE", SCRIPTED_PROVIDER_CONTROL_URL +).rstrip("/") ANOMALY_SESSIONS = int(os.environ.get("E2E_ANOMALY_SESSIONS", "6")) ANOMALY_TURNS_PER_SESSION = int(os.environ.get("E2E_ANOMALY_TURNS_PER_SESSION", "6")) ANOMALY_TURN_ATTEMPTS = int(os.environ.get("E2E_ANOMALY_TURN_ATTEMPTS", "3")) diff --git a/tests/e2e/pytest.ini b/tests/e2e/pytest.ini index 1fdd3bd28ad..7d37bcc6d3e 100644 --- a/tests/e2e/pytest.ini +++ b/tests/e2e/pytest.ini @@ -11,3 +11,4 @@ markers = managed_files: needs a proxy running with require_managed_files enabled; deselected unless E2E_MANAGED_FILES_STACK is set prompt_caching_stack: needs a proxy running with router_settings.optional_pre_call_checks including prompt_caching; deselected unless E2E_PROMPT_CACHING_STACK is set redis_chaos: load test that pauses the proxy's Redis outright mid-run; needs a proxy booted from gateway/redis_chaos_ci_config.yml on the same host, and is deselected unless E2E_REDIS_CHAOS is set + cost_map_stack: needs a proxy whose whole cost map is tests/e2e/cost_map.json (LITELLM_MODEL_COST_MAP_URL) plus a scripted-provider sidecar; deselected unless E2E_COST_MAP_STACK is set From 269afbe382df06d33780571a40a55e527afea2b7 Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 15 Sep 2026 23:28:44 +0000 Subject: [PATCH 003/210] test(e2e): apply review nits to cost calculation suite Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/cost_calculation/conftest.py | 28 +- tests/e2e/cost_calculation/cost_matrix.py | 174 ++-- tests/e2e/cost_calculation/scripted_client.py | 12 +- .../e2e/cost_calculation/scripted_provider.py | 803 +++++++++++------- .../test_token_pricing_e2e.py | 17 +- .../cost_calculation/test_wire_formats_e2e.py | 43 +- 6 files changed, 620 insertions(+), 457 deletions(-) diff --git a/tests/e2e/cost_calculation/conftest.py b/tests/e2e/cost_calculation/conftest.py index 1bba3d50e1d..345ca26f7e3 100644 --- a/tests/e2e/cost_calculation/conftest.py +++ b/tests/e2e/cost_calculation/conftest.py @@ -13,7 +13,7 @@ from __future__ import annotations import importlib.util import sys -from collections.abc import Callable +from collections.abc import Callable, Mapping from dataclasses import dataclass from pathlib import Path from types import ModuleType @@ -34,16 +34,16 @@ def _load_cost_rows() -> ModuleType: """Load quota_management/spend_tracking/cost_rows.py by path (the e2e tree has no package layout), the same trick the mcp suite uses for logging/datadog_reader.py.""" - path = ( + path: Final = ( Path(__file__).resolve().parent.parent / "quota_management" / "spend_tracking" / "cost_rows.py" ) - name = "e2e_spend_tracking_cost_rows" - spec = importlib.util.spec_from_file_location(name, path) + name: Final = "e2e_spend_tracking_cost_rows" + spec: Final = importlib.util.spec_from_file_location(name, path) assert spec is not None and spec.loader is not None - module = importlib.util.module_from_spec(spec) + module: Final = importlib.util.module_from_spec(spec) sys.modules[name] = module spec.loader.exec_module(module) return module @@ -59,7 +59,7 @@ class SpendCostBreakdown(Protocol): total_cost: float | None service_tier: str | None - def model_dump(self) -> dict[str, object]: ... + def model_dump(self) -> Mapping[str, object]: ... class SpendRowMetadata(Protocol): @@ -89,7 +89,9 @@ class CostRowsModule(Protocol): ] -cost_rows: Final[CostRowsModule] = cast(CostRowsModule, _load_cost_rows()) +cost_rows: Final[CostRowsModule] = cast( # cast-ok: cost_rows.py is loaded by path, so basedpyright has no importable name for it; its surface is declared in CostRowsModule + CostRowsModule, _load_cost_rows() +) @dataclass(frozen=True, slots=True) @@ -101,7 +103,7 @@ class CostCalcClient: @pytest.fixture(scope="session") def client() -> CostCalcClient: - proxy = build_proxy_client( + proxy: Final = build_proxy_client( base_url=COST_MAP_PROXY_URL, control_plane_base_url=COST_MAP_PROXY_URL, replica_urls=(COST_MAP_PROXY_URL,), @@ -118,18 +120,18 @@ def register_scenario_deployment( ) -> tuple[str, ScenarioHandle]: """Register the case's scenario on the sidecar plus a deployment pointed at it; both are torn down by ``resources``. Returns the callable model_name.""" - scenario: Scenario = case.scenario( + scenario: Final[Scenario] = case.scenario( scenario_id=f"sc-{marker}", model=model, text=f"scripted answer {marker}" ) - handle = register_scenario(scenario) + handle: Final = register_scenario(scenario) resources.defer(lambda: delete_scenario(handle)) - model_name = f"{model.model_name}-{marker}" - model_id = client.proxy.register_model( + model_name: Final = f"{model.model_name}-{marker}" + model_id: Final = client.proxy.register_model( ModelNewBody( model_name=model_name, litellm_params=LiteLLMParamsBody( model=model.litellm_model, - api_key="sk-scripted-provider", + api_key=model.api_key, api_base=handle.api_base(), ), model_info=ModelInfoBody(), diff --git a/tests/e2e/cost_calculation/cost_matrix.py b/tests/e2e/cost_calculation/cost_matrix.py index bc466d7d823..e8b1d249559 100644 --- a/tests/e2e/cost_calculation/cost_matrix.py +++ b/tests/e2e/cost_calculation/cost_matrix.py @@ -18,9 +18,11 @@ creation), the case is absent from the matrix rather than silently zero. from __future__ import annotations import json +from collections.abc import Mapping from dataclasses import dataclass from pathlib import Path -from typing import Final, Literal +from types import MappingProxyType +from typing import Final, Literal, TypeAlias from pydantic import BaseModel, ConfigDict, TypeAdapter @@ -64,8 +66,8 @@ class CostMapEntry(BaseModel): _COST_MAP_ADAPTER: Final = TypeAdapter(dict[str, CostMapEntry]) -_COST_MAP: Final[dict[str, CostMapEntry]] = _COST_MAP_ADAPTER.validate_python( - json.loads(COST_MAP_PATH.read_text()) +_COST_MAP: Final[Mapping[str, CostMapEntry]] = MappingProxyType( + _COST_MAP_ADAPTER.validate_python(json.loads(COST_MAP_PATH.read_text())) ) TIER_THRESHOLD_TOKENS: Final = 200_000 @@ -109,7 +111,7 @@ class FrontierModel: # Response-model override targets: emit a sibling's bare provider-facing name so # the biller's provider-prefixed lookup lands on that sibling's map key. -_OVERRIDE_MODELS: Final[dict[str, str]] = { +_OVERRIDE_MODELS: Final[Mapping[str, str]] = MappingProxyType({ "gpt-5.6": "gpt-5.4-mini", "gpt-5.5-pro": "gpt-5.3-codex", "gpt-5.3-codex": "gpt-5.5-pro", @@ -124,9 +126,9 @@ _OVERRIDE_MODELS: Final[dict[str, str]] = { "fireworks_ai/kimi-k3": "qwen3p8-max", "fireworks_ai/qwen3p8-max": "kimi-k3", "fireworks_ai/deepseek-v4p1-flash": "kimi-k3", -} +}) -_OVERRIDE_MAP_KEYS: Final[dict[str, str]] = { +_OVERRIDE_MAP_KEYS: Final[Mapping[str, str]] = MappingProxyType({ "gpt-5.4-mini": "gpt-5.4-mini", "gpt-5.6": "gpt-5.6", "gpt-5.3-codex": "gpt-5.3-codex", @@ -139,7 +141,7 @@ _OVERRIDE_MAP_KEYS: Final[dict[str, str]] = { "moonshotai/Kimi-K3": "together_ai/moonshotai/Kimi-K3", "qwen3p8-max": "fireworks_ai/qwen3p8-max", "kimi-k3": "fireworks_ai/kimi-k3", -} +}) _FRONTIER_SPECS: Final[tuple[tuple[str, str, Wire], ...]] = ( @@ -176,7 +178,7 @@ def _frontier() -> tuple[FrontierModel, ...]: FRONTIER_MODELS: Final[tuple[FrontierModel, ...]] = _frontier() # Token kinds each wire can report, gating which pricing cases apply. -_WIRE_CAPS: Final[dict[str, frozenset[str]]] = { +_WIRE_CAPS: Final[Mapping[str, frozenset[str]]] = MappingProxyType({ "openai_chat": frozenset( { "cache_read", "cache_write_5m", "cache_write_1h", "reasoning", "audio", @@ -205,9 +207,9 @@ _WIRE_CAPS: Final[dict[str, frozenset[str]]] = { "web_search", "response_model", "absent_usage", } ), -} +}) -CaseName = Literal[ +CaseName: TypeAlias = Literal[ "basic", "cache_read", "cache_write_5m", @@ -260,7 +262,7 @@ _BASIC_USAGE: Final = ScriptedUsage(fresh_input_tokens=120, output_tokens=40) def _web_search_case(model: FrontierModel) -> Case: - counts_exactly = model.wire in ("openai_responses", "anthropic_messages", "gemini_generate") + counts_exactly: Final = model.wire in ("openai_responses", "anthropic_messages", "gemini_generate") return Case( name="web_search", usage=ScriptedUsage(fresh_input_tokens=100, output_tokens=30, web_search_calls=3), @@ -269,26 +271,24 @@ def _web_search_case(model: FrontierModel) -> Case: def cases_for(model: FrontierModel) -> tuple[Case, ...]: - rates = model.rates - caps = _WIRE_CAPS[model.wire] - cases: list[Case] = [Case(name="basic", usage=_BASIC_USAGE)] - if rates.cache_read_input_token_cost is not None and "cache_read" in caps: - cases.append( + rates: Final = model.rates + caps: Final = _WIRE_CAPS[model.wire] + candidates: Final[tuple[Case | None, ...]] = ( + Case(name="basic", usage=_BASIC_USAGE), + ( Case(name="cache_read", usage=ScriptedUsage(fresh_input_tokens=100, cache_read_tokens=50, output_tokens=30)) - ) - if rates.cache_creation_input_token_cost is not None and "cache_write_5m" in caps: - cases.append( + if rates.cache_read_input_token_cost is not None and "cache_read" in caps + else None + ), + ( Case( name="cache_write_5m", usage=ScriptedUsage(fresh_input_tokens=90, cache_write_5m_tokens=60, output_tokens=30), ) - ) - if ( - rates.cache_creation_input_token_cost_above_1hr is not None - and rates.cache_creation_input_token_cost is not None - and "cache_write_1h" in caps - ): - cases.append( + if rates.cache_creation_input_token_cost is not None and "cache_write_5m" in caps + else None + ), + ( Case( name="cache_write_1h", usage=ScriptedUsage( @@ -298,52 +298,61 @@ def cases_for(model: FrontierModel) -> tuple[Case, ...]: output_tokens=30, ), ) - ) - if rates.output_cost_per_reasoning_token is not None and "reasoning" in caps: - cases.append( + if ( + rates.cache_creation_input_token_cost_above_1hr is not None + and rates.cache_creation_input_token_cost is not None + and "cache_write_1h" in caps + ) + else None + ), + ( Case( name="reasoning", usage=ScriptedUsage(fresh_input_tokens=100, output_tokens=30, reasoning_tokens=70), ) - ) - if ( - rates.input_cost_per_audio_token is not None - and rates.output_cost_per_audio_token is not None - and "audio" in caps - ): - cases.append( + if rates.output_cost_per_reasoning_token is not None and "reasoning" in caps + else None + ), + ( Case( name="audio", usage=ScriptedUsage( fresh_input_tokens=100, audio_input_tokens=25, output_tokens=30, audio_output_tokens=15 ), ) - ) - if ( - rates.input_cost_per_token_above_200k_tokens is not None - and rates.output_cost_per_token_above_200k_tokens is not None - ): - cases.append( + if ( + rates.input_cost_per_audio_token is not None + and rates.output_cost_per_audio_token is not None + and "audio" in caps + ) + else None + ), + ( Case( name="tiered", usage=ScriptedUsage( fresh_input_tokens=TIER_THRESHOLD_TOKENS + 1, output_tokens=30 ), ) - ) - if rates.input_cost_per_token_flex is not None and rates.output_cost_per_token_flex is not None: - cases.append( + if ( + rates.input_cost_per_token_above_200k_tokens is not None + and rates.output_cost_per_token_above_200k_tokens is not None + ) + else None + ), + ( Case(name="service_tier_flex", usage=_BASIC_USAGE, service_tier="flex") - ) - if rates.input_cost_per_token_priority is not None and rates.output_cost_per_token_priority is not None: - cases.append( + if rates.input_cost_per_token_flex is not None and rates.output_cost_per_token_flex is not None + else None + ), + ( Case(name="service_tier_priority", usage=_BASIC_USAGE, service_tier="priority") - ) - if rates.search_context_cost_per_query is not None and "web_search" in caps: - cases.append(_web_search_case(model)) - cases.append(Case(name="stream", usage=_BASIC_USAGE, stream=True)) - if "absent_usage" in caps: - cases.append( + if rates.input_cost_per_token_priority is not None and rates.output_cost_per_token_priority is not None + else None + ), + _web_search_case(model) if rates.search_context_cost_per_query is not None and "web_search" in caps else None, + Case(name="stream", usage=_BASIC_USAGE, stream=True), + ( Case( name="stream_no_usage", usage=_BASIC_USAGE, @@ -355,10 +364,16 @@ def cases_for(model: FrontierModel) -> tuple[Case, ...]: # wires recount tokens proxy-side and bill a nonzero amount. expect_zero_bill=model.wire == "openai_responses", ) - ) - if "response_model" in caps: - cases.append(Case(name="response_model_override", usage=_BASIC_USAGE, response_model_override=True)) - return tuple(cases) + if "absent_usage" in caps + else None + ), + ( + Case(name="response_model_override", usage=_BASIC_USAGE, response_model_override=True) + if "response_model" in caps + else None + ), + ) + return tuple(case for case in candidates if case is not None) @dataclass(frozen=True, slots=True) @@ -387,38 +402,41 @@ def expected_breakdown(model: FrontierModel, case: Case) -> ExpectedCost: to the tier's variants, falling back to the base rate when a variant is unset -- mirroring _get_token_base_cost in litellm's cost calculator. """ - rates = model.override_rates if case.response_model_override else model.rates - u = case.usage - prompt_tokens = ( + rates: Final = model.override_rates if case.response_model_override else model.rates + u: Final = case.usage + prompt_tokens: Final = ( u.fresh_input_tokens + u.cache_read_tokens + u.cache_write_5m_tokens + u.cache_write_1h_tokens + u.audio_input_tokens ) - tiered = prompt_tokens > TIER_THRESHOLD_TOKENS - in_rate = rates.input_cost_per_token or 0.0 - out_rate = rates.output_cost_per_token or 0.0 - if case.service_tier == "flex": - in_rate = rates.input_cost_per_token_flex or in_rate - out_rate = rates.output_cost_per_token_flex or out_rate - if case.service_tier == "priority": - in_rate = rates.input_cost_per_token_priority or in_rate - out_rate = rates.output_cost_per_token_priority or out_rate - if tiered: - in_rate = rates.input_cost_per_token_above_200k_tokens or in_rate - out_rate = rates.output_cost_per_token_above_200k_tokens or out_rate - input_cost = ( + tiered: Final = prompt_tokens > TIER_THRESHOLD_TOKENS + in_rate: Final = ( + (rates.input_cost_per_token_above_200k_tokens if tiered else None) + or (rates.input_cost_per_token_priority if case.service_tier == "priority" else None) + or (rates.input_cost_per_token_flex if case.service_tier == "flex" else None) + or rates.input_cost_per_token + or 0.0 + ) + out_rate: Final = ( + (rates.output_cost_per_token_above_200k_tokens if tiered else None) + or (rates.output_cost_per_token_priority if case.service_tier == "priority" else None) + or (rates.output_cost_per_token_flex if case.service_tier == "flex" else None) + or rates.output_cost_per_token + or 0.0 + ) + input_cost: Final = ( u.fresh_input_tokens * in_rate + u.cache_read_tokens * (rates.cache_read_input_token_cost or 0.0) + u.cache_write_5m_tokens * (rates.cache_creation_input_token_cost or 0.0) + u.cache_write_1h_tokens * (rates.cache_creation_input_token_cost_above_1hr or 0.0) + u.audio_input_tokens * (rates.input_cost_per_audio_token or 0.0) ) - output_cost = ( + output_cost: Final = ( u.output_tokens * out_rate + u.reasoning_tokens * (rates.output_cost_per_reasoning_token or out_rate) + u.audio_output_tokens * (rates.output_cost_per_audio_token or out_rate) ) - search = rates.search_context_cost_per_query - tool_cost = case.billed_web_search_calls * ( + search: Final = rates.search_context_cost_per_query + tool_cost: Final = case.billed_web_search_calls * ( search.search_context_size_medium if search and search.search_context_size_medium else 0.0 ) return ExpectedCost(input_cost=input_cost, output_cost=output_cost, tool_cost=tool_cost) @@ -432,7 +450,7 @@ def expected_token_columns(model: FrontierModel, case: Case) -> tuple[int, int]: """(prompt_tokens, completion_tokens) the spend row should carry, per the wire's normalization: Anthropic folds cache read/write into prompt_tokens, everyone else reports the totals the wire emitted.""" - u = case.usage + u: Final = case.usage if model.wire == "anthropic_messages": return ( u.fresh_input_tokens + u.cache_read_tokens + u.cache_write_5m_tokens + u.cache_write_1h_tokens, diff --git a/tests/e2e/cost_calculation/scripted_client.py b/tests/e2e/cost_calculation/scripted_client.py index dceec02630a..9dbf9c98986 100644 --- a/tests/e2e/cost_calculation/scripted_client.py +++ b/tests/e2e/cost_calculation/scripted_client.py @@ -12,6 +12,7 @@ from e2e_config import SCRIPTED_PROVIDER_CONTROL_URL, SCRIPTED_PROVIDER_PROXY_BA from e2e_http import URL, NoBody, unwrap, post from e2e_http import delete as http_delete from scripted_provider import ( + WIRE_MOUNTS, Scenario, ScenarioDeleted, ScenarioRegistered, @@ -29,19 +30,12 @@ class ScenarioHandle: return f"{self.proxy_base}/{self.scenario_id}/{self._mount()}" def _mount(self) -> str: - return { - "openai_chat": "openai", - "openai_responses": "openai", - "anthropic_messages": "anthropic", - "gemini_generate": "gemini", - "together_chat": "together", - "fireworks_chat": "fireworks", - }[self.wire] + return WIRE_MOUNTS[self.wire] def register_scenario(scenario: Scenario) -> ScenarioHandle: """POST the scenario to the sidecar's control API and return its handle.""" - result = unwrap( + result: Final = unwrap( post( URL(f"{SCRIPTED_PROVIDER_CONTROL_URL}/_scenarios"), headers=NoBody(), diff --git a/tests/e2e/cost_calculation/scripted_provider.py b/tests/e2e/cost_calculation/scripted_provider.py index 93a6f49ec25..f1deafd1bc5 100644 --- a/tests/e2e/cost_calculation/scripted_provider.py +++ b/tests/e2e/cost_calculation/scripted_provider.py @@ -31,14 +31,16 @@ import json import sys import threading import time +from collections.abc import Mapping from dataclasses import dataclass from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from typing import Final, Literal +from types import MappingProxyType +from typing import Final, Literal, TypeAlias from urllib.parse import urlsplit from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError -Wire = Literal[ +Wire: TypeAlias = Literal[ "openai_chat", "openai_responses", "anthropic_messages", @@ -47,17 +49,19 @@ Wire = Literal[ "fireworks_chat", ] -_WIRE_MOUNTS: Final[dict[str, str]] = { - "openai_chat": "openai", - "openai_responses": "openai", - "anthropic_messages": "anthropic", - "gemini_generate": "gemini", - "together_chat": "together", - "fireworks_chat": "fireworks", -} +WIRE_MOUNTS: Final[Mapping[str, str]] = MappingProxyType( + { + "openai_chat": "openai", + "openai_responses": "openai", + "anthropic_messages": "anthropic", + "gemini_generate": "gemini", + "together_chat": "together", + "fireworks_chat": "fireworks", + } +) -StreamUsage = Literal["final_chunk", "absent"] -ServiceTier = Literal["flex", "priority"] +StreamUsage: TypeAlias = Literal["final_chunk", "absent"] +ServiceTier: TypeAlias = Literal["flex", "priority"] class ScriptedUsage(BaseModel): @@ -106,7 +110,7 @@ class Scenario(BaseModel): @property def mount(self) -> str: - return _WIRE_MOUNTS[self.wire] + return WIRE_MOUNTS[self.wire] class ScenarioRegistered(BaseModel): @@ -128,369 +132,494 @@ class RenderedResponse: body: bytes -def _json_bytes(payload: dict[str, object]) -> bytes: - return json.dumps(payload).encode("utf-8") +def _jobj(*pairs: tuple[str, object]) -> Mapping[str, object]: + """A JSON object payload built in one shot and frozen.""" + return MappingProxyType(dict(pairs)) -def _sse(events: tuple[tuple[str | None, dict[str, object] | str], ...]) -> bytes: - frames: list[str] = [] - for event_name, data in events: - head = f"event: {event_name}\n" if event_name is not None else "" - payload = data if isinstance(data, str) else json.dumps(data) - frames.append(f"{head}data: {payload}\n\n") - return "".join(frames).encode("utf-8") +def _jobj_opt(*pairs: tuple[str, object] | None) -> Mapping[str, object]: + """``_jobj`` where a ``None`` pair means the field is absent.""" + return MappingProxyType(dict(pair for pair in pairs if pair is not None)) + + +def _json_bytes(payload: Mapping[str, object]) -> bytes: + return json.dumps(payload, default=dict).encode("utf-8") + + +def _sse_frame(event_name: str | None, data: Mapping[str, object] | str) -> str: + head: Final = f"event: {event_name}\n" if event_name is not None else "" + payload: Final = data if isinstance(data, str) else json.dumps(data, default=dict) + return f"{head}data: {payload}\n\n" + + +def _sse(events: tuple[tuple[str | None, Mapping[str, object] | str], ...]) -> bytes: + return "".join(_sse_frame(event_name, data) for event_name, data in events).encode("utf-8") # ---------- per-wire usage shapes ---------- -def _openai_usage(u: ScriptedUsage) -> dict[str, object]: - prompt_tokens = ( +def _openai_usage(u: ScriptedUsage) -> Mapping[str, object]: + prompt_tokens: Final = ( u.fresh_input_tokens + u.cache_read_tokens + u.cache_write_5m_tokens + u.cache_write_1h_tokens + u.audio_input_tokens ) - completion_tokens = u.output_tokens + u.reasoning_tokens + u.audio_output_tokens - prompt_details: dict[str, object] = {} - if u.cache_read_tokens: - prompt_details["cached_tokens"] = u.cache_read_tokens - if u.cache_write_5m_tokens or u.cache_write_1h_tokens: - prompt_details["cache_write_tokens"] = u.cache_write_5m_tokens + u.cache_write_1h_tokens - prompt_details["cache_creation_token_details"] = { - "ephemeral_5m_input_tokens": u.cache_write_5m_tokens, - "ephemeral_1h_input_tokens": u.cache_write_1h_tokens, - } - if u.audio_input_tokens: - prompt_details["audio_tokens"] = u.audio_input_tokens - completion_details: dict[str, object] = {} - if u.reasoning_tokens: - completion_details["reasoning_tokens"] = u.reasoning_tokens - if u.audio_output_tokens: - completion_details["audio_tokens"] = u.audio_output_tokens - usage: dict[str, object] = { - "prompt_tokens": prompt_tokens, - "completion_tokens": completion_tokens, - "total_tokens": prompt_tokens + completion_tokens, - } - if prompt_details: - usage["prompt_tokens_details"] = prompt_details - if completion_details: - usage["completion_tokens_details"] = completion_details - return usage + completion_tokens: Final = u.output_tokens + u.reasoning_tokens + u.audio_output_tokens + prompt_details: Final = _jobj_opt( + ("cached_tokens", u.cache_read_tokens) if u.cache_read_tokens else None, + ( + ("cache_write_tokens", u.cache_write_5m_tokens + u.cache_write_1h_tokens) + if u.cache_write_5m_tokens or u.cache_write_1h_tokens + else None + ), + ( + ( + "cache_creation_token_details", + _jobj( + ("ephemeral_5m_input_tokens", u.cache_write_5m_tokens), + ("ephemeral_1h_input_tokens", u.cache_write_1h_tokens), + ), + ) + if u.cache_write_5m_tokens or u.cache_write_1h_tokens + else None + ), + ("audio_tokens", u.audio_input_tokens) if u.audio_input_tokens else None, + ) + completion_details: Final = _jobj_opt( + ("reasoning_tokens", u.reasoning_tokens) if u.reasoning_tokens else None, + ("audio_tokens", u.audio_output_tokens) if u.audio_output_tokens else None, + ) + return _jobj_opt( + ("prompt_tokens", prompt_tokens), + ("completion_tokens", completion_tokens), + ("total_tokens", prompt_tokens + completion_tokens), + ("prompt_tokens_details", prompt_details) if prompt_details else None, + ("completion_tokens_details", completion_details) if completion_details else None, + ) -def _anthropic_usage(u: ScriptedUsage) -> dict[str, object]: +def _anthropic_usage(u: ScriptedUsage) -> Mapping[str, object]: # Anthropic reports uncached-only input_tokens; cache reads and writes ride # top-level fields, with the 5m/1h write split under cache_creation. - usage: dict[str, object] = { - "input_tokens": u.fresh_input_tokens, - "output_tokens": u.output_tokens, - } - if u.cache_read_tokens: - usage["cache_read_input_tokens"] = u.cache_read_tokens - if u.cache_write_5m_tokens or u.cache_write_1h_tokens: - usage["cache_creation_input_tokens"] = u.cache_write_5m_tokens + u.cache_write_1h_tokens - usage["cache_creation"] = { - "ephemeral_5m_input_tokens": u.cache_write_5m_tokens, - "ephemeral_1h_input_tokens": u.cache_write_1h_tokens, - } - if u.web_search_calls: - usage["server_tool_use"] = {"web_search_requests": u.web_search_calls} - return usage + return _jobj_opt( + ("input_tokens", u.fresh_input_tokens), + ("output_tokens", u.output_tokens), + ("cache_read_input_tokens", u.cache_read_tokens) if u.cache_read_tokens else None, + ( + ("cache_creation_input_tokens", u.cache_write_5m_tokens + u.cache_write_1h_tokens) + if u.cache_write_5m_tokens or u.cache_write_1h_tokens + else None + ), + ( + ( + "cache_creation", + _jobj( + ("ephemeral_5m_input_tokens", u.cache_write_5m_tokens), + ("ephemeral_1h_input_tokens", u.cache_write_1h_tokens), + ), + ) + if u.cache_write_5m_tokens or u.cache_write_1h_tokens + else None + ), + ( + ("server_tool_use", _jobj(("web_search_requests", u.web_search_calls))) + if u.web_search_calls + else None + ), + ) -def _gemini_usage(u: ScriptedUsage) -> dict[str, object]: +def _gemini_usage(u: ScriptedUsage) -> Mapping[str, object]: # promptTokenCount carries the cached count inside it; TEXT modality is the # cached-inclusive text count so litellm's implicit-caching subtraction lands # on the fresh figure. candidatesTokenCount includes reasoning + audio. - prompt_tokens = u.fresh_input_tokens + u.cache_read_tokens + u.audio_input_tokens - candidates = u.output_tokens + u.reasoning_tokens + u.audio_output_tokens - usage: dict[str, object] = { - "promptTokenCount": prompt_tokens, - "candidatesTokenCount": candidates, - "totalTokenCount": prompt_tokens + candidates, - } - if u.cache_read_tokens: - usage["cachedContentTokenCount"] = u.cache_read_tokens - if u.reasoning_tokens: - usage["thoughtsTokenCount"] = u.reasoning_tokens - prompt_details = [{"modality": "TEXT", "tokenCount": u.fresh_input_tokens + u.cache_read_tokens}] - if u.audio_input_tokens: - prompt_details.append({"modality": "AUDIO", "tokenCount": u.audio_input_tokens}) - usage["promptTokensDetails"] = prompt_details - if u.audio_output_tokens: - usage["candidatesTokensDetails"] = [ - {"modality": "TEXT", "tokenCount": u.output_tokens + u.reasoning_tokens}, - {"modality": "AUDIO", "tokenCount": u.audio_output_tokens}, - ] - return usage + prompt_tokens: Final = u.fresh_input_tokens + u.cache_read_tokens + u.audio_input_tokens + candidates: Final = u.output_tokens + u.reasoning_tokens + u.audio_output_tokens + return _jobj_opt( + ("promptTokenCount", prompt_tokens), + ("candidatesTokenCount", candidates), + ("totalTokenCount", prompt_tokens + candidates), + ("cachedContentTokenCount", u.cache_read_tokens) if u.cache_read_tokens else None, + ("thoughtsTokenCount", u.reasoning_tokens) if u.reasoning_tokens else None, + ( + "promptTokensDetails", + ( + _jobj(("modality", "TEXT"), ("tokenCount", u.fresh_input_tokens + u.cache_read_tokens)), + *( + (_jobj(("modality", "AUDIO"), ("tokenCount", u.audio_input_tokens)),) + if u.audio_input_tokens + else () + ), + ), + ), + ( + ( + "candidatesTokensDetails", + ( + _jobj(("modality", "TEXT"), ("tokenCount", u.output_tokens + u.reasoning_tokens)), + _jobj(("modality", "AUDIO"), ("tokenCount", u.audio_output_tokens)), + ), + ) + if u.audio_output_tokens + else None + ), + ) -def _responses_usage(u: ScriptedUsage) -> dict[str, object]: - input_tokens = u.fresh_input_tokens + u.cache_read_tokens + u.audio_input_tokens - output_tokens = u.output_tokens + u.reasoning_tokens + u.audio_output_tokens - usage: dict[str, object] = { - "input_tokens": input_tokens, - "output_tokens": output_tokens, - "total_tokens": input_tokens + output_tokens, - } - input_details: dict[str, object] = {} - if u.cache_read_tokens: - input_details["cached_tokens"] = u.cache_read_tokens - if input_details: - usage["input_tokens_details"] = input_details - if u.reasoning_tokens: - usage["output_tokens_details"] = {"reasoning_tokens": u.reasoning_tokens} - return usage +def _responses_usage(u: ScriptedUsage) -> Mapping[str, object]: + input_tokens: Final = u.fresh_input_tokens + u.cache_read_tokens + u.audio_input_tokens + output_tokens: Final = u.output_tokens + u.reasoning_tokens + u.audio_output_tokens + input_details: Final = _jobj_opt( + ("cached_tokens", u.cache_read_tokens) if u.cache_read_tokens else None, + ) + return _jobj_opt( + ("input_tokens", input_tokens), + ("output_tokens", output_tokens), + ("total_tokens", input_tokens + output_tokens), + ("input_tokens_details", input_details) if input_details else None, + ( + ("output_tokens_details", _jobj(("reasoning_tokens", u.reasoning_tokens))) + if u.reasoning_tokens + else None + ), + ) # ---------- per-wire responses ---------- -def _openai_message(scenario: Scenario) -> dict[str, object]: - message: dict[str, object] = {"role": "assistant", "content": scenario.output.text} - if scenario.usage.web_search_calls: - message["annotations"] = [ - { - "type": "url_citation", - "url_citation": { - "url": "https://scripted.example/source", - "title": "scripted source", - "start_index": 0, - "end_index": 1, - }, - } - for _ in range(scenario.usage.web_search_calls) - ] - return message +def _openai_message(scenario: Scenario) -> Mapping[str, object]: + return _jobj_opt( + ("role", "assistant"), + ("content", scenario.output.text), + ( + ( + "annotations", + tuple( + _jobj( + ("type", "url_citation"), + ( + "url_citation", + _jobj( + ("url", "https://scripted.example/source"), + ("title", "scripted source"), + ("start_index", 0), + ("end_index", 1), + ), + ), + ) + for _ in range(scenario.usage.web_search_calls) + ), + ) + if scenario.usage.web_search_calls + else None + ), + ) -def _openai_chat_body(scenario: Scenario, requested_model: str) -> dict[str, object]: - body: dict[str, object] = { - "id": f"chatcmpl-{scenario.scenario_id}", - "object": "chat.completion", - "created": int(time.time()), - "model": scenario.output.response_model or requested_model, - "choices": [ - { - "index": 0, - "message": _openai_message(scenario), - "finish_reason": scenario.output.finish_reason, - } - ], - "usage": _openai_usage(scenario.usage), - } - if scenario.service_tier is not None: - body["service_tier"] = scenario.service_tier - if scenario.output.provider_cost is not None: - body["cost"] = scenario.output.provider_cost - return body +def _openai_chat_body(scenario: Scenario, requested_model: str) -> Mapping[str, object]: + return _jobj_opt( + ("id", f"chatcmpl-{scenario.scenario_id}"), + ("object", "chat.completion"), + ("created", int(time.time())), + ("model", scenario.output.response_model or requested_model), + ( + "choices", + ( + _jobj( + ("index", 0), + ("message", _openai_message(scenario)), + ("finish_reason", scenario.output.finish_reason), + ), + ), + ), + ("usage", _openai_usage(scenario.usage)), + ("service_tier", scenario.service_tier) if scenario.service_tier is not None else None, + ("cost", scenario.output.provider_cost) if scenario.output.provider_cost is not None else None, + ) -def _openai_chunk(scenario: Scenario, requested_model: str, **kw: object) -> dict[str, object]: - chunk: dict[str, object] = { - "id": f"chatcmpl-{scenario.scenario_id}", - "object": "chat.completion.chunk", - "created": int(time.time()), - "model": scenario.output.response_model or requested_model, - } - chunk.update(kw) - return chunk +def _openai_chunk( + scenario: Scenario, + requested_model: str, + choices: tuple[Mapping[str, object], ...] = (), + usage: Mapping[str, object] | None = None, +) -> Mapping[str, object]: + return _jobj_opt( + ("id", f"chatcmpl-{scenario.scenario_id}"), + ("object", "chat.completion.chunk"), + ("created", int(time.time())), + ("model", scenario.output.response_model or requested_model), + ("choices", choices), + ("usage", usage), + ) def _openai_chat_sse(scenario: Scenario, requested_model: str) -> bytes: - _EMPTY_DELTA: Final[dict[str, object]] = {} - delta: dict[str, object] = {"role": "assistant", "content": scenario.output.text} - if scenario.usage.web_search_calls: - delta["annotations"] = _openai_message(scenario)["annotations"] - events: list[tuple[str | None, dict[str, object] | str]] = [ + delta: Final = _jobj_opt( + ("role", "assistant"), + ("content", scenario.output.text), ( - None, - _openai_chunk( - scenario, - requested_model, - choices=[{"index": 0, "delta": {"role": "assistant"}, "finish_reason": None}], - ), + ("annotations", _openai_message(scenario)["annotations"]) + if scenario.usage.web_search_calls + else None ), + ) + return _sse( ( - None, - _openai_chunk( - scenario, - requested_model, - choices=[{"index": 0, "delta": delta, "finish_reason": None}], + ( + None, + _openai_chunk( + scenario, + requested_model, + choices=(_jobj(("index", 0), ("delta", _jobj(("role", "assistant"))), ("finish_reason", None)),), + ), ), - ), - ( - None, - _openai_chunk( - scenario, - requested_model, - choices=[ - { - "index": 0, - "delta": _EMPTY_DELTA, - "finish_reason": scenario.output.finish_reason, - } - ], + ( + None, + _openai_chunk( + scenario, + requested_model, + choices=(_jobj(("index", 0), ("delta", delta), ("finish_reason", None)),), + ), ), - ), - ] - if scenario.stream_usage == "final_chunk": - events.append( - (None, _openai_chunk(scenario, requested_model, choices=(), usage=_openai_usage(scenario.usage))) + ( + None, + _openai_chunk( + scenario, + requested_model, + choices=( + _jobj( + ("index", 0), + ("delta", _jobj()), + ("finish_reason", scenario.output.finish_reason), + ), + ), + ), + ), + *( + ((None, _openai_chunk(scenario, requested_model, usage=_openai_usage(scenario.usage))),) + if scenario.stream_usage == "final_chunk" + else () + ), + (None, "[DONE]"), ) - events.append((None, "[DONE]")) - return _sse(tuple(events)) + ) -def _anthropic_body(scenario: Scenario, requested_model: str) -> dict[str, object]: - return { - "id": f"msg_{scenario.scenario_id}", - "type": "message", - "role": "assistant", - "model": scenario.output.response_model or requested_model, - "content": [{"type": "text", "text": scenario.output.text}], - "stop_reason": "end_turn" if scenario.output.finish_reason == "stop" else scenario.output.finish_reason, - "usage": _anthropic_usage(scenario.usage), - } +def _anthropic_body(scenario: Scenario, requested_model: str) -> Mapping[str, object]: + return _jobj( + ("id", f"msg_{scenario.scenario_id}"), + ("type", "message"), + ("role", "assistant"), + ("model", scenario.output.response_model or requested_model), + ("content", (_jobj(("type", "text"), ("text", scenario.output.text)),)), + ( + "stop_reason", + "end_turn" if scenario.output.finish_reason == "stop" else scenario.output.finish_reason, + ), + ("usage", _anthropic_usage(scenario.usage)), + ) def _anthropic_sse(scenario: Scenario, requested_model: str) -> bytes: - emit_usage = scenario.stream_usage == "final_chunk" - input_usage = {k: v for k, v in _anthropic_usage(scenario.usage).items() if k != "output_tokens"} - message_start: dict[str, object] = { - "type": "message_start", - "message": { - "id": f"msg_{scenario.scenario_id}", - "type": "message", - "role": "assistant", - "model": scenario.output.response_model or requested_model, - "content": [], - "stop_reason": None, - **({"usage": input_usage} if emit_usage else {}), - }, - } - message_delta: dict[str, object] = { - "type": "message_delta", - "delta": { - "stop_reason": "end_turn" if scenario.output.finish_reason == "stop" else scenario.output.finish_reason - }, - **({"usage": {"output_tokens": scenario.usage.output_tokens}} if emit_usage else {}), - } + emit_usage: Final = scenario.stream_usage == "final_chunk" + input_usage: Final = _jobj( + *( + (key, value) + for key, value in _anthropic_usage(scenario.usage).items() + if key != "output_tokens" + ) + ) + message_start: Final = _jobj( + ("type", "message_start"), + ( + "message", + _jobj_opt( + ("id", f"msg_{scenario.scenario_id}"), + ("type", "message"), + ("role", "assistant"), + ("model", scenario.output.response_model or requested_model), + ("content", ()), + ("stop_reason", None), + ("usage", input_usage) if emit_usage else None, + ), + ), + ) + message_delta: Final = _jobj_opt( + ("type", "message_delta"), + ( + "delta", + _jobj( + ( + "stop_reason", + "end_turn" if scenario.output.finish_reason == "stop" else scenario.output.finish_reason, + ) + ), + ), + ( + ("usage", _jobj(("output_tokens", scenario.usage.output_tokens))) + if emit_usage + else None + ), + ) return _sse( ( ("message_start", message_start), ( "content_block_start", - { - "type": "content_block_start", - "index": 0, - "content_block": {"type": "text", "text": ""}, - }, + _jobj( + ("type", "content_block_start"), + ("index", 0), + ("content_block", _jobj(("type", "text"), ("text", ""))), + ), ), ( "content_block_delta", - { - "type": "content_block_delta", - "index": 0, - "delta": {"type": "text_delta", "text": scenario.output.text}, - }, + _jobj( + ("type", "content_block_delta"), + ("index", 0), + ("delta", _jobj(("type", "text_delta"), ("text", scenario.output.text))), + ), ), - ("content_block_stop", {"type": "content_block_stop", "index": 0}), + ("content_block_stop", _jobj(("type", "content_block_stop"), ("index", 0))), ("message_delta", message_delta), - ("message_stop", {"type": "message_stop"}), + ("message_stop", _jobj(("type", "message_stop"))), ) ) -def _gemini_body(scenario: Scenario, requested_model: str) -> dict[str, object]: - candidate: dict[str, object] = { - "content": {"parts": [{"text": scenario.output.text}], "role": "model"}, - "finishReason": "STOP" if scenario.output.finish_reason == "stop" else scenario.output.finish_reason.upper(), - "index": 0, - } - if scenario.usage.web_search_calls: - candidate["groundingMetadata"] = { - "webSearchQueries": [f"query {i}" for i in range(scenario.usage.web_search_calls)] - } - return { - "candidates": [candidate], - "usageMetadata": _gemini_usage(scenario.usage), - "modelVersion": scenario.output.response_model or requested_model, - } +def _gemini_body(scenario: Scenario, requested_model: str) -> Mapping[str, object]: + return _jobj( + ( + "candidates", + ( + _jobj_opt( + ( + "content", + _jobj( + ("parts", (_jobj(("text", scenario.output.text)),)), + ("role", "model"), + ), + ), + ( + "finishReason", + "STOP" if scenario.output.finish_reason == "stop" else scenario.output.finish_reason.upper(), + ), + ("index", 0), + ( + ( + "groundingMetadata", + _jobj( + ( + "webSearchQueries", + tuple(f"query {i}" for i in range(scenario.usage.web_search_calls)), + ) + ), + ) + if scenario.usage.web_search_calls + else None + ), + ), + ), + ), + ("usageMetadata", _gemini_usage(scenario.usage)), + ("modelVersion", scenario.output.response_model or requested_model), + ) def _gemini_sse(scenario: Scenario, requested_model: str) -> bytes: - first = _gemini_body(scenario, requested_model) - if scenario.stream_usage == "absent": - first = {k: v for k, v in first.items() if k != "usageMetadata"} - events: list[tuple[str | None, dict[str, object] | str]] = [(None, first)] - if scenario.stream_usage == "final_chunk": - events.append( - ( - None, - { - "candidates": [], - "usageMetadata": _gemini_usage(scenario.usage), - "modelVersion": scenario.output.response_model or requested_model, - }, - ) - ) - return _sse(tuple(events)) - - -def _responses_body(scenario: Scenario, requested_model: str) -> dict[str, object]: - output: list[dict[str, object]] = [ - {"type": "web_search_call", "id": f"ws_{i}", "status": "completed"} - for i in range(scenario.usage.web_search_calls) - ] - output.append( - { - "type": "message", - "id": f"msg_{scenario.scenario_id}", - "status": "completed", - "role": "assistant", - "content": [ - { - "type": "output_text", - "text": scenario.output.text, - "annotations": [], - } - ], - } + emit_usage: Final = scenario.stream_usage == "final_chunk" + first: Final = ( + _jobj(*((key, value) for key, value in _gemini_body(scenario, requested_model).items() if key != "usageMetadata")) + if scenario.stream_usage == "absent" + else _gemini_body(scenario, requested_model) + ) + return _sse( + ( + (None, first), + *( + ( + ( + None, + _jobj( + ("candidates", ()), + ("usageMetadata", _gemini_usage(scenario.usage)), + ("modelVersion", scenario.output.response_model or requested_model), + ), + ), + ) + if emit_usage + else () + ), + ) + ) + + +def _responses_body(scenario: Scenario, requested_model: str) -> Mapping[str, object]: + return _jobj( + ("id", f"resp_{scenario.scenario_id}"), + ("object", "response"), + ("created_at", int(time.time())), + ("status", "completed"), + ("model", scenario.output.response_model or requested_model), + ( + "output", + ( + *( + _jobj(("type", "web_search_call"), ("id", f"ws_{i}"), ("status", "completed")) + for i in range(scenario.usage.web_search_calls) + ), + _jobj( + ("type", "message"), + ("id", f"msg_{scenario.scenario_id}"), + ("status", "completed"), + ("role", "assistant"), + ( + "content", + ( + _jobj( + ("type", "output_text"), + ("text", scenario.output.text), + ("annotations", ()), + ), + ), + ), + ), + ), + ), + ("usage", _responses_usage(scenario.usage)), ) - return { - "id": f"resp_{scenario.scenario_id}", - "object": "response", - "created_at": int(time.time()), - "status": "completed", - "model": scenario.output.response_model or requested_model, - "output": output, - "usage": _responses_usage(scenario.usage), - } def _responses_sse(scenario: Scenario, requested_model: str) -> bytes: - completed = _responses_body(scenario, requested_model) - if scenario.stream_usage == "absent": - completed = {k: v for k, v in completed.items() if k != "usage"} - created = {**completed, "status": "in_progress", "usage": None} + completed: Final = ( + _jobj(*((key, value) for key, value in _responses_body(scenario, requested_model).items() if key != "usage")) + if scenario.stream_usage == "absent" + else _responses_body(scenario, requested_model) + ) + created: Final = _jobj( + *((key, value) for key, value in completed.items() if key not in ("status", "usage")), + ("status", "in_progress"), + ("usage", None), + ) return _sse( ( - ("response.created", {"type": "response.created", "response": created}), + ("response.created", _jobj(("type", "response.created"), ("response", created))), ( "response.output_text.delta", - { - "type": "response.output_text.delta", - "item_id": f"msg_{scenario.scenario_id}", - "output_index": scenario.usage.web_search_calls, - "content_index": 0, - "delta": scenario.output.text, - }, + _jobj( + ("type", "response.output_text.delta"), + ("item_id", f"msg_{scenario.scenario_id}"), + ("output_index", scenario.usage.web_search_calls), + ("content_index", 0), + ("delta", scenario.output.text), + ), ), - ("response.completed", {"type": "response.completed", "response": completed}), + ("response.completed", _jobj(("type", "response.completed"), ("response", completed))), ) ) @@ -538,11 +667,11 @@ class _ScenarioStore: _REQUEST_BODY: Final = TypeAdapter(dict[str, object]) -def _request_body(body: bytes) -> dict[str, object]: +def _request_body(body: bytes) -> Mapping[str, object]: try: return _REQUEST_BODY.validate_json(body) except ValueError: - return {} + return MappingProxyType({}) def _request_wants_stream(path_tail: str, body: bytes) -> bool: @@ -554,52 +683,66 @@ def _request_wants_stream(path_tail: str, body: bytes) -> bool: def _request_model(body: bytes) -> str: - model = _request_body(body).get("model") + model: Final = _request_body(body).get("model") return model if isinstance(model, str) else "unknown" def handle_request(store: _ScenarioStore, method: str, raw_path: str, body: bytes) -> RenderedResponse: - path = urlsplit(raw_path).path - segments = [segment for segment in path.split("/") if segment] - if method == "GET" and segments == ["health"]: - return RenderedResponse(200, "application/json", _json_bytes({"status": "ok"})) + path: Final = urlsplit(raw_path).path + segments: Final = tuple(segment for segment in path.split("/") if segment) + if method == "GET" and segments == ("health",): + return RenderedResponse(200, "application/json", _json_bytes(_jobj(("status", "ok")))) if segments and segments[0] == "_scenarios": if method == "POST" and len(segments) == 1: try: - scenario = Scenario.model_validate_json(body) + scenario: Final = Scenario.model_validate_json(body) except ValidationError as exc: - return RenderedResponse(400, "application/json", _json_bytes({"error": str(exc)})) + return RenderedResponse( + 400, "application/json", _json_bytes(_jobj(("error", str(exc)))) + ) store.put(scenario) - return RenderedResponse(200, "application/json", _json_bytes({"scenario_id": scenario.scenario_id})) - if method == "DELETE" and len(segments) == 2: - deleted = store.drop(segments[1]) return RenderedResponse( - 200 if deleted else 404, "application/json", _json_bytes({"deleted": deleted}) + 200, "application/json", _json_bytes(_jobj(("scenario_id", scenario.scenario_id))) ) - return RenderedResponse(404, "application/json", _json_bytes({"error": "unknown control route"})) + if method == "DELETE" and len(segments) == 2: + deleted: Final = store.drop(segments[1]) + return RenderedResponse( + 200 if deleted else 404, + "application/json", + _json_bytes(_jobj(("deleted", deleted))), + ) + return RenderedResponse( + 404, "application/json", _json_bytes(_jobj(("error", "unknown control route"))) + ) if len(segments) < 2 or method != "POST": - return RenderedResponse(404, "application/json", _json_bytes({"error": f"no route for {method} {path}"})) + return RenderedResponse( + 404, "application/json", _json_bytes(_jobj(("error", f"no route for {method} {path}"))) + ) scenario_id, mount = segments[0], segments[1] - scenario = store.get(scenario_id) - if scenario is None: - return RenderedResponse(404, "application/json", _json_bytes({"error": f"unknown scenario {scenario_id}"})) - if scenario.mount != mount: + found: Final = store.get(scenario_id) + if found is None: + return RenderedResponse( + 404, "application/json", _json_bytes(_jobj(("error", f"unknown scenario {scenario_id}"))) + ) + if found.mount != mount: return RenderedResponse( 400, "application/json", - _json_bytes({"error": f"scenario {scenario_id} is wire {scenario.wire}, not mount {mount}"}), + _json_bytes( + _jobj(("error", f"scenario {scenario_id} is wire {found.wire}, not mount {mount}")) + ), ) - tail = "/".join(segments[2:]) - return _render(scenario, stream=_request_wants_stream(tail, body), requested_model=_request_model(body)) + tail: Final = "/".join(segments[2:]) + return _render(found, stream=_request_wants_stream(tail, body), requested_model=_request_model(body)) class _ScriptedHandler(BaseHTTPRequestHandler): store: Final[_ScenarioStore] = _ScenarioStore() def _dispatch(self, method: str) -> None: - length = int(self.headers.get("content-length") or 0) - body = self.rfile.read(length) if length else b"" - rendered = handle_request(self.store, method, self.path, body) + length: Final = int(self.headers.get("content-length") or 0) + body: Final = self.rfile.read(length) if length else b"" + rendered: Final = handle_request(self.store, method, self.path, body) self.send_response(rendered.status_code) self.send_header("content-type", rendered.content_type) self.send_header("content-length", str(len(rendered.body))) @@ -621,11 +764,11 @@ DEFAULT_PORT: Final = 9100 def serve(port: int = DEFAULT_PORT, bind_host: str = "127.0.0.1") -> None: - server = ThreadingHTTPServer((bind_host, port), _ScriptedHandler) + server: Final = ThreadingHTTPServer((bind_host, port), _ScriptedHandler) sys.stderr.write(f"scripted-provider listening on http://{bind_host}:{port}\n") server.serve_forever() if __name__ == "__main__": - port_arg = int(sys.argv[1]) if len(sys.argv) > 1 else DEFAULT_PORT + port_arg: Final = int(sys.argv[1]) if len(sys.argv) > 1 else DEFAULT_PORT serve(port=port_arg) diff --git a/tests/e2e/cost_calculation/test_token_pricing_e2e.py b/tests/e2e/cost_calculation/test_token_pricing_e2e.py index 8d7678cf9ca..e210dad94b1 100644 --- a/tests/e2e/cost_calculation/test_token_pricing_e2e.py +++ b/tests/e2e/cost_calculation/test_token_pricing_e2e.py @@ -11,6 +11,7 @@ tests/e2e/cost_map.json. from __future__ import annotations import pytest +from typing import Final from conftest import CostCalcClient, cost_rows, register_scenario_deployment from cost_matrix import ( @@ -25,11 +26,11 @@ from e2e_config import unique_marker from lifecycle import ResourceManager from models import ChatBody, ChatMessage, ChatStreamOptions -pytestmark = [pytest.mark.e2e, pytest.mark.cost_map_stack] +pytestmark: Final = [pytest.mark.e2e, pytest.mark.cost_map_stack] # mutable-ok: pytest only accepts a list for pytestmark -_MATRIX: list[tuple[FrontierModel, Case]] = [ +_MATRIX: Final[tuple[tuple[FrontierModel, Case], ...]] = tuple( (model, case) for model in FRONTIER_MODELS for case in cases_for(model) -] +) def _case_id(param: tuple[FrontierModel, Case]) -> str: @@ -40,7 +41,7 @@ def _case_id(param: tuple[FrontierModel, Case]) -> str: def _chat_body(model_name: str, marker: str, case: Case) -> ChatBody: return ChatBody( model=model_name, - messages=[ChatMessage(role="user", content=f"{marker} scripted pricing call")], + messages=(ChatMessage(role="user", content=f"{marker} scripted pricing call"),), stream=case.stream, stream_options=ChatStreamOptions(include_usage=True) if case.stream else None, service_tier=case.service_tier, @@ -58,9 +59,9 @@ class TestTokenPricing: model_case: tuple[FrontierModel, Case], ) -> None: model, case = model_case - marker = unique_marker() + marker: Final = unique_marker() model_name, _handle = register_scenario_deployment(client, resources, model, case, marker) - response = client.proxy.transport.send( + response: Final = client.proxy.transport.send( "/chat/completions", headers=client.proxy.transport.bearer(scoped_key), json=_chat_body(model_name, marker, case), @@ -71,7 +72,7 @@ class TestTokenPricing: ) assert response.stream_error is None, f"stream carried an error event: {response.stream_error}" - expected = expected_cost(model, case) + expected: Final = expected_cost(model, case) if case.exact_spend and not case.stream: # Streamed responses commit headers before the bill is computed, so # the x-litellm-response-cost header is asserted only on non-stream @@ -82,7 +83,7 @@ class TestTokenPricing: f"x-litellm-response-cost {response.response_cost} != expected {expected}" ) - row = cost_rows.poll_cost_row_where( + row: Final = cost_rows.poll_cost_row_where( client.proxy, scoped_key, lambda r: r.metadata is not None and r.metadata.cost_breakdown is not None, diff --git a/tests/e2e/cost_calculation/test_wire_formats_e2e.py b/tests/e2e/cost_calculation/test_wire_formats_e2e.py index b1ef675d9ef..c0276cf370c 100644 --- a/tests/e2e/cost_calculation/test_wire_formats_e2e.py +++ b/tests/e2e/cost_calculation/test_wire_formats_e2e.py @@ -12,6 +12,9 @@ the proxy to POST /responses) and a streamed Anthropic-messages case. from __future__ import annotations import pytest +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final from conftest import CostCalcClient, cost_rows, register_scenario_deployment from cost_matrix import ( @@ -26,12 +29,14 @@ from lifecycle import ResourceManager from models import ChatBody, ChatMessage, ChatStreamOptions from scripted_provider import ScriptedUsage -pytestmark = [pytest.mark.e2e, pytest.mark.cost_map_stack] +pytestmark: Final = [pytest.mark.e2e, pytest.mark.cost_map_stack] # mutable-ok: pytest only accepts a list for pytestmark -_MODELS: dict[str, FrontierModel] = {model.map_key: model for model in FRONTIER_MODELS} +_MODELS: Final[Mapping[str, FrontierModel]] = MappingProxyType( + {model.map_key: model for model in FRONTIER_MODELS} +) # One scripted usage per wire, every reportable token kind nonzero. -_WIRE_USAGE: dict[str, tuple[str, ScriptedUsage]] = { +_WIRE_USAGE: Final[Mapping[str, tuple[str, ScriptedUsage]]] = MappingProxyType({ "openai_chat": ( "gpt-5.6", ScriptedUsage( @@ -89,7 +94,7 @@ _WIRE_USAGE: dict[str, tuple[str, ScriptedUsage]] = { "fireworks_ai/kimi-k3", ScriptedUsage(fresh_input_tokens=80, cache_read_tokens=40, output_tokens=25), ), -} +}) class TestWireFormats: @@ -103,22 +108,22 @@ class TestWireFormats: wire: str, ) -> None: map_key, usage = _WIRE_USAGE[wire] - model = _MODELS[map_key] - case = Case(name="basic", usage=usage) - marker = unique_marker() + model: Final = _MODELS[map_key] + case: Final = Case(name="basic", usage=usage) + marker: Final = unique_marker() model_name, _handle = register_scenario_deployment(client, resources, model, case, marker) - response = client.proxy.transport.send( + response: Final = client.proxy.transport.send( "/chat/completions", headers=client.proxy.transport.bearer(scoped_key), json=ChatBody( model=model_name, - messages=[ChatMessage(role="user", content=f"{marker} scripted wire call")], + messages=(ChatMessage(role="user", content=f"{marker} scripted wire call"),), ), ) assert response.ok, f"{wire}: proxy returned {response.status_code}: {response.body[:400]}" - expected = expected_breakdown(model, case) - row = cost_rows.poll_cost_row_where( + expected: Final = expected_breakdown(model, case) + row: Final = cost_rows.poll_cost_row_where( client.proxy, scoped_key, lambda r: r.metadata is not None and r.metadata.cost_breakdown is not None, @@ -128,7 +133,7 @@ class TestWireFormats: f"{wire}: spend {row.spend} != expected {expected.total} " f"(breakdown {row.breakdown.model_dump()})" ) - breakdown = row.breakdown + breakdown: Final = row.breakdown assert breakdown.input_cost is not None and cost_rows.approx_equal( breakdown.input_cost, expected.input_cost ), ( @@ -153,16 +158,16 @@ class TestWireFormats: self, client: CostCalcClient, resources: ResourceManager, scoped_key: str ) -> None: map_key, usage = _WIRE_USAGE["anthropic_messages"] - model = _MODELS[map_key] - case = Case(name="stream", usage=usage, stream=True) - marker = unique_marker() + model: Final = _MODELS[map_key] + case: Final = Case(name="stream", usage=usage, stream=True) + marker: Final = unique_marker() model_name, _handle = register_scenario_deployment(client, resources, model, case, marker) - response = client.proxy.transport.send( + response: Final = client.proxy.transport.send( "/chat/completions", headers=client.proxy.transport.bearer(scoped_key), json=ChatBody( model=model_name, - messages=[ChatMessage(role="user", content=f"{marker} scripted anthropic stream")], + messages=(ChatMessage(role="user", content=f"{marker} scripted anthropic stream"),), stream=True, stream_options=ChatStreamOptions(include_usage=True), ), @@ -172,8 +177,8 @@ class TestWireFormats: assert response.stream_done, "anthropic stream did not reach its terminal event" assert response.stream_error is None, f"stream carried an error event: {response.stream_error}" - expected = expected_breakdown(model, case) - row = cost_rows.poll_cost_row_where( + expected: Final = expected_breakdown(model, case) + row: Final = cost_rows.poll_cost_row_where( client.proxy, scoped_key, lambda r: r.metadata is not None and r.metadata.cost_breakdown is not None, From 99bf8e9b2ffb6c647813029debba23c788e86b41 Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 15 Sep 2026 23:32:39 +0000 Subject: [PATCH 004/210] test(e2e): add cost calculation CI proxy config Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/CLAUDE.md | 2 +- tests/e2e/gateway/cost_calculation_ci_config.yml | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 tests/e2e/gateway/cost_calculation_ci_config.yml diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index b6c3840f626..49cfc29aa17 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -21,7 +21,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments split round robin over `/chat/completions` and `/v1/messages`, one endpoint per simulated user, with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests on every endpoint, budgeting RSS and CPU-per-request as ratios against the same run's healthy phase, and holding p50/p90/p99 latency and log-bytes-per-request to flat ceilings (a ratio cannot bound those two: an open breaker skips Redis instead of waiting on it, so the chaos phase can measure cheaper than baseline while still being far slower than a user should see); needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` and `E2E_PROXY_LOG` set, marked `redis_chaos`, deselected unless `E2E_REDIS_CHAOS` is set and excluded from the per-PR selector like the rest of `load/`, driven by `.github/workflows/test-e2e-redis-chaos.yml` and by the Buildkite `e2e-redis-chaos` step in project-releaser, which runs the proxy, Postgres and Valkey co-located with pytest in one pod and sets the opt-in), and markerless harness unit tests for the locust, process-usage, and session-anomaly aggregation logic - `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate, JWT auth (access tokens issued by a real Keycloak realm, `idp.py` plus `idp_realm.json`, whose JWKS the proxy's `JWT_PUBLIC_KEY_URL` points at; see CONTRIBUTING.md for the start command and config block), and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite - `gateway/` - proxy configuration only (`litellm-config.yml`); no tests -- `cost_calculation/` - cost accounting against a dedicated proxy whose whole model cost map is the test-owned `tests/e2e/cost_map.json` (loaded via `LITELLM_MODEL_COST_MAP_URL`), with provider calls answered by the scripted-provider sidecar in `scripted_provider.py`; asserts literal rate arithmetic on scripted usage across every provider wire and pricing component, deselected unless `E2E_COST_MAP_STACK` is set +- `cost_calculation/` - cost accounting against a dedicated proxy whose whole model cost map is the test-owned `tests/e2e/cost_map.json` (loaded via `LITELLM_MODEL_COST_MAP_URL`), with provider calls answered by the scripted-provider sidecar in `scripted_provider.py`; asserts literal rate arithmetic on scripted usage across every provider wire and pricing component, deselected unless `E2E_COST_MAP_STACK` is set, driven by the Buildkite `e2e-cost-calculation` step in project-releaser, which runs a proxy booted from `gateway/cost_calculation_ci_config.yml`, Postgres and the scripted provider co-located with pytest in one pod and sets the opt-in - `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture; ships its own driver/builder/publisher plus `_*_unit_tests/` trees. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`); the CLI-driving path stays bespoke - `ui/` - the Admin UI browser suite: Playwright in TypeScript, driving the dashboard served by a live proxy on port 4000 (seeded postgres + mock LLM upstream; see its `run_e2e.sh`). It is a self-contained npm package with its own lockfile and does not use the Python harness, pytest markers, or the shared transport; the Python rules in this file (typed models, `Result` unions, basedpyright zero-error gate) do not apply inside it. Its only Python file, `fixtures/mock_llm_server/server.py`, is excluded from the e2e basedpyright gate via the root `pyrightconfig.json` diff --git a/tests/e2e/gateway/cost_calculation_ci_config.yml b/tests/e2e/gateway/cost_calculation_ci_config.yml new file mode 100644 index 00000000000..ac0603fa7c1 --- /dev/null +++ b/tests/e2e/gateway/cost_calculation_ci_config.yml @@ -0,0 +1,7 @@ +general_settings: + master_key: os.environ/LITELLM_MASTER_KEY + database_url: os.environ/DATABASE_URL + store_model_in_db: true + proxy_batch_write_at: 5 + +model_list: [] From 415b06f5ff6d5959e2144ea826922694b2c8a60b Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 16 Sep 2026 04:42:07 +0000 Subject: [PATCH 005/210] test(e2e): assert the real bill for the four fixed cost gaps Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/cost_calculation/cost_matrix.py | 22 +++++-------------- .../test_token_pricing_e2e.py | 5 ----- tests/e2e/cost_map.json | 15 +++++++++++++ 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/tests/e2e/cost_calculation/cost_matrix.py b/tests/e2e/cost_calculation/cost_matrix.py index e8b1d249559..8f39e89a358 100644 --- a/tests/e2e/cost_calculation/cost_matrix.py +++ b/tests/e2e/cost_calculation/cost_matrix.py @@ -186,15 +186,12 @@ _WIRE_CAPS: Final[Mapping[str, frozenset[str]]] = MappingProxyType({ } ), "openai_responses": frozenset({"cache_read", "reasoning", "web_search", "response_model", "absent_usage"}), - # Product gap: litellm hard-indexes message_delta["usage"] in - # anthropic/chat/handler.py, so a usage-absent anthropic stream raises - # KeyError; the real wire always carries it, so the case cannot be - # represented. - "anthropic_messages": frozenset({"cache_read", "cache_write_5m", "cache_write_1h", "web_search", "response_model"}), - # Product gap: the gemini transform sets ModelResponse.model from the - # request and drops the provider's modelVersion, so a response-model - # override can never be priced on this wire. - "gemini_generate": frozenset({"cache_read", "reasoning", "audio", "web_search", "absent_usage"}), + "anthropic_messages": frozenset( + {"cache_read", "cache_write_5m", "cache_write_1h", "web_search", "response_model", "absent_usage"} + ), + "gemini_generate": frozenset( + {"cache_read", "reasoning", "audio", "web_search", "response_model", "absent_usage"} + ), "together_chat": frozenset( { "cache_read", "cache_write_5m", "cache_write_1h", "reasoning", "audio", @@ -240,9 +237,6 @@ class Case: billed_web_search_calls: int = 0 response_model_override: bool = False exact_spend: bool = True - # stream_usage=absent on a wire with no proxy-side token recount means the - # bill is exactly zero; asserted as such rather than skipped. - expect_zero_bill: bool = False def scenario(self, scenario_id: str, model: FrontierModel, text: str) -> Scenario: return Scenario( @@ -359,10 +353,6 @@ def cases_for(model: FrontierModel) -> tuple[Case, ...]: stream=True, stream_usage="absent", exact_spend=False, - # The responses surface bills only provider-reported usage; - # with no usage in the stream the spend row is zero. Other - # wires recount tokens proxy-side and bill a nonzero amount. - expect_zero_bill=model.wire == "openai_responses", ) if "absent_usage" in caps else None diff --git a/tests/e2e/cost_calculation/test_token_pricing_e2e.py b/tests/e2e/cost_calculation/test_token_pricing_e2e.py index e210dad94b1..ead86931424 100644 --- a/tests/e2e/cost_calculation/test_token_pricing_e2e.py +++ b/tests/e2e/cost_calculation/test_token_pricing_e2e.py @@ -90,11 +90,6 @@ class TestTokenPricing: ) assert row is not None, f"no spend row with a cost breakdown landed for {model.map_key}/{case.name}" - if not case.exact_spend and case.expect_zero_bill: - # The provider reported no usage and this wire has no proxy-side - # recount, so the bill is exactly zero. - assert row.spend is not None and row.spend == 0, f"no-usage stream billed {row.spend}: {row}" - return if not case.exact_spend: # stream_usage=absent: the provider reported no usage, so the row's # token counts are the proxy's own recount; only assert a bill landed. diff --git a/tests/e2e/cost_map.json b/tests/e2e/cost_map.json index b761710bae3..68d840870c9 100644 --- a/tests/e2e/cost_map.json +++ b/tests/e2e/cost_map.json @@ -63,13 +63,18 @@ "supports_web_search": true }, "fireworks_ai/deepseek-v4p1-flash": { + "cache_creation_input_token_cost": 0.00033, + "cache_creation_input_token_cost_above_1hr": 0.00044, "cache_read_input_token_cost": 1.4e-05, + "input_cost_per_audio_token": 0.00066, "input_cost_per_token": 0.00014000000000000001, "litellm_provider": "fireworks_ai", "max_input_tokens": 2000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", + "output_cost_per_audio_token": 0.00077, + "output_cost_per_reasoning_token": 0.00055, "output_cost_per_token": 0.00028000000000000003, "search_context_cost_per_query": { "search_context_size_high": 0.03, @@ -82,13 +87,18 @@ "supports_web_search": true }, "fireworks_ai/kimi-k3": { + "cache_creation_input_token_cost": 0.00033, + "cache_creation_input_token_cost_above_1hr": 0.00044, "cache_read_input_token_cost": 1.2e-05, + "input_cost_per_audio_token": 0.00066, "input_cost_per_token": 0.00012000000000000002, "litellm_provider": "fireworks_ai", "max_input_tokens": 2000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", + "output_cost_per_audio_token": 0.00077, + "output_cost_per_reasoning_token": 0.00055, "output_cost_per_token": 0.00024000000000000003, "search_context_cost_per_query": { "search_context_size_high": 0.03, @@ -101,13 +111,18 @@ "supports_web_search": true }, "fireworks_ai/qwen3p8-max": { + "cache_creation_input_token_cost": 0.00033, + "cache_creation_input_token_cost_above_1hr": 0.00044, "cache_read_input_token_cost": 1.3e-05, + "input_cost_per_audio_token": 0.00066, "input_cost_per_token": 0.00013000000000000002, "litellm_provider": "fireworks_ai", "max_input_tokens": 2000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", + "output_cost_per_audio_token": 0.00077, + "output_cost_per_reasoning_token": 0.00055, "output_cost_per_token": 0.00026000000000000003, "search_context_cost_per_query": { "search_context_size_high": 0.03, From 67778cfe2625eb97fd3d4733f1fae41aed7599fb Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 16 Sep 2026 14:18:08 +0000 Subject: [PATCH 006/210] fix(cost): resolve dated openai/azure snapshots to their undated cost map entry Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/utils.py | 9 +++++++- tests/test_litellm/test_cost_calculator.py | 24 ++++++++++++++++++++++ tests/test_litellm/test_utils.py | 19 +++++++++++++++++ 3 files changed, 51 insertions(+), 1 deletion(-) diff --git a/litellm/utils.py b/litellm/utils.py index 734522c0c6a..33fdfcc36ba 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5286,6 +5286,13 @@ def _strip_stable_vertex_version(model_name) -> str: return re.sub(r"-\d+$", "", model_name) +_DATED_SNAPSHOT_SUFFIX: Final = re.compile(r"-\d{4}-\d{2}-\d{2}$") + + +def _strip_dated_snapshot_suffix(model_name: str) -> str: + return _DATED_SNAPSHOT_SUFFIX.sub("", model_name) + + def _get_base_bedrock_model(model_name) -> str: """ Get the base model from the given model name. @@ -5333,7 +5340,7 @@ def _strip_model_name(model: str, custom_llm_provider: str | None) -> str: strip_finetune: Final = _strip_openai_finetune_model_name(model_name=model) return strip_finetune else: - return model + return _strip_dated_snapshot_suffix(model_name=model) # Global case-insensitive lookup map for model_cost (built eagerly at module import) diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index 7b53d3a58df..c5bc8d80fed 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -21,7 +21,9 @@ from litellm.types.llms.openai import OpenAIRealtimeStreamList, ResponseAPIUsage from litellm.types.rerank import RerankResponse from litellm.types.utils import ( CallTypes, + Choices, LiteLLMRealtimeStreamLoggingObject, + Message, ModelInfo, ModelResponse, PromptTokensDetailsWrapper, @@ -110,6 +112,28 @@ def test_completion_cost_uses_response_model_for_dynamic_routing(_local_model_co assert cost > 0, "Cost should be calculated using response model" +def test_completion_cost_strips_dated_azure_snapshot_model(_local_model_cost_map: None) -> None: + dated_response = ModelResponse( + model="gpt-5.6-luna-2026-07-09", + choices=[Choices(index=0, message=Message(role="assistant", content="hi"), finish_reason="stop")], + usage=Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150), + ) + dated_response._hidden_params = {"custom_llm_provider": "azure"} + + undated_response = ModelResponse( + model="gpt-5.6-luna", + choices=[Choices(index=0, message=Message(role="assistant", content="hi"), finish_reason="stop")], + usage=Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150), + ) + undated_response._hidden_params = {"custom_llm_provider": "azure"} + + dated_cost = litellm.completion_cost(completion_response=dated_response) + undated_cost = litellm.completion_cost(completion_response=undated_response) + + assert dated_cost == undated_cost + assert dated_cost > 0 + + def test_cost_calculator_with_response_cost_in_additional_headers(): class MockResponse(BaseModel): _hidden_params = {"additional_headers": {"llm_provider-x-litellm-response-cost": 1000}} diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 46149589371..2f9e27af797 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -186,6 +186,25 @@ def test_get_model_info_strips_openai_finetune_ids_without_a_custom_suffix(local assert info["key"] == "ft:gpt-4o-2024-08-06" +@pytest.mark.parametrize( + ("model", "custom_llm_provider", "expected_key"), + [ + ("gpt-5.6-luna-2026-07-09", "openai", "gpt-5.6-luna"), + ("gpt-5.6-luna-2026-07-09", "azure", "azure/gpt-5.6-luna"), + ], +) +def test_get_model_info_falls_back_from_dated_snapshot_to_undated_entry( + local_model_cost_map, model, custom_llm_provider, expected_key +): + info = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider) + assert info["key"] == expected_key + + +def test_get_model_info_prefers_exact_dated_key_over_stripped(local_model_cost_map): + info = litellm.get_model_info(model="gpt-4o-2024-08-06", custom_llm_provider="openai") + assert info["key"] == "gpt-4o-2024-08-06" + + def test_check_provider_match_azure_ai_allows_openai_and_azure(): """ Test that azure_ai provider can match openai and azure models. From feb69c5f789d44a65dbbfa348ce39eaa3874b37f Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 16 Sep 2026 16:38:15 +0000 Subject: [PATCH 007/210] test(e2e): add tool-call, terminal, and image-input shapes to the cost matrix Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/cost_calculation/cost_matrix.py | 184 +++++++- .../e2e/cost_calculation/scripted_provider.py | 408 +++++++++++++++--- .../test_token_pricing_e2e.py | 61 ++- .../cost_calculation/test_wire_formats_e2e.py | 111 ++++- 4 files changed, 688 insertions(+), 76 deletions(-) diff --git a/tests/e2e/cost_calculation/cost_matrix.py b/tests/e2e/cost_calculation/cost_matrix.py index 8f39e89a358..5f634712778 100644 --- a/tests/e2e/cost_calculation/cost_matrix.py +++ b/tests/e2e/cost_calculation/cost_matrix.py @@ -17,7 +17,11 @@ creation), the case is absent from the matrix rather than silently zero. from __future__ import annotations +import base64 import json +import random +import struct +import zlib from collections.abc import Mapping from dataclasses import dataclass from pathlib import Path @@ -26,7 +30,7 @@ from typing import Final, Literal, TypeAlias from pydantic import BaseModel, ConfigDict, TypeAdapter -from scripted_provider import Scenario, ScriptedOutput, ScriptedUsage, Wire +from scripted_provider import Scenario, ScriptedOutput, ScriptedToolCall, ScriptedUsage, Wire COST_MAP_PATH: Final = Path(__file__).resolve().parent.parent / "cost_map.json" @@ -182,26 +186,37 @@ _WIRE_CAPS: Final[Mapping[str, frozenset[str]]] = MappingProxyType({ "openai_chat": frozenset( { "cache_read", "cache_write_5m", "cache_write_1h", "reasoning", "audio", - "web_search", "response_model", "absent_usage", + "web_search", "response_model", "absent_usage", "tool_call", "image_input", + } + ), + "openai_responses": frozenset( + { + "cache_read", "reasoning", "web_search", "response_model", "absent_usage", + "tool_call", "image_input", "responses_terminal", } ), - "openai_responses": frozenset({"cache_read", "reasoning", "web_search", "response_model", "absent_usage"}), "anthropic_messages": frozenset( - {"cache_read", "cache_write_5m", "cache_write_1h", "web_search", "response_model", "absent_usage"} + { + "cache_read", "cache_write_5m", "cache_write_1h", "web_search", + "response_model", "absent_usage", "tool_call", "image_input", + } ), "gemini_generate": frozenset( - {"cache_read", "reasoning", "audio", "web_search", "response_model", "absent_usage"} + { + "cache_read", "reasoning", "audio", "web_search", "response_model", + "absent_usage", "tool_call", "image_input", "prompt_blocked", + } ), "together_chat": frozenset( { "cache_read", "cache_write_5m", "cache_write_1h", "reasoning", "audio", - "web_search", "response_model", "absent_usage", + "web_search", "response_model", "absent_usage", "tool_call", "image_input", } ), "fireworks_chat": frozenset( { "cache_read", "cache_write_5m", "cache_write_1h", "reasoning", "audio", - "web_search", "response_model", "absent_usage", + "web_search", "response_model", "absent_usage", "tool_call", "image_input", } ), }) @@ -220,6 +235,15 @@ CaseName: TypeAlias = Literal[ "stream", "stream_no_usage", "response_model_override", + "stream_response_model_override", + "tool_call", + "stream_no_usage_tool_call", + "stream_no_usage_image_input", + "stream_no_usage_incomplete", + "stream_unvalidated", + "stream_no_usage_unvalidated", + "prompt_blocked", + "stream_prompt_blocked", ] @@ -237,6 +261,9 @@ class Case: billed_web_search_calls: int = 0 response_model_override: bool = False exact_spend: bool = True + tool_call: bool = False + image_input: bool = False + terminal: Literal["completed", "incomplete", "unvalidated", "prompt_blocked"] = "completed" def scenario(self, scenario_id: str, model: FrontierModel, text: str) -> Scenario: return Scenario( @@ -246,6 +273,10 @@ class Case: output=ScriptedOutput( text=text, response_model=model.override_model if self.response_model_override else None, + tool_call=ScriptedToolCall(name="get_weather", arguments=TOOL_CALL_ARGUMENTS) + if self.tool_call + else None, + terminal=self.terminal, ), stream_usage=self.stream_usage, service_tier=self.service_tier, @@ -254,6 +285,15 @@ class Case: _BASIC_USAGE: Final = ScriptedUsage(fresh_input_tokens=120, output_tokens=40) +TOOL_CALL_ARGUMENTS: Final = json.dumps({ + "city": "Berlin", + "days": 7, + "units": "metric", + "notes": "filler " * 30, +}) + +_PROMPT_BLOCKED_USAGE: Final = ScriptedUsage(fresh_input_tokens=1000, output_tokens=0) + def _web_search_case(model: FrontierModel) -> Case: counts_exactly: Final = model.wire in ("openai_responses", "anthropic_messages", "gemini_generate") @@ -362,6 +402,100 @@ def cases_for(model: FrontierModel) -> tuple[Case, ...]: if "response_model" in caps else None ), + ( + Case( + name="stream_response_model_override", + usage=_BASIC_USAGE, + stream=True, + response_model_override=True, + ) + if "response_model" in caps + else None + ), + ( + Case(name="tool_call", usage=_BASIC_USAGE, tool_call=True) + if "tool_call" in caps + else None + ), + ( + Case( + name="stream_no_usage_tool_call", + usage=_BASIC_USAGE, + stream=True, + stream_usage="absent", + tool_call=True, + exact_spend=False, + ) + if "absent_usage" in caps and "tool_call" in caps + else None + ), + ( + Case( + name="stream_no_usage_image_input", + usage=_BASIC_USAGE, + stream=True, + stream_usage="absent", + image_input=True, + exact_spend=False, + ) + if "absent_usage" in caps and "image_input" in caps + else None + ), + ( + Case( + name="stream_no_usage_incomplete", + usage=_BASIC_USAGE, + stream=True, + stream_usage="absent", + terminal="incomplete", + exact_spend=False, + ) + if "responses_terminal" in caps + else None + ), + ( + Case( + name="stream_unvalidated", + usage=_BASIC_USAGE, + stream=True, + terminal="unvalidated", + ) + if "responses_terminal" in caps + else None + ), + ( + Case( + name="stream_no_usage_unvalidated", + usage=_BASIC_USAGE, + stream=True, + stream_usage="absent", + terminal="unvalidated", + exact_spend=False, + ) + if "responses_terminal" in caps + else None + ), + ( + Case( + name="prompt_blocked", + usage=_PROMPT_BLOCKED_USAGE, + terminal="prompt_blocked", + response_model_override=True, + ) + if "prompt_blocked" in caps + else None + ), + ( + Case( + name="stream_prompt_blocked", + usage=_PROMPT_BLOCKED_USAGE, + stream=True, + terminal="prompt_blocked", + response_model_override=True, + ) + if "prompt_blocked" in caps + else None + ), ) return tuple(case for case in candidates if case is not None) @@ -436,6 +570,42 @@ def expected_cost(model: FrontierModel, case: Case) -> float: return expected_breakdown(model, case).total +def recount_cost( + model: FrontierModel, case: Case, prompt_tokens: int, completion_tokens: int +) -> float: + """What the proxy's own token recount should cost at the case's rates, + without pinning the tokenizer's exact counts.""" + rates: Final = model.override_rates if case.response_model_override else model.rates + return prompt_tokens * (rates.input_cost_per_token or 0.0) + completion_tokens * ( + rates.output_cost_per_token or 0.0 + ) + + +def _png_chunk(tag: bytes, payload: bytes) -> bytes: + return struct.pack(">I", len(payload)) + tag + payload + struct.pack(">I", zlib.crc32(tag + payload)) + + +def image_input_data_url() -> str: + """A deterministic 256x256 RGB noise PNG as a data URL; noise compresses + poorly on purpose so the base64 payload stays well above 100 KB and would + blow up the prompt recount if the URL were ever tokenized as text.""" + rng: Final = random.Random(0) + side: Final = 256 + raw: Final = b"".join( + b"\x00" + rng.randbytes(side * 3) for _ in range(side) + ) + png: Final = ( + b"\x89PNG\r\n\x1a\n" + + _png_chunk(b"IHDR", struct.pack(">IIBBBBB", side, side, 8, 2, 0, 0, 0)) + + _png_chunk(b"IDAT", zlib.compress(raw)) + + _png_chunk(b"IEND", b"") + ) + return "data:image/png;base64," + base64.b64encode(png).decode() + + +IMAGE_INPUT_DATA_URL: Final = image_input_data_url() + + def expected_token_columns(model: FrontierModel, case: Case) -> tuple[int, int]: """(prompt_tokens, completion_tokens) the spend row should carry, per the wire's normalization: Anthropic folds cache read/write into prompt_tokens, diff --git a/tests/e2e/cost_calculation/scripted_provider.py b/tests/e2e/cost_calculation/scripted_provider.py index f1deafd1bc5..e1a6c430307 100644 --- a/tests/e2e/cost_calculation/scripted_provider.py +++ b/tests/e2e/cost_calculation/scripted_provider.py @@ -38,7 +38,7 @@ from types import MappingProxyType from typing import Final, Literal, TypeAlias from urllib.parse import urlsplit -from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError, model_validator Wire: TypeAlias = Literal[ "openai_chat", @@ -62,6 +62,26 @@ WIRE_MOUNTS: Final[Mapping[str, str]] = MappingProxyType( StreamUsage: TypeAlias = Literal["final_chunk", "absent"] ServiceTier: TypeAlias = Literal["flex", "priority"] +TerminalKind: TypeAlias = Literal["completed", "incomplete", "unvalidated", "prompt_blocked"] + +# Which terminal variant each wire can represent. +_TERMINAL_CAPS: Final[Mapping[str, frozenset[str]]] = MappingProxyType( + { + "openai_responses": frozenset({"incomplete", "unvalidated"}), + "gemini_generate": frozenset({"prompt_blocked"}), + } +) + + +class ScriptedToolCall(BaseModel): + """A single function call the scripted output emits instead of text. + ``arguments`` is the wire's JSON string (~250 chars), sliced into deltas + for streams.""" + + model_config = ConfigDict(frozen=True) + + name: str + arguments: str class ScriptedUsage(BaseModel): @@ -96,6 +116,12 @@ class ScriptedOutput(BaseModel): # OpenAI-compatible providers can report a provider-computed cost; emitted as # the top-level "cost" field on the together/fireworks wire. provider_cost: float | None = None + # When set, the response is a tool call only: no text content on any wire. + tool_call: ScriptedToolCall | None = None + # Terminal shape: "unvalidated" makes the Responses terminal response fail + # pydantic validation so the proxy takes its model_construct dict path; + # "prompt_blocked" is a Gemini promptFeedback-only body. + terminal: TerminalKind = "completed" class Scenario(BaseModel): @@ -108,6 +134,17 @@ class Scenario(BaseModel): stream_usage: StreamUsage = "final_chunk" service_tier: ServiceTier | None = None + @model_validator(mode="after") + def _check_terminal_supported(self) -> Scenario: + if ( + self.output.terminal != "completed" + and self.output.terminal not in _TERMINAL_CAPS.get(self.wire, frozenset()) + ): + raise ValueError( + f"wire {self.wire} cannot emit terminal={self.output.terminal}" + ) + return self + @property def mount(self) -> str: return WIRE_MOUNTS[self.wire] @@ -291,10 +328,38 @@ def _responses_usage(u: ScriptedUsage) -> Mapping[str, object]: # ---------- per-wire responses ---------- +def _split_arguments(arguments: str) -> tuple[str, ...]: + """Slice a tool-call arguments JSON string into 2-3 streamed deltas.""" + third: Final = max(1, len(arguments) // 3) + return tuple( + slice_ + for slice_ in (arguments[:third], arguments[third : 2 * third], arguments[2 * third :]) + if slice_ + ) + + def _openai_message(scenario: Scenario) -> Mapping[str, object]: + tool_call: Final = scenario.output.tool_call return _jobj_opt( ("role", "assistant"), - ("content", scenario.output.text), + ("content", None if tool_call is not None else scenario.output.text), + ( + ( + "tool_calls", + ( + _jobj( + ("id", f"call_{scenario.scenario_id}"), + ("type", "function"), + ( + "function", + _jobj(("name", tool_call.name), ("arguments", tool_call.arguments)), + ), + ), + ), + ) + if tool_call is not None + else None + ), ( ( "annotations", @@ -332,7 +397,12 @@ def _openai_chat_body(scenario: Scenario, requested_model: str) -> Mapping[str, _jobj( ("index", 0), ("message", _openai_message(scenario)), - ("finish_reason", scenario.output.finish_reason), + ( + "finish_reason", + "tool_calls" + if scenario.output.tool_call is not None + else scenario.output.finish_reason, + ), ), ), ), @@ -359,6 +429,7 @@ def _openai_chunk( def _openai_chat_sse(scenario: Scenario, requested_model: str) -> bytes: + tool_call: Final = scenario.output.tool_call delta: Final = _jobj_opt( ("role", "assistant"), ("content", scenario.output.text), @@ -368,6 +439,43 @@ def _openai_chat_sse(scenario: Scenario, requested_model: str) -> bytes: else None ), ) + body_deltas: Final[tuple[Mapping[str, object], ...]] = ( + ( + _jobj( + ("role", "assistant"), + ( + "tool_calls", + ( + _jobj( + ("index", 0), + ("id", f"call_{scenario.scenario_id}"), + ("type", "function"), + ( + "function", + _jobj(("name", tool_call.name), ("arguments", "")), + ), + ), + ), + ), + ), + *( + _jobj( + ( + "tool_calls", + ( + _jobj( + ("index", 0), + ("function", _jobj(("arguments", arguments_slice))), + ), + ), + ) + ) + for arguments_slice in _split_arguments(tool_call.arguments) + ), + ) + if tool_call is not None + else (delta,) + ) return _sse( ( ( @@ -378,13 +486,16 @@ def _openai_chat_sse(scenario: Scenario, requested_model: str) -> bytes: choices=(_jobj(("index", 0), ("delta", _jobj(("role", "assistant"))), ("finish_reason", None)),), ), ), - ( - None, - _openai_chunk( - scenario, - requested_model, - choices=(_jobj(("index", 0), ("delta", delta), ("finish_reason", None)),), - ), + *( + ( + None, + _openai_chunk( + scenario, + requested_model, + choices=(_jobj(("index", 0), ("delta", body_delta), ("finish_reason", None)),), + ), + ) + for body_delta in body_deltas ), ( None, @@ -395,7 +506,12 @@ def _openai_chat_sse(scenario: Scenario, requested_model: str) -> bytes: _jobj( ("index", 0), ("delta", _jobj()), - ("finish_reason", scenario.output.finish_reason), + ( + "finish_reason", + "tool_calls" + if tool_call is not None + else scenario.output.finish_reason, + ), ), ), ), @@ -410,17 +526,34 @@ def _openai_chat_sse(scenario: Scenario, requested_model: str) -> bytes: ) +def _anthropic_content(scenario: Scenario) -> tuple[Mapping[str, object], ...]: + tool_call: Final = scenario.output.tool_call + if tool_call is not None: + return ( + _jobj( + ("type", "tool_use"), + ("id", f"toolu_{scenario.scenario_id}"), + ("name", tool_call.name), + ("input", json.loads(tool_call.arguments)), + ), + ) + return (_jobj(("type", "text"), ("text", scenario.output.text)),) + + +def _anthropic_stop_reason(scenario: Scenario) -> str: + if scenario.output.tool_call is not None: + return "tool_use" + return "end_turn" if scenario.output.finish_reason == "stop" else scenario.output.finish_reason + + def _anthropic_body(scenario: Scenario, requested_model: str) -> Mapping[str, object]: return _jobj( ("id", f"msg_{scenario.scenario_id}"), ("type", "message"), ("role", "assistant"), ("model", scenario.output.response_model or requested_model), - ("content", (_jobj(("type", "text"), ("text", scenario.output.text)),)), - ( - "stop_reason", - "end_turn" if scenario.output.finish_reason == "stop" else scenario.output.finish_reason, - ), + ("content", _anthropic_content(scenario)), + ("stop_reason", _anthropic_stop_reason(scenario)), ("usage", _anthropic_usage(scenario.usage)), ) @@ -453,12 +586,7 @@ def _anthropic_sse(scenario: Scenario, requested_model: str) -> bytes: ("type", "message_delta"), ( "delta", - _jobj( - ( - "stop_reason", - "end_turn" if scenario.output.finish_reason == "stop" else scenario.output.finish_reason, - ) - ), + _jobj(("stop_reason", _anthropic_stop_reason(scenario))), ), ( ("usage", _jobj(("output_tokens", scenario.usage.output_tokens))) @@ -474,16 +602,45 @@ def _anthropic_sse(scenario: Scenario, requested_model: str) -> bytes: _jobj( ("type", "content_block_start"), ("index", 0), - ("content_block", _jobj(("type", "text"), ("text", ""))), + ( + "content_block", + _jobj( + ("type", "tool_use"), + ("id", f"toolu_{scenario.scenario_id}"), + ("name", scenario.output.tool_call.name), + ("input", _jobj()), + ) + if scenario.output.tool_call is not None + else _jobj(("type", "text"), ("text", "")), + ), ), ), - ( - "content_block_delta", - _jobj( - ("type", "content_block_delta"), - ("index", 0), - ("delta", _jobj(("type", "text_delta"), ("text", scenario.output.text))), - ), + *( + tuple( + ( + "content_block_delta", + _jobj( + ("type", "content_block_delta"), + ("index", 0), + ( + "delta", + _jobj(("type", "input_json_delta"), ("partial_json", arguments_slice)), + ), + ), + ) + for arguments_slice in _split_arguments(scenario.output.tool_call.arguments) + ) + if scenario.output.tool_call is not None + else ( + ( + "content_block_delta", + _jobj( + ("type", "content_block_delta"), + ("index", 0), + ("delta", _jobj(("type", "text_delta"), ("text", scenario.output.text))), + ), + ), + ) ), ("content_block_stop", _jobj(("type", "content_block_stop"), ("index", 0))), ("message_delta", message_delta), @@ -492,7 +649,49 @@ def _anthropic_sse(scenario: Scenario, requested_model: str) -> bytes: ) +def _gemini_prompt_blocked_body(scenario: Scenario, requested_model: str) -> Mapping[str, object]: + return _jobj( + ( + "promptFeedback", + _jobj( + ("blockReason", "SAFETY"), + ( + "safetyRatings", + ( + _jobj( + ("category", "HARM_CATEGORY_HARASSMENT"), + ("probability", "HIGH"), + ("blocked", True), + ), + ), + ), + ), + ), + ("usageMetadata", _gemini_usage(scenario.usage)), + ("modelVersion", scenario.output.response_model or requested_model), + ) + + +def _gemini_parts(scenario: Scenario) -> tuple[Mapping[str, object], ...]: + tool_call: Final = scenario.output.tool_call + if tool_call is not None: + return ( + _jobj( + ( + "functionCall", + _jobj( + ("name", tool_call.name), + ("args", json.loads(tool_call.arguments)), + ), + ) + ), + ) + return (_jobj(("text", scenario.output.text)),) + + def _gemini_body(scenario: Scenario, requested_model: str) -> Mapping[str, object]: + if scenario.output.terminal == "prompt_blocked": + return _gemini_prompt_blocked_body(scenario, requested_model) return _jobj( ( "candidates", @@ -501,7 +700,7 @@ def _gemini_body(scenario: Scenario, requested_model: str) -> Mapping[str, objec ( "content", _jobj( - ("parts", (_jobj(("text", scenario.output.text)),)), + ("parts", _gemini_parts(scenario)), ("role", "model"), ), ), @@ -559,67 +758,148 @@ def _gemini_sse(scenario: Scenario, requested_model: str) -> bytes: ) -def _responses_body(scenario: Scenario, requested_model: str) -> Mapping[str, object]: - return _jobj( - ("id", f"resp_{scenario.scenario_id}"), - ("object", "response"), - ("created_at", int(time.time())), - ("status", "completed"), - ("model", scenario.output.response_model or requested_model), - ( - "output", +def _responses_output(scenario: Scenario) -> tuple[Mapping[str, object], ...]: + tool_call: Final = scenario.output.tool_call + return ( + *( ( - *( - _jobj(("type", "web_search_call"), ("id", f"ws_{i}"), ("status", "completed")) - for i in range(scenario.usage.web_search_calls) - ), - _jobj( - ("type", "message"), - ("id", f"msg_{scenario.scenario_id}"), - ("status", "completed"), - ("role", "assistant"), - ( - "content", - ( - _jobj( - ("type", "output_text"), - ("text", scenario.output.text), - ("annotations", ()), - ), - ), + _jobj(("type", "scripted_future_item"), ("id", f"fut_{scenario.scenario_id}"), ("status", "completed")), + ) + if scenario.output.terminal == "unvalidated" + else () + ), + *( + _jobj(("type", "web_search_call"), ("id", f"ws_{i}"), ("status", "completed")) + for i in range(scenario.usage.web_search_calls) + ), + _jobj( + ("type", "function_call"), + ("id", f"fc_{scenario.scenario_id}"), + ("call_id", f"call_{scenario.scenario_id}"), + ("name", tool_call.name), + ("arguments", tool_call.arguments), + ("status", "completed"), + ) + if tool_call is not None + else _jobj( + ("type", "message"), + ("id", f"msg_{scenario.scenario_id}"), + ("status", "completed"), + ("role", "assistant"), + ( + "content", + ( + _jobj( + ("type", "output_text"), + ("text", scenario.output.text), + ("annotations", ()), ), ), ), ), + ) + + +def _responses_body(scenario: Scenario, requested_model: str) -> Mapping[str, object]: + incomplete: Final = scenario.output.terminal == "incomplete" + return _jobj_opt( + ("id", f"resp_{scenario.scenario_id}"), + ("object", "response"), + ( + "created_at", + "not-a-number" if scenario.output.terminal == "unvalidated" else int(time.time()), + ), + ("status", "incomplete" if incomplete else "completed"), + ( + ("incomplete_details", _jobj(("reason", "max_output_tokens"))) + if incomplete + else None + ), + ("model", scenario.output.response_model or requested_model), + ("output", _responses_output(scenario)), ("usage", _responses_usage(scenario.usage)), ) def _responses_sse(scenario: Scenario, requested_model: str) -> bytes: - completed: Final = ( + tool_call: Final = scenario.output.tool_call + terminal: Final = ( _jobj(*((key, value) for key, value in _responses_body(scenario, requested_model).items() if key != "usage")) if scenario.stream_usage == "absent" else _responses_body(scenario, requested_model) ) created: Final = _jobj( - *((key, value) for key, value in completed.items() if key not in ("status", "usage")), + *((key, value) for key, value in terminal.items() if key not in ("status", "usage")), ("status", "in_progress"), ("usage", None), ) - return _sse( + terminal_event: Final = ( + "response.incomplete" if scenario.output.terminal == "incomplete" else "response.completed" + ) + output_index: Final = ( + scenario.usage.web_search_calls + (1 if scenario.output.terminal == "unvalidated" else 0) + ) + middle_events: Final[tuple[tuple[str, Mapping[str, object]], ...]] = ( ( - ("response.created", _jobj(("type", "response.created"), ("response", created))), + ( + "response.output_item.added", + _jobj( + ("type", "response.output_item.added"), + ("output_index", output_index), + ( + "item", + _jobj( + ("type", "function_call"), + ("id", f"fc_{scenario.scenario_id}"), + ("call_id", f"call_{scenario.scenario_id}"), + ("name", tool_call.name), + ("arguments", ""), + ("status", "in_progress"), + ), + ), + ), + ), + *( + ( + "response.function_call_arguments.delta", + _jobj( + ("type", "response.function_call_arguments.delta"), + ("item_id", f"fc_{scenario.scenario_id}"), + ("output_index", output_index), + ("delta", arguments_slice), + ), + ) + for arguments_slice in _split_arguments(tool_call.arguments) + ), + ( + "response.function_call_arguments.done", + _jobj( + ("type", "response.function_call_arguments.done"), + ("item_id", f"fc_{scenario.scenario_id}"), + ("output_index", output_index), + ("arguments", tool_call.arguments), + ), + ), + ) + if tool_call is not None + else ( ( "response.output_text.delta", _jobj( ("type", "response.output_text.delta"), ("item_id", f"msg_{scenario.scenario_id}"), - ("output_index", scenario.usage.web_search_calls), + ("output_index", output_index), ("content_index", 0), ("delta", scenario.output.text), ), ), - ("response.completed", _jobj(("type", "response.completed"), ("response", completed))), + ) + ) + return _sse( + ( + ("response.created", _jobj(("type", "response.created"), ("response", created))), + *middle_events, + (terminal_event, _jobj(("type", terminal_event), ("response", terminal))), ) ) diff --git a/tests/e2e/cost_calculation/test_token_pricing_e2e.py b/tests/e2e/cost_calculation/test_token_pricing_e2e.py index ead86931424..0b4f3e1fd37 100644 --- a/tests/e2e/cost_calculation/test_token_pricing_e2e.py +++ b/tests/e2e/cost_calculation/test_token_pricing_e2e.py @@ -16,15 +16,26 @@ from typing import Final from conftest import CostCalcClient, cost_rows, register_scenario_deployment from cost_matrix import ( FRONTIER_MODELS, + IMAGE_INPUT_DATA_URL, Case, FrontierModel, cases_for, expected_cost, expected_token_columns, + recount_cost, ) from e2e_config import unique_marker from lifecycle import ResourceManager -from models import ChatBody, ChatMessage, ChatStreamOptions +from models import ( + ChatBody, + ChatMessage, + ChatStreamOptions, + ChatTool, + ChatToolFunction, + ImageContentPart, + ImageUrl, + TextContentPart, +) pytestmark: Final = [pytest.mark.e2e, pytest.mark.cost_map_stack] # mutable-ok: pytest only accepts a list for pytestmark @@ -41,10 +52,37 @@ def _case_id(param: tuple[FrontierModel, Case]) -> str: def _chat_body(model_name: str, marker: str, case: Case) -> ChatBody: return ChatBody( model=model_name, - messages=(ChatMessage(role="user", content=f"{marker} scripted pricing call"),), + messages=( + ChatMessage( + role="user", + content=( + [ + TextContentPart(text=f"{marker} scripted pricing call"), + ImageContentPart(image_url=ImageUrl(url=IMAGE_INPUT_DATA_URL)), + ] + if case.image_input + else f"{marker} scripted pricing call" + ), + ), + ), stream=case.stream, stream_options=ChatStreamOptions(include_usage=True) if case.stream else None, service_tier=case.service_tier, + tools=( + ( + ChatTool( + function=ChatToolFunction( + name="get_weather", + parameters={ + "type": "object", + "properties": {"city": {"type": "string"}}, + }, + ) + ), + ) + if case.tool_call + else None + ), ) @@ -92,8 +130,23 @@ class TestTokenPricing: if not case.exact_spend: # stream_usage=absent: the provider reported no usage, so the row's - # token counts are the proxy's own recount; only assert a bill landed. - assert row.spend is not None and row.spend > 0, f"no-usage stream billed nothing: {row}" + # token counts are the proxy's own recount; assert the recount + # billed both directions at the case's rates. + assert row.prompt_tokens is not None and row.prompt_tokens > 0, ( + f"no-usage stream counted no input tokens: {row}" + ) + assert row.completion_tokens is not None and row.completion_tokens > 0, ( + f"no-usage stream counted no output tokens: {row}" + ) + if case.image_input: + assert row.prompt_tokens < 4000, ( + f"image data URL looks tokenized as text: prompt_tokens={row.prompt_tokens}" + ) + assert row.spend is not None and cost_rows.approx_equal( + row.spend, + recount_cost(model, case, row.prompt_tokens, row.completion_tokens), + ), f"no-usage stream spend {row.spend} != recount at map rates: {row}" + cost_rows.assert_total_is_sum_of_components(row) return assert row.spend is not None and cost_rows.approx_equal(row.spend, expected), ( diff --git a/tests/e2e/cost_calculation/test_wire_formats_e2e.py b/tests/e2e/cost_calculation/test_wire_formats_e2e.py index c0276cf370c..3c6c34b24fb 100644 --- a/tests/e2e/cost_calculation/test_wire_formats_e2e.py +++ b/tests/e2e/cost_calculation/test_wire_formats_e2e.py @@ -26,7 +26,7 @@ from cost_matrix import ( ) from e2e_config import unique_marker from lifecycle import ResourceManager -from models import ChatBody, ChatMessage, ChatStreamOptions +from models import ChatBody, ChatMessage, ChatStreamOptions, ChatTool, ChatToolFunction from scripted_provider import ScriptedUsage pytestmark: Final = [pytest.mark.e2e, pytest.mark.cost_map_stack] # mutable-ok: pytest only accepts a list for pytestmark @@ -96,6 +96,53 @@ _WIRE_USAGE: Final[Mapping[str, tuple[str, ScriptedUsage]]] = MappingProxyType({ ), }) +_SHAPE_USAGE: Final = ScriptedUsage(fresh_input_tokens=80, output_tokens=25) + +# Renderer-level shapes the pricing matrix gates per cap, pinned here once per +# wire so the sidecar emits prove they survive the proxy end to end. +_SHAPES: Final[tuple[tuple[str, str, Case], ...]] = ( + *( + ( + f"tool_call_{'stream' if stream else 'sync'}", + wire, + Case(name="tool_call", usage=_SHAPE_USAGE, stream=stream, tool_call=True), + ) + for wire in _WIRE_USAGE + for stream in (False, True) + ), + ( + "responses_incomplete", + "openai_responses", + Case(name="stream_no_usage_incomplete", usage=_SHAPE_USAGE, stream=True, terminal="incomplete"), + ), + ( + "responses_unvalidated", + "openai_responses", + Case(name="stream_unvalidated", usage=_SHAPE_USAGE, stream=True, terminal="unvalidated"), + ), + ( + "gemini_prompt_blocked", + "gemini_generate", + Case( + name="prompt_blocked", + usage=ScriptedUsage(fresh_input_tokens=1000, output_tokens=0), + terminal="prompt_blocked", + response_model_override=True, + ), + ), + ( + "gemini_prompt_blocked_stream", + "gemini_generate", + Case( + name="stream_prompt_blocked", + usage=ScriptedUsage(fresh_input_tokens=1000, output_tokens=0), + stream=True, + terminal="prompt_blocked", + response_model_override=True, + ), + ), +) + class TestWireFormats: @pytest.mark.parametrize("wire", tuple(_WIRE_USAGE)) @@ -189,3 +236,65 @@ class TestWireFormats: f"(breakdown {row.breakdown.model_dump()})" ) cost_rows.assert_total_is_sum_of_components(row) + + @pytest.mark.parametrize("shape_wire_case", _SHAPES, ids=lambda entry: entry[0]) + @pytest.mark.covers("quota_management.spend_tracking.scripted_wire.logs_cost") + def test_response_shape_bills_reported_usage( + self, + client: CostCalcClient, + resources: ResourceManager, + scoped_key: str, + shape_wire_case: tuple[str, str, Case], + ) -> None: + shape, wire, case = shape_wire_case + map_key, _usage = _WIRE_USAGE[wire] + model: Final = _MODELS[map_key] + marker: Final = unique_marker() + model_name, _handle = register_scenario_deployment(client, resources, model, case, marker) + response: Final = client.proxy.transport.send( + "/chat/completions", + headers=client.proxy.transport.bearer(scoped_key), + json=ChatBody( + model=model_name, + messages=(ChatMessage(role="user", content=f"{marker} scripted {shape}"),), + stream=case.stream, + stream_options=ChatStreamOptions(include_usage=True) if case.stream else None, + tools=( + ( + ChatTool( + function=ChatToolFunction( + name="get_weather", + parameters={"type": "object", "properties": {"city": {"type": "string"}}}, + ) + ), + ) + if case.tool_call + else None + ), + ), + stream=case.stream, + ) + assert response.ok, f"{shape}: proxy returned {response.status_code}: {response.body[:400]}" + if case.stream: + assert response.stream_done, f"{shape}: stream did not reach its terminal event" + assert response.stream_error is None, f"{shape}: stream error: {response.stream_error}" + + expected: Final = expected_breakdown(model, case) + row: Final = cost_rows.poll_cost_row_where( + client.proxy, + scoped_key, + lambda r: r.metadata is not None and r.metadata.cost_breakdown is not None, + ) + assert row is not None, f"{shape}: no spend row landed" + assert row.spend is not None and cost_rows.approx_equal(row.spend, expected.total), ( + f"{shape}: spend {row.spend} != expected {expected.total} " + f"(breakdown {row.breakdown.model_dump()})" + ) + prompt_tokens, completion_tokens = expected_token_columns(model, case) + assert row.prompt_tokens == prompt_tokens, ( + f"{shape}: prompt_tokens {row.prompt_tokens} != {prompt_tokens}" + ) + assert row.completion_tokens == completion_tokens, ( + f"{shape}: completion_tokens {row.completion_tokens} != {completion_tokens}" + ) + cost_rows.assert_total_is_sum_of_components(row) From 5507de326e3e98f9069af5f9d1315c89bb3c3e25 Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 16 Sep 2026 16:45:09 +0000 Subject: [PATCH 008/210] test(e2e): type the wire-shape parametrize ids callback Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/cost_calculation/test_wire_formats_e2e.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/e2e/cost_calculation/test_wire_formats_e2e.py b/tests/e2e/cost_calculation/test_wire_formats_e2e.py index 3c6c34b24fb..4da7b31a6ef 100644 --- a/tests/e2e/cost_calculation/test_wire_formats_e2e.py +++ b/tests/e2e/cost_calculation/test_wire_formats_e2e.py @@ -144,6 +144,10 @@ _SHAPES: Final[tuple[tuple[str, str, Case], ...]] = ( ) +def _shape_id(entry: tuple[str, str, Case]) -> str: + return entry[0] + + class TestWireFormats: @pytest.mark.parametrize("wire", tuple(_WIRE_USAGE)) @pytest.mark.covers("quota_management.spend_tracking.scripted_wire.logs_cost") @@ -237,7 +241,7 @@ class TestWireFormats: ) cost_rows.assert_total_is_sum_of_components(row) - @pytest.mark.parametrize("shape_wire_case", _SHAPES, ids=lambda entry: entry[0]) + @pytest.mark.parametrize("shape_wire_case", _SHAPES, ids=_shape_id) @pytest.mark.covers("quota_management.spend_tracking.scripted_wire.logs_cost") def test_response_shape_bills_reported_usage( self, From 9885dc89621697e235fc65e0e86e147da87398c1 Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 16 Sep 2026 22:52:42 +0000 Subject: [PATCH 009/210] test(e2e): add azure, bedrock converse and vertex wires to the cost suite Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/cost_calculation/conftest.py | 54 +++- tests/e2e/cost_calculation/cost_matrix.py | 144 ++++++++- .../e2e/cost_calculation/scripted_provider.py | 284 +++++++++++++++++- .../cost_calculation/test_wire_formats_e2e.py | 64 ++++ tests/e2e/cost_map.json | 158 ++++++++++ tests/e2e/models.py | 1 + 6 files changed, 681 insertions(+), 24 deletions(-) diff --git a/tests/e2e/cost_calculation/conftest.py b/tests/e2e/cost_calculation/conftest.py index 345ca26f7e3..8c6db7c0010 100644 --- a/tests/e2e/cost_calculation/conftest.py +++ b/tests/e2e/cost_calculation/conftest.py @@ -12,6 +12,7 @@ Deselected unless E2E_COST_MAP_STACK is set (marker `cost_map_stack`). from __future__ import annotations import importlib.util +import json import sys from collections.abc import Callable, Mapping from dataclasses import dataclass @@ -22,7 +23,7 @@ from typing import Final, Protocol, cast import pytest from cost_matrix import Case, FrontierModel -from e2e_config import COST_MAP_PROXY_URL +from e2e_config import COST_MAP_PROXY_URL, SCRIPTED_PROVIDER_PROXY_BASE from lifecycle import ResourceManager from models import LiteLLMParamsBody, ModelInfoBody, ModelNewBody from proxy_client import ProxyClient, build_proxy_client @@ -111,6 +112,41 @@ def client() -> CostCalcClient: return CostCalcClient(proxy=proxy) +_vertex_key_pem: str | None = None + + +def _vertex_service_account_json() -> str: + """A service-account credential JSON whose token_uri is the sidecar's + /_oauth/token route: the proxy's google-auth refresh then gets a scripted + access token without touching Google. One generated RSA key per process.""" + global _vertex_key_pem # mutable-ok: session-scoped key generation cached for reuse + if _vertex_key_pem is None: + from cryptography.hazmat.primitives import serialization + from cryptography.hazmat.primitives.asymmetric import rsa + + _vertex_key_pem = ( + rsa.generate_private_key(public_exponent=65537, key_size=2048) + .private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, + serialization.NoEncryption(), + ) + .decode() + ) + return json.dumps( + { + "type": "service_account", + "project_id": "cc-scripted-project", + "private_key_id": "scripted", + "private_key": _vertex_key_pem, + "client_email": "scripted@cc-scripted-project.iam.gserviceaccount.com", + "client_id": "0", + "auth_uri": f"{SCRIPTED_PROVIDER_PROXY_BASE}/_oauth/authorize", + "token_uri": f"{SCRIPTED_PROVIDER_PROXY_BASE}/_oauth/token", + } + ) + + def register_scenario_deployment( client: CostCalcClient, resources: ResourceManager, @@ -126,15 +162,21 @@ def register_scenario_deployment( handle: Final = register_scenario(scenario) resources.defer(lambda: delete_scenario(handle)) model_name: Final = f"{model.model_name}-{marker}" + extra_params: Final[dict[str, str]] = dict(model.litellm_params) + if model.wire == "vertex_generate": + extra_params["vertex_credentials"] = _vertex_service_account_json() model_id: Final = client.proxy.register_model( ModelNewBody( model_name=model_name, - litellm_params=LiteLLMParamsBody( - model=model.litellm_model, - api_key=model.api_key, - api_base=handle.api_base(), + litellm_params=LiteLLMParamsBody.model_validate( + { + "model": model.litellm_model, + "api_key": model.api_key, + "api_base": handle.api_base(), + **extra_params, + } ), - model_info=ModelInfoBody(), + model_info=ModelInfoBody(base_model=model.base_model), ) ) resources.defer(lambda: client.proxy.delete_model(model_id)) diff --git a/tests/e2e/cost_calculation/cost_matrix.py b/tests/e2e/cost_calculation/cost_matrix.py index 5f634712778..37495f37b0b 100644 --- a/tests/e2e/cost_calculation/cost_matrix.py +++ b/tests/e2e/cost_calculation/cost_matrix.py @@ -88,7 +88,14 @@ class FrontierModel: litellm_model: str wire: Wire map_key: str - override_model: str + override_model: str | None = None + override_map_key: str | None = None + # Registered as model_info.base_model; when set, the provider-reported + # model loses to it and every case bills at this deployment's own rates. + base_model: str | None = None + # Extra litellm_params merged into the /model/new registration (api_version, + # aws_* credentials, vertex_* auth). + litellm_params: Mapping[str, str] = MappingProxyType({}) @property def rates(self) -> CostMapEntry: @@ -96,11 +103,16 @@ class FrontierModel: @property def override_rates(self) -> CostMapEntry: + if self.base_model is not None or self.override_map_key is None: + return self.rates return _COST_MAP[self.override_map_key] @property - def override_map_key(self) -> str: - return _OVERRIDE_MAP_KEYS[self.override_model] + def provider_model(self) -> str: + """The bare provider-facing model name: litellm_model minus the provider + prefix and any routing segment (converse/, responses/).""" + tail: Final = self.litellm_model.split("/")[1:] + return "/".join(tail[1:] if tail and tail[0] in ("converse", "responses") else tail) @property def provider(self) -> str: @@ -166,6 +178,92 @@ _FRONTIER_SPECS: Final[tuple[tuple[str, str, Wire], ...]] = ( ) +@dataclass(frozen=True, slots=True) +class _ExtendedSpec: + """A frontier entry whose override target, model_info.base_model or extra + litellm_params can't be derived from the map key alone.""" + + map_key: str + litellm_model: str + wire: Wire + override_model: str | None = None + override_map_key: str | None = None + base_model: str | None = None + litellm_params: Mapping[str, str] = MappingProxyType({}) + + +_AZURE_PARAMS: Final[Mapping[str, str]] = MappingProxyType({"api_version": "2025-04-01-preview"}) +_BEDROCK_PARAMS: Final[Mapping[str, str]] = MappingProxyType( + { + "aws_access_key_id": "AKIASCRIPTEDPROVIDER", + "aws_secret_access_key": "scripted-secret", + "aws_region_name": "us-east-1", + } +) +_VERTEX_PARAMS: Final[Mapping[str, str]] = MappingProxyType( + { + "vertex_project": "cc-scripted-project", + "vertex_location": "us-central1", + } +) + +_EXTENDED_SPECS: Final[tuple[_ExtendedSpec, ...]] = ( + _ExtendedSpec( + map_key="azure/gpt-5.6", + litellm_model="azure/gpt-5.6", + wire="azure_chat", + override_model="gpt-5.4-mini", + override_map_key="azure/gpt-5.4-mini", + litellm_params=_AZURE_PARAMS, + ), + _ExtendedSpec( + # Deployment name is not a model; base_model pins billing so the + # response's model field loses, proving base_model wins. + map_key="azure/gpt-5.4-mini", + litellm_model="azure/cc-pinned-deployment", + wire="azure_chat", + override_model="gpt-5.6", + override_map_key="azure/gpt-5.6", + base_model="azure/gpt-5.4-mini", + litellm_params=_AZURE_PARAMS, + ), + _ExtendedSpec( + map_key="anthropic.claude-sonnet-5-v1:0", + litellm_model="bedrock/converse/anthropic.claude-sonnet-5-v1:0", + wire="bedrock_converse", + litellm_params=_BEDROCK_PARAMS, + ), + _ExtendedSpec( + map_key="us.anthropic.claude-opus-5-v1:0", + litellm_model="bedrock/converse/us.anthropic.claude-opus-5-v1:0", + wire="bedrock_converse", + litellm_params=_BEDROCK_PARAMS, + ), + _ExtendedSpec( + map_key="meta.llama4-maverick-17b-instruct-v1:0", + litellm_model="bedrock/converse/meta.llama4-maverick-17b-instruct-v1:0", + wire="bedrock_converse", + litellm_params=_BEDROCK_PARAMS, + ), + _ExtendedSpec( + map_key="gemini-3.8-flash", + litellm_model="vertex_ai/gemini-3.8-flash", + wire="vertex_generate", + override_model="gemini-3.1-pro-preview", + override_map_key="gemini-3.1-pro-preview", + litellm_params=_VERTEX_PARAMS, + ), + _ExtendedSpec( + map_key="gemini-3.1-pro-preview", + litellm_model="vertex_ai/gemini-3.1-pro-preview", + wire="vertex_generate", + override_model="gemini-3.8-flash", + override_map_key="gemini-3.8-flash", + litellm_params=_VERTEX_PARAMS, + ), +) + + def _frontier() -> tuple[FrontierModel, ...]: return tuple( FrontierModel( @@ -174,8 +272,21 @@ def _frontier() -> tuple[FrontierModel, ...]: wire=wire, map_key=map_key, override_model=_OVERRIDE_MODELS[map_key], + override_map_key=_OVERRIDE_MAP_KEYS[_OVERRIDE_MODELS[map_key]], ) for map_key, litellm_model, wire in _FRONTIER_SPECS + ) + tuple( + FrontierModel( + model_name=f"cc-{spec.map_key.replace('/', '-').replace(':', '-').replace('.', '-').lower()}", + litellm_model=spec.litellm_model, + wire=spec.wire, + map_key=spec.map_key, + override_model=spec.override_model, + override_map_key=spec.override_map_key, + base_model=spec.base_model, + litellm_params=spec.litellm_params, + ) + for spec in _EXTENDED_SPECS ) @@ -219,6 +330,24 @@ _WIRE_CAPS: Final[Mapping[str, frozenset[str]]] = MappingProxyType({ "web_search", "response_model", "absent_usage", "tool_call", "image_input", } ), + "azure_chat": frozenset( + { + "cache_read", "cache_write_5m", "cache_write_1h", "reasoning", "audio", + "web_search", "response_model", "absent_usage", "tool_call", "image_input", + } + ), + "bedrock_converse": frozenset( + { + "cache_read", "cache_write_5m", "cache_write_1h", "absent_usage", + "tool_call", "image_input", + } + ), + "vertex_generate": frozenset( + { + "cache_read", "reasoning", "audio", "web_search", "response_model", + "absent_usage", "tool_call", "image_input", "prompt_blocked", + } + ), }) CaseName: TypeAlias = Literal[ @@ -270,6 +399,7 @@ class Case: scenario_id=scenario_id, wire=model.wire, usage=self.usage, + model=model.provider_model, output=ScriptedOutput( text=text, response_model=model.override_model if self.response_model_override else None, @@ -296,7 +426,9 @@ _PROMPT_BLOCKED_USAGE: Final = ScriptedUsage(fresh_input_tokens=1000, output_tok def _web_search_case(model: FrontierModel) -> Case: - counts_exactly: Final = model.wire in ("openai_responses", "anthropic_messages", "gemini_generate") + counts_exactly: Final = model.wire in ( + "openai_responses", "anthropic_messages", "gemini_generate", "vertex_generate" + ) return Case( name="web_search", usage=ScriptedUsage(fresh_input_tokens=100, output_tokens=30, web_search_calls=3), @@ -611,12 +743,12 @@ def expected_token_columns(model: FrontierModel, case: Case) -> tuple[int, int]: wire's normalization: Anthropic folds cache read/write into prompt_tokens, everyone else reports the totals the wire emitted.""" u: Final = case.usage - if model.wire == "anthropic_messages": + if model.wire in ("anthropic_messages", "bedrock_converse"): return ( u.fresh_input_tokens + u.cache_read_tokens + u.cache_write_5m_tokens + u.cache_write_1h_tokens, u.output_tokens, ) - if model.wire == "gemini_generate": + if model.wire in ("gemini_generate", "vertex_generate"): return ( u.fresh_input_tokens + u.cache_read_tokens + u.audio_input_tokens, u.output_tokens + u.reasoning_tokens + u.audio_output_tokens, diff --git a/tests/e2e/cost_calculation/scripted_provider.py b/tests/e2e/cost_calculation/scripted_provider.py index e1a6c430307..00230fabeba 100644 --- a/tests/e2e/cost_calculation/scripted_provider.py +++ b/tests/e2e/cost_calculation/scripted_provider.py @@ -15,10 +15,15 @@ Layout on one port: - ``GET /health`` liveness - ``POST /_scenarios`` register a Scenario JSON, returns its id - ``DELETE /_scenarios/`` remove it +- ``POST /_oauth/token`` fake Google OAuth token endpoint for the + Vertex service-account credential's refresh call - ``POST ///`` provider wire; mount is one of - ``openai``, ``anthropic``, ``gemini``, ``together``, ``fireworks`` and the - remainder is whatever path the provider client appends (``chat/completions``, - ``responses``, ``v1/messages``, ``models/:generateContent`` ...) + ``openai``, ``anthropic``, ``gemini``, ``together``, ``fireworks``, ``azure``, + ``bedrock``, ``vertex`` and the remainder is whatever path the provider + client appends (``chat/completions``, ``responses``, ``v1/messages``, + ``models/:generateContent`` ...). Vertex appends ``:generateContent`` / + ``:streamGenerateContent`` to the mount segment itself, and Bedrock Converse + targets ``model//converse`` / ``converse-stream`` A request carrying ``"stream": true`` (or the ``:streamGenerateContent`` Gemini verb) gets an SSE answer; ``stream_usage`` on the Scenario decides whether the @@ -28,15 +33,17 @@ final stream chunk carries usage or the provider reports none. from __future__ import annotations import json +import struct import sys import threading import time +import zlib from collections.abc import Mapping from dataclasses import dataclass from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from types import MappingProxyType from typing import Final, Literal, TypeAlias -from urllib.parse import urlsplit +from urllib.parse import unquote, urlsplit from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError, model_validator @@ -47,6 +54,9 @@ Wire: TypeAlias = Literal[ "gemini_generate", "together_chat", "fireworks_chat", + "azure_chat", + "bedrock_converse", + "vertex_generate", ] WIRE_MOUNTS: Final[Mapping[str, str]] = MappingProxyType( @@ -57,6 +67,9 @@ WIRE_MOUNTS: Final[Mapping[str, str]] = MappingProxyType( "gemini_generate": "gemini", "together_chat": "together", "fireworks_chat": "fireworks", + "azure_chat": "azure", + "bedrock_converse": "bedrock", + "vertex_generate": "vertex", } ) @@ -69,6 +82,7 @@ _TERMINAL_CAPS: Final[Mapping[str, frozenset[str]]] = MappingProxyType( { "openai_responses": frozenset({"incomplete", "unvalidated"}), "gemini_generate": frozenset({"prompt_blocked"}), + "vertex_generate": frozenset({"prompt_blocked"}), } ) @@ -131,6 +145,10 @@ class Scenario(BaseModel): wire: Wire usage: ScriptedUsage output: ScriptedOutput + # The bare provider-facing model name the renderer echoes when the request + # carries no model of its own (Vertex and Bedrock name the model in the URL + # path, not the body). + model: str stream_usage: StreamUsage = "final_chunk" service_tier: ServiceTier | None = None @@ -904,7 +922,208 @@ def _responses_sse(scenario: Scenario, requested_model: str) -> bytes: ) -def _render(scenario: Scenario, *, stream: bool, requested_model: str) -> RenderedResponse: +def _bedrock_usage(u: ScriptedUsage) -> Mapping[str, object]: + # Converse reports uncached input in inputTokens and rides cache reads and + # writes on top-level fields; totalTokens covers every input kind + output. + cache_writes: Final = u.cache_write_5m_tokens + u.cache_write_1h_tokens + return _jobj_opt( + ("inputTokens", u.fresh_input_tokens), + ("outputTokens", u.output_tokens), + ( + "totalTokens", + u.fresh_input_tokens + u.cache_read_tokens + cache_writes + u.output_tokens, + ), + ("cacheReadInputTokens", u.cache_read_tokens) if u.cache_read_tokens else None, + ("cacheWriteInputTokens", cache_writes) if cache_writes else None, + ( + ( + "cacheDetails", + tuple( + _jobj(("inputTokens", count), ("ttl", ttl)) + for count, ttl in ( + (u.cache_write_5m_tokens, "5m"), + (u.cache_write_1h_tokens, "1h"), + ) + if count + ), + ) + if cache_writes + else None + ), + ) + + +def _bedrock_stop_reason(scenario: Scenario) -> str: + if scenario.output.tool_call is not None: + return "tool_use" + return "end_turn" if scenario.output.finish_reason == "stop" else scenario.output.finish_reason + + +def _bedrock_content(scenario: Scenario) -> tuple[Mapping[str, object], ...]: + tool_call: Final = scenario.output.tool_call + if tool_call is not None: + return ( + _jobj( + ( + "toolUse", + _jobj( + ("toolUseId", f"tooluse_{scenario.scenario_id}"), + ("name", tool_call.name), + ("input", json.loads(tool_call.arguments)), + ), + ), + ), + ) + return (_jobj(("text", scenario.output.text)),) + + +def _bedrock_body(scenario: Scenario) -> Mapping[str, object]: + return _jobj( + ( + "output", + _jobj( + ( + "message", + _jobj( + ("role", "assistant"), + ("content", _bedrock_content(scenario)), + ), + ), + ), + ), + ("stopReason", _bedrock_stop_reason(scenario)), + ("usage", _bedrock_usage(scenario.usage)), + ("metrics", _jobj(("latencyMs", 42))), + ) + + +def _aws_event_frame(event_type: str, payload: Mapping[str, object]) -> bytes: + """One application/vnd.amazon.eventstream frame: prelude + prelude CRC32 + + headers + JSON payload + message CRC32, matching botocore EventStreamBuffer.""" + try: + from botocore.eventstream import crc32 as _crc32 + except ImportError: + _crc32 = zlib.crc32 + + def _str_header(name: str, value: str) -> bytes: + name_b: Final = name.encode() + value_b: Final = value.encode() + return ( + struct.pack("!B", len(name_b)) + + name_b + + struct.pack("!B", 7) + + struct.pack("!H", len(value_b)) + + value_b + ) + + payload_bytes: Final = json.dumps(payload, default=dict, separators=(",", ":")).encode() + headers_bytes: Final = ( + _str_header(":event-type", event_type) + + _str_header(":content-type", "application/json") + + _str_header(":message-type", "event") + ) + total_length: Final = 12 + len(headers_bytes) + len(payload_bytes) + 4 + prelude: Final = struct.pack("!II", total_length, len(headers_bytes)) + prelude_crc: Final = struct.pack("!I", _crc32(prelude) & 0xFFFFFFFF) + message: Final = prelude + prelude_crc + headers_bytes + payload_bytes + return message + struct.pack("!I", _crc32(message, 0) & 0xFFFFFFFF) + + +def _bedrock_eventstream(scenario: Scenario) -> bytes: + tool_call: Final = scenario.output.tool_call + block_start: Final[tuple[bytes, ...]] = ( + ( + _aws_event_frame( + "contentBlockStart", + _jobj( + ( + "start", + _jobj( + ( + "toolUse", + _jobj( + ("toolUseId", f"tooluse_{scenario.scenario_id}"), + ("name", tool_call.name), + ), + ), + ), + ), + ("contentBlockIndex", 0), + ), + ), + ) + if tool_call is not None + else () + ) + deltas: Final[tuple[bytes, ...]] = ( + tuple( + _aws_event_frame( + "contentBlockDelta", + _jobj( + ("delta", _jobj(("toolUse", _jobj(("input", arguments_slice))))), + ("contentBlockIndex", 0), + ), + ) + for arguments_slice in _split_arguments(tool_call.arguments) + ) + if tool_call is not None + else ( + _aws_event_frame( + "contentBlockDelta", + _jobj( + ("delta", _jobj(("text", scenario.output.text))), + ("contentBlockIndex", 0), + ), + ), + ) + ) + return b"".join( + ( + _aws_event_frame("messageStart", _jobj(("role", "assistant"))), + *block_start, + *deltas, + _aws_event_frame("contentBlockStop", _jobj(("contentBlockIndex", 0))), + _aws_event_frame("messageStop", _jobj(("stopReason", _bedrock_stop_reason(scenario)))), + *( + ( + _aws_event_frame( + "metadata", + _jobj( + ("usage", _bedrock_usage(scenario.usage)), + ("metrics", _jobj(("latencyMs", 42))), + ), + ), + ) + if scenario.stream_usage == "final_chunk" + else () + ), + ) + ) + + +def _render( + scenario: Scenario, *, stream: bool, requested_model: str, path_tail: str +) -> RenderedResponse: + # Azure bridges gpt-5.4+ chat requests carrying function tools onto the + # Responses API, which lands on the same mount at openai/responses. + if scenario.wire == "azure_chat" and path_tail.endswith("openai/responses"): + if stream: + return RenderedResponse( + 200, "text/event-stream", _responses_sse(scenario, requested_model) + ) + return RenderedResponse( + 200, "application/json", _json_bytes(_responses_body(scenario, requested_model)) + ) + if scenario.wire == "bedrock_converse": + if stream: + return RenderedResponse( + 200, "application/vnd.amazon.eventstream", _bedrock_eventstream(scenario) + ) + return RenderedResponse(200, "application/json", _json_bytes(_bedrock_body(scenario))) + if scenario.wire == "vertex_generate": + if stream: + return RenderedResponse(200, "text/event-stream", _gemini_sse(scenario, requested_model)) + return RenderedResponse(200, "application/json", _json_bytes(_gemini_body(scenario, requested_model))) if scenario.wire == "anthropic_messages": if stream: return RenderedResponse(200, "text/event-stream", _anthropic_sse(scenario, requested_model)) @@ -917,7 +1136,8 @@ def _render(scenario: Scenario, *, stream: bool, requested_model: str) -> Render if stream: return RenderedResponse(200, "text/event-stream", _responses_sse(scenario, requested_model)) return RenderedResponse(200, "application/json", _json_bytes(_responses_body(scenario, requested_model))) - # openai_chat, together_chat, fireworks_chat share the OpenAI chat shape. + # openai_chat, together_chat, fireworks_chat and azure_chat share the + # OpenAI chat shape. if stream: return RenderedResponse(200, "text/event-stream", _openai_chat_sse(scenario, requested_model)) return RenderedResponse(200, "application/json", _json_bytes(_openai_chat_body(scenario, requested_model))) @@ -954,17 +1174,28 @@ def _request_body(body: bytes) -> Mapping[str, object]: return MappingProxyType({}) -def _request_wants_stream(path_tail: str, body: bytes) -> bool: - if ":streamGenerateContent" in path_tail: +def _request_wants_stream(mount_endpoint: str | None, path_tail: str, body: bytes) -> bool: + if mount_endpoint == "streamGenerateContent" or ":streamGenerateContent" in path_tail: + return True + if path_tail.endswith("converse-stream"): return True if not body: return False return _request_body(body).get("stream") is True -def _request_model(body: bytes) -> str: +def _request_model(body: bytes, path_tail: str, scenario: Scenario) -> str: model: Final = _request_body(body).get("model") - return model if isinstance(model, str) else "unknown" + if isinstance(model, str): + return model + # Bedrock Converse names the model in the path: model//converse[-stream]. + if path_tail.startswith("model/"): + path_model: Final = path_tail.split("/", 2)[1] if path_tail.count("/") >= 2 else "" + if path_model: + return unquote(path_model) + # Vertex names it in the URL too, but the mount segment swallowed it when + # the api_base carried a path; fall back to the scenario's declared model. + return scenario.model def handle_request(store: _ScenarioStore, method: str, raw_path: str, body: bytes) -> RenderedResponse: @@ -972,6 +1203,22 @@ def handle_request(store: _ScenarioStore, method: str, raw_path: str, body: byte segments: Final = tuple(segment for segment in path.split("/") if segment) if method == "GET" and segments == ("health",): return RenderedResponse(200, "application/json", _json_bytes(_jobj(("status", "ok")))) + if segments and segments[0] == "_oauth": + if method == "POST" and segments == ("_oauth", "token"): + return RenderedResponse( + 200, + "application/json", + _json_bytes( + _jobj( + ("access_token", "scripted-token"), + ("token_type", "Bearer"), + ("expires_in", 3600), + ) + ), + ) + return RenderedResponse( + 404, "application/json", _json_bytes(_jobj(("error", "unknown control route"))) + ) if segments and segments[0] == "_scenarios": if method == "POST" and len(segments) == 1: try: @@ -998,7 +1245,15 @@ def handle_request(store: _ScenarioStore, method: str, raw_path: str, body: byte return RenderedResponse( 404, "application/json", _json_bytes(_jobj(("error", f"no route for {method} {path}"))) ) - scenario_id, mount = segments[0], segments[1] + scenario_id: Final = segments[0] + # Vertex builds {api_base}:{endpoint}, so the mount segment can carry a + # :generateContent / :streamGenerateContent suffix. + mount_segment: Final = segments[1] + mount, mount_endpoint = ( + mount_segment.split(":", 1) + if ":" in mount_segment + else (mount_segment, None) + ) found: Final = store.get(scenario_id) if found is None: return RenderedResponse( @@ -1013,7 +1268,12 @@ def handle_request(store: _ScenarioStore, method: str, raw_path: str, body: byte ), ) tail: Final = "/".join(segments[2:]) - return _render(found, stream=_request_wants_stream(tail, body), requested_model=_request_model(body)) + return _render( + found, + stream=_request_wants_stream(mount_endpoint, tail, body), + requested_model=_request_model(body, tail, found), + path_tail=tail, + ) class _ScriptedHandler(BaseHTTPRequestHandler): diff --git a/tests/e2e/cost_calculation/test_wire_formats_e2e.py b/tests/e2e/cost_calculation/test_wire_formats_e2e.py index 4da7b31a6ef..a36bb1a8662 100644 --- a/tests/e2e/cost_calculation/test_wire_formats_e2e.py +++ b/tests/e2e/cost_calculation/test_wire_formats_e2e.py @@ -94,6 +94,40 @@ _WIRE_USAGE: Final[Mapping[str, tuple[str, ScriptedUsage]]] = MappingProxyType({ "fireworks_ai/kimi-k3", ScriptedUsage(fresh_input_tokens=80, cache_read_tokens=40, output_tokens=25), ), + "azure_chat": ( + "azure/gpt-5.6", + ScriptedUsage( + fresh_input_tokens=80, + cache_read_tokens=40, + cache_write_5m_tokens=20, + cache_write_1h_tokens=10, + output_tokens=25, + reasoning_tokens=15, + audio_input_tokens=5, + audio_output_tokens=3, + ), + ), + "bedrock_converse": ( + "anthropic.claude-sonnet-5-v1:0", + ScriptedUsage( + fresh_input_tokens=80, + cache_read_tokens=40, + cache_write_5m_tokens=20, + cache_write_1h_tokens=10, + output_tokens=25, + ), + ), + "vertex_generate": ( + "gemini-3.8-flash", + ScriptedUsage( + fresh_input_tokens=80, + cache_read_tokens=40, + output_tokens=25, + reasoning_tokens=15, + audio_input_tokens=5, + audio_output_tokens=3, + ), + ), }) _SHAPE_USAGE: Final = ScriptedUsage(fresh_input_tokens=80, output_tokens=25) @@ -141,6 +175,36 @@ _SHAPES: Final[tuple[tuple[str, str, Case], ...]] = ( response_model_override=True, ), ), + ( + "vertex_prompt_blocked", + "vertex_generate", + Case( + name="prompt_blocked", + usage=ScriptedUsage(fresh_input_tokens=1000, output_tokens=0), + terminal="prompt_blocked", + response_model_override=True, + ), + ), + ( + "vertex_prompt_blocked_stream", + "vertex_generate", + Case( + name="stream_prompt_blocked", + usage=ScriptedUsage(fresh_input_tokens=1000, output_tokens=0), + stream=True, + terminal="prompt_blocked", + response_model_override=True, + ), + ), + ( + "azure_served_model_override", + "azure_chat", + Case( + name="response_model_override", + usage=_SHAPE_USAGE, + response_model_override=True, + ), + ), ) diff --git a/tests/e2e/cost_map.json b/tests/e2e/cost_map.json index 68d840870c9..4fba337b701 100644 --- a/tests/e2e/cost_map.json +++ b/tests/e2e/cost_map.json @@ -304,6 +304,149 @@ "supports_reasoning": true, "supports_web_search": true }, + "anthropic.claude-sonnet-5-v1:0": { + "cache_creation_input_token_cost": 0.00051, + "cache_creation_input_token_cost_above_1hr": 0.00068, + "cache_read_input_token_cost": 1.7e-05, + "input_cost_per_token": 0.00017, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 0.00034, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true + }, + "azure/gpt-5.4-mini": { + "cache_creation_input_token_cost": 0.00048, + "cache_creation_input_token_cost_above_1hr": 0.00064, + "cache_read_input_token_cost": 1.6e-05, + "input_cost_per_audio_token": 0.00096, + "input_cost_per_token": 0.00016, + "input_cost_per_token_above_200k_tokens": 0.00128, + "input_cost_per_token_flex": 0.00024, + "input_cost_per_token_priority": 0.000272, + "litellm_provider": "azure", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_audio_token": 0.00112, + "output_cost_per_reasoning_token": 0.0008, + "output_cost_per_token": 0.00032, + "output_cost_per_token_above_200k_tokens": 0.00144, + "output_cost_per_token_flex": 0.0004, + "output_cost_per_token_priority": 0.000432, + "search_context_cost_per_query": { + "search_context_size_high": 0.03, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.02 + }, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_web_search": true + }, + "azure/gpt-5.6": { + "cache_creation_input_token_cost": 0.00044999999999999996, + "cache_creation_input_token_cost_above_1hr": 0.0006000000000000001, + "cache_read_input_token_cost": 1.5e-05, + "input_cost_per_audio_token": 0.0009000000000000001, + "input_cost_per_token": 0.00015000000000000001, + "input_cost_per_token_above_200k_tokens": 0.0012000000000000001, + "input_cost_per_token_flex": 0.000225, + "input_cost_per_token_priority": 0.000255, + "litellm_provider": "azure", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_audio_token": 0.0010500000000000002, + "output_cost_per_reasoning_token": 0.00075, + "output_cost_per_token": 0.00030000000000000003, + "output_cost_per_token_above_200k_tokens": 0.00135, + "output_cost_per_token_flex": 0.000375, + "output_cost_per_token_priority": 0.00040499999999999996, + "search_context_cost_per_query": { + "search_context_size_high": 0.03, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.02 + }, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_web_search": true + }, + "gemini-3.1-pro-preview": { + "cache_read_input_token_cost": 2.1e-05, + "input_cost_per_audio_token": 0.00126, + "input_cost_per_token": 0.00021, + "input_cost_per_token_above_200k_tokens": 0.00168, + "input_cost_per_token_flex": 0.000315, + "input_cost_per_token_priority": 0.000357, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_audio_token": 0.00147, + "output_cost_per_reasoning_token": 0.0010500000000000002, + "output_cost_per_token": 0.00042, + "output_cost_per_token_above_200k_tokens": 0.0018900000000000001, + "output_cost_per_token_flex": 0.000525, + "output_cost_per_token_priority": 0.000567, + "search_context_cost_per_query": { + "search_context_size_high": 0.03, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.02 + }, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_web_search": true, + "web_search_billing_unit": "per_query" + }, + "gemini-3.8-flash": { + "cache_read_input_token_cost": 2e-05, + "input_cost_per_audio_token": 0.0012, + "input_cost_per_token": 0.0002, + "input_cost_per_token_above_200k_tokens": 0.0016, + "input_cost_per_token_flex": 0.0003, + "input_cost_per_token_priority": 0.00034, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_audio_token": 0.0014000000000000002, + "output_cost_per_reasoning_token": 0.001, + "output_cost_per_token": 0.0004, + "output_cost_per_token_above_200k_tokens": 0.0018000000000000001, + "output_cost_per_token_flex": 0.0005, + "output_cost_per_token_priority": 0.00054, + "search_context_cost_per_query": { + "search_context_size_high": 0.03, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.02 + }, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_web_search": true, + "web_search_billing_unit": "per_query" + }, + "meta.llama4-maverick-17b-instruct-v1:0": { + "input_cost_per_token": 0.00019, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 0.00038, + "supports_function_calling": true + }, "together_ai/moonshotai/Kimi-K3": { "cache_creation_input_token_cost": 0.00030000000000000003, "cache_creation_input_token_cost_above_1hr": 0.0004, @@ -363,5 +506,20 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_web_search": true + }, + "us.anthropic.claude-opus-5-v1:0": { + "cache_creation_input_token_cost": 0.0005400000000000001, + "cache_creation_input_token_cost_above_1hr": 0.00072, + "cache_read_input_token_cost": 1.8e-05, + "input_cost_per_token": 0.00018, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 0.00036000000000000004, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true } } diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 7101438c5f8..d96478de1c4 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -1000,6 +1000,7 @@ class ModelInfoBody(BaseModel): access_groups: list[str] | None = None team_id: str | None = None allowed_fails_policy: dict[str, int] | None = None + base_model: str | None = None class ModelNewBody(BaseModel): From 2466975d290576de9e89a1d5d69c9ca9a6aab1ab Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 16 Sep 2026 22:57:23 +0000 Subject: [PATCH 010/210] test(e2e): clean cost map decimals and simplify scripted wire helpers Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/cost_calculation/conftest.py | 52 ++-- .../e2e/cost_calculation/scripted_provider.py | 39 ++- tests/e2e/cost_map.json | 270 +++++++++--------- 3 files changed, 177 insertions(+), 184 deletions(-) diff --git a/tests/e2e/cost_calculation/conftest.py b/tests/e2e/cost_calculation/conftest.py index 8c6db7c0010..3f3e9fd9243 100644 --- a/tests/e2e/cost_calculation/conftest.py +++ b/tests/e2e/cost_calculation/conftest.py @@ -11,6 +11,7 @@ Deselected unless E2E_COST_MAP_STACK is set (marker `cost_map_stack`). from __future__ import annotations +import functools import importlib.util import json import sys @@ -21,6 +22,8 @@ from types import ModuleType from typing import Final, Protocol, cast import pytest +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import rsa from cost_matrix import Case, FrontierModel from e2e_config import COST_MAP_PROXY_URL, SCRIPTED_PROVIDER_PROXY_BASE @@ -112,33 +115,25 @@ def client() -> CostCalcClient: return CostCalcClient(proxy=proxy) -_vertex_key_pem: str | None = None +@functools.cache +def _vertex_private_key_pem() -> str: + return rsa.generate_private_key(public_exponent=65537, key_size=2048).private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, + serialization.NoEncryption(), + ).decode() def _vertex_service_account_json() -> str: """A service-account credential JSON whose token_uri is the sidecar's /_oauth/token route: the proxy's google-auth refresh then gets a scripted - access token without touching Google. One generated RSA key per process.""" - global _vertex_key_pem # mutable-ok: session-scoped key generation cached for reuse - if _vertex_key_pem is None: - from cryptography.hazmat.primitives import serialization - from cryptography.hazmat.primitives.asymmetric import rsa - - _vertex_key_pem = ( - rsa.generate_private_key(public_exponent=65537, key_size=2048) - .private_bytes( - serialization.Encoding.PEM, - serialization.PrivateFormat.PKCS8, - serialization.NoEncryption(), - ) - .decode() - ) + access token without touching Google.""" return json.dumps( { "type": "service_account", "project_id": "cc-scripted-project", "private_key_id": "scripted", - "private_key": _vertex_key_pem, + "private_key": _vertex_private_key_pem(), "client_email": "scripted@cc-scripted-project.iam.gserviceaccount.com", "client_id": "0", "auth_uri": f"{SCRIPTED_PROVIDER_PROXY_BASE}/_oauth/authorize", @@ -162,20 +157,21 @@ def register_scenario_deployment( handle: Final = register_scenario(scenario) resources.defer(lambda: delete_scenario(handle)) model_name: Final = f"{model.model_name}-{marker}" - extra_params: Final[dict[str, str]] = dict(model.litellm_params) - if model.wire == "vertex_generate": - extra_params["vertex_credentials"] = _vertex_service_account_json() + params: Final = { + "model": model.litellm_model, + "api_key": model.api_key, + "api_base": handle.api_base(), + **model.litellm_params, + **( + {"vertex_credentials": _vertex_service_account_json()} + if model.wire == "vertex_generate" + else {} + ), + } model_id: Final = client.proxy.register_model( ModelNewBody( model_name=model_name, - litellm_params=LiteLLMParamsBody.model_validate( - { - "model": model.litellm_model, - "api_key": model.api_key, - "api_base": handle.api_base(), - **extra_params, - } - ), + litellm_params=LiteLLMParamsBody.model_validate(params), model_info=ModelInfoBody(base_model=model.base_model), ) ) diff --git a/tests/e2e/cost_calculation/scripted_provider.py b/tests/e2e/cost_calculation/scripted_provider.py index 00230fabeba..982132ed8df 100644 --- a/tests/e2e/cost_calculation/scripted_provider.py +++ b/tests/e2e/cost_calculation/scripted_provider.py @@ -997,36 +997,33 @@ def _bedrock_body(scenario: Scenario) -> Mapping[str, object]: ) +def _aws_str_header(name: str, value: str) -> bytes: + """One eventstream header: 1-byte name len + name + type-7 marker + value.""" + name_b: Final = name.encode() + value_b: Final = value.encode() + return ( + struct.pack("!B", len(name_b)) + + name_b + + struct.pack("!B", 7) + + struct.pack("!H", len(value_b)) + + value_b + ) + + def _aws_event_frame(event_type: str, payload: Mapping[str, object]) -> bytes: """One application/vnd.amazon.eventstream frame: prelude + prelude CRC32 + headers + JSON payload + message CRC32, matching botocore EventStreamBuffer.""" - try: - from botocore.eventstream import crc32 as _crc32 - except ImportError: - _crc32 = zlib.crc32 - - def _str_header(name: str, value: str) -> bytes: - name_b: Final = name.encode() - value_b: Final = value.encode() - return ( - struct.pack("!B", len(name_b)) - + name_b - + struct.pack("!B", 7) - + struct.pack("!H", len(value_b)) - + value_b - ) - payload_bytes: Final = json.dumps(payload, default=dict, separators=(",", ":")).encode() headers_bytes: Final = ( - _str_header(":event-type", event_type) - + _str_header(":content-type", "application/json") - + _str_header(":message-type", "event") + _aws_str_header(":event-type", event_type) + + _aws_str_header(":content-type", "application/json") + + _aws_str_header(":message-type", "event") ) total_length: Final = 12 + len(headers_bytes) + len(payload_bytes) + 4 prelude: Final = struct.pack("!II", total_length, len(headers_bytes)) - prelude_crc: Final = struct.pack("!I", _crc32(prelude) & 0xFFFFFFFF) + prelude_crc: Final = struct.pack("!I", zlib.crc32(prelude) & 0xFFFFFFFF) message: Final = prelude + prelude_crc + headers_bytes + payload_bytes - return message + struct.pack("!I", _crc32(message, 0) & 0xFFFFFFFF) + return message + struct.pack("!I", zlib.crc32(message) & 0xFFFFFFFF) def _bedrock_eventstream(scenario: Scenario) -> bytes: diff --git a/tests/e2e/cost_map.json b/tests/e2e/cost_map.json index 4fba337b701..85cd5ade3d5 100644 --- a/tests/e2e/cost_map.json +++ b/tests/e2e/cost_map.json @@ -1,4 +1,79 @@ { + "anthropic.claude-sonnet-5-v1:0": { + "cache_creation_input_token_cost": 0.00051, + "cache_creation_input_token_cost_above_1hr": 0.00068, + "cache_read_input_token_cost": 1.7e-05, + "input_cost_per_token": 0.00017, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 0.00034, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true + }, + "azure/gpt-5.4-mini": { + "cache_creation_input_token_cost": 0.00048, + "cache_creation_input_token_cost_above_1hr": 0.00064, + "cache_read_input_token_cost": 1.6e-05, + "input_cost_per_audio_token": 0.00096, + "input_cost_per_token": 0.00016, + "input_cost_per_token_above_200k_tokens": 0.00128, + "input_cost_per_token_flex": 0.00024, + "input_cost_per_token_priority": 0.000272, + "litellm_provider": "azure", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_audio_token": 0.00112, + "output_cost_per_reasoning_token": 0.0008, + "output_cost_per_token": 0.00032, + "output_cost_per_token_above_200k_tokens": 0.00144, + "output_cost_per_token_flex": 0.0004, + "output_cost_per_token_priority": 0.000432, + "search_context_cost_per_query": { + "search_context_size_high": 0.03, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.02 + }, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_web_search": true + }, + "azure/gpt-5.6": { + "cache_creation_input_token_cost": 0.00045, + "cache_creation_input_token_cost_above_1hr": 0.0006, + "cache_read_input_token_cost": 1.5e-05, + "input_cost_per_audio_token": 0.0009, + "input_cost_per_token": 0.00015, + "input_cost_per_token_above_200k_tokens": 0.0012, + "input_cost_per_token_flex": 0.000225, + "input_cost_per_token_priority": 0.000255, + "litellm_provider": "azure", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_audio_token": 0.00105, + "output_cost_per_reasoning_token": 0.00075, + "output_cost_per_token": 0.0003, + "output_cost_per_token_above_200k_tokens": 0.00135, + "output_cost_per_token_flex": 0.000375, + "output_cost_per_token_priority": 0.000405, + "search_context_cost_per_query": { + "search_context_size_high": 0.03, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.02 + }, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_web_search": true + }, "claude-haiku-4-5": { "cache_creation_input_token_cost": 0.00021, "cache_creation_input_token_cost_above_1hr": 0.00028000000000000003, @@ -134,6 +209,64 @@ "supports_reasoning": true, "supports_web_search": true }, + "gemini-3.1-pro-preview": { + "cache_read_input_token_cost": 2.1e-05, + "input_cost_per_audio_token": 0.00126, + "input_cost_per_token": 0.00021, + "input_cost_per_token_above_200k_tokens": 0.00168, + "input_cost_per_token_flex": 0.000315, + "input_cost_per_token_priority": 0.000357, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_audio_token": 0.00147, + "output_cost_per_reasoning_token": 0.00105, + "output_cost_per_token": 0.00042, + "output_cost_per_token_above_200k_tokens": 0.00189, + "output_cost_per_token_flex": 0.000525, + "output_cost_per_token_priority": 0.000567, + "search_context_cost_per_query": { + "search_context_size_high": 0.03, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.02 + }, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_web_search": true, + "web_search_billing_unit": "per_query" + }, + "gemini-3.8-flash": { + "cache_read_input_token_cost": 2e-05, + "input_cost_per_audio_token": 0.0012, + "input_cost_per_token": 0.0002, + "input_cost_per_token_above_200k_tokens": 0.0016, + "input_cost_per_token_flex": 0.0003, + "input_cost_per_token_priority": 0.00034, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_audio_token": 0.0014, + "output_cost_per_reasoning_token": 0.001, + "output_cost_per_token": 0.0004, + "output_cost_per_token_above_200k_tokens": 0.0018, + "output_cost_per_token_flex": 0.0005, + "output_cost_per_token_priority": 0.00054, + "search_context_cost_per_query": { + "search_context_size_high": 0.03, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.02 + }, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_web_search": true, + "web_search_billing_unit": "per_query" + }, "gemini/gemini-3.1-pro-preview": { "cache_read_input_token_cost": 9e-06, "input_cost_per_audio_token": 0.00054, @@ -304,139 +437,6 @@ "supports_reasoning": true, "supports_web_search": true }, - "anthropic.claude-sonnet-5-v1:0": { - "cache_creation_input_token_cost": 0.00051, - "cache_creation_input_token_cost_above_1hr": 0.00068, - "cache_read_input_token_cost": 1.7e-05, - "input_cost_per_token": 0.00017, - "litellm_provider": "bedrock_converse", - "max_input_tokens": 2000000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 0.00034, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true - }, - "azure/gpt-5.4-mini": { - "cache_creation_input_token_cost": 0.00048, - "cache_creation_input_token_cost_above_1hr": 0.00064, - "cache_read_input_token_cost": 1.6e-05, - "input_cost_per_audio_token": 0.00096, - "input_cost_per_token": 0.00016, - "input_cost_per_token_above_200k_tokens": 0.00128, - "input_cost_per_token_flex": 0.00024, - "input_cost_per_token_priority": 0.000272, - "litellm_provider": "azure", - "max_input_tokens": 2000000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_audio_token": 0.00112, - "output_cost_per_reasoning_token": 0.0008, - "output_cost_per_token": 0.00032, - "output_cost_per_token_above_200k_tokens": 0.00144, - "output_cost_per_token_flex": 0.0004, - "output_cost_per_token_priority": 0.000432, - "search_context_cost_per_query": { - "search_context_size_high": 0.03, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.02 - }, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_web_search": true - }, - "azure/gpt-5.6": { - "cache_creation_input_token_cost": 0.00044999999999999996, - "cache_creation_input_token_cost_above_1hr": 0.0006000000000000001, - "cache_read_input_token_cost": 1.5e-05, - "input_cost_per_audio_token": 0.0009000000000000001, - "input_cost_per_token": 0.00015000000000000001, - "input_cost_per_token_above_200k_tokens": 0.0012000000000000001, - "input_cost_per_token_flex": 0.000225, - "input_cost_per_token_priority": 0.000255, - "litellm_provider": "azure", - "max_input_tokens": 2000000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_audio_token": 0.0010500000000000002, - "output_cost_per_reasoning_token": 0.00075, - "output_cost_per_token": 0.00030000000000000003, - "output_cost_per_token_above_200k_tokens": 0.00135, - "output_cost_per_token_flex": 0.000375, - "output_cost_per_token_priority": 0.00040499999999999996, - "search_context_cost_per_query": { - "search_context_size_high": 0.03, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.02 - }, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_web_search": true - }, - "gemini-3.1-pro-preview": { - "cache_read_input_token_cost": 2.1e-05, - "input_cost_per_audio_token": 0.00126, - "input_cost_per_token": 0.00021, - "input_cost_per_token_above_200k_tokens": 0.00168, - "input_cost_per_token_flex": 0.000315, - "input_cost_per_token_priority": 0.000357, - "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 2000000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_audio_token": 0.00147, - "output_cost_per_reasoning_token": 0.0010500000000000002, - "output_cost_per_token": 0.00042, - "output_cost_per_token_above_200k_tokens": 0.0018900000000000001, - "output_cost_per_token_flex": 0.000525, - "output_cost_per_token_priority": 0.000567, - "search_context_cost_per_query": { - "search_context_size_high": 0.03, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.02 - }, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_web_search": true, - "web_search_billing_unit": "per_query" - }, - "gemini-3.8-flash": { - "cache_read_input_token_cost": 2e-05, - "input_cost_per_audio_token": 0.0012, - "input_cost_per_token": 0.0002, - "input_cost_per_token_above_200k_tokens": 0.0016, - "input_cost_per_token_flex": 0.0003, - "input_cost_per_token_priority": 0.00034, - "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 2000000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_audio_token": 0.0014000000000000002, - "output_cost_per_reasoning_token": 0.001, - "output_cost_per_token": 0.0004, - "output_cost_per_token_above_200k_tokens": 0.0018000000000000001, - "output_cost_per_token_flex": 0.0005, - "output_cost_per_token_priority": 0.00054, - "search_context_cost_per_query": { - "search_context_size_high": 0.03, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.02 - }, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_web_search": true, - "web_search_billing_unit": "per_query" - }, "meta.llama4-maverick-17b-instruct-v1:0": { "input_cost_per_token": 0.00019, "litellm_provider": "bedrock_converse", @@ -508,7 +508,7 @@ "supports_web_search": true }, "us.anthropic.claude-opus-5-v1:0": { - "cache_creation_input_token_cost": 0.0005400000000000001, + "cache_creation_input_token_cost": 0.00054, "cache_creation_input_token_cost_above_1hr": 0.00072, "cache_read_input_token_cost": 1.8e-05, "input_cost_per_token": 0.00018, @@ -517,7 +517,7 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 0.00036000000000000004, + "output_cost_per_token": 0.00036, "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true From 813d96f26ea6780bacb4b5ad562f1aecc3cb5069 Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 16 Sep 2026 23:18:26 +0000 Subject: [PATCH 011/210] fix(e2e): resolve remaining merge markers in e2e_config Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/e2e_config.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py index b34eadd8744..e19cfaa684f 100644 --- a/tests/e2e/e2e_config.py +++ b/tests/e2e/e2e_config.py @@ -145,7 +145,6 @@ WEEKLY_ANOMALY_OPT_IN_ENV = "E2E_WEEKLY_ANOMALY" MANAGED_FILES_OPT_IN_ENV = "E2E_MANAGED_FILES_STACK" PROMPT_CACHING_OPT_IN_ENV = "E2E_PROMPT_CACHING_STACK" REDIS_CHAOS_OPT_IN_ENV = "E2E_REDIS_CHAOS" -<<<<<<< HEAD # The cost_calculation suite needs a proxy booted with LITELLM_MODEL_COST_MAP_URL # pointing at tests/e2e/cost_map.json (its whole map is test-owned rates) plus a # scripted-provider sidecar; deselected unless the opt-in env var is set. @@ -162,9 +161,6 @@ SCRIPTED_PROVIDER_CONTROL_URL = os.environ.get( SCRIPTED_PROVIDER_PROXY_BASE = os.environ.get( "E2E_SCRIPTED_PROVIDER_PROXY_BASE", SCRIPTED_PROVIDER_CONTROL_URL ).rstrip("/") -||||||| 930ec9643a -======= -CLI_DETERMINISM_OPT_IN_ENV = "E2E_CLI_DETERMINISM" CLI_DETERMINISM_OPT_IN_ENV = "E2E_CLI_DETERMINISM" ANOMALY_SESSIONS = int(os.environ.get("E2E_ANOMALY_SESSIONS", "6")) ANOMALY_TURNS_PER_SESSION = int(os.environ.get("E2E_ANOMALY_TURNS_PER_SESSION", "6")) From bdfff602fb0325f88768f7c4411cce921ab28fcb Mon Sep 17 00:00:00 2001 From: kerry Date: Thu, 17 Sep 2026 00:47:55 +0000 Subject: [PATCH 012/210] test(e2e): drive the cost matrix from cases.json and expected.json goldens Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/CLAUDE.md | 2 +- tests/e2e/cost_calculation/cases.json | 231 ++ tests/e2e/cost_calculation/conftest.py | 10 +- tests/e2e/cost_calculation/cost_matrix.py | 788 ++----- tests/e2e/cost_calculation/expected.json | 2004 +++++++++++++++++ .../e2e/cost_calculation/generate_expected.py | 189 ++ .../e2e/cost_calculation/test_matrix_data.py | 64 + .../test_token_pricing_e2e.py | 62 +- .../cost_calculation/test_wire_formats_e2e.py | 368 --- 9 files changed, 2761 insertions(+), 957 deletions(-) create mode 100644 tests/e2e/cost_calculation/cases.json create mode 100644 tests/e2e/cost_calculation/expected.json create mode 100644 tests/e2e/cost_calculation/generate_expected.py create mode 100644 tests/e2e/cost_calculation/test_matrix_data.py delete mode 100644 tests/e2e/cost_calculation/test_wire_formats_e2e.py diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index 49cfc29aa17..707d35b4aa6 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -21,7 +21,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments split round robin over `/chat/completions` and `/v1/messages`, one endpoint per simulated user, with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests on every endpoint, budgeting RSS and CPU-per-request as ratios against the same run's healthy phase, and holding p50/p90/p99 latency and log-bytes-per-request to flat ceilings (a ratio cannot bound those two: an open breaker skips Redis instead of waiting on it, so the chaos phase can measure cheaper than baseline while still being far slower than a user should see); needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` and `E2E_PROXY_LOG` set, marked `redis_chaos`, deselected unless `E2E_REDIS_CHAOS` is set and excluded from the per-PR selector like the rest of `load/`, driven by `.github/workflows/test-e2e-redis-chaos.yml` and by the Buildkite `e2e-redis-chaos` step in project-releaser, which runs the proxy, Postgres and Valkey co-located with pytest in one pod and sets the opt-in), and markerless harness unit tests for the locust, process-usage, and session-anomaly aggregation logic - `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate, JWT auth (access tokens issued by a real Keycloak realm, `idp.py` plus `idp_realm.json`, whose JWKS the proxy's `JWT_PUBLIC_KEY_URL` points at; see CONTRIBUTING.md for the start command and config block), and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite - `gateway/` - proxy configuration only (`litellm-config.yml`); no tests -- `cost_calculation/` - cost accounting against a dedicated proxy whose whole model cost map is the test-owned `tests/e2e/cost_map.json` (loaded via `LITELLM_MODEL_COST_MAP_URL`), with provider calls answered by the scripted-provider sidecar in `scripted_provider.py`; asserts literal rate arithmetic on scripted usage across every provider wire and pricing component, deselected unless `E2E_COST_MAP_STACK` is set, driven by the Buildkite `e2e-cost-calculation` step in project-releaser, which runs a proxy booted from `gateway/cost_calculation_ci_config.yml`, Postgres and the scripted provider co-located with pytest in one pod and sets the opt-in +- `cost_calculation/` - cost accounting against a dedicated proxy whose whole model cost map is the test-owned `tests/e2e/cost_map.json` (loaded via `LITELLM_MODEL_COST_MAP_URL`), with provider calls answered by the scripted-provider sidecar in `scripted_provider.py`; every cost-map entry is a deployment and the cases plus asserted goldens are data in `cases.json` and `expected.json` (regenerate with `generate_expected.py`), deselected unless `E2E_COST_MAP_STACK` is set, driven by the Buildkite `e2e-cost-calculation` step in project-releaser, which runs a proxy booted from `gateway/cost_calculation_ci_config.yml`, Postgres and the scripted provider co-located with pytest in one pod and sets the opt-in - `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture; ships its own driver/builder/publisher plus `_*_unit_tests/` trees. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`); the CLI-driving path stays bespoke - `ui/` - the Admin UI browser suite: Playwright in TypeScript, driving the dashboard served by a live proxy on port 4000 (seeded postgres + mock LLM upstream; see its `run_e2e.sh`). It is a self-contained npm package with its own lockfile and does not use the Python harness, pytest markers, or the shared transport; the Python rules in this file (typed models, `Result` unions, basedpyright zero-error gate) do not apply inside it. Its only Python file, `fixtures/mock_llm_server/server.py`, is excluded from the e2e basedpyright gate via the root `pyrightconfig.json` diff --git a/tests/e2e/cost_calculation/cases.json b/tests/e2e/cost_calculation/cases.json new file mode 100644 index 00000000000..e898557ea35 --- /dev/null +++ b/tests/e2e/cost_calculation/cases.json @@ -0,0 +1,231 @@ +{ + "deployments": [ + { + "map_key": "azure/gpt-5.4-mini", + "litellm_model": "azure/cc-pinned-deployment", + "base_model": "azure/gpt-5.4-mini" + } + ], + "cases": [ + { + "name": "basic", + "usage": {"fresh_input_tokens": 120, "output_tokens": 40} + }, + { + "name": "cache_read", + "usage": {"fresh_input_tokens": 100, "cache_read_tokens": 50, "output_tokens": 30}, + "requires_rates": ["cache_read_input_token_cost"], + "requires_caps": ["cache_read"] + }, + { + "name": "cache_write_5m", + "usage": {"fresh_input_tokens": 90, "cache_write_5m_tokens": 60, "output_tokens": 30}, + "requires_rates": ["cache_creation_input_token_cost"], + "requires_caps": ["cache_write_5m"] + }, + { + "name": "cache_write_1h", + "usage": {"fresh_input_tokens": 90, "cache_write_5m_tokens": 20, "cache_write_1h_tokens": 40, "output_tokens": 30}, + "requires_rates": ["cache_creation_input_token_cost_above_1hr", "cache_creation_input_token_cost"], + "requires_caps": ["cache_write_1h"] + }, + { + "name": "reasoning", + "usage": {"fresh_input_tokens": 100, "output_tokens": 30, "reasoning_tokens": 70}, + "requires_rates": ["output_cost_per_reasoning_token"], + "requires_caps": ["reasoning"] + }, + { + "name": "audio", + "usage": {"fresh_input_tokens": 100, "audio_input_tokens": 25, "output_tokens": 30, "audio_output_tokens": 15}, + "requires_rates": ["input_cost_per_audio_token", "output_cost_per_audio_token"], + "requires_caps": ["audio"] + }, + { + "name": "tiered", + "usage": {"fresh_input_tokens": 200001, "output_tokens": 30}, + "requires_rates": ["input_cost_per_token_above_200k_tokens", "output_cost_per_token_above_200k_tokens"] + }, + { + "name": "service_tier_flex", + "usage": {"fresh_input_tokens": 120, "output_tokens": 40}, + "service_tier": "flex", + "requires_rates": ["input_cost_per_token_flex", "output_cost_per_token_flex"] + }, + { + "name": "service_tier_priority", + "usage": {"fresh_input_tokens": 120, "output_tokens": 40}, + "service_tier": "priority", + "requires_rates": ["input_cost_per_token_priority", "output_cost_per_token_priority"] + }, + { + "name": "web_search", + "usage": {"fresh_input_tokens": 100, "output_tokens": 30, "web_search_calls": 3}, + "requires_rates": ["search_context_cost_per_query"], + "requires_caps": ["web_search"] + }, + { + "name": "stream", + "usage": {"fresh_input_tokens": 120, "output_tokens": 40}, + "stream": true + }, + { + "name": "stream_no_usage", + "usage": {"fresh_input_tokens": 120, "output_tokens": 40}, + "stream": true, + "stream_usage": "absent", + "exact_spend": false, + "requires_caps": ["absent_usage"] + }, + { + "name": "response_model_override", + "usage": {"fresh_input_tokens": 120, "output_tokens": 40}, + "response_model_override": true, + "requires_caps": ["response_model"] + }, + { + "name": "stream_response_model_override", + "usage": {"fresh_input_tokens": 120, "output_tokens": 40}, + "stream": true, + "response_model_override": true, + "requires_caps": ["response_model"] + }, + { + "name": "tool_call", + "usage": {"fresh_input_tokens": 120, "output_tokens": 40}, + "tool_call": true, + "requires_caps": ["tool_call"] + }, + { + "name": "stream_tool_call", + "usage": {"fresh_input_tokens": 80, "output_tokens": 25}, + "stream": true, + "tool_call": true, + "requires_caps": ["tool_call"] + }, + { + "name": "stream_no_usage_tool_call", + "usage": {"fresh_input_tokens": 120, "output_tokens": 40}, + "stream": true, + "stream_usage": "absent", + "tool_call": true, + "exact_spend": false, + "requires_caps": ["absent_usage", "tool_call"] + }, + { + "name": "stream_no_usage_image_input", + "usage": {"fresh_input_tokens": 120, "output_tokens": 40}, + "stream": true, + "stream_usage": "absent", + "image_input": true, + "exact_spend": false, + "requires_caps": ["absent_usage", "image_input"] + }, + { + "name": "stream_incomplete", + "usage": {"fresh_input_tokens": 120, "output_tokens": 40}, + "stream": true, + "terminal": "incomplete", + "requires_caps": ["responses_terminal"] + }, + { + "name": "stream_no_usage_incomplete", + "usage": {"fresh_input_tokens": 120, "output_tokens": 40}, + "stream": true, + "stream_usage": "absent", + "terminal": "incomplete", + "exact_spend": false, + "requires_caps": ["responses_terminal"] + }, + { + "name": "stream_unvalidated", + "usage": {"fresh_input_tokens": 120, "output_tokens": 40}, + "stream": true, + "terminal": "unvalidated", + "requires_caps": ["responses_terminal"] + }, + { + "name": "stream_no_usage_unvalidated", + "usage": {"fresh_input_tokens": 120, "output_tokens": 40}, + "stream": true, + "stream_usage": "absent", + "terminal": "unvalidated", + "exact_spend": false, + "requires_caps": ["responses_terminal"] + }, + { + "name": "prompt_blocked", + "usage": {"fresh_input_tokens": 1000, "output_tokens": 0}, + "terminal": "prompt_blocked", + "response_model_override": true, + "requires_caps": ["prompt_blocked"] + }, + { + "name": "stream_prompt_blocked", + "usage": {"fresh_input_tokens": 1000, "output_tokens": 0}, + "stream": true, + "terminal": "prompt_blocked", + "response_model_override": true, + "requires_caps": ["prompt_blocked"] + }, + { + "name": "all_components_chat", + "usage": { + "fresh_input_tokens": 80, + "cache_read_tokens": 40, + "cache_write_5m_tokens": 20, + "cache_write_1h_tokens": 10, + "output_tokens": 25, + "reasoning_tokens": 15, + "audio_input_tokens": 5, + "audio_output_tokens": 3 + }, + "wires": ["openai_chat", "azure_chat", "together_chat"] + }, + { + "name": "all_components_fireworks", + "usage": {"fresh_input_tokens": 80, "cache_read_tokens": 40, "output_tokens": 25}, + "wires": ["fireworks_chat"] + }, + { + "name": "all_components_anthropic", + "usage": { + "fresh_input_tokens": 80, + "cache_read_tokens": 40, + "cache_write_5m_tokens": 20, + "cache_write_1h_tokens": 10, + "output_tokens": 25 + }, + "wires": ["anthropic_messages", "bedrock_converse"] + }, + { + "name": "all_components_anthropic_stream", + "usage": { + "fresh_input_tokens": 80, + "cache_read_tokens": 40, + "cache_write_5m_tokens": 20, + "cache_write_1h_tokens": 10, + "output_tokens": 25 + }, + "stream": true, + "wires": ["anthropic_messages"] + }, + { + "name": "all_components_gemini", + "usage": { + "fresh_input_tokens": 80, + "cache_read_tokens": 40, + "output_tokens": 25, + "reasoning_tokens": 15, + "audio_input_tokens": 5, + "audio_output_tokens": 3 + }, + "wires": ["gemini_generate", "vertex_generate"] + }, + { + "name": "all_components_responses", + "usage": {"fresh_input_tokens": 80, "cache_read_tokens": 40, "output_tokens": 25, "reasoning_tokens": 15}, + "wires": ["openai_responses"] + } + ] +} diff --git a/tests/e2e/cost_calculation/conftest.py b/tests/e2e/cost_calculation/conftest.py index 3f3e9fd9243..3de9786854e 100644 --- a/tests/e2e/cost_calculation/conftest.py +++ b/tests/e2e/cost_calculation/conftest.py @@ -1,10 +1,12 @@ """Cost-calculation suite fixtures. Runs against a dedicated proxy whose whole model cost map is the test-owned -``tests/e2e/cost_map.json`` (LITELLM_MODEL_COST_MAP_URL), so every deployment -bills at rates the test asserts literal arithmetic on. Provider calls are -answered by the scripted-provider sidecar (``scripted_provider.py``), registered -per scenario over its control API. +``tests/e2e/cost_map.json`` (LITELLM_MODEL_COST_MAP_URL); every map entry is a +deployment under test, the request shapes live in ``cases.json``, and the +asserted goldens live in ``expected.json`` (regenerate proposals with +``generate_expected.py``). Provider calls are answered by the +scripted-provider sidecar (``scripted_provider.py``), registered per scenario +over its control API. Deselected unless E2E_COST_MAP_STACK is set (marker `cost_map_stack`). """ diff --git a/tests/e2e/cost_calculation/cost_matrix.py b/tests/e2e/cost_calculation/cost_matrix.py index 37495f37b0b..b03c851d208 100644 --- a/tests/e2e/cost_calculation/cost_matrix.py +++ b/tests/e2e/cost_calculation/cost_matrix.py @@ -1,18 +1,16 @@ -"""The cost-calculation matrix: frontier model set, the pricing-component cases -each model runs, and the expected-cost arithmetic. +"""The cost-calculation matrix: the model set derived from the test cost map, +the request/response cases from ``cases.json``, and the loaders both use. -Rates come from ``tests/e2e/cost_map.json``, which the proxy under test loads as -its ENTIRE model cost map (LITELLM_MODEL_COST_MAP_URL), so an entry's rates are -exactly what the proxy bills and nothing in the suite depends on the bundled -map. Each model's rates are a distinct multiple of a shared base set, so a -component billed at the wrong model's rate (or the wrong case's rate) can never -coincidentally match. - -Case applicability is pricing-field-gated AND wire-gated: a case runs for a -model only when the entry carries the rate the case exercises and the wire can -report the token kind that rate prices. When the wire cannot report a kind -(e.g. Anthropic has no reasoning-token field, Responses reports no cache -creation), the case is absent from the matrix rather than silently zero. +Three data files drive the suite; nothing in Python lists models or cases: +- ``tests/e2e/cost_map.json`` is the proxy's ENTIRE model cost map + (LITELLM_MODEL_COST_MAP_URL); every entry becomes a deployment under test. +- ``tests/e2e/cost_calculation/cases.json`` is the case list; each case runs + for a model when the entry carries the rates it exercises (``requires_rates``) + and the wire can report the token kinds involved (``requires_caps`` / + ``wires``). +- ``tests/e2e/cost_calculation/expected.json`` holds the reviewed goldens; the + tests assert them verbatim and never compute a price themselves. The rate + arithmetic that proposes goldens lives in ``generate_expected.py``, not here. """ from __future__ import annotations @@ -26,13 +24,15 @@ from collections.abc import Mapping from dataclasses import dataclass from pathlib import Path from types import MappingProxyType -from typing import Final, Literal, TypeAlias +from typing import Final, Literal from pydantic import BaseModel, ConfigDict, TypeAdapter from scripted_provider import Scenario, ScriptedOutput, ScriptedToolCall, ScriptedUsage, Wire COST_MAP_PATH: Final = Path(__file__).resolve().parent.parent / "cost_map.json" +CASES_PATH: Final = Path(__file__).resolve().parent / "cases.json" +EXPECTED_PATH: Final = Path(__file__).resolve().parent / "expected.json" class SearchContextCostPerQuery(BaseModel): @@ -77,119 +77,90 @@ _COST_MAP: Final[Mapping[str, CostMapEntry]] = MappingProxyType( TIER_THRESHOLD_TOKENS: Final = 200_000 -@dataclass(frozen=True, slots=True) -class FrontierModel: - """One deployment under test: the model_name the suite registers, the - provider-prefixed litellm model string, the wire the scripted upstream - speaks, its cost-map key, and the sibling map model the response_model - override case reports.""" +class DeploymentSpec(BaseModel): + """A deployment-level fact from cases.json: when a map key needs a + registered deployment name that is not its provider model (or a + model_info.base_model pin), the matrix uses these instead of the defaults.""" + + model_config = ConfigDict(frozen=True) - model_name: str - litellm_model: str - wire: Wire map_key: str - override_model: str | None = None - override_map_key: str | None = None - # Registered as model_info.base_model; when set, the provider-reported - # model loses to it and every case bills at this deployment's own rates. + litellm_model: str | None = None base_model: str | None = None - # Extra litellm_params merged into the /model/new registration (api_version, - # aws_* credentials, vertex_* auth). - litellm_params: Mapping[str, str] = MappingProxyType({}) - - @property - def rates(self) -> CostMapEntry: - return _COST_MAP[self.map_key] - - @property - def override_rates(self) -> CostMapEntry: - if self.base_model is not None or self.override_map_key is None: - return self.rates - return _COST_MAP[self.override_map_key] - - @property - def provider_model(self) -> str: - """The bare provider-facing model name: litellm_model minus the provider - prefix and any routing segment (converse/, responses/).""" - tail: Final = self.litellm_model.split("/")[1:] - return "/".join(tail[1:] if tail and tail[0] in ("converse", "responses") else tail) - - @property - def provider(self) -> str: - return self.rates.litellm_provider - - @property - def api_key(self) -> str: - # The scripted upstream ignores auth; a fixed bogus key proves the suite - # spends zero real provider calls. - return "sk-scripted-provider" -# Response-model override targets: emit a sibling's bare provider-facing name so -# the biller's provider-prefixed lookup lands on that sibling's map key. -_OVERRIDE_MODELS: Final[Mapping[str, str]] = MappingProxyType({ - "gpt-5.6": "gpt-5.4-mini", - "gpt-5.5-pro": "gpt-5.3-codex", - "gpt-5.3-codex": "gpt-5.5-pro", - "gpt-5.4-mini": "gpt-5.6", - "claude-opus-5": "claude-sonnet-5", - "claude-sonnet-5": "claude-opus-5", - "claude-haiku-4-5": "claude-sonnet-5", - "gemini/gemini-3.8-flash": "gemini-3.1-pro-preview", - "gemini/gemini-3.1-pro-preview": "gemini-3.8-flash", - "together_ai/moonshotai/Kimi-K3": "zai-org/GLM-5.3", - "together_ai/zai-org/GLM-5.3": "moonshotai/Kimi-K3", - "fireworks_ai/kimi-k3": "qwen3p8-max", - "fireworks_ai/qwen3p8-max": "kimi-k3", - "fireworks_ai/deepseek-v4p1-flash": "kimi-k3", -}) +class Case(BaseModel): + """One request/response shape from cases.json; gated onto a model by + ``requires_rates`` (entry must carry each rate field), ``requires_caps`` + (the wire must report the token kind) and ``wires`` (shape is wire-specific).""" -_OVERRIDE_MAP_KEYS: Final[Mapping[str, str]] = MappingProxyType({ - "gpt-5.4-mini": "gpt-5.4-mini", - "gpt-5.6": "gpt-5.6", - "gpt-5.3-codex": "gpt-5.3-codex", - "gpt-5.5-pro": "gpt-5.5-pro", - "claude-sonnet-5": "claude-sonnet-5", - "claude-opus-5": "claude-opus-5", - "gemini-3.1-pro-preview": "gemini/gemini-3.1-pro-preview", - "gemini-3.8-flash": "gemini/gemini-3.8-flash", - "zai-org/GLM-5.3": "together_ai/zai-org/GLM-5.3", - "moonshotai/Kimi-K3": "together_ai/moonshotai/Kimi-K3", - "qwen3p8-max": "fireworks_ai/qwen3p8-max", - "kimi-k3": "fireworks_ai/kimi-k3", -}) + model_config = ConfigDict(frozen=True) + + name: str + usage: ScriptedUsage + stream: bool = False + stream_usage: Literal["final_chunk", "absent"] = "final_chunk" + service_tier: Literal["flex", "priority"] | None = None + response_model_override: bool = False + exact_spend: bool = True + tool_call: bool = False + image_input: bool = False + terminal: Literal["completed", "incomplete", "unvalidated", "prompt_blocked"] = "completed" + requires_rates: tuple[str, ...] = () + requires_caps: tuple[str, ...] = () + wires: tuple[Wire, ...] | None = None + + def applies_to(self, model: FrontierModel) -> bool: + if self.wires is not None and model.wire not in self.wires: + return False + caps: Final = _WIRE_CAPS[model.wire] + if not frozenset(self.requires_caps) <= caps: + return False + return all( + getattr(model.rates, field, None) is not None for field in self.requires_rates + ) + + def scenario(self, scenario_id: str, model: FrontierModel, text: str) -> Scenario: + return Scenario( + scenario_id=scenario_id, + wire=model.wire, + usage=self.usage, + model=model.provider_model, + output=ScriptedOutput( + text=text, + response_model=model.override_model if self.response_model_override else None, + tool_call=ScriptedToolCall(name="get_weather", arguments=TOOL_CALL_ARGUMENTS) + if self.tool_call + else None, + terminal=self.terminal, + ), + stream_usage=self.stream_usage, + service_tier=self.service_tier, + ) -_FRONTIER_SPECS: Final[tuple[tuple[str, str, Wire], ...]] = ( - ("gpt-5.6", "openai/gpt-5.6", "openai_chat"), - ("gpt-5.5-pro", "openai/gpt-5.5-pro", "openai_responses"), - ("gpt-5.3-codex", "openai/gpt-5.3-codex", "openai_responses"), - ("gpt-5.4-mini", "openai/gpt-5.4-mini", "openai_chat"), - ("claude-opus-5", "anthropic/claude-opus-5", "anthropic_messages"), - ("claude-sonnet-5", "anthropic/claude-sonnet-5", "anthropic_messages"), - ("claude-haiku-4-5", "anthropic/claude-haiku-4-5", "anthropic_messages"), - ("gemini/gemini-3.8-flash", "gemini/gemini-3.8-flash", "gemini_generate"), - ("gemini/gemini-3.1-pro-preview", "gemini/gemini-3.1-pro-preview", "gemini_generate"), - ("together_ai/moonshotai/Kimi-K3", "together_ai/moonshotai/Kimi-K3", "together_chat"), - ("together_ai/zai-org/GLM-5.3", "together_ai/zai-org/GLM-5.3", "together_chat"), - ("fireworks_ai/kimi-k3", "fireworks_ai/kimi-k3", "fireworks_chat"), - ("fireworks_ai/qwen3p8-max", "fireworks_ai/qwen3p8-max", "fireworks_chat"), - ("fireworks_ai/deepseek-v4p1-flash", "fireworks_ai/deepseek-v4p1-flash", "fireworks_chat"), +class _CasesFile(BaseModel): + model_config = ConfigDict(frozen=True) + + deployments: tuple[DeploymentSpec, ...] = () + cases: tuple[Case, ...] = () + + +_CASES_FILE: Final = _CasesFile.model_validate(json.loads(CASES_PATH.read_text())) +CASES: Final[tuple[Case, ...]] = _CASES_FILE.cases +_DEPLOYMENTS: Final[Mapping[str, DeploymentSpec]] = MappingProxyType( + {spec.map_key: spec for spec in _CASES_FILE.deployments} ) @dataclass(frozen=True, slots=True) -class _ExtendedSpec: - """A frontier entry whose override target, model_info.base_model or extra - litellm_params can't be derived from the map key alone.""" +class _ProviderWiring: + """How a (litellm_provider, mode) pair maps to a sidecar wire, the provider + prefix on the registered litellm model string, and extra litellm_params.""" - map_key: str - litellm_model: str wire: Wire - override_model: str | None = None - override_map_key: str | None = None - base_model: str | None = None - litellm_params: Mapping[str, str] = MappingProxyType({}) + model_prefix: str | None + litellm_params: Mapping[str, str] _AZURE_PARAMS: Final[Mapping[str, str]] = MappingProxyType({"api_version": "2025-04-01-preview"}) @@ -207,87 +178,134 @@ _VERTEX_PARAMS: Final[Mapping[str, str]] = MappingProxyType( } ) -_EXTENDED_SPECS: Final[tuple[_ExtendedSpec, ...]] = ( - _ExtendedSpec( - map_key="azure/gpt-5.6", - litellm_model="azure/gpt-5.6", - wire="azure_chat", - override_model="gpt-5.4-mini", - override_map_key="azure/gpt-5.4-mini", - litellm_params=_AZURE_PARAMS, - ), - _ExtendedSpec( - # Deployment name is not a model; base_model pins billing so the - # response's model field loses, proving base_model wins. - map_key="azure/gpt-5.4-mini", - litellm_model="azure/cc-pinned-deployment", - wire="azure_chat", - override_model="gpt-5.6", - override_map_key="azure/gpt-5.6", - base_model="azure/gpt-5.4-mini", - litellm_params=_AZURE_PARAMS, - ), - _ExtendedSpec( - map_key="anthropic.claude-sonnet-5-v1:0", - litellm_model="bedrock/converse/anthropic.claude-sonnet-5-v1:0", - wire="bedrock_converse", - litellm_params=_BEDROCK_PARAMS, - ), - _ExtendedSpec( - map_key="us.anthropic.claude-opus-5-v1:0", - litellm_model="bedrock/converse/us.anthropic.claude-opus-5-v1:0", - wire="bedrock_converse", - litellm_params=_BEDROCK_PARAMS, - ), - _ExtendedSpec( - map_key="meta.llama4-maverick-17b-instruct-v1:0", - litellm_model="bedrock/converse/meta.llama4-maverick-17b-instruct-v1:0", - wire="bedrock_converse", - litellm_params=_BEDROCK_PARAMS, - ), - _ExtendedSpec( - map_key="gemini-3.8-flash", - litellm_model="vertex_ai/gemini-3.8-flash", - wire="vertex_generate", - override_model="gemini-3.1-pro-preview", - override_map_key="gemini-3.1-pro-preview", - litellm_params=_VERTEX_PARAMS, - ), - _ExtendedSpec( - map_key="gemini-3.1-pro-preview", - litellm_model="vertex_ai/gemini-3.1-pro-preview", - wire="vertex_generate", - override_model="gemini-3.8-flash", - override_map_key="gemini-3.8-flash", - litellm_params=_VERTEX_PARAMS, - ), +_PROVIDER_WIRING: Final[Mapping[tuple[str, str], _ProviderWiring]] = MappingProxyType( + { + ("openai", "chat"): _ProviderWiring("openai_chat", "openai", MappingProxyType({})), + ("openai", "responses"): _ProviderWiring( + "openai_responses", "openai", MappingProxyType({}) + ), + ("anthropic", "chat"): _ProviderWiring( + "anthropic_messages", "anthropic", MappingProxyType({}) + ), + ("gemini", "chat"): _ProviderWiring("gemini_generate", None, MappingProxyType({})), + ("together_ai", "chat"): _ProviderWiring("together_chat", None, MappingProxyType({})), + ("fireworks_ai", "chat"): _ProviderWiring("fireworks_chat", None, MappingProxyType({})), + ("azure", "chat"): _ProviderWiring("azure_chat", None, _AZURE_PARAMS), + ("bedrock_converse", "chat"): _ProviderWiring( + "bedrock_converse", "bedrock/converse", _BEDROCK_PARAMS + ), + ("vertex_ai-language-models", "chat"): _ProviderWiring( + "vertex_generate", "vertex_ai", _VERTEX_PARAMS + ), + } ) +@dataclass(frozen=True, slots=True) +class FrontierModel: + """One deployment under test, derived from a cost-map entry: the model_name + the suite registers, the provider-prefixed litellm model string, the wire + the scripted upstream speaks, and the sibling map model the response_model + override case reports.""" + + model_name: str + litellm_model: str + wire: Wire + map_key: str + override_model: str | None = None + override_map_key: str | None = None + # Registered as model_info.base_model; when set, the provider-reported + # model loses to it and every case bills at this deployment's own rates. + base_model: str | None = None + litellm_params: Mapping[str, str] = MappingProxyType({}) + + @property + def rates(self) -> CostMapEntry: + return _COST_MAP[self.map_key] + + @property + def override_rates(self) -> CostMapEntry: + if self.base_model is not None or self.override_map_key is None: + return self.rates + return _COST_MAP[self.override_map_key] + + @property + def provider_model(self) -> str: + """The bare provider-facing model name: litellm_model minus the provider + prefix and any routing segment (converse/, responses/).""" + return _provider_model(self.litellm_model) + + @property + def provider(self) -> str: + return self.rates.litellm_provider + + @property + def api_key(self) -> str: + # The scripted upstream ignores auth; a fixed bogus key proves the suite + # spends zero real provider calls. + return "sk-scripted-provider" + + +def _provider_model(litellm_model: str) -> str: + tail: Final = litellm_model.split("/")[1:] + return "/".join(tail[1:] if tail and tail[0] in ("converse", "responses") else tail) + + +def _litellm_model_for(map_key: str, wiring: _ProviderWiring) -> str: + if wiring.model_prefix is None: + return map_key + if map_key.startswith(f"{wiring.model_prefix}/"): + return map_key + return f"{wiring.model_prefix}/{map_key}" + + def _frontier() -> tuple[FrontierModel, ...]: - return tuple( - FrontierModel( - model_name=f"cc-{map_key.replace('/', '-').lower()}", - litellm_model=litellm_model, - wire=wire, - map_key=map_key, - override_model=_OVERRIDE_MODELS[map_key], - override_map_key=_OVERRIDE_MAP_KEYS[_OVERRIDE_MODELS[map_key]], - ) - for map_key, litellm_model, wire in _FRONTIER_SPECS - ) + tuple( - FrontierModel( - model_name=f"cc-{spec.map_key.replace('/', '-').replace(':', '-').replace('.', '-').lower()}", - litellm_model=spec.litellm_model, - wire=spec.wire, - map_key=spec.map_key, - override_model=spec.override_model, - override_map_key=spec.override_map_key, - base_model=spec.base_model, - litellm_params=spec.litellm_params, - ) - for spec in _EXTENDED_SPECS + groups: Final[Mapping[tuple[str, str], tuple[str, ...]]] = MappingProxyType( + { + pair: tuple(sorted(k for k, e in _COST_MAP.items() if (e.litellm_provider, e.mode) == pair)) + for pair in {(e.litellm_provider, e.mode) for e in _COST_MAP.values()} + } ) + models: list[FrontierModel] = [] # mutable-ok: accumulated once at import into a tuple + for map_key in sorted(_COST_MAP): + entry: Final = _COST_MAP[map_key] + pair: Final = (entry.litellm_provider, entry.mode) + wiring: Final = _PROVIDER_WIRING.get(pair) + if wiring is None: + raise ValueError( + f"cost_map entry {map_key} has no wiring for " + f"(litellm_provider={pair[0]}, mode={pair[1]}); add a " + f"_ProviderWiring row in cost_matrix.py" + ) + siblings: Final = groups[pair] + override_key: Final = ( + siblings[(siblings.index(map_key) + 1) % len(siblings)] if len(siblings) > 1 else None + ) + override_litellm: Final = ( + _litellm_model_for(override_key, wiring) if override_key is not None else None + ) + deployment: Final = _DEPLOYMENTS.get(map_key) + models.append( + FrontierModel( + model_name=f"cc-{map_key.replace('/', '-').replace(':', '-').replace('.', '-').lower()}", + litellm_model=( + deployment.litellm_model + if deployment is not None and deployment.litellm_model is not None + else _litellm_model_for(map_key, wiring) + ), + wire=wiring.wire, + map_key=map_key, + override_model=( + _provider_model(override_litellm) + if override_litellm is not None + else None + ), + override_map_key=override_key, + base_model=deployment.base_model if deployment is not None else None, + litellm_params=wiring.litellm_params, + ) + ) + return tuple(models) FRONTIER_MODELS: Final[tuple[FrontierModel, ...]] = _frontier() @@ -350,71 +368,6 @@ _WIRE_CAPS: Final[Mapping[str, frozenset[str]]] = MappingProxyType({ ), }) -CaseName: TypeAlias = Literal[ - "basic", - "cache_read", - "cache_write_5m", - "cache_write_1h", - "reasoning", - "audio", - "tiered", - "service_tier_flex", - "service_tier_priority", - "web_search", - "stream", - "stream_no_usage", - "response_model_override", - "stream_response_model_override", - "tool_call", - "stream_no_usage_tool_call", - "stream_no_usage_image_input", - "stream_no_usage_incomplete", - "stream_unvalidated", - "stream_no_usage_unvalidated", - "prompt_blocked", - "stream_prompt_blocked", -] - - -@dataclass(frozen=True, slots=True) -class Case: - name: CaseName - usage: ScriptedUsage - stream: bool = False - stream_usage: Literal["final_chunk", "absent"] = "final_chunk" - service_tier: Literal["flex", "priority"] | None = None - # For web_search the wire's reported call count is not always what gets - # billed: chat-completions surfaces only expose url_citation annotations, so - # the biller floors to one call; responses/messages/gemini report a real - # count. - billed_web_search_calls: int = 0 - response_model_override: bool = False - exact_spend: bool = True - tool_call: bool = False - image_input: bool = False - terminal: Literal["completed", "incomplete", "unvalidated", "prompt_blocked"] = "completed" - - def scenario(self, scenario_id: str, model: FrontierModel, text: str) -> Scenario: - return Scenario( - scenario_id=scenario_id, - wire=model.wire, - usage=self.usage, - model=model.provider_model, - output=ScriptedOutput( - text=text, - response_model=model.override_model if self.response_model_override else None, - tool_call=ScriptedToolCall(name="get_weather", arguments=TOOL_CALL_ARGUMENTS) - if self.tool_call - else None, - terminal=self.terminal, - ), - stream_usage=self.stream_usage, - service_tier=self.service_tier, - ) - - -_BASIC_USAGE: Final = ScriptedUsage(fresh_input_tokens=120, output_tokens=40) - TOOL_CALL_ARGUMENTS: Final = json.dumps({ "city": "Berlin", "days": 7, @@ -422,284 +375,9 @@ TOOL_CALL_ARGUMENTS: Final = json.dumps({ "notes": "filler " * 30, }) -_PROMPT_BLOCKED_USAGE: Final = ScriptedUsage(fresh_input_tokens=1000, output_tokens=0) - - -def _web_search_case(model: FrontierModel) -> Case: - counts_exactly: Final = model.wire in ( - "openai_responses", "anthropic_messages", "gemini_generate", "vertex_generate" - ) - return Case( - name="web_search", - usage=ScriptedUsage(fresh_input_tokens=100, output_tokens=30, web_search_calls=3), - billed_web_search_calls=3 if counts_exactly else 1, - ) - def cases_for(model: FrontierModel) -> tuple[Case, ...]: - rates: Final = model.rates - caps: Final = _WIRE_CAPS[model.wire] - candidates: Final[tuple[Case | None, ...]] = ( - Case(name="basic", usage=_BASIC_USAGE), - ( - Case(name="cache_read", usage=ScriptedUsage(fresh_input_tokens=100, cache_read_tokens=50, output_tokens=30)) - if rates.cache_read_input_token_cost is not None and "cache_read" in caps - else None - ), - ( - Case( - name="cache_write_5m", - usage=ScriptedUsage(fresh_input_tokens=90, cache_write_5m_tokens=60, output_tokens=30), - ) - if rates.cache_creation_input_token_cost is not None and "cache_write_5m" in caps - else None - ), - ( - Case( - name="cache_write_1h", - usage=ScriptedUsage( - fresh_input_tokens=90, - cache_write_5m_tokens=20, - cache_write_1h_tokens=40, - output_tokens=30, - ), - ) - if ( - rates.cache_creation_input_token_cost_above_1hr is not None - and rates.cache_creation_input_token_cost is not None - and "cache_write_1h" in caps - ) - else None - ), - ( - Case( - name="reasoning", - usage=ScriptedUsage(fresh_input_tokens=100, output_tokens=30, reasoning_tokens=70), - ) - if rates.output_cost_per_reasoning_token is not None and "reasoning" in caps - else None - ), - ( - Case( - name="audio", - usage=ScriptedUsage( - fresh_input_tokens=100, audio_input_tokens=25, output_tokens=30, audio_output_tokens=15 - ), - ) - if ( - rates.input_cost_per_audio_token is not None - and rates.output_cost_per_audio_token is not None - and "audio" in caps - ) - else None - ), - ( - Case( - name="tiered", - usage=ScriptedUsage( - fresh_input_tokens=TIER_THRESHOLD_TOKENS + 1, output_tokens=30 - ), - ) - if ( - rates.input_cost_per_token_above_200k_tokens is not None - and rates.output_cost_per_token_above_200k_tokens is not None - ) - else None - ), - ( - Case(name="service_tier_flex", usage=_BASIC_USAGE, service_tier="flex") - if rates.input_cost_per_token_flex is not None and rates.output_cost_per_token_flex is not None - else None - ), - ( - Case(name="service_tier_priority", usage=_BASIC_USAGE, service_tier="priority") - if rates.input_cost_per_token_priority is not None and rates.output_cost_per_token_priority is not None - else None - ), - _web_search_case(model) if rates.search_context_cost_per_query is not None and "web_search" in caps else None, - Case(name="stream", usage=_BASIC_USAGE, stream=True), - ( - Case( - name="stream_no_usage", - usage=_BASIC_USAGE, - stream=True, - stream_usage="absent", - exact_spend=False, - ) - if "absent_usage" in caps - else None - ), - ( - Case(name="response_model_override", usage=_BASIC_USAGE, response_model_override=True) - if "response_model" in caps - else None - ), - ( - Case( - name="stream_response_model_override", - usage=_BASIC_USAGE, - stream=True, - response_model_override=True, - ) - if "response_model" in caps - else None - ), - ( - Case(name="tool_call", usage=_BASIC_USAGE, tool_call=True) - if "tool_call" in caps - else None - ), - ( - Case( - name="stream_no_usage_tool_call", - usage=_BASIC_USAGE, - stream=True, - stream_usage="absent", - tool_call=True, - exact_spend=False, - ) - if "absent_usage" in caps and "tool_call" in caps - else None - ), - ( - Case( - name="stream_no_usage_image_input", - usage=_BASIC_USAGE, - stream=True, - stream_usage="absent", - image_input=True, - exact_spend=False, - ) - if "absent_usage" in caps and "image_input" in caps - else None - ), - ( - Case( - name="stream_no_usage_incomplete", - usage=_BASIC_USAGE, - stream=True, - stream_usage="absent", - terminal="incomplete", - exact_spend=False, - ) - if "responses_terminal" in caps - else None - ), - ( - Case( - name="stream_unvalidated", - usage=_BASIC_USAGE, - stream=True, - terminal="unvalidated", - ) - if "responses_terminal" in caps - else None - ), - ( - Case( - name="stream_no_usage_unvalidated", - usage=_BASIC_USAGE, - stream=True, - stream_usage="absent", - terminal="unvalidated", - exact_spend=False, - ) - if "responses_terminal" in caps - else None - ), - ( - Case( - name="prompt_blocked", - usage=_PROMPT_BLOCKED_USAGE, - terminal="prompt_blocked", - response_model_override=True, - ) - if "prompt_blocked" in caps - else None - ), - ( - Case( - name="stream_prompt_blocked", - usage=_PROMPT_BLOCKED_USAGE, - stream=True, - terminal="prompt_blocked", - response_model_override=True, - ) - if "prompt_blocked" in caps - else None - ), - ) - return tuple(case for case in candidates if case is not None) - - -@dataclass(frozen=True, slots=True) -class ExpectedCost: - """The expected bill split the way the spend row's cost_breakdown reports - it: the gross input component (cache reads/writes folded in), the output - component, and the tool-usage component.""" - - input_cost: float - output_cost: float - tool_cost: float - - @property - def total(self) -> float: - return self.input_cost + self.output_cost + self.tool_cost - - -def expected_breakdown(model: FrontierModel, case: Case) -> ExpectedCost: - """Literal arithmetic on the test-map rates over the scripted token counts. - - Input = fresh*in + read*read + 5m*create + 1h*create_1h + audio_in*audio_in; - output = text*out + reasoning*reasoning + audio_out*audio_out; plus the - billed web-search calls at the medium search-context rate. Above-threshold - swaps every input/output rate to its ``_above_200k_tokens`` variant when - total prompt tokens exceed the threshold; a service tier swaps input/output - to the tier's variants, falling back to the base rate when a variant is - unset -- mirroring _get_token_base_cost in litellm's cost calculator. - """ - rates: Final = model.override_rates if case.response_model_override else model.rates - u: Final = case.usage - prompt_tokens: Final = ( - u.fresh_input_tokens + u.cache_read_tokens + u.cache_write_5m_tokens - + u.cache_write_1h_tokens + u.audio_input_tokens - ) - tiered: Final = prompt_tokens > TIER_THRESHOLD_TOKENS - in_rate: Final = ( - (rates.input_cost_per_token_above_200k_tokens if tiered else None) - or (rates.input_cost_per_token_priority if case.service_tier == "priority" else None) - or (rates.input_cost_per_token_flex if case.service_tier == "flex" else None) - or rates.input_cost_per_token - or 0.0 - ) - out_rate: Final = ( - (rates.output_cost_per_token_above_200k_tokens if tiered else None) - or (rates.output_cost_per_token_priority if case.service_tier == "priority" else None) - or (rates.output_cost_per_token_flex if case.service_tier == "flex" else None) - or rates.output_cost_per_token - or 0.0 - ) - input_cost: Final = ( - u.fresh_input_tokens * in_rate - + u.cache_read_tokens * (rates.cache_read_input_token_cost or 0.0) - + u.cache_write_5m_tokens * (rates.cache_creation_input_token_cost or 0.0) - + u.cache_write_1h_tokens * (rates.cache_creation_input_token_cost_above_1hr or 0.0) - + u.audio_input_tokens * (rates.input_cost_per_audio_token or 0.0) - ) - output_cost: Final = ( - u.output_tokens * out_rate - + u.reasoning_tokens * (rates.output_cost_per_reasoning_token or out_rate) - + u.audio_output_tokens * (rates.output_cost_per_audio_token or out_rate) - ) - search: Final = rates.search_context_cost_per_query - tool_cost: Final = case.billed_web_search_calls * ( - search.search_context_size_medium if search and search.search_context_size_medium else 0.0 - ) - return ExpectedCost(input_cost=input_cost, output_cost=output_cost, tool_cost=tool_cost) - - -def expected_cost(model: FrontierModel, case: Case) -> float: - return expected_breakdown(model, case).total + return tuple(case for case in CASES if case.applies_to(model)) def recount_cost( @@ -738,31 +416,23 @@ def image_input_data_url() -> str: IMAGE_INPUT_DATA_URL: Final = image_input_data_url() -def expected_token_columns(model: FrontierModel, case: Case) -> tuple[int, int]: - """(prompt_tokens, completion_tokens) the spend row should carry, per the - wire's normalization: Anthropic folds cache read/write into prompt_tokens, - everyone else reports the totals the wire emitted.""" - u: Final = case.usage - if model.wire in ("anthropic_messages", "bedrock_converse"): - return ( - u.fresh_input_tokens + u.cache_read_tokens + u.cache_write_5m_tokens + u.cache_write_1h_tokens, - u.output_tokens, - ) - if model.wire in ("gemini_generate", "vertex_generate"): - return ( - u.fresh_input_tokens + u.cache_read_tokens + u.audio_input_tokens, - u.output_tokens + u.reasoning_tokens + u.audio_output_tokens, - ) - if model.wire == "openai_responses": - return ( - u.fresh_input_tokens + u.cache_read_tokens, - u.output_tokens + u.reasoning_tokens, - ) - return ( - u.fresh_input_tokens - + u.cache_read_tokens - + u.cache_write_5m_tokens - + u.cache_write_1h_tokens - + u.audio_input_tokens, - u.output_tokens + u.reasoning_tokens + u.audio_output_tokens, - ) +class _ExpectedCell(BaseModel): + model_config = ConfigDict(frozen=True) + + spend: float + input_cost: float + output_cost: float + prompt_tokens: int + completion_tokens: int + + +_EXPECTED_ADAPTER: Final = TypeAdapter(dict[str, _ExpectedCell]) +EXPECTED: Final[Mapping[str, _ExpectedCell]] = MappingProxyType( + _EXPECTED_ADAPTER.validate_python(json.loads(EXPECTED_PATH.read_text())) + if EXPECTED_PATH.exists() + else {} +) + + +def expected_key(model: FrontierModel, case: Case) -> str: + return f"{model.map_key}|{case.name}" diff --git a/tests/e2e/cost_calculation/expected.json b/tests/e2e/cost_calculation/expected.json new file mode 100644 index 00000000000..7a92fb2476f --- /dev/null +++ b/tests/e2e/cost_calculation/expected.json @@ -0,0 +1,2004 @@ +{ + "anthropic.claude-sonnet-5-v1:0|all_components_anthropic": { + "completion_tokens": 25, + "input_cost": 0.03128, + "output_cost": 0.0085, + "prompt_tokens": 150, + "spend": 0.03978 + }, + "anthropic.claude-sonnet-5-v1:0|basic": { + "completion_tokens": 40, + "input_cost": 0.0204, + "output_cost": 0.013600000000000001, + "prompt_tokens": 120, + "spend": 0.034 + }, + "anthropic.claude-sonnet-5-v1:0|cache_read": { + "completion_tokens": 30, + "input_cost": 0.01785, + "output_cost": 0.0102, + "prompt_tokens": 150, + "spend": 0.028050000000000002 + }, + "anthropic.claude-sonnet-5-v1:0|cache_write_1h": { + "completion_tokens": 30, + "input_cost": 0.052700000000000004, + "output_cost": 0.0102, + "prompt_tokens": 150, + "spend": 0.06290000000000001 + }, + "anthropic.claude-sonnet-5-v1:0|cache_write_5m": { + "completion_tokens": 30, + "input_cost": 0.0459, + "output_cost": 0.0102, + "prompt_tokens": 150, + "spend": 0.056100000000000004 + }, + "anthropic.claude-sonnet-5-v1:0|stream": { + "completion_tokens": 40, + "input_cost": 0.0204, + "output_cost": 0.013600000000000001, + "prompt_tokens": 120, + "spend": 0.034 + }, + "anthropic.claude-sonnet-5-v1:0|stream_tool_call": { + "completion_tokens": 25, + "input_cost": 0.013600000000000001, + "output_cost": 0.0085, + "prompt_tokens": 80, + "spend": 0.0221 + }, + "anthropic.claude-sonnet-5-v1:0|tool_call": { + "completion_tokens": 40, + "input_cost": 0.0204, + "output_cost": 0.013600000000000001, + "prompt_tokens": 120, + "spend": 0.034 + }, + "azure/gpt-5.4-mini|all_components_chat": { + "completion_tokens": 43, + "input_cost": 0.03424, + "output_cost": 0.02336, + "prompt_tokens": 155, + "spend": 0.0576 + }, + "azure/gpt-5.4-mini|audio": { + "completion_tokens": 45, + "input_cost": 0.04, + "output_cost": 0.0264, + "prompt_tokens": 125, + "spend": 0.0664 + }, + "azure/gpt-5.4-mini|basic": { + "completion_tokens": 40, + "input_cost": 0.019200000000000002, + "output_cost": 0.0128, + "prompt_tokens": 120, + "spend": 0.032 + }, + "azure/gpt-5.4-mini|cache_read": { + "completion_tokens": 30, + "input_cost": 0.0168, + "output_cost": 0.009600000000000001, + "prompt_tokens": 150, + "spend": 0.0264 + }, + "azure/gpt-5.4-mini|cache_write_1h": { + "completion_tokens": 30, + "input_cost": 0.049600000000000005, + "output_cost": 0.009600000000000001, + "prompt_tokens": 150, + "spend": 0.0592 + }, + "azure/gpt-5.4-mini|cache_write_5m": { + "completion_tokens": 30, + "input_cost": 0.0432, + "output_cost": 0.009600000000000001, + "prompt_tokens": 150, + "spend": 0.0528 + }, + "azure/gpt-5.4-mini|reasoning": { + "completion_tokens": 100, + "input_cost": 0.016, + "output_cost": 0.0656, + "prompt_tokens": 100, + "spend": 0.0816 + }, + "azure/gpt-5.4-mini|response_model_override": { + "completion_tokens": 40, + "input_cost": 0.019200000000000002, + "output_cost": 0.0128, + "prompt_tokens": 120, + "spend": 0.032 + }, + "azure/gpt-5.4-mini|service_tier_flex": { + "completion_tokens": 40, + "input_cost": 0.0288, + "output_cost": 0.016, + "prompt_tokens": 120, + "spend": 0.0448 + }, + "azure/gpt-5.4-mini|service_tier_priority": { + "completion_tokens": 40, + "input_cost": 0.03264, + "output_cost": 0.01728, + "prompt_tokens": 120, + "spend": 0.049920000000000006 + }, + "azure/gpt-5.4-mini|stream": { + "completion_tokens": 40, + "input_cost": 0.019200000000000002, + "output_cost": 0.0128, + "prompt_tokens": 120, + "spend": 0.032 + }, + "azure/gpt-5.4-mini|stream_response_model_override": { + "completion_tokens": 40, + "input_cost": 0.019200000000000002, + "output_cost": 0.0128, + "prompt_tokens": 120, + "spend": 0.032 + }, + "azure/gpt-5.4-mini|stream_tool_call": { + "completion_tokens": 25, + "input_cost": 0.0128, + "output_cost": 0.008, + "prompt_tokens": 80, + "spend": 0.0208 + }, + "azure/gpt-5.4-mini|tiered": { + "completion_tokens": 30, + "input_cost": 256.00128, + "output_cost": 0.0432, + "prompt_tokens": 200001, + "spend": 256.04448 + }, + "azure/gpt-5.4-mini|tool_call": { + "completion_tokens": 40, + "input_cost": 0.019200000000000002, + "output_cost": 0.0128, + "prompt_tokens": 120, + "spend": 0.032 + }, + "azure/gpt-5.4-mini|web_search": { + "completion_tokens": 30, + "input_cost": 0.016, + "output_cost": 0.009600000000000001, + "prompt_tokens": 100, + "spend": 0.0456 + }, + "azure/gpt-5.6|all_components_chat": { + "completion_tokens": 43, + "input_cost": 0.0321, + "output_cost": 0.0219, + "prompt_tokens": 155, + "spend": 0.05399999999999999 + }, + "azure/gpt-5.6|audio": { + "completion_tokens": 45, + "input_cost": 0.0375, + "output_cost": 0.02475, + "prompt_tokens": 125, + "spend": 0.06225 + }, + "azure/gpt-5.6|basic": { + "completion_tokens": 40, + "input_cost": 0.018, + "output_cost": 0.011999999999999999, + "prompt_tokens": 120, + "spend": 0.03 + }, + "azure/gpt-5.6|cache_read": { + "completion_tokens": 30, + "input_cost": 0.01575, + "output_cost": 0.009, + "prompt_tokens": 150, + "spend": 0.02475 + }, + "azure/gpt-5.6|cache_write_1h": { + "completion_tokens": 30, + "input_cost": 0.0465, + "output_cost": 0.009, + "prompt_tokens": 150, + "spend": 0.0555 + }, + "azure/gpt-5.6|cache_write_5m": { + "completion_tokens": 30, + "input_cost": 0.040499999999999994, + "output_cost": 0.009, + "prompt_tokens": 150, + "spend": 0.049499999999999995 + }, + "azure/gpt-5.6|reasoning": { + "completion_tokens": 100, + "input_cost": 0.015, + "output_cost": 0.0615, + "prompt_tokens": 100, + "spend": 0.0765 + }, + "azure/gpt-5.6|response_model_override": { + "completion_tokens": 40, + "input_cost": 0.019200000000000002, + "output_cost": 0.0128, + "prompt_tokens": 120, + "spend": 0.032 + }, + "azure/gpt-5.6|service_tier_flex": { + "completion_tokens": 40, + "input_cost": 0.027, + "output_cost": 0.015, + "prompt_tokens": 120, + "spend": 0.041999999999999996 + }, + "azure/gpt-5.6|service_tier_priority": { + "completion_tokens": 40, + "input_cost": 0.030600000000000002, + "output_cost": 0.0162, + "prompt_tokens": 120, + "spend": 0.0468 + }, + "azure/gpt-5.6|stream": { + "completion_tokens": 40, + "input_cost": 0.018, + "output_cost": 0.011999999999999999, + "prompt_tokens": 120, + "spend": 0.03 + }, + "azure/gpt-5.6|stream_response_model_override": { + "completion_tokens": 40, + "input_cost": 0.019200000000000002, + "output_cost": 0.0128, + "prompt_tokens": 120, + "spend": 0.032 + }, + "azure/gpt-5.6|stream_tool_call": { + "completion_tokens": 25, + "input_cost": 0.011999999999999999, + "output_cost": 0.0075, + "prompt_tokens": 80, + "spend": 0.019499999999999997 + }, + "azure/gpt-5.6|tiered": { + "completion_tokens": 30, + "input_cost": 240.00119999999998, + "output_cost": 0.0405, + "prompt_tokens": 200001, + "spend": 240.0417 + }, + "azure/gpt-5.6|tool_call": { + "completion_tokens": 40, + "input_cost": 0.018, + "output_cost": 0.011999999999999999, + "prompt_tokens": 120, + "spend": 0.03 + }, + "azure/gpt-5.6|web_search": { + "completion_tokens": 30, + "input_cost": 0.015, + "output_cost": 0.009, + "prompt_tokens": 100, + "spend": 0.044 + }, + "claude-haiku-4-5|all_components_anthropic": { + "completion_tokens": 25, + "input_cost": 0.012880000000000003, + "output_cost": 0.0035000000000000005, + "prompt_tokens": 150, + "spend": 0.016380000000000002 + }, + "claude-haiku-4-5|all_components_anthropic_stream": { + "completion_tokens": 25, + "input_cost": 0.012880000000000003, + "output_cost": 0.0035000000000000005, + "prompt_tokens": 150, + "spend": 0.016380000000000002 + }, + "claude-haiku-4-5|basic": { + "completion_tokens": 40, + "input_cost": 0.008400000000000001, + "output_cost": 0.005600000000000001, + "prompt_tokens": 120, + "spend": 0.014000000000000002 + }, + "claude-haiku-4-5|cache_read": { + "completion_tokens": 30, + "input_cost": 0.007350000000000001, + "output_cost": 0.004200000000000001, + "prompt_tokens": 150, + "spend": 0.011550000000000001 + }, + "claude-haiku-4-5|cache_write_1h": { + "completion_tokens": 30, + "input_cost": 0.021700000000000004, + "output_cost": 0.004200000000000001, + "prompt_tokens": 150, + "spend": 0.025900000000000006 + }, + "claude-haiku-4-5|cache_write_5m": { + "completion_tokens": 30, + "input_cost": 0.0189, + "output_cost": 0.004200000000000001, + "prompt_tokens": 150, + "spend": 0.023100000000000002 + }, + "claude-haiku-4-5|response_model_override": { + "completion_tokens": 40, + "input_cost": 0.006, + "output_cost": 0.004, + "prompt_tokens": 120, + "spend": 0.01 + }, + "claude-haiku-4-5|stream": { + "completion_tokens": 40, + "input_cost": 0.008400000000000001, + "output_cost": 0.005600000000000001, + "prompt_tokens": 120, + "spend": 0.014000000000000002 + }, + "claude-haiku-4-5|stream_response_model_override": { + "completion_tokens": 40, + "input_cost": 0.006, + "output_cost": 0.004, + "prompt_tokens": 120, + "spend": 0.01 + }, + "claude-haiku-4-5|stream_tool_call": { + "completion_tokens": 25, + "input_cost": 0.005600000000000001, + "output_cost": 0.0035000000000000005, + "prompt_tokens": 80, + "spend": 0.0091 + }, + "claude-haiku-4-5|tool_call": { + "completion_tokens": 40, + "input_cost": 0.008400000000000001, + "output_cost": 0.005600000000000001, + "prompt_tokens": 120, + "spend": 0.014000000000000002 + }, + "claude-haiku-4-5|web_search": { + "completion_tokens": 30, + "input_cost": 0.007000000000000001, + "output_cost": 0.004200000000000001, + "prompt_tokens": 100, + "spend": 0.0712 + }, + "claude-opus-5|all_components_anthropic": { + "completion_tokens": 25, + "input_cost": 0.0092, + "output_cost": 0.0025, + "prompt_tokens": 150, + "spend": 0.0117 + }, + "claude-opus-5|all_components_anthropic_stream": { + "completion_tokens": 25, + "input_cost": 0.0092, + "output_cost": 0.0025, + "prompt_tokens": 150, + "spend": 0.0117 + }, + "claude-opus-5|basic": { + "completion_tokens": 40, + "input_cost": 0.006, + "output_cost": 0.004, + "prompt_tokens": 120, + "spend": 0.01 + }, + "claude-opus-5|cache_read": { + "completion_tokens": 30, + "input_cost": 0.00525, + "output_cost": 0.003, + "prompt_tokens": 150, + "spend": 0.00825 + }, + "claude-opus-5|cache_write_1h": { + "completion_tokens": 30, + "input_cost": 0.0155, + "output_cost": 0.003, + "prompt_tokens": 150, + "spend": 0.0185 + }, + "claude-opus-5|cache_write_5m": { + "completion_tokens": 30, + "input_cost": 0.013500000000000002, + "output_cost": 0.003, + "prompt_tokens": 150, + "spend": 0.0165 + }, + "claude-opus-5|response_model_override": { + "completion_tokens": 40, + "input_cost": 0.007200000000000001, + "output_cost": 0.0048000000000000004, + "prompt_tokens": 120, + "spend": 0.012 + }, + "claude-opus-5|stream": { + "completion_tokens": 40, + "input_cost": 0.006, + "output_cost": 0.004, + "prompt_tokens": 120, + "spend": 0.01 + }, + "claude-opus-5|stream_response_model_override": { + "completion_tokens": 40, + "input_cost": 0.007200000000000001, + "output_cost": 0.0048000000000000004, + "prompt_tokens": 120, + "spend": 0.012 + }, + "claude-opus-5|stream_tool_call": { + "completion_tokens": 25, + "input_cost": 0.004, + "output_cost": 0.0025, + "prompt_tokens": 80, + "spend": 0.006500000000000001 + }, + "claude-opus-5|tool_call": { + "completion_tokens": 40, + "input_cost": 0.006, + "output_cost": 0.004, + "prompt_tokens": 120, + "spend": 0.01 + }, + "claude-opus-5|web_search": { + "completion_tokens": 30, + "input_cost": 0.005, + "output_cost": 0.003, + "prompt_tokens": 100, + "spend": 0.068 + }, + "claude-sonnet-5|all_components_anthropic": { + "completion_tokens": 25, + "input_cost": 0.011040000000000001, + "output_cost": 0.0030000000000000005, + "prompt_tokens": 150, + "spend": 0.014040000000000002 + }, + "claude-sonnet-5|all_components_anthropic_stream": { + "completion_tokens": 25, + "input_cost": 0.011040000000000001, + "output_cost": 0.0030000000000000005, + "prompt_tokens": 150, + "spend": 0.014040000000000002 + }, + "claude-sonnet-5|basic": { + "completion_tokens": 40, + "input_cost": 0.007200000000000001, + "output_cost": 0.0048000000000000004, + "prompt_tokens": 120, + "spend": 0.012 + }, + "claude-sonnet-5|cache_read": { + "completion_tokens": 30, + "input_cost": 0.006300000000000001, + "output_cost": 0.0036000000000000003, + "prompt_tokens": 150, + "spend": 0.0099 + }, + "claude-sonnet-5|cache_write_1h": { + "completion_tokens": 30, + "input_cost": 0.018600000000000002, + "output_cost": 0.0036000000000000003, + "prompt_tokens": 150, + "spend": 0.0222 + }, + "claude-sonnet-5|cache_write_5m": { + "completion_tokens": 30, + "input_cost": 0.016200000000000003, + "output_cost": 0.0036000000000000003, + "prompt_tokens": 150, + "spend": 0.0198 + }, + "claude-sonnet-5|response_model_override": { + "completion_tokens": 40, + "input_cost": 0.008400000000000001, + "output_cost": 0.005600000000000001, + "prompt_tokens": 120, + "spend": 0.014000000000000002 + }, + "claude-sonnet-5|stream": { + "completion_tokens": 40, + "input_cost": 0.007200000000000001, + "output_cost": 0.0048000000000000004, + "prompt_tokens": 120, + "spend": 0.012 + }, + "claude-sonnet-5|stream_response_model_override": { + "completion_tokens": 40, + "input_cost": 0.008400000000000001, + "output_cost": 0.005600000000000001, + "prompt_tokens": 120, + "spend": 0.014000000000000002 + }, + "claude-sonnet-5|stream_tool_call": { + "completion_tokens": 25, + "input_cost": 0.0048000000000000004, + "output_cost": 0.0030000000000000005, + "prompt_tokens": 80, + "spend": 0.007800000000000001 + }, + "claude-sonnet-5|tool_call": { + "completion_tokens": 40, + "input_cost": 0.007200000000000001, + "output_cost": 0.0048000000000000004, + "prompt_tokens": 120, + "spend": 0.012 + }, + "claude-sonnet-5|web_search": { + "completion_tokens": 30, + "input_cost": 0.006000000000000001, + "output_cost": 0.0036000000000000003, + "prompt_tokens": 100, + "spend": 0.0696 + }, + "fireworks_ai/deepseek-v4p1-flash|all_components_fireworks": { + "completion_tokens": 25, + "input_cost": 0.011760000000000001, + "output_cost": 0.007000000000000001, + "prompt_tokens": 120, + "spend": 0.018760000000000002 + }, + "fireworks_ai/deepseek-v4p1-flash|audio": { + "completion_tokens": 45, + "input_cost": 0.030500000000000003, + "output_cost": 0.019950000000000002, + "prompt_tokens": 125, + "spend": 0.05045000000000001 + }, + "fireworks_ai/deepseek-v4p1-flash|basic": { + "completion_tokens": 40, + "input_cost": 0.016800000000000002, + "output_cost": 0.011200000000000002, + "prompt_tokens": 120, + "spend": 0.028000000000000004 + }, + "fireworks_ai/deepseek-v4p1-flash|cache_read": { + "completion_tokens": 30, + "input_cost": 0.014700000000000001, + "output_cost": 0.008400000000000001, + "prompt_tokens": 150, + "spend": 0.023100000000000002 + }, + "fireworks_ai/deepseek-v4p1-flash|cache_write_1h": { + "completion_tokens": 30, + "input_cost": 0.0368, + "output_cost": 0.008400000000000001, + "prompt_tokens": 150, + "spend": 0.045200000000000004 + }, + "fireworks_ai/deepseek-v4p1-flash|cache_write_5m": { + "completion_tokens": 30, + "input_cost": 0.0324, + "output_cost": 0.008400000000000001, + "prompt_tokens": 150, + "spend": 0.0408 + }, + "fireworks_ai/deepseek-v4p1-flash|reasoning": { + "completion_tokens": 100, + "input_cost": 0.014000000000000002, + "output_cost": 0.0469, + "prompt_tokens": 100, + "spend": 0.060899999999999996 + }, + "fireworks_ai/deepseek-v4p1-flash|response_model_override": { + "completion_tokens": 40, + "input_cost": 0.014400000000000001, + "output_cost": 0.009600000000000001, + "prompt_tokens": 120, + "spend": 0.024 + }, + "fireworks_ai/deepseek-v4p1-flash|stream": { + "completion_tokens": 40, + "input_cost": 0.016800000000000002, + "output_cost": 0.011200000000000002, + "prompt_tokens": 120, + "spend": 0.028000000000000004 + }, + "fireworks_ai/deepseek-v4p1-flash|stream_response_model_override": { + "completion_tokens": 40, + "input_cost": 0.014400000000000001, + "output_cost": 0.009600000000000001, + "prompt_tokens": 120, + "spend": 0.024 + }, + "fireworks_ai/deepseek-v4p1-flash|stream_tool_call": { + "completion_tokens": 25, + "input_cost": 0.011200000000000002, + "output_cost": 0.007000000000000001, + "prompt_tokens": 80, + "spend": 0.0182 + }, + "fireworks_ai/deepseek-v4p1-flash|tool_call": { + "completion_tokens": 40, + "input_cost": 0.016800000000000002, + "output_cost": 0.011200000000000002, + "prompt_tokens": 120, + "spend": 0.028000000000000004 + }, + "fireworks_ai/deepseek-v4p1-flash|web_search": { + "completion_tokens": 30, + "input_cost": 0.014000000000000002, + "output_cost": 0.008400000000000001, + "prompt_tokens": 100, + "spend": 0.04240000000000001 + }, + "fireworks_ai/kimi-k3|all_components_fireworks": { + "completion_tokens": 25, + "input_cost": 0.01008, + "output_cost": 0.006000000000000001, + "prompt_tokens": 120, + "spend": 0.01608 + }, + "fireworks_ai/kimi-k3|audio": { + "completion_tokens": 45, + "input_cost": 0.028500000000000004, + "output_cost": 0.01875, + "prompt_tokens": 125, + "spend": 0.04725 + }, + "fireworks_ai/kimi-k3|basic": { + "completion_tokens": 40, + "input_cost": 0.014400000000000001, + "output_cost": 0.009600000000000001, + "prompt_tokens": 120, + "spend": 0.024 + }, + "fireworks_ai/kimi-k3|cache_read": { + "completion_tokens": 30, + "input_cost": 0.012600000000000002, + "output_cost": 0.007200000000000001, + "prompt_tokens": 150, + "spend": 0.0198 + }, + "fireworks_ai/kimi-k3|cache_write_1h": { + "completion_tokens": 30, + "input_cost": 0.035, + "output_cost": 0.007200000000000001, + "prompt_tokens": 150, + "spend": 0.0422 + }, + "fireworks_ai/kimi-k3|cache_write_5m": { + "completion_tokens": 30, + "input_cost": 0.030600000000000002, + "output_cost": 0.007200000000000001, + "prompt_tokens": 150, + "spend": 0.0378 + }, + "fireworks_ai/kimi-k3|reasoning": { + "completion_tokens": 100, + "input_cost": 0.012000000000000002, + "output_cost": 0.0457, + "prompt_tokens": 100, + "spend": 0.0577 + }, + "fireworks_ai/kimi-k3|response_model_override": { + "completion_tokens": 40, + "input_cost": 0.015600000000000003, + "output_cost": 0.010400000000000001, + "prompt_tokens": 120, + "spend": 0.026000000000000002 + }, + "fireworks_ai/kimi-k3|stream": { + "completion_tokens": 40, + "input_cost": 0.014400000000000001, + "output_cost": 0.009600000000000001, + "prompt_tokens": 120, + "spend": 0.024 + }, + "fireworks_ai/kimi-k3|stream_response_model_override": { + "completion_tokens": 40, + "input_cost": 0.015600000000000003, + "output_cost": 0.010400000000000001, + "prompt_tokens": 120, + "spend": 0.026000000000000002 + }, + "fireworks_ai/kimi-k3|stream_tool_call": { + "completion_tokens": 25, + "input_cost": 0.009600000000000001, + "output_cost": 0.006000000000000001, + "prompt_tokens": 80, + "spend": 0.015600000000000003 + }, + "fireworks_ai/kimi-k3|tool_call": { + "completion_tokens": 40, + "input_cost": 0.014400000000000001, + "output_cost": 0.009600000000000001, + "prompt_tokens": 120, + "spend": 0.024 + }, + "fireworks_ai/kimi-k3|web_search": { + "completion_tokens": 30, + "input_cost": 0.012000000000000002, + "output_cost": 0.007200000000000001, + "prompt_tokens": 100, + "spend": 0.0392 + }, + "fireworks_ai/qwen3p8-max|all_components_fireworks": { + "completion_tokens": 25, + "input_cost": 0.010920000000000001, + "output_cost": 0.006500000000000001, + "prompt_tokens": 120, + "spend": 0.01742 + }, + "fireworks_ai/qwen3p8-max|audio": { + "completion_tokens": 45, + "input_cost": 0.029500000000000002, + "output_cost": 0.01935, + "prompt_tokens": 125, + "spend": 0.048850000000000005 + }, + "fireworks_ai/qwen3p8-max|basic": { + "completion_tokens": 40, + "input_cost": 0.015600000000000003, + "output_cost": 0.010400000000000001, + "prompt_tokens": 120, + "spend": 0.026000000000000002 + }, + "fireworks_ai/qwen3p8-max|cache_read": { + "completion_tokens": 30, + "input_cost": 0.01365, + "output_cost": 0.007800000000000001, + "prompt_tokens": 150, + "spend": 0.021450000000000004 + }, + "fireworks_ai/qwen3p8-max|cache_write_1h": { + "completion_tokens": 30, + "input_cost": 0.0359, + "output_cost": 0.007800000000000001, + "prompt_tokens": 150, + "spend": 0.0437 + }, + "fireworks_ai/qwen3p8-max|cache_write_5m": { + "completion_tokens": 30, + "input_cost": 0.0315, + "output_cost": 0.007800000000000001, + "prompt_tokens": 150, + "spend": 0.0393 + }, + "fireworks_ai/qwen3p8-max|reasoning": { + "completion_tokens": 100, + "input_cost": 0.013000000000000001, + "output_cost": 0.0463, + "prompt_tokens": 100, + "spend": 0.059300000000000005 + }, + "fireworks_ai/qwen3p8-max|response_model_override": { + "completion_tokens": 40, + "input_cost": 0.016800000000000002, + "output_cost": 0.011200000000000002, + "prompt_tokens": 120, + "spend": 0.028000000000000004 + }, + "fireworks_ai/qwen3p8-max|stream": { + "completion_tokens": 40, + "input_cost": 0.015600000000000003, + "output_cost": 0.010400000000000001, + "prompt_tokens": 120, + "spend": 0.026000000000000002 + }, + "fireworks_ai/qwen3p8-max|stream_response_model_override": { + "completion_tokens": 40, + "input_cost": 0.016800000000000002, + "output_cost": 0.011200000000000002, + "prompt_tokens": 120, + "spend": 0.028000000000000004 + }, + "fireworks_ai/qwen3p8-max|stream_tool_call": { + "completion_tokens": 25, + "input_cost": 0.010400000000000001, + "output_cost": 0.006500000000000001, + "prompt_tokens": 80, + "spend": 0.016900000000000002 + }, + "fireworks_ai/qwen3p8-max|tool_call": { + "completion_tokens": 40, + "input_cost": 0.015600000000000003, + "output_cost": 0.010400000000000001, + "prompt_tokens": 120, + "spend": 0.026000000000000002 + }, + "fireworks_ai/qwen3p8-max|web_search": { + "completion_tokens": 30, + "input_cost": 0.013000000000000001, + "output_cost": 0.007800000000000001, + "prompt_tokens": 100, + "spend": 0.0408 + }, + "gemini-3.1-pro-preview|all_components_gemini": { + "completion_tokens": 43, + "input_cost": 0.023940000000000003, + "output_cost": 0.030660000000000003, + "prompt_tokens": 125, + "spend": 0.05460000000000001 + }, + "gemini-3.1-pro-preview|audio": { + "completion_tokens": 45, + "input_cost": 0.052500000000000005, + "output_cost": 0.03465, + "prompt_tokens": 125, + "spend": 0.08715 + }, + "gemini-3.1-pro-preview|basic": { + "completion_tokens": 40, + "input_cost": 0.0252, + "output_cost": 0.016800000000000002, + "prompt_tokens": 120, + "spend": 0.042 + }, + "gemini-3.1-pro-preview|cache_read": { + "completion_tokens": 30, + "input_cost": 0.02205, + "output_cost": 0.0126, + "prompt_tokens": 150, + "spend": 0.03465 + }, + "gemini-3.1-pro-preview|prompt_blocked": { + "completion_tokens": 0, + "input_cost": 0.2, + "output_cost": 0.0, + "prompt_tokens": 1000, + "spend": 0.2 + }, + "gemini-3.1-pro-preview|reasoning": { + "completion_tokens": 100, + "input_cost": 0.021, + "output_cost": 0.0861, + "prompt_tokens": 100, + "spend": 0.1071 + }, + "gemini-3.1-pro-preview|response_model_override": { + "completion_tokens": 40, + "input_cost": 0.024, + "output_cost": 0.016, + "prompt_tokens": 120, + "spend": 0.04 + }, + "gemini-3.1-pro-preview|service_tier_flex": { + "completion_tokens": 40, + "input_cost": 0.0378, + "output_cost": 0.020999999999999998, + "prompt_tokens": 120, + "spend": 0.0588 + }, + "gemini-3.1-pro-preview|service_tier_priority": { + "completion_tokens": 40, + "input_cost": 0.04284, + "output_cost": 0.02268, + "prompt_tokens": 120, + "spend": 0.06552 + }, + "gemini-3.1-pro-preview|stream": { + "completion_tokens": 40, + "input_cost": 0.0252, + "output_cost": 0.016800000000000002, + "prompt_tokens": 120, + "spend": 0.042 + }, + "gemini-3.1-pro-preview|stream_prompt_blocked": { + "completion_tokens": 0, + "input_cost": 0.2, + "output_cost": 0.0, + "prompt_tokens": 1000, + "spend": 0.2 + }, + "gemini-3.1-pro-preview|stream_response_model_override": { + "completion_tokens": 40, + "input_cost": 0.024, + "output_cost": 0.016, + "prompt_tokens": 120, + "spend": 0.04 + }, + "gemini-3.1-pro-preview|stream_tool_call": { + "completion_tokens": 25, + "input_cost": 0.016800000000000002, + "output_cost": 0.0105, + "prompt_tokens": 80, + "spend": 0.027300000000000005 + }, + "gemini-3.1-pro-preview|tiered": { + "completion_tokens": 30, + "input_cost": 336.00168, + "output_cost": 0.0567, + "prompt_tokens": 200001, + "spend": 336.05838 + }, + "gemini-3.1-pro-preview|tool_call": { + "completion_tokens": 40, + "input_cost": 0.0252, + "output_cost": 0.016800000000000002, + "prompt_tokens": 120, + "spend": 0.042 + }, + "gemini-3.1-pro-preview|web_search": { + "completion_tokens": 30, + "input_cost": 0.021, + "output_cost": 0.0126, + "prompt_tokens": 100, + "spend": 0.0936 + }, + "gemini-3.8-flash|all_components_gemini": { + "completion_tokens": 43, + "input_cost": 0.022799999999999997, + "output_cost": 0.0292, + "prompt_tokens": 125, + "spend": 0.052 + }, + "gemini-3.8-flash|audio": { + "completion_tokens": 45, + "input_cost": 0.05, + "output_cost": 0.033, + "prompt_tokens": 125, + "spend": 0.083 + }, + "gemini-3.8-flash|basic": { + "completion_tokens": 40, + "input_cost": 0.024, + "output_cost": 0.016, + "prompt_tokens": 120, + "spend": 0.04 + }, + "gemini-3.8-flash|cache_read": { + "completion_tokens": 30, + "input_cost": 0.021, + "output_cost": 0.012, + "prompt_tokens": 150, + "spend": 0.033 + }, + "gemini-3.8-flash|prompt_blocked": { + "completion_tokens": 0, + "input_cost": 0.21000000000000002, + "output_cost": 0.0, + "prompt_tokens": 1000, + "spend": 0.21000000000000002 + }, + "gemini-3.8-flash|reasoning": { + "completion_tokens": 100, + "input_cost": 0.02, + "output_cost": 0.082, + "prompt_tokens": 100, + "spend": 0.10200000000000001 + }, + "gemini-3.8-flash|response_model_override": { + "completion_tokens": 40, + "input_cost": 0.0252, + "output_cost": 0.016800000000000002, + "prompt_tokens": 120, + "spend": 0.042 + }, + "gemini-3.8-flash|service_tier_flex": { + "completion_tokens": 40, + "input_cost": 0.036, + "output_cost": 0.02, + "prompt_tokens": 120, + "spend": 0.055999999999999994 + }, + "gemini-3.8-flash|service_tier_priority": { + "completion_tokens": 40, + "input_cost": 0.0408, + "output_cost": 0.0216, + "prompt_tokens": 120, + "spend": 0.062400000000000004 + }, + "gemini-3.8-flash|stream": { + "completion_tokens": 40, + "input_cost": 0.024, + "output_cost": 0.016, + "prompt_tokens": 120, + "spend": 0.04 + }, + "gemini-3.8-flash|stream_prompt_blocked": { + "completion_tokens": 0, + "input_cost": 0.21000000000000002, + "output_cost": 0.0, + "prompt_tokens": 1000, + "spend": 0.21000000000000002 + }, + "gemini-3.8-flash|stream_response_model_override": { + "completion_tokens": 40, + "input_cost": 0.0252, + "output_cost": 0.016800000000000002, + "prompt_tokens": 120, + "spend": 0.042 + }, + "gemini-3.8-flash|stream_tool_call": { + "completion_tokens": 25, + "input_cost": 0.016, + "output_cost": 0.01, + "prompt_tokens": 80, + "spend": 0.026000000000000002 + }, + "gemini-3.8-flash|tiered": { + "completion_tokens": 30, + "input_cost": 320.0016, + "output_cost": 0.054, + "prompt_tokens": 200001, + "spend": 320.05559999999997 + }, + "gemini-3.8-flash|tool_call": { + "completion_tokens": 40, + "input_cost": 0.024, + "output_cost": 0.016, + "prompt_tokens": 120, + "spend": 0.04 + }, + "gemini-3.8-flash|web_search": { + "completion_tokens": 30, + "input_cost": 0.02, + "output_cost": 0.012, + "prompt_tokens": 100, + "spend": 0.092 + }, + "gemini/gemini-3.1-pro-preview|all_components_gemini": { + "completion_tokens": 43, + "input_cost": 0.010260000000000002, + "output_cost": 0.01314, + "prompt_tokens": 125, + "spend": 0.023400000000000004 + }, + "gemini/gemini-3.1-pro-preview|audio": { + "completion_tokens": 45, + "input_cost": 0.0225, + "output_cost": 0.014849999999999999, + "prompt_tokens": 125, + "spend": 0.037349999999999994 + }, + "gemini/gemini-3.1-pro-preview|basic": { + "completion_tokens": 40, + "input_cost": 0.0108, + "output_cost": 0.007200000000000001, + "prompt_tokens": 120, + "spend": 0.018000000000000002 + }, + "gemini/gemini-3.1-pro-preview|cache_read": { + "completion_tokens": 30, + "input_cost": 0.009450000000000002, + "output_cost": 0.0054, + "prompt_tokens": 150, + "spend": 0.014850000000000002 + }, + "gemini/gemini-3.1-pro-preview|prompt_blocked": { + "completion_tokens": 0, + "input_cost": 0.08, + "output_cost": 0.0, + "prompt_tokens": 1000, + "spend": 0.08 + }, + "gemini/gemini-3.1-pro-preview|reasoning": { + "completion_tokens": 100, + "input_cost": 0.009000000000000001, + "output_cost": 0.0369, + "prompt_tokens": 100, + "spend": 0.0459 + }, + "gemini/gemini-3.1-pro-preview|response_model_override": { + "completion_tokens": 40, + "input_cost": 0.009600000000000001, + "output_cost": 0.0064, + "prompt_tokens": 120, + "spend": 0.016 + }, + "gemini/gemini-3.1-pro-preview|service_tier_flex": { + "completion_tokens": 40, + "input_cost": 0.0162, + "output_cost": 0.009000000000000001, + "prompt_tokens": 120, + "spend": 0.0252 + }, + "gemini/gemini-3.1-pro-preview|service_tier_priority": { + "completion_tokens": 40, + "input_cost": 0.01836, + "output_cost": 0.00972, + "prompt_tokens": 120, + "spend": 0.02808 + }, + "gemini/gemini-3.1-pro-preview|stream": { + "completion_tokens": 40, + "input_cost": 0.0108, + "output_cost": 0.007200000000000001, + "prompt_tokens": 120, + "spend": 0.018000000000000002 + }, + "gemini/gemini-3.1-pro-preview|stream_prompt_blocked": { + "completion_tokens": 0, + "input_cost": 0.08, + "output_cost": 0.0, + "prompt_tokens": 1000, + "spend": 0.08 + }, + "gemini/gemini-3.1-pro-preview|stream_response_model_override": { + "completion_tokens": 40, + "input_cost": 0.009600000000000001, + "output_cost": 0.0064, + "prompt_tokens": 120, + "spend": 0.016 + }, + "gemini/gemini-3.1-pro-preview|stream_tool_call": { + "completion_tokens": 25, + "input_cost": 0.007200000000000001, + "output_cost": 0.0045000000000000005, + "prompt_tokens": 80, + "spend": 0.011700000000000002 + }, + "gemini/gemini-3.1-pro-preview|tiered": { + "completion_tokens": 30, + "input_cost": 144.00072, + "output_cost": 0.024300000000000002, + "prompt_tokens": 200001, + "spend": 144.02502 + }, + "gemini/gemini-3.1-pro-preview|tool_call": { + "completion_tokens": 40, + "input_cost": 0.0108, + "output_cost": 0.007200000000000001, + "prompt_tokens": 120, + "spend": 0.018000000000000002 + }, + "gemini/gemini-3.1-pro-preview|web_search": { + "completion_tokens": 30, + "input_cost": 0.009000000000000001, + "output_cost": 0.0054, + "prompt_tokens": 100, + "spend": 0.0744 + }, + "gemini/gemini-3.8-flash|all_components_gemini": { + "completion_tokens": 43, + "input_cost": 0.00912, + "output_cost": 0.01168, + "prompt_tokens": 125, + "spend": 0.0208 + }, + "gemini/gemini-3.8-flash|audio": { + "completion_tokens": 45, + "input_cost": 0.02, + "output_cost": 0.0132, + "prompt_tokens": 125, + "spend": 0.0332 + }, + "gemini/gemini-3.8-flash|basic": { + "completion_tokens": 40, + "input_cost": 0.009600000000000001, + "output_cost": 0.0064, + "prompt_tokens": 120, + "spend": 0.016 + }, + "gemini/gemini-3.8-flash|cache_read": { + "completion_tokens": 30, + "input_cost": 0.0084, + "output_cost": 0.0048000000000000004, + "prompt_tokens": 150, + "spend": 0.0132 + }, + "gemini/gemini-3.8-flash|prompt_blocked": { + "completion_tokens": 0, + "input_cost": 0.09000000000000001, + "output_cost": 0.0, + "prompt_tokens": 1000, + "spend": 0.09000000000000001 + }, + "gemini/gemini-3.8-flash|reasoning": { + "completion_tokens": 100, + "input_cost": 0.008, + "output_cost": 0.0328, + "prompt_tokens": 100, + "spend": 0.0408 + }, + "gemini/gemini-3.8-flash|response_model_override": { + "completion_tokens": 40, + "input_cost": 0.0108, + "output_cost": 0.007200000000000001, + "prompt_tokens": 120, + "spend": 0.018000000000000002 + }, + "gemini/gemini-3.8-flash|service_tier_flex": { + "completion_tokens": 40, + "input_cost": 0.0144, + "output_cost": 0.008, + "prompt_tokens": 120, + "spend": 0.0224 + }, + "gemini/gemini-3.8-flash|service_tier_priority": { + "completion_tokens": 40, + "input_cost": 0.01632, + "output_cost": 0.00864, + "prompt_tokens": 120, + "spend": 0.024960000000000003 + }, + "gemini/gemini-3.8-flash|stream": { + "completion_tokens": 40, + "input_cost": 0.009600000000000001, + "output_cost": 0.0064, + "prompt_tokens": 120, + "spend": 0.016 + }, + "gemini/gemini-3.8-flash|stream_prompt_blocked": { + "completion_tokens": 0, + "input_cost": 0.09000000000000001, + "output_cost": 0.0, + "prompt_tokens": 1000, + "spend": 0.09000000000000001 + }, + "gemini/gemini-3.8-flash|stream_response_model_override": { + "completion_tokens": 40, + "input_cost": 0.0108, + "output_cost": 0.007200000000000001, + "prompt_tokens": 120, + "spend": 0.018000000000000002 + }, + "gemini/gemini-3.8-flash|stream_tool_call": { + "completion_tokens": 25, + "input_cost": 0.0064, + "output_cost": 0.004, + "prompt_tokens": 80, + "spend": 0.0104 + }, + "gemini/gemini-3.8-flash|tiered": { + "completion_tokens": 30, + "input_cost": 128.00064, + "output_cost": 0.0216, + "prompt_tokens": 200001, + "spend": 128.02224 + }, + "gemini/gemini-3.8-flash|tool_call": { + "completion_tokens": 40, + "input_cost": 0.009600000000000001, + "output_cost": 0.0064, + "prompt_tokens": 120, + "spend": 0.016 + }, + "gemini/gemini-3.8-flash|web_search": { + "completion_tokens": 30, + "input_cost": 0.008, + "output_cost": 0.0048000000000000004, + "prompt_tokens": 100, + "spend": 0.0728 + }, + "gpt-5.3-codex|all_components_responses": { + "completion_tokens": 40, + "input_cost": 0.00252, + "output_cost": 0.0037500000000000007, + "prompt_tokens": 120, + "spend": 0.006270000000000001 + }, + "gpt-5.3-codex|basic": { + "completion_tokens": 40, + "input_cost": 0.0036000000000000003, + "output_cost": 0.0024000000000000002, + "prompt_tokens": 120, + "spend": 0.006 + }, + "gpt-5.3-codex|cache_read": { + "completion_tokens": 30, + "input_cost": 0.0031500000000000005, + "output_cost": 0.0018000000000000002, + "prompt_tokens": 150, + "spend": 0.00495 + }, + "gpt-5.3-codex|reasoning": { + "completion_tokens": 100, + "input_cost": 0.0030000000000000005, + "output_cost": 0.0123, + "prompt_tokens": 100, + "spend": 0.015300000000000001 + }, + "gpt-5.3-codex|response_model_override": { + "completion_tokens": 40, + "input_cost": 0.0024000000000000002, + "output_cost": 0.0016, + "prompt_tokens": 120, + "spend": 0.004 + }, + "gpt-5.3-codex|service_tier_flex": { + "completion_tokens": 40, + "input_cost": 0.0054, + "output_cost": 0.003, + "prompt_tokens": 120, + "spend": 0.008400000000000001 + }, + "gpt-5.3-codex|service_tier_priority": { + "completion_tokens": 40, + "input_cost": 0.00612, + "output_cost": 0.00324, + "prompt_tokens": 120, + "spend": 0.00936 + }, + "gpt-5.3-codex|stream": { + "completion_tokens": 40, + "input_cost": 0.0036000000000000003, + "output_cost": 0.0024000000000000002, + "prompt_tokens": 120, + "spend": 0.006 + }, + "gpt-5.3-codex|stream_incomplete": { + "completion_tokens": 40, + "input_cost": 0.0036000000000000003, + "output_cost": 0.0024000000000000002, + "prompt_tokens": 120, + "spend": 0.006 + }, + "gpt-5.3-codex|stream_response_model_override": { + "completion_tokens": 40, + "input_cost": 0.0024000000000000002, + "output_cost": 0.0016, + "prompt_tokens": 120, + "spend": 0.004 + }, + "gpt-5.3-codex|stream_tool_call": { + "completion_tokens": 25, + "input_cost": 0.0024000000000000002, + "output_cost": 0.0015000000000000002, + "prompt_tokens": 80, + "spend": 0.0039000000000000007 + }, + "gpt-5.3-codex|stream_unvalidated": { + "completion_tokens": 40, + "input_cost": 0.0036000000000000003, + "output_cost": 0.0024000000000000002, + "prompt_tokens": 120, + "spend": 0.006 + }, + "gpt-5.3-codex|tiered": { + "completion_tokens": 30, + "input_cost": 48.000240000000005, + "output_cost": 0.0081, + "prompt_tokens": 200001, + "spend": 48.008340000000004 + }, + "gpt-5.3-codex|tool_call": { + "completion_tokens": 40, + "input_cost": 0.0036000000000000003, + "output_cost": 0.0024000000000000002, + "prompt_tokens": 120, + "spend": 0.006 + }, + "gpt-5.3-codex|web_search": { + "completion_tokens": 30, + "input_cost": 0.0030000000000000005, + "output_cost": 0.0018000000000000002, + "prompt_tokens": 100, + "spend": 0.0648 + }, + "gpt-5.4-mini|all_components_chat": { + "completion_tokens": 43, + "input_cost": 0.00856, + "output_cost": 0.00584, + "prompt_tokens": 155, + "spend": 0.0144 + }, + "gpt-5.4-mini|audio": { + "completion_tokens": 45, + "input_cost": 0.01, + "output_cost": 0.0066, + "prompt_tokens": 125, + "spend": 0.0166 + }, + "gpt-5.4-mini|basic": { + "completion_tokens": 40, + "input_cost": 0.0048000000000000004, + "output_cost": 0.0032, + "prompt_tokens": 120, + "spend": 0.008 + }, + "gpt-5.4-mini|cache_read": { + "completion_tokens": 30, + "input_cost": 0.0042, + "output_cost": 0.0024000000000000002, + "prompt_tokens": 150, + "spend": 0.0066 + }, + "gpt-5.4-mini|cache_write_1h": { + "completion_tokens": 30, + "input_cost": 0.012400000000000001, + "output_cost": 0.0024000000000000002, + "prompt_tokens": 150, + "spend": 0.0148 + }, + "gpt-5.4-mini|cache_write_5m": { + "completion_tokens": 30, + "input_cost": 0.0108, + "output_cost": 0.0024000000000000002, + "prompt_tokens": 150, + "spend": 0.0132 + }, + "gpt-5.4-mini|reasoning": { + "completion_tokens": 100, + "input_cost": 0.004, + "output_cost": 0.0164, + "prompt_tokens": 100, + "spend": 0.0204 + }, + "gpt-5.4-mini|response_model_override": { + "completion_tokens": 40, + "input_cost": 0.0012000000000000001, + "output_cost": 0.0008, + "prompt_tokens": 120, + "spend": 0.002 + }, + "gpt-5.4-mini|service_tier_flex": { + "completion_tokens": 40, + "input_cost": 0.0072, + "output_cost": 0.004, + "prompt_tokens": 120, + "spend": 0.0112 + }, + "gpt-5.4-mini|service_tier_priority": { + "completion_tokens": 40, + "input_cost": 0.00816, + "output_cost": 0.00432, + "prompt_tokens": 120, + "spend": 0.012480000000000002 + }, + "gpt-5.4-mini|stream": { + "completion_tokens": 40, + "input_cost": 0.0048000000000000004, + "output_cost": 0.0032, + "prompt_tokens": 120, + "spend": 0.008 + }, + "gpt-5.4-mini|stream_response_model_override": { + "completion_tokens": 40, + "input_cost": 0.0012000000000000001, + "output_cost": 0.0008, + "prompt_tokens": 120, + "spend": 0.002 + }, + "gpt-5.4-mini|stream_tool_call": { + "completion_tokens": 25, + "input_cost": 0.0032, + "output_cost": 0.002, + "prompt_tokens": 80, + "spend": 0.0052 + }, + "gpt-5.4-mini|tiered": { + "completion_tokens": 30, + "input_cost": 64.00032, + "output_cost": 0.0108, + "prompt_tokens": 200001, + "spend": 64.01112 + }, + "gpt-5.4-mini|tool_call": { + "completion_tokens": 40, + "input_cost": 0.0048000000000000004, + "output_cost": 0.0032, + "prompt_tokens": 120, + "spend": 0.008 + }, + "gpt-5.4-mini|web_search": { + "completion_tokens": 30, + "input_cost": 0.004, + "output_cost": 0.0024000000000000002, + "prompt_tokens": 100, + "spend": 0.0264 + }, + "gpt-5.5-pro|all_components_responses": { + "completion_tokens": 40, + "input_cost": 0.00168, + "output_cost": 0.0025, + "prompt_tokens": 120, + "spend": 0.00418 + }, + "gpt-5.5-pro|basic": { + "completion_tokens": 40, + "input_cost": 0.0024000000000000002, + "output_cost": 0.0016, + "prompt_tokens": 120, + "spend": 0.004 + }, + "gpt-5.5-pro|cache_read": { + "completion_tokens": 30, + "input_cost": 0.0021, + "output_cost": 0.0012000000000000001, + "prompt_tokens": 150, + "spend": 0.0033 + }, + "gpt-5.5-pro|reasoning": { + "completion_tokens": 100, + "input_cost": 0.002, + "output_cost": 0.0082, + "prompt_tokens": 100, + "spend": 0.0102 + }, + "gpt-5.5-pro|response_model_override": { + "completion_tokens": 40, + "input_cost": 0.0036000000000000003, + "output_cost": 0.0024000000000000002, + "prompt_tokens": 120, + "spend": 0.006 + }, + "gpt-5.5-pro|service_tier_flex": { + "completion_tokens": 40, + "input_cost": 0.0036, + "output_cost": 0.002, + "prompt_tokens": 120, + "spend": 0.0056 + }, + "gpt-5.5-pro|service_tier_priority": { + "completion_tokens": 40, + "input_cost": 0.00408, + "output_cost": 0.00216, + "prompt_tokens": 120, + "spend": 0.006240000000000001 + }, + "gpt-5.5-pro|stream": { + "completion_tokens": 40, + "input_cost": 0.0024000000000000002, + "output_cost": 0.0016, + "prompt_tokens": 120, + "spend": 0.004 + }, + "gpt-5.5-pro|stream_incomplete": { + "completion_tokens": 40, + "input_cost": 0.0024000000000000002, + "output_cost": 0.0016, + "prompt_tokens": 120, + "spend": 0.004 + }, + "gpt-5.5-pro|stream_response_model_override": { + "completion_tokens": 40, + "input_cost": 0.0036000000000000003, + "output_cost": 0.0024000000000000002, + "prompt_tokens": 120, + "spend": 0.006 + }, + "gpt-5.5-pro|stream_tool_call": { + "completion_tokens": 25, + "input_cost": 0.0016, + "output_cost": 0.001, + "prompt_tokens": 80, + "spend": 0.0026 + }, + "gpt-5.5-pro|stream_unvalidated": { + "completion_tokens": 40, + "input_cost": 0.0024000000000000002, + "output_cost": 0.0016, + "prompt_tokens": 120, + "spend": 0.004 + }, + "gpt-5.5-pro|tiered": { + "completion_tokens": 30, + "input_cost": 32.00016, + "output_cost": 0.0054, + "prompt_tokens": 200001, + "spend": 32.00556 + }, + "gpt-5.5-pro|tool_call": { + "completion_tokens": 40, + "input_cost": 0.0024000000000000002, + "output_cost": 0.0016, + "prompt_tokens": 120, + "spend": 0.004 + }, + "gpt-5.5-pro|web_search": { + "completion_tokens": 30, + "input_cost": 0.002, + "output_cost": 0.0012000000000000001, + "prompt_tokens": 100, + "spend": 0.06319999999999999 + }, + "gpt-5.6|all_components_chat": { + "completion_tokens": 43, + "input_cost": 0.00214, + "output_cost": 0.00146, + "prompt_tokens": 155, + "spend": 0.0036 + }, + "gpt-5.6|audio": { + "completion_tokens": 45, + "input_cost": 0.0025, + "output_cost": 0.00165, + "prompt_tokens": 125, + "spend": 0.00415 + }, + "gpt-5.6|basic": { + "completion_tokens": 40, + "input_cost": 0.0012000000000000001, + "output_cost": 0.0008, + "prompt_tokens": 120, + "spend": 0.002 + }, + "gpt-5.6|cache_read": { + "completion_tokens": 30, + "input_cost": 0.00105, + "output_cost": 0.0006000000000000001, + "prompt_tokens": 150, + "spend": 0.00165 + }, + "gpt-5.6|cache_write_1h": { + "completion_tokens": 30, + "input_cost": 0.0031000000000000003, + "output_cost": 0.0006000000000000001, + "prompt_tokens": 150, + "spend": 0.0037 + }, + "gpt-5.6|cache_write_5m": { + "completion_tokens": 30, + "input_cost": 0.0027, + "output_cost": 0.0006000000000000001, + "prompt_tokens": 150, + "spend": 0.0033 + }, + "gpt-5.6|reasoning": { + "completion_tokens": 100, + "input_cost": 0.001, + "output_cost": 0.0041, + "prompt_tokens": 100, + "spend": 0.0051 + }, + "gpt-5.6|response_model_override": { + "completion_tokens": 40, + "input_cost": 0.0048000000000000004, + "output_cost": 0.0032, + "prompt_tokens": 120, + "spend": 0.008 + }, + "gpt-5.6|service_tier_flex": { + "completion_tokens": 40, + "input_cost": 0.0018, + "output_cost": 0.001, + "prompt_tokens": 120, + "spend": 0.0028 + }, + "gpt-5.6|service_tier_priority": { + "completion_tokens": 40, + "input_cost": 0.00204, + "output_cost": 0.00108, + "prompt_tokens": 120, + "spend": 0.0031200000000000004 + }, + "gpt-5.6|stream": { + "completion_tokens": 40, + "input_cost": 0.0012000000000000001, + "output_cost": 0.0008, + "prompt_tokens": 120, + "spend": 0.002 + }, + "gpt-5.6|stream_response_model_override": { + "completion_tokens": 40, + "input_cost": 0.0048000000000000004, + "output_cost": 0.0032, + "prompt_tokens": 120, + "spend": 0.008 + }, + "gpt-5.6|stream_tool_call": { + "completion_tokens": 25, + "input_cost": 0.0008, + "output_cost": 0.0005, + "prompt_tokens": 80, + "spend": 0.0013 + }, + "gpt-5.6|tiered": { + "completion_tokens": 30, + "input_cost": 16.00008, + "output_cost": 0.0027, + "prompt_tokens": 200001, + "spend": 16.00278 + }, + "gpt-5.6|tool_call": { + "completion_tokens": 40, + "input_cost": 0.0012000000000000001, + "output_cost": 0.0008, + "prompt_tokens": 120, + "spend": 0.002 + }, + "gpt-5.6|web_search": { + "completion_tokens": 30, + "input_cost": 0.001, + "output_cost": 0.0006000000000000001, + "prompt_tokens": 100, + "spend": 0.0216 + }, + "meta.llama4-maverick-17b-instruct-v1:0|all_components_anthropic": { + "completion_tokens": 25, + "input_cost": 0.020900000000000002, + "output_cost": 0.0095, + "prompt_tokens": 150, + "spend": 0.030400000000000003 + }, + "meta.llama4-maverick-17b-instruct-v1:0|basic": { + "completion_tokens": 40, + "input_cost": 0.0228, + "output_cost": 0.015200000000000002, + "prompt_tokens": 120, + "spend": 0.038000000000000006 + }, + "meta.llama4-maverick-17b-instruct-v1:0|stream": { + "completion_tokens": 40, + "input_cost": 0.0228, + "output_cost": 0.015200000000000002, + "prompt_tokens": 120, + "spend": 0.038000000000000006 + }, + "meta.llama4-maverick-17b-instruct-v1:0|stream_tool_call": { + "completion_tokens": 25, + "input_cost": 0.015200000000000002, + "output_cost": 0.0095, + "prompt_tokens": 80, + "spend": 0.0247 + }, + "meta.llama4-maverick-17b-instruct-v1:0|tool_call": { + "completion_tokens": 40, + "input_cost": 0.0228, + "output_cost": 0.015200000000000002, + "prompt_tokens": 120, + "spend": 0.038000000000000006 + }, + "together_ai/moonshotai/Kimi-K3|all_components_chat": { + "completion_tokens": 43, + "input_cost": 0.0214, + "output_cost": 0.0146, + "prompt_tokens": 155, + "spend": 0.036 + }, + "together_ai/moonshotai/Kimi-K3|audio": { + "completion_tokens": 45, + "input_cost": 0.025, + "output_cost": 0.0165, + "prompt_tokens": 125, + "spend": 0.0415 + }, + "together_ai/moonshotai/Kimi-K3|basic": { + "completion_tokens": 40, + "input_cost": 0.012, + "output_cost": 0.008, + "prompt_tokens": 120, + "spend": 0.02 + }, + "together_ai/moonshotai/Kimi-K3|cache_read": { + "completion_tokens": 30, + "input_cost": 0.0105, + "output_cost": 0.006, + "prompt_tokens": 150, + "spend": 0.0165 + }, + "together_ai/moonshotai/Kimi-K3|cache_write_1h": { + "completion_tokens": 30, + "input_cost": 0.031, + "output_cost": 0.006, + "prompt_tokens": 150, + "spend": 0.037 + }, + "together_ai/moonshotai/Kimi-K3|cache_write_5m": { + "completion_tokens": 30, + "input_cost": 0.027000000000000003, + "output_cost": 0.006, + "prompt_tokens": 150, + "spend": 0.033 + }, + "together_ai/moonshotai/Kimi-K3|reasoning": { + "completion_tokens": 100, + "input_cost": 0.01, + "output_cost": 0.041, + "prompt_tokens": 100, + "spend": 0.051000000000000004 + }, + "together_ai/moonshotai/Kimi-K3|response_model_override": { + "completion_tokens": 40, + "input_cost": 0.0132, + "output_cost": 0.0088, + "prompt_tokens": 120, + "spend": 0.022 + }, + "together_ai/moonshotai/Kimi-K3|service_tier_flex": { + "completion_tokens": 40, + "input_cost": 0.018000000000000002, + "output_cost": 0.01, + "prompt_tokens": 120, + "spend": 0.028000000000000004 + }, + "together_ai/moonshotai/Kimi-K3|service_tier_priority": { + "completion_tokens": 40, + "input_cost": 0.0204, + "output_cost": 0.0108, + "prompt_tokens": 120, + "spend": 0.031200000000000002 + }, + "together_ai/moonshotai/Kimi-K3|stream": { + "completion_tokens": 40, + "input_cost": 0.012, + "output_cost": 0.008, + "prompt_tokens": 120, + "spend": 0.02 + }, + "together_ai/moonshotai/Kimi-K3|stream_response_model_override": { + "completion_tokens": 40, + "input_cost": 0.0132, + "output_cost": 0.0088, + "prompt_tokens": 120, + "spend": 0.022 + }, + "together_ai/moonshotai/Kimi-K3|stream_tool_call": { + "completion_tokens": 25, + "input_cost": 0.008, + "output_cost": 0.005, + "prompt_tokens": 80, + "spend": 0.013000000000000001 + }, + "together_ai/moonshotai/Kimi-K3|tiered": { + "completion_tokens": 30, + "input_cost": 160.0008, + "output_cost": 0.027000000000000003, + "prompt_tokens": 200001, + "spend": 160.02779999999998 + }, + "together_ai/moonshotai/Kimi-K3|tool_call": { + "completion_tokens": 40, + "input_cost": 0.012, + "output_cost": 0.008, + "prompt_tokens": 120, + "spend": 0.02 + }, + "together_ai/moonshotai/Kimi-K3|web_search": { + "completion_tokens": 30, + "input_cost": 0.01, + "output_cost": 0.006, + "prompt_tokens": 100, + "spend": 0.036000000000000004 + }, + "together_ai/zai-org/GLM-5.3|all_components_chat": { + "completion_tokens": 43, + "input_cost": 0.023540000000000002, + "output_cost": 0.01606, + "prompt_tokens": 155, + "spend": 0.0396 + }, + "together_ai/zai-org/GLM-5.3|audio": { + "completion_tokens": 45, + "input_cost": 0.027500000000000004, + "output_cost": 0.01815, + "prompt_tokens": 125, + "spend": 0.04565 + }, + "together_ai/zai-org/GLM-5.3|basic": { + "completion_tokens": 40, + "input_cost": 0.0132, + "output_cost": 0.0088, + "prompt_tokens": 120, + "spend": 0.022 + }, + "together_ai/zai-org/GLM-5.3|cache_read": { + "completion_tokens": 30, + "input_cost": 0.011550000000000001, + "output_cost": 0.0066, + "prompt_tokens": 150, + "spend": 0.01815 + }, + "together_ai/zai-org/GLM-5.3|cache_write_1h": { + "completion_tokens": 30, + "input_cost": 0.034100000000000005, + "output_cost": 0.0066, + "prompt_tokens": 150, + "spend": 0.04070000000000001 + }, + "together_ai/zai-org/GLM-5.3|cache_write_5m": { + "completion_tokens": 30, + "input_cost": 0.029699999999999997, + "output_cost": 0.0066, + "prompt_tokens": 150, + "spend": 0.0363 + }, + "together_ai/zai-org/GLM-5.3|reasoning": { + "completion_tokens": 100, + "input_cost": 0.011000000000000001, + "output_cost": 0.0451, + "prompt_tokens": 100, + "spend": 0.056100000000000004 + }, + "together_ai/zai-org/GLM-5.3|response_model_override": { + "completion_tokens": 40, + "input_cost": 0.012, + "output_cost": 0.008, + "prompt_tokens": 120, + "spend": 0.02 + }, + "together_ai/zai-org/GLM-5.3|service_tier_flex": { + "completion_tokens": 40, + "input_cost": 0.019799999999999998, + "output_cost": 0.011000000000000001, + "prompt_tokens": 120, + "spend": 0.0308 + }, + "together_ai/zai-org/GLM-5.3|service_tier_priority": { + "completion_tokens": 40, + "input_cost": 0.022439999999999998, + "output_cost": 0.01188, + "prompt_tokens": 120, + "spend": 0.034319999999999996 + }, + "together_ai/zai-org/GLM-5.3|stream": { + "completion_tokens": 40, + "input_cost": 0.0132, + "output_cost": 0.0088, + "prompt_tokens": 120, + "spend": 0.022 + }, + "together_ai/zai-org/GLM-5.3|stream_response_model_override": { + "completion_tokens": 40, + "input_cost": 0.012, + "output_cost": 0.008, + "prompt_tokens": 120, + "spend": 0.02 + }, + "together_ai/zai-org/GLM-5.3|stream_tool_call": { + "completion_tokens": 25, + "input_cost": 0.0088, + "output_cost": 0.0055000000000000005, + "prompt_tokens": 80, + "spend": 0.0143 + }, + "together_ai/zai-org/GLM-5.3|tiered": { + "completion_tokens": 30, + "input_cost": 176.00088, + "output_cost": 0.0297, + "prompt_tokens": 200001, + "spend": 176.03058 + }, + "together_ai/zai-org/GLM-5.3|tool_call": { + "completion_tokens": 40, + "input_cost": 0.0132, + "output_cost": 0.0088, + "prompt_tokens": 120, + "spend": 0.022 + }, + "together_ai/zai-org/GLM-5.3|web_search": { + "completion_tokens": 30, + "input_cost": 0.011000000000000001, + "output_cost": 0.0066, + "prompt_tokens": 100, + "spend": 0.0376 + }, + "us.anthropic.claude-opus-5-v1:0|all_components_anthropic": { + "completion_tokens": 25, + "input_cost": 0.033120000000000004, + "output_cost": 0.009000000000000001, + "prompt_tokens": 150, + "spend": 0.042120000000000005 + }, + "us.anthropic.claude-opus-5-v1:0|basic": { + "completion_tokens": 40, + "input_cost": 0.0216, + "output_cost": 0.014400000000000001, + "prompt_tokens": 120, + "spend": 0.036000000000000004 + }, + "us.anthropic.claude-opus-5-v1:0|cache_read": { + "completion_tokens": 30, + "input_cost": 0.018900000000000004, + "output_cost": 0.0108, + "prompt_tokens": 150, + "spend": 0.029700000000000004 + }, + "us.anthropic.claude-opus-5-v1:0|cache_write_1h": { + "completion_tokens": 30, + "input_cost": 0.0558, + "output_cost": 0.0108, + "prompt_tokens": 150, + "spend": 0.0666 + }, + "us.anthropic.claude-opus-5-v1:0|cache_write_5m": { + "completion_tokens": 30, + "input_cost": 0.048600000000000004, + "output_cost": 0.0108, + "prompt_tokens": 150, + "spend": 0.05940000000000001 + }, + "us.anthropic.claude-opus-5-v1:0|stream": { + "completion_tokens": 40, + "input_cost": 0.0216, + "output_cost": 0.014400000000000001, + "prompt_tokens": 120, + "spend": 0.036000000000000004 + }, + "us.anthropic.claude-opus-5-v1:0|stream_tool_call": { + "completion_tokens": 25, + "input_cost": 0.014400000000000001, + "output_cost": 0.009000000000000001, + "prompt_tokens": 80, + "spend": 0.023400000000000004 + }, + "us.anthropic.claude-opus-5-v1:0|tool_call": { + "completion_tokens": 40, + "input_cost": 0.0216, + "output_cost": 0.014400000000000001, + "prompt_tokens": 120, + "spend": 0.036000000000000004 + } +} diff --git a/tests/e2e/cost_calculation/generate_expected.py b/tests/e2e/cost_calculation/generate_expected.py new file mode 100644 index 00000000000..de979f272fe --- /dev/null +++ b/tests/e2e/cost_calculation/generate_expected.py @@ -0,0 +1,189 @@ +"""Golden generator for the cost suite. Run: + + uv run python tests/e2e/cost_calculation/generate_expected.py + +Loads the derived matrix (models x applicable cases), computes the golden for +each exact-spend cell from the rate arithmetic, and writes ``expected.json`` +with sorted keys. Default behaviour adds missing cells and drops stale cells +but never overwrites an existing cell's values (a reviewed golden is +authoritative); ``--rewrite`` recomputes everything. Prints added/removed/kept +counts. +""" + +from __future__ import annotations + +import json +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from cost_matrix import ( # noqa: E402 # path bootstrap before package-local imports + EXPECTED_PATH, + FRONTIER_MODELS, + TIER_THRESHOLD_TOKENS, + Case, + CostMapEntry, + FrontierModel, + cases_for, + expected_key, +) + +# Wires whose response surface reports a real web-search call count; the +# chat-completions wires only expose url_citation annotations, so their billed +# count floors to one. +_EXACT_WEB_SEARCH_WIRES: Final = frozenset( + {"openai_responses", "anthropic_messages", "gemini_generate", "vertex_generate"} +) + + +def billed_web_search_calls(model: FrontierModel, case: Case) -> int: + if case.usage.web_search_calls == 0: + return 0 + return case.usage.web_search_calls if model.wire in _EXACT_WEB_SEARCH_WIRES else 1 + + +@dataclass(frozen=True, slots=True) +class ExpectedCost: + """The expected bill split the way the spend row's cost_breakdown reports + it: the gross input component (cache reads/writes folded in), the output + component, and the tool-usage component.""" + + input_cost: float + output_cost: float + tool_cost: float + + @property + def total(self) -> float: + return self.input_cost + self.output_cost + self.tool_cost + + +def expected_breakdown(model: FrontierModel, case: Case) -> ExpectedCost: + """Literal arithmetic on the test-map rates over the scripted token counts. + + Input = fresh*in + read*read + 5m*create + 1h*create_1h + audio_in*audio_in; + output = text*out + reasoning*reasoning + audio_out*audio_out; plus the + billed web-search calls at the medium search-context rate. Above-threshold + swaps every input/output rate to its ``_above_200k_tokens`` variant when + total prompt tokens exceed the threshold; a service tier swaps input/output + to the tier's variants, falling back to the base rate when a variant is + unset -- mirroring _get_token_base_cost in litellm's cost calculator. + """ + rates: Final[CostMapEntry] = model.override_rates if case.response_model_override else model.rates + u: Final = case.usage + prompt_tokens: Final = ( + u.fresh_input_tokens + u.cache_read_tokens + u.cache_write_5m_tokens + + u.cache_write_1h_tokens + u.audio_input_tokens + ) + tiered: Final = prompt_tokens > TIER_THRESHOLD_TOKENS + in_rate: Final = ( + (rates.input_cost_per_token_above_200k_tokens if tiered else None) + or (rates.input_cost_per_token_priority if case.service_tier == "priority" else None) + or (rates.input_cost_per_token_flex if case.service_tier == "flex" else None) + or rates.input_cost_per_token + or 0.0 + ) + out_rate: Final = ( + (rates.output_cost_per_token_above_200k_tokens if tiered else None) + or (rates.output_cost_per_token_priority if case.service_tier == "priority" else None) + or (rates.output_cost_per_token_flex if case.service_tier == "flex" else None) + or rates.output_cost_per_token + or 0.0 + ) + # The biller charges cache writes at the input rate when the entry carries + # no cache_creation rate (cost_calculator.py:2452), and at the 5m write + # rate when the 1h variant is unset; cache reads bill only at their own + # rate (zero when the entry lacks one). + write_5m_rate: Final = rates.cache_creation_input_token_cost or in_rate + input_cost: Final = ( + u.fresh_input_tokens * in_rate + + u.cache_read_tokens * (rates.cache_read_input_token_cost or 0.0) + + u.cache_write_5m_tokens * write_5m_rate + + u.cache_write_1h_tokens * (rates.cache_creation_input_token_cost_above_1hr or write_5m_rate) + + u.audio_input_tokens * (rates.input_cost_per_audio_token or 0.0) + ) + output_cost: Final = ( + u.output_tokens * out_rate + + u.reasoning_tokens * (rates.output_cost_per_reasoning_token or out_rate) + + u.audio_output_tokens * (rates.output_cost_per_audio_token or out_rate) + ) + search: Final = rates.search_context_cost_per_query + tool_cost: Final = billed_web_search_calls(model, case) * ( + search.search_context_size_medium if search and search.search_context_size_medium else 0.0 + ) + return ExpectedCost(input_cost=input_cost, output_cost=output_cost, tool_cost=tool_cost) + + +def expected_token_columns(model: FrontierModel, case: Case) -> tuple[int, int]: + """(prompt_tokens, completion_tokens) the spend row should carry, per the + wire's normalization: Anthropic folds cache read/write into prompt_tokens, + everyone else reports the totals the wire emitted.""" + u: Final = case.usage + if model.wire in ("anthropic_messages", "bedrock_converse"): + return ( + u.fresh_input_tokens + u.cache_read_tokens + u.cache_write_5m_tokens + u.cache_write_1h_tokens, + u.output_tokens, + ) + if model.wire in ("gemini_generate", "vertex_generate"): + return ( + u.fresh_input_tokens + u.cache_read_tokens + u.audio_input_tokens, + u.output_tokens + u.reasoning_tokens + u.audio_output_tokens, + ) + if model.wire == "openai_responses": + return ( + u.fresh_input_tokens + u.cache_read_tokens, + u.output_tokens + u.reasoning_tokens, + ) + return ( + u.fresh_input_tokens + + u.cache_read_tokens + + u.cache_write_5m_tokens + + u.cache_write_1h_tokens + + u.audio_input_tokens, + u.output_tokens + u.reasoning_tokens + u.audio_output_tokens, + ) + + +def _proposed() -> dict[str, dict[str, object]]: + return { + expected_key(model, case): ( + lambda breakdown, tokens: { + "spend": breakdown.total, + "input_cost": breakdown.input_cost, + "output_cost": breakdown.output_cost, + "prompt_tokens": tokens[0], + "completion_tokens": tokens[1], + } + )(expected_breakdown(model, case), expected_token_columns(model, case)) + for model in FRONTIER_MODELS + for case in cases_for(model) + if case.exact_spend + } + + +def main() -> None: + rewrite: Final = "--rewrite" in sys.argv[1:] + proposed: Final = _proposed() + existing: Final = ( + json.loads(EXPECTED_PATH.read_text()) if EXPECTED_PATH.exists() else {} + ) + merged: Final = { + key: (proposed[key] if rewrite or key not in existing else existing[key]) + for key in sorted(proposed) + } + added: Final = sum(1 for key in proposed if key not in existing) + removed: Final = sum(1 for key in existing if key not in proposed) + kept: Final = sum(1 for key in proposed if key in existing and not rewrite) + rewritten: Final = sum(1 for key in proposed if key in existing and rewrite) + EXPECTED_PATH.write_text(json.dumps(merged, indent=2, sort_keys=True) + "\n") + print( + f"expected.json: {added} added, {removed} removed, {kept} kept, " + f"{rewritten} rewritten ({len(merged)} cells)" + ) + + +if __name__ == "__main__": + main() diff --git a/tests/e2e/cost_calculation/test_matrix_data.py b/tests/e2e/cost_calculation/test_matrix_data.py new file mode 100644 index 00000000000..fdbb6ddd293 --- /dev/null +++ b/tests/e2e/cost_calculation/test_matrix_data.py @@ -0,0 +1,64 @@ +"""Freshness checks for the cost suite's data files; markerless, so it runs on +any pytest invocation of the folder without the stack. expected.json is the +oracle: these tests check its key set against the derived matrix, never its +values (the generator proposes, the file decides).""" + +from __future__ import annotations + +from typing import Final + +import pytest + +from cost_matrix import ( + _CASES_FILE, + _COST_MAP, + CASES, + EXPECTED, + FRONTIER_MODELS, + CostMapEntry, + cases_for, + expected_key, +) + + +def test_expected_keys_match_derived_exact_cells() -> None: + derived: Final = { + expected_key(model, case) + for model in FRONTIER_MODELS + for case in cases_for(model) + if case.exact_spend + } + golden: Final = set(EXPECTED) + if derived != golden: + missing: Final = sorted(derived - golden) + stale: Final = sorted(golden - derived) + pytest.fail( + "expected.json is out of sync with the derived matrix; run " + "uv run python tests/e2e/cost_calculation/generate_expected.py " + f"(missing: {missing}; stale: {stale})" + ) + + +def test_deployments_reference_existing_map_keys() -> None: + unknown: Final = sorted( + spec.map_key for spec in _CASES_FILE.deployments if spec.map_key not in _COST_MAP + ) + assert not unknown, f"deployments entries name map keys absent from cost_map.json: {unknown}" + + +def test_requires_rates_are_cost_map_fields() -> None: + fields: Final = set(CostMapEntry.model_fields) + unknown: Final = sorted( + {field for case in CASES for field in case.requires_rates} - fields + ) + assert not unknown, f"requires_rates names that are not CostMapEntry fields: {unknown}" + + +def test_no_two_entries_share_input_rate() -> None: + rates: Final = [ + entry.input_cost_per_token for entry in _COST_MAP.values() + ] + assert len(rates) == len(set(rates)), ( + "two cost_map entries share input_cost_per_token; the suite relies on " + "distinct rates so a wrong-model bill can never coincidentally match" + ) diff --git a/tests/e2e/cost_calculation/test_token_pricing_e2e.py b/tests/e2e/cost_calculation/test_token_pricing_e2e.py index 0b4f3e1fd37..7cd128ad6fb 100644 --- a/tests/e2e/cost_calculation/test_token_pricing_e2e.py +++ b/tests/e2e/cost_calculation/test_token_pricing_e2e.py @@ -1,7 +1,7 @@ -"""Token-pricing e2e: every (frontier model, pricing-component case) cell runs a -scripted-usage call through a deployment registered on the cost-map proxy, and -the spend row plus response-cost header must equal literal arithmetic on the -test map's rates. +"""Token-pricing e2e: every (map entry, case) cell derived from cost_map.json x +cases.json runs a scripted-usage call through a deployment registered on the +cost-map proxy, and the spend row plus response-cost header must equal the +reviewed golden in expected.json verbatim -- no rate arithmetic lives here. Nothing here touches a real provider or the bundled cost map: the proxy's upstream is the scripted-provider sidecar and its entire cost map is @@ -15,13 +15,13 @@ from typing import Final from conftest import CostCalcClient, cost_rows, register_scenario_deployment from cost_matrix import ( + EXPECTED, FRONTIER_MODELS, IMAGE_INPUT_DATA_URL, Case, FrontierModel, cases_for, - expected_cost, - expected_token_columns, + expected_key, recount_cost, ) from e2e_config import unique_marker @@ -110,17 +110,6 @@ class TestTokenPricing: ) assert response.stream_error is None, f"stream carried an error event: {response.stream_error}" - expected: Final = expected_cost(model, case) - if case.exact_spend and not case.stream: - # Streamed responses commit headers before the bill is computed, so - # the x-litellm-response-cost header is asserted only on non-stream - # calls. - assert response.response_cost is not None and cost_rows.approx_equal( - response.response_cost, expected - ), ( - f"x-litellm-response-cost {response.response_cost} != expected {expected}" - ) - row: Final = cost_rows.poll_cost_row_where( client.proxy, scoped_key, @@ -149,16 +138,39 @@ class TestTokenPricing: cost_rows.assert_total_is_sum_of_components(row) return - assert row.spend is not None and cost_rows.approx_equal(row.spend, expected), ( - f"{model.map_key}/{case.name}: spend {row.spend} != expected {expected} " + golden: Final = EXPECTED[expected_key(model, case)] + + if not case.stream: + # Streamed responses commit headers before the bill is computed, so + # the x-litellm-response-cost header is asserted only on non-stream + # calls. + assert response.response_cost is not None and cost_rows.approx_equal( + response.response_cost, golden.spend + ), ( + f"x-litellm-response-cost {response.response_cost} != golden {golden.spend}" + ) + + assert row.spend is not None and cost_rows.approx_equal(row.spend, golden.spend), ( + f"{model.map_key}/{case.name}: spend {row.spend} != golden {golden.spend} " f"(breakdown {row.breakdown.model_dump()})" ) - - prompt_tokens, completion_tokens = expected_token_columns(model, case) - assert row.prompt_tokens == prompt_tokens, ( - f"prompt_tokens {row.prompt_tokens} != {prompt_tokens}" + breakdown: Final = row.breakdown + assert breakdown.input_cost is not None and cost_rows.approx_equal( + breakdown.input_cost, golden.input_cost + ), ( + f"{model.map_key}/{case.name}: gross input_cost {breakdown.input_cost} " + f"!= golden {golden.input_cost}; cached/written tokens billed at the input rate" ) - assert row.completion_tokens == completion_tokens, ( - f"completion_tokens {row.completion_tokens} != {completion_tokens}" + assert breakdown.output_cost is not None and cost_rows.approx_equal( + breakdown.output_cost, golden.output_cost + ), ( + f"{model.map_key}/{case.name}: output_cost {breakdown.output_cost} " + f"!= golden {golden.output_cost}" + ) + assert row.prompt_tokens == golden.prompt_tokens, ( + f"prompt_tokens {row.prompt_tokens} != {golden.prompt_tokens}" + ) + assert row.completion_tokens == golden.completion_tokens, ( + f"completion_tokens {row.completion_tokens} != {golden.completion_tokens}" ) cost_rows.assert_total_is_sum_of_components(row) diff --git a/tests/e2e/cost_calculation/test_wire_formats_e2e.py b/tests/e2e/cost_calculation/test_wire_formats_e2e.py deleted file mode 100644 index a36bb1a8662..00000000000 --- a/tests/e2e/cost_calculation/test_wire_formats_e2e.py +++ /dev/null @@ -1,368 +0,0 @@ -"""Wire-format e2e: one scripted upstream per provider wire, answering with a -usage payload where every token kind the wire can report is nonzero. The spend -row's gross input cost must equal fresh tokens at the input rate plus each cache -and audio component at its own rate -- proving the wire's usage shape landed the -cached tokens inside the total (OpenAI/Gemini) or as separate fields -(Anthropic), and that the biller subtracted them before billing fresh tokens. - -Also covers the Responses API wire (an openai/gpt-5.5-pro deployment bridged by -the proxy to POST /responses) and a streamed Anthropic-messages case. -""" - -from __future__ import annotations - -import pytest -from collections.abc import Mapping -from types import MappingProxyType -from typing import Final - -from conftest import CostCalcClient, cost_rows, register_scenario_deployment -from cost_matrix import ( - FRONTIER_MODELS, - Case, - FrontierModel, - expected_breakdown, - expected_token_columns, -) -from e2e_config import unique_marker -from lifecycle import ResourceManager -from models import ChatBody, ChatMessage, ChatStreamOptions, ChatTool, ChatToolFunction -from scripted_provider import ScriptedUsage - -pytestmark: Final = [pytest.mark.e2e, pytest.mark.cost_map_stack] # mutable-ok: pytest only accepts a list for pytestmark - -_MODELS: Final[Mapping[str, FrontierModel]] = MappingProxyType( - {model.map_key: model for model in FRONTIER_MODELS} -) - -# One scripted usage per wire, every reportable token kind nonzero. -_WIRE_USAGE: Final[Mapping[str, tuple[str, ScriptedUsage]]] = MappingProxyType({ - "openai_chat": ( - "gpt-5.6", - ScriptedUsage( - fresh_input_tokens=80, - cache_read_tokens=40, - cache_write_5m_tokens=20, - cache_write_1h_tokens=10, - output_tokens=25, - reasoning_tokens=15, - audio_input_tokens=5, - audio_output_tokens=3, - ), - ), - "openai_responses": ( - "gpt-5.5-pro", - ScriptedUsage( - fresh_input_tokens=80, cache_read_tokens=40, output_tokens=25, reasoning_tokens=15 - ), - ), - "anthropic_messages": ( - "claude-sonnet-5", - ScriptedUsage( - fresh_input_tokens=80, - cache_read_tokens=40, - cache_write_5m_tokens=20, - cache_write_1h_tokens=10, - output_tokens=25, - ), - ), - "gemini_generate": ( - "gemini/gemini-3.8-flash", - ScriptedUsage( - fresh_input_tokens=80, - cache_read_tokens=40, - output_tokens=25, - reasoning_tokens=15, - audio_input_tokens=5, - audio_output_tokens=3, - ), - ), - "together_chat": ( - "together_ai/moonshotai/Kimi-K3", - ScriptedUsage( - fresh_input_tokens=80, - cache_read_tokens=40, - cache_write_5m_tokens=20, - cache_write_1h_tokens=10, - output_tokens=25, - reasoning_tokens=15, - audio_input_tokens=5, - audio_output_tokens=3, - ), - ), - "fireworks_chat": ( - "fireworks_ai/kimi-k3", - ScriptedUsage(fresh_input_tokens=80, cache_read_tokens=40, output_tokens=25), - ), - "azure_chat": ( - "azure/gpt-5.6", - ScriptedUsage( - fresh_input_tokens=80, - cache_read_tokens=40, - cache_write_5m_tokens=20, - cache_write_1h_tokens=10, - output_tokens=25, - reasoning_tokens=15, - audio_input_tokens=5, - audio_output_tokens=3, - ), - ), - "bedrock_converse": ( - "anthropic.claude-sonnet-5-v1:0", - ScriptedUsage( - fresh_input_tokens=80, - cache_read_tokens=40, - cache_write_5m_tokens=20, - cache_write_1h_tokens=10, - output_tokens=25, - ), - ), - "vertex_generate": ( - "gemini-3.8-flash", - ScriptedUsage( - fresh_input_tokens=80, - cache_read_tokens=40, - output_tokens=25, - reasoning_tokens=15, - audio_input_tokens=5, - audio_output_tokens=3, - ), - ), -}) - -_SHAPE_USAGE: Final = ScriptedUsage(fresh_input_tokens=80, output_tokens=25) - -# Renderer-level shapes the pricing matrix gates per cap, pinned here once per -# wire so the sidecar emits prove they survive the proxy end to end. -_SHAPES: Final[tuple[tuple[str, str, Case], ...]] = ( - *( - ( - f"tool_call_{'stream' if stream else 'sync'}", - wire, - Case(name="tool_call", usage=_SHAPE_USAGE, stream=stream, tool_call=True), - ) - for wire in _WIRE_USAGE - for stream in (False, True) - ), - ( - "responses_incomplete", - "openai_responses", - Case(name="stream_no_usage_incomplete", usage=_SHAPE_USAGE, stream=True, terminal="incomplete"), - ), - ( - "responses_unvalidated", - "openai_responses", - Case(name="stream_unvalidated", usage=_SHAPE_USAGE, stream=True, terminal="unvalidated"), - ), - ( - "gemini_prompt_blocked", - "gemini_generate", - Case( - name="prompt_blocked", - usage=ScriptedUsage(fresh_input_tokens=1000, output_tokens=0), - terminal="prompt_blocked", - response_model_override=True, - ), - ), - ( - "gemini_prompt_blocked_stream", - "gemini_generate", - Case( - name="stream_prompt_blocked", - usage=ScriptedUsage(fresh_input_tokens=1000, output_tokens=0), - stream=True, - terminal="prompt_blocked", - response_model_override=True, - ), - ), - ( - "vertex_prompt_blocked", - "vertex_generate", - Case( - name="prompt_blocked", - usage=ScriptedUsage(fresh_input_tokens=1000, output_tokens=0), - terminal="prompt_blocked", - response_model_override=True, - ), - ), - ( - "vertex_prompt_blocked_stream", - "vertex_generate", - Case( - name="stream_prompt_blocked", - usage=ScriptedUsage(fresh_input_tokens=1000, output_tokens=0), - stream=True, - terminal="prompt_blocked", - response_model_override=True, - ), - ), - ( - "azure_served_model_override", - "azure_chat", - Case( - name="response_model_override", - usage=_SHAPE_USAGE, - response_model_override=True, - ), - ), -) - - -def _shape_id(entry: tuple[str, str, Case]) -> str: - return entry[0] - - -class TestWireFormats: - @pytest.mark.parametrize("wire", tuple(_WIRE_USAGE)) - @pytest.mark.covers("quota_management.spend_tracking.scripted_wire.logs_cost") - def test_wire_usage_shape_bills_each_component( - self, - client: CostCalcClient, - resources: ResourceManager, - scoped_key: str, - wire: str, - ) -> None: - map_key, usage = _WIRE_USAGE[wire] - model: Final = _MODELS[map_key] - case: Final = Case(name="basic", usage=usage) - marker: Final = unique_marker() - model_name, _handle = register_scenario_deployment(client, resources, model, case, marker) - response: Final = client.proxy.transport.send( - "/chat/completions", - headers=client.proxy.transport.bearer(scoped_key), - json=ChatBody( - model=model_name, - messages=(ChatMessage(role="user", content=f"{marker} scripted wire call"),), - ), - ) - assert response.ok, f"{wire}: proxy returned {response.status_code}: {response.body[:400]}" - - expected: Final = expected_breakdown(model, case) - row: Final = cost_rows.poll_cost_row_where( - client.proxy, - scoped_key, - lambda r: r.metadata is not None and r.metadata.cost_breakdown is not None, - ) - assert row is not None, f"{wire}: no spend row landed" - assert row.spend is not None and cost_rows.approx_equal(row.spend, expected.total), ( - f"{wire}: spend {row.spend} != expected {expected.total} " - f"(breakdown {row.breakdown.model_dump()})" - ) - breakdown: Final = row.breakdown - assert breakdown.input_cost is not None and cost_rows.approx_equal( - breakdown.input_cost, expected.input_cost - ), ( - f"{wire}: gross input_cost {breakdown.input_cost} != expected {expected.input_cost}; " - "cached/written tokens billed at the input rate" - ) - assert breakdown.output_cost is not None and cost_rows.approx_equal( - breakdown.output_cost, expected.output_cost - ), f"{wire}: output_cost {breakdown.output_cost} != expected {expected.output_cost}" - - prompt_tokens, completion_tokens = expected_token_columns(model, case) - assert row.prompt_tokens == prompt_tokens, ( - f"{wire}: prompt_tokens {row.prompt_tokens} != {prompt_tokens}" - ) - assert row.completion_tokens == completion_tokens, ( - f"{wire}: completion_tokens {row.completion_tokens} != {completion_tokens}" - ) - cost_rows.assert_total_is_sum_of_components(row) - - @pytest.mark.covers("quota_management.spend_tracking.scripted_wire.logs_cost") - def test_anthropic_streamed_usage_bills_each_component( - self, client: CostCalcClient, resources: ResourceManager, scoped_key: str - ) -> None: - map_key, usage = _WIRE_USAGE["anthropic_messages"] - model: Final = _MODELS[map_key] - case: Final = Case(name="stream", usage=usage, stream=True) - marker: Final = unique_marker() - model_name, _handle = register_scenario_deployment(client, resources, model, case, marker) - response: Final = client.proxy.transport.send( - "/chat/completions", - headers=client.proxy.transport.bearer(scoped_key), - json=ChatBody( - model=model_name, - messages=(ChatMessage(role="user", content=f"{marker} scripted anthropic stream"),), - stream=True, - stream_options=ChatStreamOptions(include_usage=True), - ), - stream=True, - ) - assert response.ok, f"anthropic stream: proxy returned {response.status_code}: {response.body[:400]}" - assert response.stream_done, "anthropic stream did not reach its terminal event" - assert response.stream_error is None, f"stream carried an error event: {response.stream_error}" - - expected: Final = expected_breakdown(model, case) - row: Final = cost_rows.poll_cost_row_where( - client.proxy, - scoped_key, - lambda r: r.metadata is not None and r.metadata.cost_breakdown is not None, - ) - assert row is not None, "anthropic stream: no spend row landed" - assert row.spend is not None and cost_rows.approx_equal(row.spend, expected.total), ( - f"anthropic stream: spend {row.spend} != expected {expected.total} " - f"(breakdown {row.breakdown.model_dump()})" - ) - cost_rows.assert_total_is_sum_of_components(row) - - @pytest.mark.parametrize("shape_wire_case", _SHAPES, ids=_shape_id) - @pytest.mark.covers("quota_management.spend_tracking.scripted_wire.logs_cost") - def test_response_shape_bills_reported_usage( - self, - client: CostCalcClient, - resources: ResourceManager, - scoped_key: str, - shape_wire_case: tuple[str, str, Case], - ) -> None: - shape, wire, case = shape_wire_case - map_key, _usage = _WIRE_USAGE[wire] - model: Final = _MODELS[map_key] - marker: Final = unique_marker() - model_name, _handle = register_scenario_deployment(client, resources, model, case, marker) - response: Final = client.proxy.transport.send( - "/chat/completions", - headers=client.proxy.transport.bearer(scoped_key), - json=ChatBody( - model=model_name, - messages=(ChatMessage(role="user", content=f"{marker} scripted {shape}"),), - stream=case.stream, - stream_options=ChatStreamOptions(include_usage=True) if case.stream else None, - tools=( - ( - ChatTool( - function=ChatToolFunction( - name="get_weather", - parameters={"type": "object", "properties": {"city": {"type": "string"}}}, - ) - ), - ) - if case.tool_call - else None - ), - ), - stream=case.stream, - ) - assert response.ok, f"{shape}: proxy returned {response.status_code}: {response.body[:400]}" - if case.stream: - assert response.stream_done, f"{shape}: stream did not reach its terminal event" - assert response.stream_error is None, f"{shape}: stream error: {response.stream_error}" - - expected: Final = expected_breakdown(model, case) - row: Final = cost_rows.poll_cost_row_where( - client.proxy, - scoped_key, - lambda r: r.metadata is not None and r.metadata.cost_breakdown is not None, - ) - assert row is not None, f"{shape}: no spend row landed" - assert row.spend is not None and cost_rows.approx_equal(row.spend, expected.total), ( - f"{shape}: spend {row.spend} != expected {expected.total} " - f"(breakdown {row.breakdown.model_dump()})" - ) - prompt_tokens, completion_tokens = expected_token_columns(model, case) - assert row.prompt_tokens == prompt_tokens, ( - f"{shape}: prompt_tokens {row.prompt_tokens} != {prompt_tokens}" - ) - assert row.completion_tokens == completion_tokens, ( - f"{shape}: completion_tokens {row.completion_tokens} != {completion_tokens}" - ) - cost_rows.assert_total_is_sum_of_components(row) From 522a7f569283b9a3bc0fed2a66f1c7545f30b12b Mon Sep 17 00:00:00 2001 From: kerry Date: Thu, 17 Sep 2026 00:51:43 +0000 Subject: [PATCH 013/210] test(e2e): gate all_components cases by rates and tidy cost matrix names Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/cost_calculation/cases.json | 26 ++++++++ tests/e2e/cost_calculation/cost_matrix.py | 31 +++++---- tests/e2e/cost_calculation/expected.json | 7 --- .../e2e/cost_calculation/generate_expected.py | 63 ++++++++++--------- .../e2e/cost_calculation/test_matrix_data.py | 11 ++-- 5 files changed, 79 insertions(+), 59 deletions(-) diff --git a/tests/e2e/cost_calculation/cases.json b/tests/e2e/cost_calculation/cases.json index e898557ea35..3dc4fc4d99c 100644 --- a/tests/e2e/cost_calculation/cases.json +++ b/tests/e2e/cost_calculation/cases.json @@ -180,11 +180,20 @@ "audio_input_tokens": 5, "audio_output_tokens": 3 }, + "requires_rates": [ + "cache_read_input_token_cost", + "cache_creation_input_token_cost", + "cache_creation_input_token_cost_above_1hr", + "output_cost_per_reasoning_token", + "input_cost_per_audio_token", + "output_cost_per_audio_token" + ], "wires": ["openai_chat", "azure_chat", "together_chat"] }, { "name": "all_components_fireworks", "usage": {"fresh_input_tokens": 80, "cache_read_tokens": 40, "output_tokens": 25}, + "requires_rates": ["cache_read_input_token_cost"], "wires": ["fireworks_chat"] }, { @@ -196,6 +205,11 @@ "cache_write_1h_tokens": 10, "output_tokens": 25 }, + "requires_rates": [ + "cache_read_input_token_cost", + "cache_creation_input_token_cost", + "cache_creation_input_token_cost_above_1hr" + ], "wires": ["anthropic_messages", "bedrock_converse"] }, { @@ -208,6 +222,11 @@ "output_tokens": 25 }, "stream": true, + "requires_rates": [ + "cache_read_input_token_cost", + "cache_creation_input_token_cost", + "cache_creation_input_token_cost_above_1hr" + ], "wires": ["anthropic_messages"] }, { @@ -220,11 +239,18 @@ "audio_input_tokens": 5, "audio_output_tokens": 3 }, + "requires_rates": [ + "cache_read_input_token_cost", + "output_cost_per_reasoning_token", + "input_cost_per_audio_token", + "output_cost_per_audio_token" + ], "wires": ["gemini_generate", "vertex_generate"] }, { "name": "all_components_responses", "usage": {"fresh_input_tokens": 80, "cache_read_tokens": 40, "output_tokens": 25, "reasoning_tokens": 15}, + "requires_rates": ["cache_read_input_token_cost", "output_cost_per_reasoning_token"], "wires": ["openai_responses"] } ] diff --git a/tests/e2e/cost_calculation/cost_matrix.py b/tests/e2e/cost_calculation/cost_matrix.py index b03c851d208..a8f60b79ae7 100644 --- a/tests/e2e/cost_calculation/cost_matrix.py +++ b/tests/e2e/cost_calculation/cost_matrix.py @@ -27,7 +27,6 @@ from types import MappingProxyType from typing import Final, Literal from pydantic import BaseModel, ConfigDict, TypeAdapter - from scripted_provider import Scenario, ScriptedOutput, ScriptedToolCall, ScriptedUsage, Wire COST_MAP_PATH: Final = Path(__file__).resolve().parent.parent / "cost_map.json" @@ -69,9 +68,9 @@ class CostMapEntry(BaseModel): web_search_billing_unit: str | None = None -_COST_MAP_ADAPTER: Final = TypeAdapter(dict[str, CostMapEntry]) -_COST_MAP: Final[Mapping[str, CostMapEntry]] = MappingProxyType( - _COST_MAP_ADAPTER.validate_python(json.loads(COST_MAP_PATH.read_text())) +COST_MAP_ADAPTER: Final = TypeAdapter(dict[str, CostMapEntry]) +COST_MAP: Final[Mapping[str, CostMapEntry]] = MappingProxyType( + COST_MAP_ADAPTER.validate_python(json.loads(COST_MAP_PATH.read_text())) ) TIER_THRESHOLD_TOKENS: Final = 200_000 @@ -146,10 +145,10 @@ class _CasesFile(BaseModel): cases: tuple[Case, ...] = () -_CASES_FILE: Final = _CasesFile.model_validate(json.loads(CASES_PATH.read_text())) -CASES: Final[tuple[Case, ...]] = _CASES_FILE.cases +CASES_FILE: Final = _CasesFile.model_validate(json.loads(CASES_PATH.read_text())) +CASES: Final[tuple[Case, ...]] = CASES_FILE.cases _DEPLOYMENTS: Final[Mapping[str, DeploymentSpec]] = MappingProxyType( - {spec.map_key: spec for spec in _CASES_FILE.deployments} + {spec.map_key: spec for spec in CASES_FILE.deployments} ) @@ -221,13 +220,13 @@ class FrontierModel: @property def rates(self) -> CostMapEntry: - return _COST_MAP[self.map_key] + return COST_MAP[self.map_key] @property def override_rates(self) -> CostMapEntry: if self.base_model is not None or self.override_map_key is None: return self.rates - return _COST_MAP[self.override_map_key] + return COST_MAP[self.override_map_key] @property def provider_model(self) -> str: @@ -262,13 +261,13 @@ def _litellm_model_for(map_key: str, wiring: _ProviderWiring) -> str: def _frontier() -> tuple[FrontierModel, ...]: groups: Final[Mapping[tuple[str, str], tuple[str, ...]]] = MappingProxyType( { - pair: tuple(sorted(k for k, e in _COST_MAP.items() if (e.litellm_provider, e.mode) == pair)) - for pair in {(e.litellm_provider, e.mode) for e in _COST_MAP.values()} + pair: tuple(sorted(k for k, e in COST_MAP.items() if (e.litellm_provider, e.mode) == pair)) + for pair in {(e.litellm_provider, e.mode) for e in COST_MAP.values()} } ) models: list[FrontierModel] = [] # mutable-ok: accumulated once at import into a tuple - for map_key in sorted(_COST_MAP): - entry: Final = _COST_MAP[map_key] + for map_key in sorted(COST_MAP): + entry: Final = COST_MAP[map_key] pair: Final = (entry.litellm_provider, entry.mode) wiring: Final = _PROVIDER_WIRING.get(pair) if wiring is None: @@ -416,7 +415,7 @@ def image_input_data_url() -> str: IMAGE_INPUT_DATA_URL: Final = image_input_data_url() -class _ExpectedCell(BaseModel): +class ExpectedCell(BaseModel): model_config = ConfigDict(frozen=True) spend: float @@ -426,8 +425,8 @@ class _ExpectedCell(BaseModel): completion_tokens: int -_EXPECTED_ADAPTER: Final = TypeAdapter(dict[str, _ExpectedCell]) -EXPECTED: Final[Mapping[str, _ExpectedCell]] = MappingProxyType( +_EXPECTED_ADAPTER: Final = TypeAdapter(dict[str, ExpectedCell]) +EXPECTED: Final[Mapping[str, ExpectedCell]] = MappingProxyType( _EXPECTED_ADAPTER.validate_python(json.loads(EXPECTED_PATH.read_text())) if EXPECTED_PATH.exists() else {} diff --git a/tests/e2e/cost_calculation/expected.json b/tests/e2e/cost_calculation/expected.json index 7a92fb2476f..3b18e9ed9f4 100644 --- a/tests/e2e/cost_calculation/expected.json +++ b/tests/e2e/cost_calculation/expected.json @@ -1686,13 +1686,6 @@ "prompt_tokens": 100, "spend": 0.0216 }, - "meta.llama4-maverick-17b-instruct-v1:0|all_components_anthropic": { - "completion_tokens": 25, - "input_cost": 0.020900000000000002, - "output_cost": 0.0095, - "prompt_tokens": 150, - "spend": 0.030400000000000003 - }, "meta.llama4-maverick-17b-instruct-v1:0|basic": { "completion_tokens": 40, "input_cost": 0.0228, diff --git a/tests/e2e/cost_calculation/generate_expected.py b/tests/e2e/cost_calculation/generate_expected.py index de979f272fe..c093ecbe0ea 100644 --- a/tests/e2e/cost_calculation/generate_expected.py +++ b/tests/e2e/cost_calculation/generate_expected.py @@ -14,8 +14,10 @@ from __future__ import annotations import json import sys +from collections.abc import Mapping from dataclasses import dataclass from pathlib import Path +from types import MappingProxyType from typing import Final sys.path.insert(0, str(Path(__file__).resolve().parent)) @@ -27,6 +29,7 @@ from cost_matrix import ( # noqa: E402 # path bootstrap before package-local i TIER_THRESHOLD_TOKENS, Case, CostMapEntry, + ExpectedCell, FrontierModel, cases_for, expected_key, @@ -93,16 +96,11 @@ def expected_breakdown(model: FrontierModel, case: Case) -> ExpectedCost: or rates.output_cost_per_token or 0.0 ) - # The biller charges cache writes at the input rate when the entry carries - # no cache_creation rate (cost_calculator.py:2452), and at the 5m write - # rate when the 1h variant is unset; cache reads bill only at their own - # rate (zero when the entry lacks one). - write_5m_rate: Final = rates.cache_creation_input_token_cost or in_rate input_cost: Final = ( u.fresh_input_tokens * in_rate + u.cache_read_tokens * (rates.cache_read_input_token_cost or 0.0) - + u.cache_write_5m_tokens * write_5m_rate - + u.cache_write_1h_tokens * (rates.cache_creation_input_token_cost_above_1hr or write_5m_rate) + + u.cache_write_5m_tokens * (rates.cache_creation_input_token_cost or 0.0) + + u.cache_write_1h_tokens * (rates.cache_creation_input_token_cost_above_1hr or 0.0) + u.audio_input_tokens * (rates.input_cost_per_audio_token or 0.0) ) output_cost: Final = ( @@ -147,39 +145,46 @@ def expected_token_columns(model: FrontierModel, case: Case) -> tuple[int, int]: ) -def _proposed() -> dict[str, dict[str, object]]: - return { - expected_key(model, case): ( - lambda breakdown, tokens: { - "spend": breakdown.total, - "input_cost": breakdown.input_cost, - "output_cost": breakdown.output_cost, - "prompt_tokens": tokens[0], - "completion_tokens": tokens[1], - } - )(expected_breakdown(model, case), expected_token_columns(model, case)) - for model in FRONTIER_MODELS - for case in cases_for(model) - if case.exact_spend - } +def _cell(model: FrontierModel, case: Case) -> ExpectedCell: + breakdown: Final = expected_breakdown(model, case) + prompt_tokens, completion_tokens = expected_token_columns(model, case) + return ExpectedCell( + spend=breakdown.total, + input_cost=breakdown.input_cost, + output_cost=breakdown.output_cost, + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + ) + + +def _proposed() -> Mapping[str, ExpectedCell]: + return MappingProxyType( + { + expected_key(model, case): _cell(model, case) + for model in FRONTIER_MODELS + for case in cases_for(model) + if case.exact_spend + } + ) def main() -> None: rewrite: Final = "--rewrite" in sys.argv[1:] proposed: Final = _proposed() + proposed_values: Final = {key: cell.model_dump() for key, cell in proposed.items()} existing: Final = ( json.loads(EXPECTED_PATH.read_text()) if EXPECTED_PATH.exists() else {} ) merged: Final = { - key: (proposed[key] if rewrite or key not in existing else existing[key]) - for key in sorted(proposed) + key: (proposed_values[key] if rewrite or key not in existing else existing[key]) + for key in sorted(proposed_values) } - added: Final = sum(1 for key in proposed if key not in existing) - removed: Final = sum(1 for key in existing if key not in proposed) - kept: Final = sum(1 for key in proposed if key in existing and not rewrite) - rewritten: Final = sum(1 for key in proposed if key in existing and rewrite) + added: Final = sum(1 for key in proposed_values if key not in existing) + removed: Final = sum(1 for key in existing if key not in proposed_values) + kept: Final = sum(1 for key in proposed_values if key in existing and not rewrite) + rewritten: Final = sum(1 for key in proposed_values if key in existing and rewrite) EXPECTED_PATH.write_text(json.dumps(merged, indent=2, sort_keys=True) + "\n") - print( + print( # noqa: T201 # CLI summary is the tool output f"expected.json: {added} added, {removed} removed, {kept} kept, " f"{rewritten} rewritten ({len(merged)} cells)" ) diff --git a/tests/e2e/cost_calculation/test_matrix_data.py b/tests/e2e/cost_calculation/test_matrix_data.py index fdbb6ddd293..8340257939c 100644 --- a/tests/e2e/cost_calculation/test_matrix_data.py +++ b/tests/e2e/cost_calculation/test_matrix_data.py @@ -8,11 +8,10 @@ from __future__ import annotations from typing import Final import pytest - from cost_matrix import ( - _CASES_FILE, - _COST_MAP, CASES, + CASES_FILE, + COST_MAP, EXPECTED, FRONTIER_MODELS, CostMapEntry, @@ -41,7 +40,7 @@ def test_expected_keys_match_derived_exact_cells() -> None: def test_deployments_reference_existing_map_keys() -> None: unknown: Final = sorted( - spec.map_key for spec in _CASES_FILE.deployments if spec.map_key not in _COST_MAP + spec.map_key for spec in CASES_FILE.deployments if spec.map_key not in COST_MAP ) assert not unknown, f"deployments entries name map keys absent from cost_map.json: {unknown}" @@ -55,9 +54,7 @@ def test_requires_rates_are_cost_map_fields() -> None: def test_no_two_entries_share_input_rate() -> None: - rates: Final = [ - entry.input_cost_per_token for entry in _COST_MAP.values() - ] + rates: Final = tuple(entry.input_cost_per_token for entry in COST_MAP.values()) assert len(rates) == len(set(rates)), ( "two cost_map entries share input_cost_per_token; the suite relies on " "distinct rates so a wrong-model bill can never coincidentally match" From e1c9ae5ae45e3b66041a25a6ae6ca9c7633944b7 Mon Sep 17 00:00:00 2001 From: kerry Date: Thu, 17 Sep 2026 00:56:39 +0000 Subject: [PATCH 014/210] test(e2e): drop needless sys.path bootstrap from golden generator Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/cost_calculation/generate_expected.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/e2e/cost_calculation/generate_expected.py b/tests/e2e/cost_calculation/generate_expected.py index c093ecbe0ea..e243e477839 100644 --- a/tests/e2e/cost_calculation/generate_expected.py +++ b/tests/e2e/cost_calculation/generate_expected.py @@ -16,14 +16,10 @@ import json import sys from collections.abc import Mapping from dataclasses import dataclass -from pathlib import Path from types import MappingProxyType from typing import Final -sys.path.insert(0, str(Path(__file__).resolve().parent)) -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) - -from cost_matrix import ( # noqa: E402 # path bootstrap before package-local imports +from cost_matrix import ( EXPECTED_PATH, FRONTIER_MODELS, TIER_THRESHOLD_TOKENS, From 3e11c986766ed7a32ead704e5284fbfeaf889c6b Mon Sep 17 00:00:00 2001 From: kerry Date: Thu, 17 Sep 2026 01:02:41 +0000 Subject: [PATCH 015/210] test(e2e): satisfy pyright in cost matrix derivation and golden generator Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/cost_calculation/cost_matrix.py | 14 +++++++------- tests/e2e/cost_calculation/generate_expected.py | 16 +++++++++++++--- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/tests/e2e/cost_calculation/cost_matrix.py b/tests/e2e/cost_calculation/cost_matrix.py index a8f60b79ae7..35344a099be 100644 --- a/tests/e2e/cost_calculation/cost_matrix.py +++ b/tests/e2e/cost_calculation/cost_matrix.py @@ -267,23 +267,23 @@ def _frontier() -> tuple[FrontierModel, ...]: ) models: list[FrontierModel] = [] # mutable-ok: accumulated once at import into a tuple for map_key in sorted(COST_MAP): - entry: Final = COST_MAP[map_key] - pair: Final = (entry.litellm_provider, entry.mode) - wiring: Final = _PROVIDER_WIRING.get(pair) + entry = COST_MAP[map_key] + pair = (entry.litellm_provider, entry.mode) + wiring = _PROVIDER_WIRING.get(pair) if wiring is None: raise ValueError( f"cost_map entry {map_key} has no wiring for " f"(litellm_provider={pair[0]}, mode={pair[1]}); add a " f"_ProviderWiring row in cost_matrix.py" ) - siblings: Final = groups[pair] - override_key: Final = ( + siblings = groups[pair] + override_key = ( siblings[(siblings.index(map_key) + 1) % len(siblings)] if len(siblings) > 1 else None ) - override_litellm: Final = ( + override_litellm = ( _litellm_model_for(override_key, wiring) if override_key is not None else None ) - deployment: Final = _DEPLOYMENTS.get(map_key) + deployment = _DEPLOYMENTS.get(map_key) models.append( FrontierModel( model_name=f"cc-{map_key.replace('/', '-').replace(':', '-').replace('.', '-').lower()}", diff --git a/tests/e2e/cost_calculation/generate_expected.py b/tests/e2e/cost_calculation/generate_expected.py index e243e477839..f5514092c88 100644 --- a/tests/e2e/cost_calculation/generate_expected.py +++ b/tests/e2e/cost_calculation/generate_expected.py @@ -19,6 +19,8 @@ from dataclasses import dataclass from types import MappingProxyType from typing import Final +from pydantic import TypeAdapter + from cost_matrix import ( EXPECTED_PATH, FRONTIER_MODELS, @@ -168,11 +170,19 @@ def main() -> None: rewrite: Final = "--rewrite" in sys.argv[1:] proposed: Final = _proposed() proposed_values: Final = {key: cell.model_dump() for key, cell in proposed.items()} - existing: Final = ( - json.loads(EXPECTED_PATH.read_text()) if EXPECTED_PATH.exists() else {} + existing: Final[Mapping[str, ExpectedCell]] = ( + TypeAdapter(dict[str, ExpectedCell]).validate_python( + json.loads(EXPECTED_PATH.read_text()) + ) + if EXPECTED_PATH.exists() + else {} ) merged: Final = { - key: (proposed_values[key] if rewrite or key not in existing else existing[key]) + key: ( + proposed_values[key] + if rewrite or key not in existing + else existing[key].model_dump() + ) for key in sorted(proposed_values) } added: Final = sum(1 for key in proposed_values if key not in existing) From fc0cce553a631e912e7892893bde188e9716b415 Mon Sep 17 00:00:00 2001 From: kerry Date: Thu, 17 Sep 2026 01:13:36 +0000 Subject: [PATCH 016/210] test(e2e): derive cache rates from first principles and ungate all_components cases Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/cost_calculation/cases.json | 17 +---------------- tests/e2e/cost_calculation/expected.json | 7 +++++++ tests/e2e/cost_calculation/generate_expected.py | 17 ++++++++++++++--- 3 files changed, 22 insertions(+), 19 deletions(-) diff --git a/tests/e2e/cost_calculation/cases.json b/tests/e2e/cost_calculation/cases.json index 3dc4fc4d99c..e01ac97e9ff 100644 --- a/tests/e2e/cost_calculation/cases.json +++ b/tests/e2e/cost_calculation/cases.json @@ -181,9 +181,6 @@ "audio_output_tokens": 3 }, "requires_rates": [ - "cache_read_input_token_cost", - "cache_creation_input_token_cost", - "cache_creation_input_token_cost_above_1hr", "output_cost_per_reasoning_token", "input_cost_per_audio_token", "output_cost_per_audio_token" @@ -193,7 +190,6 @@ { "name": "all_components_fireworks", "usage": {"fresh_input_tokens": 80, "cache_read_tokens": 40, "output_tokens": 25}, - "requires_rates": ["cache_read_input_token_cost"], "wires": ["fireworks_chat"] }, { @@ -205,11 +201,6 @@ "cache_write_1h_tokens": 10, "output_tokens": 25 }, - "requires_rates": [ - "cache_read_input_token_cost", - "cache_creation_input_token_cost", - "cache_creation_input_token_cost_above_1hr" - ], "wires": ["anthropic_messages", "bedrock_converse"] }, { @@ -222,11 +213,6 @@ "output_tokens": 25 }, "stream": true, - "requires_rates": [ - "cache_read_input_token_cost", - "cache_creation_input_token_cost", - "cache_creation_input_token_cost_above_1hr" - ], "wires": ["anthropic_messages"] }, { @@ -240,7 +226,6 @@ "audio_output_tokens": 3 }, "requires_rates": [ - "cache_read_input_token_cost", "output_cost_per_reasoning_token", "input_cost_per_audio_token", "output_cost_per_audio_token" @@ -250,7 +235,7 @@ { "name": "all_components_responses", "usage": {"fresh_input_tokens": 80, "cache_read_tokens": 40, "output_tokens": 25, "reasoning_tokens": 15}, - "requires_rates": ["cache_read_input_token_cost", "output_cost_per_reasoning_token"], + "requires_rates": ["output_cost_per_reasoning_token"], "wires": ["openai_responses"] } ] diff --git a/tests/e2e/cost_calculation/expected.json b/tests/e2e/cost_calculation/expected.json index 3b18e9ed9f4..caea2c3c764 100644 --- a/tests/e2e/cost_calculation/expected.json +++ b/tests/e2e/cost_calculation/expected.json @@ -1686,6 +1686,13 @@ "prompt_tokens": 100, "spend": 0.0216 }, + "meta.llama4-maverick-17b-instruct-v1:0|all_components_anthropic": { + "completion_tokens": 25, + "input_cost": 0.0285, + "output_cost": 0.0095, + "prompt_tokens": 150, + "spend": 0.038 + }, "meta.llama4-maverick-17b-instruct-v1:0|basic": { "completion_tokens": 40, "input_cost": 0.0228, diff --git a/tests/e2e/cost_calculation/generate_expected.py b/tests/e2e/cost_calculation/generate_expected.py index f5514092c88..a6eabcc7286 100644 --- a/tests/e2e/cost_calculation/generate_expected.py +++ b/tests/e2e/cost_calculation/generate_expected.py @@ -94,11 +94,22 @@ def expected_breakdown(model: FrontierModel, case: Case) -> ExpectedCost: or rates.output_cost_per_token or 0.0 ) + write_rate: Final = ( + rates.cache_creation_input_token_cost + if rates.cache_creation_input_token_cost is not None + else in_rate + ) input_cost: Final = ( u.fresh_input_tokens * in_rate - + u.cache_read_tokens * (rates.cache_read_input_token_cost or 0.0) - + u.cache_write_5m_tokens * (rates.cache_creation_input_token_cost or 0.0) - + u.cache_write_1h_tokens * (rates.cache_creation_input_token_cost_above_1hr or 0.0) + + u.cache_read_tokens + * (rates.cache_read_input_token_cost if rates.cache_read_input_token_cost is not None else in_rate) + + u.cache_write_5m_tokens * write_rate + + u.cache_write_1h_tokens + * ( + rates.cache_creation_input_token_cost_above_1hr + if rates.cache_creation_input_token_cost_above_1hr is not None + else write_rate + ) + u.audio_input_tokens * (rates.input_cost_per_audio_token or 0.0) ) output_cost: Final = ( From 072b32baf2097e5421672956ce34809106f592aa Mon Sep 17 00:00:00 2001 From: kerry Date: Thu, 17 Sep 2026 01:18:02 +0000 Subject: [PATCH 017/210] test(e2e): derive goldens from first-principles rate selection Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/cost_calculation/cases.json | 10 ++- tests/e2e/cost_calculation/expected.json | 18 ++-- .../e2e/cost_calculation/generate_expected.py | 86 +++++++++---------- 3 files changed, 61 insertions(+), 53 deletions(-) diff --git a/tests/e2e/cost_calculation/cases.json b/tests/e2e/cost_calculation/cases.json index e01ac97e9ff..49eebc85231 100644 --- a/tests/e2e/cost_calculation/cases.json +++ b/tests/e2e/cost_calculation/cases.json @@ -62,7 +62,15 @@ "name": "web_search", "usage": {"fresh_input_tokens": 100, "output_tokens": 30, "web_search_calls": 3}, "requires_rates": ["search_context_cost_per_query"], - "requires_caps": ["web_search"] + "requires_caps": ["web_search"], + "wires": ["openai_responses", "anthropic_messages", "gemini_generate", "vertex_generate"] + }, + { + "name": "web_search_single", + "usage": {"fresh_input_tokens": 100, "output_tokens": 30, "web_search_calls": 1}, + "requires_rates": ["search_context_cost_per_query"], + "requires_caps": ["web_search"], + "wires": ["openai_chat", "together_chat", "fireworks_chat", "azure_chat"] }, { "name": "stream", diff --git a/tests/e2e/cost_calculation/expected.json b/tests/e2e/cost_calculation/expected.json index caea2c3c764..984b670a82c 100644 --- a/tests/e2e/cost_calculation/expected.json +++ b/tests/e2e/cost_calculation/expected.json @@ -160,7 +160,7 @@ "prompt_tokens": 120, "spend": 0.032 }, - "azure/gpt-5.4-mini|web_search": { + "azure/gpt-5.4-mini|web_search_single": { "completion_tokens": 30, "input_cost": 0.016, "output_cost": 0.009600000000000001, @@ -272,7 +272,7 @@ "prompt_tokens": 120, "spend": 0.03 }, - "azure/gpt-5.6|web_search": { + "azure/gpt-5.6|web_search_single": { "completion_tokens": 30, "input_cost": 0.015, "output_cost": 0.009, @@ -615,7 +615,7 @@ "prompt_tokens": 120, "spend": 0.028000000000000004 }, - "fireworks_ai/deepseek-v4p1-flash|web_search": { + "fireworks_ai/deepseek-v4p1-flash|web_search_single": { "completion_tokens": 30, "input_cost": 0.014000000000000002, "output_cost": 0.008400000000000001, @@ -706,7 +706,7 @@ "prompt_tokens": 120, "spend": 0.024 }, - "fireworks_ai/kimi-k3|web_search": { + "fireworks_ai/kimi-k3|web_search_single": { "completion_tokens": 30, "input_cost": 0.012000000000000002, "output_cost": 0.007200000000000001, @@ -797,7 +797,7 @@ "prompt_tokens": 120, "spend": 0.026000000000000002 }, - "fireworks_ai/qwen3p8-max|web_search": { + "fireworks_ai/qwen3p8-max|web_search_single": { "completion_tokens": 30, "input_cost": 0.013000000000000001, "output_cost": 0.007800000000000001, @@ -1462,7 +1462,7 @@ "prompt_tokens": 120, "spend": 0.008 }, - "gpt-5.4-mini|web_search": { + "gpt-5.4-mini|web_search_single": { "completion_tokens": 30, "input_cost": 0.004, "output_cost": 0.0024000000000000002, @@ -1679,7 +1679,7 @@ "prompt_tokens": 120, "spend": 0.002 }, - "gpt-5.6|web_search": { + "gpt-5.6|web_search_single": { "completion_tokens": 30, "input_cost": 0.001, "output_cost": 0.0006000000000000001, @@ -1826,7 +1826,7 @@ "prompt_tokens": 120, "spend": 0.02 }, - "together_ai/moonshotai/Kimi-K3|web_search": { + "together_ai/moonshotai/Kimi-K3|web_search_single": { "completion_tokens": 30, "input_cost": 0.01, "output_cost": 0.006, @@ -1938,7 +1938,7 @@ "prompt_tokens": 120, "spend": 0.022 }, - "together_ai/zai-org/GLM-5.3|web_search": { + "together_ai/zai-org/GLM-5.3|web_search_single": { "completion_tokens": 30, "input_cost": 0.011000000000000001, "output_cost": 0.0066, diff --git a/tests/e2e/cost_calculation/generate_expected.py b/tests/e2e/cost_calculation/generate_expected.py index a6eabcc7286..64abdb14c99 100644 --- a/tests/e2e/cost_calculation/generate_expected.py +++ b/tests/e2e/cost_calculation/generate_expected.py @@ -19,8 +19,6 @@ from dataclasses import dataclass from types import MappingProxyType from typing import Final -from pydantic import TypeAdapter - from cost_matrix import ( EXPECTED_PATH, FRONTIER_MODELS, @@ -32,19 +30,11 @@ from cost_matrix import ( cases_for, expected_key, ) - -# Wires whose response surface reports a real web-search call count; the -# chat-completions wires only expose url_citation annotations, so their billed -# count floors to one. -_EXACT_WEB_SEARCH_WIRES: Final = frozenset( - {"openai_responses", "anthropic_messages", "gemini_generate", "vertex_generate"} -) +from pydantic import TypeAdapter -def billed_web_search_calls(model: FrontierModel, case: Case) -> int: - if case.usage.web_search_calls == 0: - return 0 - return case.usage.web_search_calls if model.wire in _EXACT_WEB_SEARCH_WIRES else 1 +def _first_present(*rates: float | None) -> float | None: + return next((rate for rate in rates if rate is not None), None) @dataclass(frozen=True, slots=True) @@ -67,11 +57,14 @@ def expected_breakdown(model: FrontierModel, case: Case) -> ExpectedCost: Input = fresh*in + read*read + 5m*create + 1h*create_1h + audio_in*audio_in; output = text*out + reasoning*reasoning + audio_out*audio_out; plus the - billed web-search calls at the medium search-context rate. Above-threshold - swaps every input/output rate to its ``_above_200k_tokens`` variant when - total prompt tokens exceed the threshold; a service tier swaps input/output - to the tier's variants, falling back to the base rate when a variant is - unset -- mirroring _get_token_base_cost in litellm's cost calculator. + billed web-search calls at the medium search-context rate. Every billed + token is a token the provider charged for: a component whose entry has no + dedicated rate bills at the ordinary input or output rate, and a present + rate (including an explicit 0.0) is authoritative. When the total prompt + tokens exceed the threshold, input/output rates come from the + ``_above_200k_tokens`` variants; a service tier takes its ``_priority`` or + ``_flex`` variant when the entry carries one, and otherwise bills at the + base rate. """ rates: Final[CostMapEntry] = model.override_rates if case.response_model_override else model.rates u: Final = case.usage @@ -81,46 +74,53 @@ def expected_breakdown(model: FrontierModel, case: Case) -> ExpectedCost: ) tiered: Final = prompt_tokens > TIER_THRESHOLD_TOKENS in_rate: Final = ( - (rates.input_cost_per_token_above_200k_tokens if tiered else None) - or (rates.input_cost_per_token_priority if case.service_tier == "priority" else None) - or (rates.input_cost_per_token_flex if case.service_tier == "flex" else None) - or rates.input_cost_per_token + _first_present( + rates.input_cost_per_token_above_200k_tokens if tiered else None, + rates.input_cost_per_token_priority if case.service_tier == "priority" else None, + rates.input_cost_per_token_flex if case.service_tier == "flex" else None, + rates.input_cost_per_token, + ) or 0.0 ) out_rate: Final = ( - (rates.output_cost_per_token_above_200k_tokens if tiered else None) - or (rates.output_cost_per_token_priority if case.service_tier == "priority" else None) - or (rates.output_cost_per_token_flex if case.service_tier == "flex" else None) - or rates.output_cost_per_token + _first_present( + rates.output_cost_per_token_above_200k_tokens if tiered else None, + rates.output_cost_per_token_priority if case.service_tier == "priority" else None, + rates.output_cost_per_token_flex if case.service_tier == "flex" else None, + rates.output_cost_per_token, + ) or 0.0 ) - write_rate: Final = ( - rates.cache_creation_input_token_cost - if rates.cache_creation_input_token_cost is not None - else in_rate + read_rate: Final = _first_present(rates.cache_read_input_token_cost, in_rate) or 0.0 + write_rate: Final = _first_present(rates.cache_creation_input_token_cost, in_rate) or 0.0 + write_1h_rate: Final = ( + _first_present(rates.cache_creation_input_token_cost_above_1hr, write_rate) or 0.0 ) + audio_in_rate: Final = _first_present(rates.input_cost_per_audio_token, in_rate) or 0.0 + reasoning_rate: Final = _first_present(rates.output_cost_per_reasoning_token, out_rate) or 0.0 + audio_out_rate: Final = _first_present(rates.output_cost_per_audio_token, out_rate) or 0.0 input_cost: Final = ( u.fresh_input_tokens * in_rate - + u.cache_read_tokens - * (rates.cache_read_input_token_cost if rates.cache_read_input_token_cost is not None else in_rate) + + u.cache_read_tokens * read_rate + u.cache_write_5m_tokens * write_rate - + u.cache_write_1h_tokens - * ( - rates.cache_creation_input_token_cost_above_1hr - if rates.cache_creation_input_token_cost_above_1hr is not None - else write_rate - ) - + u.audio_input_tokens * (rates.input_cost_per_audio_token or 0.0) + + u.cache_write_1h_tokens * write_1h_rate + + u.audio_input_tokens * audio_in_rate ) output_cost: Final = ( u.output_tokens * out_rate - + u.reasoning_tokens * (rates.output_cost_per_reasoning_token or out_rate) - + u.audio_output_tokens * (rates.output_cost_per_audio_token or out_rate) + + u.reasoning_tokens * reasoning_rate + + u.audio_output_tokens * audio_out_rate ) search: Final = rates.search_context_cost_per_query - tool_cost: Final = billed_web_search_calls(model, case) * ( - search.search_context_size_medium if search and search.search_context_size_medium else 0.0 + medium_rate: Final = ( + search.search_context_size_medium if search is not None else None ) + if u.web_search_calls and medium_rate is None: + raise ValueError( + f"{model.map_key}: case {case.name} bills {u.web_search_calls} web-search " + "calls but the entry has no search_context_cost_per_query medium rate" + ) + tool_cost: Final = u.web_search_calls * (medium_rate if medium_rate is not None else 0.0) return ExpectedCost(input_cost=input_cost, output_cost=output_cost, tool_cost=tool_cost) From 1de633ac36644c5774cf629a793b6716a98b7580 Mon Sep 17 00:00:00 2001 From: kerry Date: Thu, 17 Sep 2026 01:22:01 +0000 Subject: [PATCH 018/210] test(e2e): move matrix data freshness checks to collection time Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/CLAUDE.md | 2 +- tests/e2e/cost_calculation/cost_matrix.py | 48 +++++++++++++++ .../e2e/cost_calculation/test_matrix_data.py | 61 ------------------- .../test_token_pricing_e2e.py | 4 ++ 4 files changed, 53 insertions(+), 62 deletions(-) delete mode 100644 tests/e2e/cost_calculation/test_matrix_data.py diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index 707d35b4aa6..f89b3203622 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -21,7 +21,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments split round robin over `/chat/completions` and `/v1/messages`, one endpoint per simulated user, with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests on every endpoint, budgeting RSS and CPU-per-request as ratios against the same run's healthy phase, and holding p50/p90/p99 latency and log-bytes-per-request to flat ceilings (a ratio cannot bound those two: an open breaker skips Redis instead of waiting on it, so the chaos phase can measure cheaper than baseline while still being far slower than a user should see); needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` and `E2E_PROXY_LOG` set, marked `redis_chaos`, deselected unless `E2E_REDIS_CHAOS` is set and excluded from the per-PR selector like the rest of `load/`, driven by `.github/workflows/test-e2e-redis-chaos.yml` and by the Buildkite `e2e-redis-chaos` step in project-releaser, which runs the proxy, Postgres and Valkey co-located with pytest in one pod and sets the opt-in), and markerless harness unit tests for the locust, process-usage, and session-anomaly aggregation logic - `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate, JWT auth (access tokens issued by a real Keycloak realm, `idp.py` plus `idp_realm.json`, whose JWKS the proxy's `JWT_PUBLIC_KEY_URL` points at; see CONTRIBUTING.md for the start command and config block), and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite - `gateway/` - proxy configuration only (`litellm-config.yml`); no tests -- `cost_calculation/` - cost accounting against a dedicated proxy whose whole model cost map is the test-owned `tests/e2e/cost_map.json` (loaded via `LITELLM_MODEL_COST_MAP_URL`), with provider calls answered by the scripted-provider sidecar in `scripted_provider.py`; every cost-map entry is a deployment and the cases plus asserted goldens are data in `cases.json` and `expected.json` (regenerate with `generate_expected.py`), deselected unless `E2E_COST_MAP_STACK` is set, driven by the Buildkite `e2e-cost-calculation` step in project-releaser, which runs a proxy booted from `gateway/cost_calculation_ci_config.yml`, Postgres and the scripted provider co-located with pytest in one pod and sets the opt-in +- `cost_calculation/` - cost accounting against a dedicated proxy whose whole model cost map is the test-owned `tests/e2e/cost_map.json` (loaded via `LITELLM_MODEL_COST_MAP_URL`), with provider calls answered by the scripted-provider sidecar in `scripted_provider.py`; every cost-map entry is a deployment and the cases plus asserted goldens are data in `cases.json` and `expected.json` (regenerate with `generate_expected.py`; `cost_matrix.matrix_data_errors()` runs at collection time so a stale key set fails the suite's collection loudly), deselected unless `E2E_COST_MAP_STACK` is set, driven by the Buildkite `e2e-cost-calculation` step in project-releaser, which runs a proxy booted from `gateway/cost_calculation_ci_config.yml`, Postgres and the scripted provider co-located with pytest in one pod and sets the opt-in - `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture; ships its own driver/builder/publisher plus `_*_unit_tests/` trees. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`); the CLI-driving path stays bespoke - `ui/` - the Admin UI browser suite: Playwright in TypeScript, driving the dashboard served by a live proxy on port 4000 (seeded postgres + mock LLM upstream; see its `run_e2e.sh`). It is a self-contained npm package with its own lockfile and does not use the Python harness, pytest markers, or the shared transport; the Python rules in this file (typed models, `Result` unions, basedpyright zero-error gate) do not apply inside it. Its only Python file, `fixtures/mock_llm_server/server.py`, is excluded from the e2e basedpyright gate via the root `pyrightconfig.json` diff --git a/tests/e2e/cost_calculation/cost_matrix.py b/tests/e2e/cost_calculation/cost_matrix.py index 35344a099be..68f3186809d 100644 --- a/tests/e2e/cost_calculation/cost_matrix.py +++ b/tests/e2e/cost_calculation/cost_matrix.py @@ -435,3 +435,51 @@ EXPECTED: Final[Mapping[str, ExpectedCell]] = MappingProxyType( def expected_key(model: FrontierModel, case: Case) -> str: return f"{model.map_key}|{case.name}" + + +def matrix_data_errors() -> tuple[str, ...]: + """Freshness findings for the data files, as human-readable strings. + + Called at collection time by the e2e suite; also usable from + generate_expected.py's context without importing pytest. + """ + derived: Final = { + expected_key(model, case) + for model in FRONTIER_MODELS + for case in cases_for(model) + if case.exact_spend + } + golden: Final = set(EXPECTED) + unknown_deployments: Final = sorted( + spec.map_key for spec in CASES_FILE.deployments if spec.map_key not in COST_MAP + ) + unknown_rates: Final = sorted( + {field for case in CASES for field in case.requires_rates} - set(CostMapEntry.model_fields) + ) + input_rates: Final = tuple(entry.input_cost_per_token for entry in COST_MAP.values()) + findings: Final = ( + ( + "expected.json is out of sync with the derived matrix; run " + "uv run python tests/e2e/cost_calculation/generate_expected.py " + f"(missing: {sorted(derived - golden)}; stale: {sorted(golden - derived)})" + ) + if derived != golden + else None, + ( + f"deployments entries name map keys absent from cost_map.json: {unknown_deployments}" + if unknown_deployments + else None + ), + ( + f"requires_rates names that are not CostMapEntry fields: {unknown_rates}" + if unknown_rates + else None + ), + ( + "two cost_map entries share input_cost_per_token; the suite relies on " + "distinct rates so a wrong-model bill can never coincidentally match" + if len(input_rates) != len(set(input_rates)) + else None + ), + ) + return tuple(finding for finding in findings if finding is not None) diff --git a/tests/e2e/cost_calculation/test_matrix_data.py b/tests/e2e/cost_calculation/test_matrix_data.py deleted file mode 100644 index 8340257939c..00000000000 --- a/tests/e2e/cost_calculation/test_matrix_data.py +++ /dev/null @@ -1,61 +0,0 @@ -"""Freshness checks for the cost suite's data files; markerless, so it runs on -any pytest invocation of the folder without the stack. expected.json is the -oracle: these tests check its key set against the derived matrix, never its -values (the generator proposes, the file decides).""" - -from __future__ import annotations - -from typing import Final - -import pytest -from cost_matrix import ( - CASES, - CASES_FILE, - COST_MAP, - EXPECTED, - FRONTIER_MODELS, - CostMapEntry, - cases_for, - expected_key, -) - - -def test_expected_keys_match_derived_exact_cells() -> None: - derived: Final = { - expected_key(model, case) - for model in FRONTIER_MODELS - for case in cases_for(model) - if case.exact_spend - } - golden: Final = set(EXPECTED) - if derived != golden: - missing: Final = sorted(derived - golden) - stale: Final = sorted(golden - derived) - pytest.fail( - "expected.json is out of sync with the derived matrix; run " - "uv run python tests/e2e/cost_calculation/generate_expected.py " - f"(missing: {missing}; stale: {stale})" - ) - - -def test_deployments_reference_existing_map_keys() -> None: - unknown: Final = sorted( - spec.map_key for spec in CASES_FILE.deployments if spec.map_key not in COST_MAP - ) - assert not unknown, f"deployments entries name map keys absent from cost_map.json: {unknown}" - - -def test_requires_rates_are_cost_map_fields() -> None: - fields: Final = set(CostMapEntry.model_fields) - unknown: Final = sorted( - {field for case in CASES for field in case.requires_rates} - fields - ) - assert not unknown, f"requires_rates names that are not CostMapEntry fields: {unknown}" - - -def test_no_two_entries_share_input_rate() -> None: - rates: Final = tuple(entry.input_cost_per_token for entry in COST_MAP.values()) - assert len(rates) == len(set(rates)), ( - "two cost_map entries share input_cost_per_token; the suite relies on " - "distinct rates so a wrong-model bill can never coincidentally match" - ) diff --git a/tests/e2e/cost_calculation/test_token_pricing_e2e.py b/tests/e2e/cost_calculation/test_token_pricing_e2e.py index 7cd128ad6fb..346a55aa22d 100644 --- a/tests/e2e/cost_calculation/test_token_pricing_e2e.py +++ b/tests/e2e/cost_calculation/test_token_pricing_e2e.py @@ -22,6 +22,7 @@ from cost_matrix import ( FrontierModel, cases_for, expected_key, + matrix_data_errors, recount_cost, ) from e2e_config import unique_marker @@ -39,6 +40,9 @@ from models import ( pytestmark: Final = [pytest.mark.e2e, pytest.mark.cost_map_stack] # mutable-ok: pytest only accepts a list for pytestmark +if _data_errors := matrix_data_errors(): + raise ValueError("\n".join(_data_errors)) + _MATRIX: Final[tuple[tuple[FrontierModel, Case], ...]] = tuple( (model, case) for model in FRONTIER_MODELS for case in cases_for(model) ) From ef1f306a7dc0777276166859b3fa4d2e6272cefb Mon Sep 17 00:00:00 2001 From: kerry Date: Thu, 17 Sep 2026 01:53:52 +0000 Subject: [PATCH 019/210] test(e2e): emit gemini stream usage only on the final chunk Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/cost_calculation/scripted_provider.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/e2e/cost_calculation/scripted_provider.py b/tests/e2e/cost_calculation/scripted_provider.py index 982132ed8df..90d95441e5c 100644 --- a/tests/e2e/cost_calculation/scripted_provider.py +++ b/tests/e2e/cost_calculation/scripted_provider.py @@ -750,10 +750,8 @@ def _gemini_body(scenario: Scenario, requested_model: str) -> Mapping[str, objec def _gemini_sse(scenario: Scenario, requested_model: str) -> bytes: emit_usage: Final = scenario.stream_usage == "final_chunk" - first: Final = ( - _jobj(*((key, value) for key, value in _gemini_body(scenario, requested_model).items() if key != "usageMetadata")) - if scenario.stream_usage == "absent" - else _gemini_body(scenario, requested_model) + first: Final = _jobj( + *((key, value) for key, value in _gemini_body(scenario, requested_model).items() if key != "usageMetadata") ) return _sse( ( From e254377049ca6f7087693cff4b99b291c9ba6010 Mon Sep 17 00:00:00 2001 From: David Steele Date: Thu, 17 Sep 2026 07:07:57 +0100 Subject: [PATCH 020/210] fix(azure): drop tool_choice without tools DEVX-829 --- litellm/llms/azure/chat/gpt_transformation.py | 9 +- .../test_azure_chat_gpt_transformation.py | 123 ++++++++++++++++++ ...test_azure_chat_o_series_transformation.py | 3 +- 3 files changed, 133 insertions(+), 2 deletions(-) diff --git a/litellm/llms/azure/chat/gpt_transformation.py b/litellm/llms/azure/chat/gpt_transformation.py index 6d17a1359bc..0debbe4f74d 100644 --- a/litellm/llms/azure/chat/gpt_transformation.py +++ b/litellm/llms/azure/chat/gpt_transformation.py @@ -280,10 +280,17 @@ class AzureOpenAIConfig(BaseConfig): ordered_messages: Final = system_messages_first(messages) if litellm.openai_system_messages_first else messages stripped_messages: Final = drop_tool_reference_parts_from_tool_messages(ordered_messages) azure_messages: Final = convert_to_azure_openai_messages(hoist_images_from_tool_messages(stripped_messages)) + request_params: Final = { + key: value + for key, value in optional_params.items() + if key != "tool_choice" + or optional_params.get("tools") + or optional_params.get("functions") + } return { "model": model, "messages": azure_messages, - **optional_params, + **request_params, **sanitized_tools_update(optional_params), } diff --git a/tests/test_litellm/llms/azure/chat/test_azure_chat_gpt_transformation.py b/tests/test_litellm/llms/azure/chat/test_azure_chat_gpt_transformation.py index e8b98c696e1..92a8124d8a9 100644 --- a/tests/test_litellm/llms/azure/chat/test_azure_chat_gpt_transformation.py +++ b/tests/test_litellm/llms/azure/chat/test_azure_chat_gpt_transformation.py @@ -333,3 +333,126 @@ class TestAzureToolSchemaCombinatorFlattening: ) assert "tools" not in request assert request["temperature"] == 0.2 + + +@pytest.mark.parametrize("tool_choice", ["none", "auto"]) +def test_azure_drops_tool_choice_without_tools_or_functions(tool_choice: str) -> None: + optional_params = {"tool_choice": tool_choice, "temperature": 0.2} + request = AzureOpenAIConfig().transform_request( + model="gpt-4o", + messages=[{"role": "user", "content": "hi"}], + optional_params=optional_params, + litellm_params={"custom_llm_provider": "azure"}, + headers={}, + ) + + assert "tool_choice" not in request + assert request["temperature"] == 0.2 + assert optional_params["tool_choice"] == tool_choice + + +def test_azure_tools_empty_drops_tool_choice() -> None: + request = AzureOpenAIConfig().transform_request( + model="gpt-4o", + messages=[{"role": "user", "content": "hi"}], + optional_params={"tools": [], "tool_choice": "auto"}, + litellm_params={"custom_llm_provider": "azure"}, + headers={}, + ) + + assert request["tools"] == [] + assert "tool_choice" not in request + + +def test_azure_functions_empty_drops_tool_choice() -> None: + request = AzureOpenAIConfig().transform_request( + model="gpt-4o", + messages=[{"role": "user", "content": "hi"}], + optional_params={"functions": [], "tool_choice": "none"}, + litellm_params={"custom_llm_provider": "azure"}, + headers={}, + ) + + assert request["functions"] == [] + assert "tool_choice" not in request + + +def test_azure_preserves_tool_choice_with_tools() -> None: + tools = [{"type": "function", "function": {"name": "get_weather", "parameters": {}}}] + request = AzureOpenAIConfig().transform_request( + model="gpt-4o", + messages=[{"role": "user", "content": "hi"}], + optional_params={"tools": tools, "tool_choice": "auto"}, + litellm_params={"custom_llm_provider": "azure"}, + headers={}, + ) + + assert request["tools"] == tools + assert request["tool_choice"] == "auto" + + +def test_azure_preserves_tool_choice_with_legacy_functions() -> None: + functions = [{"name": "get_weather", "parameters": {}}] + request = AzureOpenAIConfig().transform_request( + model="gpt-4o", + messages=[{"role": "user", "content": "hi"}], + optional_params={"functions": functions, "tool_choice": "auto"}, + litellm_params={"custom_llm_provider": "azure"}, + headers={}, + ) + + assert request["functions"] == functions + assert request["tool_choice"] == "auto" + + +def test_azure_preserves_function_call_without_tools() -> None: + request = AzureOpenAIConfig().transform_request( + model="gpt-4o", + messages=[{"role": "user", "content": "hi"}], + optional_params={"function_call": "none", "tool_choice": "auto"}, + litellm_params={"custom_llm_provider": "azure"}, + headers={}, + ) + + assert request["function_call"] == "none" + assert "tool_choice" not in request + + +def test_azure_gpt5_drops_tool_choice_without_tools() -> None: + request = AzureOpenAIGPT5Config().transform_request( + model="gpt5_series/gpt-5.6-sol", + messages=[{"role": "user", "content": "hi"}], + optional_params={"tool_choice": "none"}, + litellm_params={"custom_llm_provider": "azure"}, + headers={}, + ) + + assert request["model"] == "gpt-5.6-sol" + assert "tool_choice" not in request + + +@pytest.mark.asyncio +async def test_azure_async_transform_drops_tool_choice_without_tools() -> None: + request = await AzureOpenAIConfig().async_transform_request( + model="gpt-4o", + messages=[{"role": "user", "content": "hi"}], + optional_params={"tool_choice": "none"}, + litellm_params={"custom_llm_provider": "azure"}, + headers={}, + ) + + assert "tool_choice" not in request + + +@pytest.mark.asyncio +async def test_azure_gpt5_async_transform_drops_tool_choice_without_tools() -> None: + request = await AzureOpenAIGPT5Config().async_transform_request( + model="gpt5_series/gpt-5.6-sol", + messages=[{"role": "user", "content": "hi"}], + optional_params={"tool_choice": "auto"}, + litellm_params={"custom_llm_provider": "azure"}, + headers={}, + ) + + assert request["model"] == "gpt-5.6-sol" + assert "tool_choice" not in request diff --git a/tests/test_litellm/llms/azure/chat/test_azure_chat_o_series_transformation.py b/tests/test_litellm/llms/azure/chat/test_azure_chat_o_series_transformation.py index 9db9ab971a0..57d60df3a11 100644 --- a/tests/test_litellm/llms/azure/chat/test_azure_chat_o_series_transformation.py +++ b/tests/test_litellm/llms/azure/chat/test_azure_chat_o_series_transformation.py @@ -14,7 +14,7 @@ async def test_azure_chat_o_series_transformation(): provider_config = AzureOpenAIO1Config() model = "o_series/web-interface-o1-mini" messages = [{"role": "user", "content": "Hello, how are you?"}] - optional_params = {} + optional_params = {"tool_choice": "none"} litellm_params = {} headers = {} @@ -23,6 +23,7 @@ async def test_azure_chat_o_series_transformation(): ) print(response) assert response["model"] == "web-interface-o1-mini" + assert "tool_choice" not in response def test_azure_o_series_transform_request_flattens_top_level_anyof(): From 48712f733a641f16b0b4fe60c221fd9f1c7076fa Mon Sep 17 00:00:00 2001 From: David Steele Date: Thu, 17 Sep 2026 07:30:06 +0100 Subject: [PATCH 021/210] style(azure): format request parameter filter DEVX-829 Co-Authored-By: Claude Code --- litellm/llms/azure/chat/gpt_transformation.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/litellm/llms/azure/chat/gpt_transformation.py b/litellm/llms/azure/chat/gpt_transformation.py index 0debbe4f74d..7cb50ee5348 100644 --- a/litellm/llms/azure/chat/gpt_transformation.py +++ b/litellm/llms/azure/chat/gpt_transformation.py @@ -283,9 +283,7 @@ class AzureOpenAIConfig(BaseConfig): request_params: Final = { key: value for key, value in optional_params.items() - if key != "tool_choice" - or optional_params.get("tools") - or optional_params.get("functions") + if key != "tool_choice" or optional_params.get("tools") or optional_params.get("functions") } return { "model": model, From bd222bd8d9f6d083b8058c5fef3e998b4f92b3af Mon Sep 17 00:00:00 2001 From: David Steele Date: Thu, 17 Sep 2026 08:18:36 +0100 Subject: [PATCH 022/210] fix(azure): avoid mutable request mapping DEVX-829 Co-Authored-By: Claude Code --- litellm/llms/azure/chat/gpt_transformation.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/litellm/llms/azure/chat/gpt_transformation.py b/litellm/llms/azure/chat/gpt_transformation.py index 7cb50ee5348..424422612db 100644 --- a/litellm/llms/azure/chat/gpt_transformation.py +++ b/litellm/llms/azure/chat/gpt_transformation.py @@ -280,11 +280,13 @@ class AzureOpenAIConfig(BaseConfig): ordered_messages: Final = system_messages_first(messages) if litellm.openai_system_messages_first else messages stripped_messages: Final = drop_tool_reference_parts_from_tool_messages(ordered_messages) azure_messages: Final = convert_to_azure_openai_messages(hoist_images_from_tool_messages(stripped_messages)) - request_params: Final = { - key: value - for key, value in optional_params.items() - if key != "tool_choice" or optional_params.get("tools") or optional_params.get("functions") - } + request_params: Final = MappingProxyType( + { + key: value + for key, value in optional_params.items() + if key != "tool_choice" or optional_params.get("tools") or optional_params.get("functions") + } + ) return { "model": model, "messages": azure_messages, From 2fa115db2bac68ac90b0b900dc9eecb728c3bd4d Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Thu, 17 Sep 2026 12:53:15 -0400 Subject: [PATCH 023/210] feat(router): add maintained Fuse model and harness presets --- .../public_endpoints/public_endpoints.py | 9 + .../complexity_router/README.md | 45 +++++ .../complexity_router/fuse_presets.json | 100 +++++++++++ .../complexity_router/fuse_presets.py | 52 ++++++ .../complexity_router/llm_v2.py | 37 +++- pyproject.toml | 1 + .../public_endpoints/test_public_endpoints.py | 11 ++ .../router_strategy/test_fuse_presets.py | 43 +++++ .../router_strategy/test_llm_v2.py | 110 ++++++++++++ .../test_auto_router_model_naming.py | 49 ++++++ ...ecastClassifierConfig.integration.test.tsx | 163 +++++++++++++++++- .../add_model/ForecastClassifierConfig.tsx | 26 +-- .../add_model/FuseProfilePresets.tsx | 117 +++++++++++++ .../build_complexity_router_config.test.ts | 17 ++ .../forecast_classifier_config.test.ts | 65 ++++++- .../add_model/forecast_classifier_config.ts | 33 +++- ui/litellm-dashboard/src/lib/http/schema.d.ts | 82 ++++++++- 17 files changed, 917 insertions(+), 43 deletions(-) create mode 100644 litellm/router_strategy/complexity_router/fuse_presets.json create mode 100644 litellm/router_strategy/complexity_router/fuse_presets.py create mode 100644 tests/test_litellm/router_strategy/test_fuse_presets.py create mode 100644 ui/litellm-dashboard/src/components/add_model/FuseProfilePresets.tsx diff --git a/litellm/proxy/public_endpoints/public_endpoints.py b/litellm/proxy/public_endpoints/public_endpoints.py index 94a59828451..e395f56194f 100644 --- a/litellm/proxy/public_endpoints/public_endpoints.py +++ b/litellm/proxy/public_endpoints/public_endpoints.py @@ -23,6 +23,7 @@ from litellm.proxy._types import ( ) from litellm.proxy.utils import get_custom_url from litellm.repositories.table_repositories import ClaudeCodePluginRepository +from litellm.router_strategy.complexity_router.fuse_presets import FusePresetCatalog, get_fuse_presets from litellm.types.agents import AgentCard from litellm.types.mcp import MCPPublicServer from litellm.types.proxy.management_endpoints.model_management_endpoints import ( @@ -424,6 +425,14 @@ async def get_complexity_scorer_defaults() -> ComplexityScorerDefaults: ) +@router.get( + "/public/complexity_router/fuse_presets", + response_model=FusePresetCatalog, +) +async def get_public_fuse_presets() -> FusePresetCatalog: + return get_fuse_presets() + + @router.get( "/public/litellm_model_cost_map", tags=["public", "model management"], diff --git a/litellm/router_strategy/complexity_router/README.md b/litellm/router_strategy/complexity_router/README.md index 6505746bca1..d9159cea426 100644 --- a/litellm/router_strategy/complexity_router/README.md +++ b/litellm/router_strategy/complexity_router/README.md @@ -179,6 +179,51 @@ Configure capability forecasting through YAML or the model-management API. The dashboard preserves its classifier and calibration on an untouched save; it does not provide a capability-card editor +### Fuse v2 profile presets + +Fuse v2 accepts maintained model and runtime descriptions instead of requiring +custom prose for both solvers and the harness. Select profiles explicitly for +all deployments behind your configured model groups and their actual settings. +Group names do not select profiles automatically + +```yaml +complexity_router_config: + classifier_type: llm_v2 + classifier_llm_config: + model: your-judge-group + tiers: + SIMPLE: your-efficient-group + REASONING: your-capable-group + llm_v2_config: + efficient_profile_preset: claude-sonnet-5-v1 + capable_profile_preset: claude-fable-5-1-v1 + harness_preset: claude-code-v1 + max_quality_gap: 0.05 +``` + +`GET /public/complexity_router/fuse_presets` returns the catalog version, model +profiles, and runtime descriptions, including source URLs. The bundled catalog +is loaded once per process without network requests. Sources are citations only + +Each of `efficient_profile`, `capable_profile`, and `harness` requires either +nonblank custom text or its corresponding preset reference. Custom text wins +when both are supplied, but an unknown or wrong-kind preset is still rejected. +Explicit blank text is invalid even with a valid preset. Custom text remains +limited to 4000 characters + +Saved configurations retain preset references and explicit text separately. +Preset text is resolved when building the classifier prompt, not copied into +stored custom fields. Existing all-custom configurations keep the same prompt. +Versioned preset IDs identify immutable content: revised wording receives a new +ID, and older referenced entries must remain available + +The runtime presets do not imply a repository, runnable tests, network access, +additional tools, or a step, time, or spending budget. mini-SWE-agent describes +an agent interface, not a SWE-bench task. Model descriptions summarize provider +positioning without solve rates or guaranteed rankings. Wording is an evaluation +input, not a calibrated quality claim. Existing Fuse licensing, policy, +calibration, and prompt version are unchanged + ### Heuristic v2 Set `classifier_type: heuristic_v2` to classify with the bundled calibrated diff --git a/litellm/router_strategy/complexity_router/fuse_presets.json b/litellm/router_strategy/complexity_router/fuse_presets.json new file mode 100644 index 00000000000..4006366dc25 --- /dev/null +++ b/litellm/router_strategy/complexity_router/fuse_presets.json @@ -0,0 +1,100 @@ +{ + "version": "2026-09-17-v1", + "models": [ + { + "id": "gpt-6-astra-v1", + "label": "GPT-6 Astra", + "model": "gpt-6-astra", + "text": "OpenAI model for demanding end-to-end work, including reasoning, coding, research, and document tasks", + "sources": ["https://developers.openai.com/api/docs/models/gpt-6-astra"] + }, + { + "id": "gpt-5.6-sol-v1", + "label": "GPT-5.6 Sol", + "model": "gpt-5.6-sol", + "text": "OpenAI model for complex professional work, supporting reasoning and tool calling", + "sources": ["https://developers.openai.com/api/docs/models/gpt-5.6-sol"] + }, + { + "id": "gpt-5.6-luna-v1", + "label": "GPT-5.6 Luna", + "model": "gpt-5.6-luna", + "text": "OpenAI model for high-volume workloads, supporting reasoning and tool calling", + "sources": ["https://developers.openai.com/api/docs/models/gpt-5.6-luna"] + }, + { + "id": "gpt-5.6-terra-v1", + "label": "GPT-5.6 Terra", + "model": "gpt-5.6-terra", + "text": "OpenAI general-purpose model supporting reasoning, text and image input, and tool calling", + "sources": ["https://developers.openai.com/api/docs/models/gpt-5.6-terra"] + }, + { + "id": "claude-haiku-4-5-v1", + "label": "Claude Haiku 4.5", + "model": "claude-haiku-4-5", + "text": "Anthropic latency-focused model supporting text and image input, tool use, and extended thinking", + "sources": ["https://platform.claude.com/docs/en/models/haiku-4-5/overview"] + }, + { + "id": "claude-sonnet-5-v1", + "label": "Claude Sonnet 5", + "model": "claude-sonnet-5", + "text": "Anthropic model balancing speed and capability, with adaptive thinking and tool use", + "sources": ["https://platform.claude.com/docs/en/models/sonnet-5/overview"] + }, + { + "id": "claude-opus-5-v1", + "label": "Claude Opus 5", + "model": "claude-opus-5", + "text": "Anthropic model for complex agentic coding and enterprise work, with adaptive thinking", + "sources": ["https://platform.claude.com/docs/en/models/opus-5/overview"] + }, + { + "id": "claude-fable-5-v1", + "label": "Claude Fable 5", + "model": "claude-fable-5", + "text": "Anthropic model for demanding reasoning and long-running agent tasks, with always-on adaptive thinking", + "sources": ["https://platform.claude.com/docs/en/models/fable-5/introducing-claude-fable-5-and-claude-mythos-5"] + }, + { + "id": "claude-fable-5-1-v1", + "label": "Claude Fable 5.1", + "model": "claude-fable-5-1", + "text": "Anthropic model for demanding reasoning, long-running agentic coding, and multistep research, with always-on adaptive thinking", + "sources": ["https://platform.claude.com/docs/en/models/fable-5-1/overview"] + } + ], + "harnesses": [ + { + "id": "unspecified-v1", + "label": "Unspecified runtime", + "text": "Agent runtime is unspecified. Assess the task using the supplied context without assuming repository access, runnable tests, network access, additional tools, or a step, time, or spending budget", + "sources": ["https://code.claude.com/docs/en/how-claude-code-works", "https://mini-swe-agent.com/latest/faq/"] + }, + { + "id": "claude-code-v1", + "label": "Claude Code", + "text": "Claude Code supplies an agent loop with context management and configured tools. Available actions depend on the session's tools, permissions, and execution environment. The runtime name alone does not establish repository access, runnable tests, network access, additional tools, or a step, time, or spending budget", + "sources": ["https://code.claude.com/docs/en/how-claude-code-works"] + }, + { + "id": "codex-cli-v1", + "label": "Codex CLI", + "text": "Codex CLI supplies a terminal-based coding agent. File operations, command execution, and integrations depend on the session's tools, permissions, and sandbox. The runtime name alone does not establish repository access, runnable tests, network access, additional tools, or a step, time, or spending budget", + "sources": ["https://learn.chatgpt.com/docs/codex/cli", "https://learn.chatgpt.com/codex/permissions"] + }, + { + "id": "opencode-v1", + "label": "OpenCode", + "text": "OpenCode supplies a configurable agent runtime. Available actions depend on the selected agent, tools, permissions, and execution environment. The runtime name alone does not establish repository access, runnable tests, network access, additional tools, or a step, time, or spending budget", + "sources": ["https://opencode.ai/docs/agents/"] + }, + { + "id": "mini-swe-agent-v1", + "label": "mini-SWE-agent", + "text": "The standard mini-SWE-agent setup uses a bash-only action interface and separate command executions. Available commands and resources depend on its configured environment. The runtime name alone does not establish repository access, runnable tests, network access, additional tools, or a step, time, or spending budget", + "sources": ["https://mini-swe-agent.com/latest/faq/"] + } + ] +} diff --git a/litellm/router_strategy/complexity_router/fuse_presets.py b/litellm/router_strategy/complexity_router/fuse_presets.py new file mode 100644 index 00000000000..66a96ec5ad5 --- /dev/null +++ b/litellm/router_strategy/complexity_router/fuse_presets.py @@ -0,0 +1,52 @@ +from functools import lru_cache +from importlib.resources import files +from typing import Annotated, Final, Literal, TypeAlias + +from pydantic import BaseModel, ConfigDict, Field, StringConstraints + +ProfileText: TypeAlias = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1, max_length=4000)] + + +class FuseModelPreset(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + id: str + label: str + text: ProfileText + sources: tuple[str, ...] = Field(min_length=1) + model: str + + +class FuseHarnessPreset(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + id: str + label: str + text: ProfileText + sources: tuple[str, ...] = Field(min_length=1) + + +class FusePresetCatalog(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + version: str + models: tuple[FuseModelPreset, ...] + harnesses: tuple[FuseHarnessPreset, ...] + + +@lru_cache(maxsize=1) +def get_fuse_presets() -> FusePresetCatalog: + return FusePresetCatalog.model_validate_json( + files(__package__).joinpath("fuse_presets.json").read_text(encoding="utf-8") + ) + + +def resolve_fuse_profile(text: str | None, preset_id: str | None, kind: Literal["model", "harness"]) -> str | None: + if preset_id is None: + return text + catalog: Final = get_fuse_presets() + presets: Final = catalog.models if kind == "model" else catalog.harnesses + preset: Final = next((entry for entry in presets if entry.id == preset_id), None) + if preset is None: + return None + return text if text is not None else preset.text diff --git a/litellm/router_strategy/complexity_router/llm_v2.py b/litellm/router_strategy/complexity_router/llm_v2.py index 2f545a65aaa..18351237e65 100644 --- a/litellm/router_strategy/complexity_router/llm_v2.py +++ b/litellm/router_strategy/complexity_router/llm_v2.py @@ -7,15 +7,15 @@ from dataclasses import dataclass from sys import float_info from typing import Annotated, Final, Literal, TypeAlias -from pydantic import BaseModel, ConfigDict, Field, StrictFloat, StringConstraints, TypeAdapter +from pydantic import BaseModel, ConfigDict, Field, StrictFloat, StringConstraints, TypeAdapter, model_validator from typing_extensions import ReadOnly, TypedDict from litellm.llms.base_llm.base_utils import ( type_to_response_format_param, # pyright: ignore[reportUnknownVariableType] # legacy output validated below ) +from litellm.router_strategy.complexity_router.fuse_presets import ProfileText, resolve_fuse_profile ShortText: TypeAlias = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1, max_length=512)] -ProfileText: TypeAlias = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1, max_length=4000)] class _SolverProfile(TypedDict): @@ -139,20 +139,41 @@ class LLMV2Config(BaseModel): efficient_tier: str = "SIMPLE" capable_tier: str = "REASONING" - efficient_profile: ProfileText - capable_profile: ProfileText - harness: ProfileText + efficient_profile: ProfileText | None = None + capable_profile: ProfileText | None = None + harness: ProfileText | None = None + efficient_profile_preset: str | None = None + capable_profile_preset: str | None = None + harness_preset: str | None = None max_quality_gap: float = Field(ge=0.0, le=1.0, description="Maximum estimated success loss allowed for efficient.") max_output_tokens: int = Field(default=1024, ge=1) response_format: Literal["json_schema", "json_object"] = "json_schema" calibration: LLMV2Calibration | None = None + @model_validator(mode="after") + def validate_profiles(self) -> LLMV2Config: + self._profile_texts() + return self + + def _profile_texts(self) -> tuple[str, str, str]: + efficient: Final = resolve_fuse_profile(self.efficient_profile, self.efficient_profile_preset, "model") + capable: Final = resolve_fuse_profile(self.capable_profile, self.capable_profile_preset, "model") + harness: Final = resolve_fuse_profile(self.harness, self.harness_preset, "harness") + if efficient is None: + raise ValueError("efficient_profile requires text or a known efficient_profile_preset") + if capable is None: + raise ValueError("capable_profile requires text or a known capable_profile_preset") + if harness is None: + raise ValueError("harness requires text or a known harness_preset") + return efficient, capable, harness + def system_prompt(self, efficient_model: str, capable_model: str) -> str: + efficient, capable, harness = self._profile_texts() profiles: Final[_SolverProfiles] = { "prompt_version": LLM_V2_PROMPT_VERSION, - "harness": self.harness, - "efficient": {"model": efficient_model, "profile": self.efficient_profile}, - "capable": {"model": capable_model, "profile": self.capable_profile}, + "harness": harness, + "efficient": {"model": efficient_model, "profile": efficient}, + "capable": {"model": capable_model, "profile": capable}, } schema: Final = ( "\n\nResponse JSON schema:\n" + json.dumps(LLMV2Verdict.model_json_schema()) diff --git a/pyproject.toml b/pyproject.toml index 93ff55c4069..2d6d133cd0e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -290,6 +290,7 @@ editable-profile = "dev" include = [ "litellm/proxy/_experimental/out/**", "litellm/router_strategy/complexity_router/artifacts/*.json", + "litellm/router_strategy/complexity_router/fuse_presets.json", "litellm/proxy/client/cli/commands/codex_base_instructions.md", ] exclude = [ diff --git a/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py b/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py index 0d82ed778f5..b830eb588a5 100644 --- a/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py @@ -11,12 +11,23 @@ from fastapi.testclient import TestClient from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.public_endpoints import router +from litellm.router_strategy.complexity_router.fuse_presets import get_fuse_presets from litellm.types.proxy.management_endpoints.model_management_endpoints import ( ModelGroupInfoProxy, ) from litellm.types.utils import LlmProviders +def test_fuse_presets_route_serves_the_shared_catalog_without_authentication() -> None: + app: Final = FastAPI() + app.include_router(router) + client: Final = TestClient(app) + response: Final = client.get("/public/complexity_router/fuse_presets") + assert response.status_code == 200 + assert response.json() == get_fuse_presets().model_dump(mode="json") + assert client.get("/public/complexity_router/fuse_presets").json() == response.json() + + def test_get_supported_providers_returns_enum_values(): app_instance = FastAPI() app_instance.include_router(router) diff --git a/tests/test_litellm/router_strategy/test_fuse_presets.py b/tests/test_litellm/router_strategy/test_fuse_presets.py new file mode 100644 index 00000000000..0b8d936383b --- /dev/null +++ b/tests/test_litellm/router_strategy/test_fuse_presets.py @@ -0,0 +1,43 @@ +import json +from importlib.resources import files +from typing import Final + +import pytest +from pydantic import ValidationError + +from litellm.router_strategy.complexity_router.fuse_presets import get_fuse_presets, resolve_fuse_profile + + +def test_catalog_is_loaded_once_and_preserves_bundled_content() -> None: + get_fuse_presets.cache_clear() + first: Final = get_fuse_presets() + second: Final = get_fuse_presets() + assert first is second + bundled: Final = json.loads( + files("litellm.router_strategy.complexity_router").joinpath("fuse_presets.json").read_text(encoding="utf-8") + ) + assert first.model_dump(mode="json") == bundled + entries: Final = (*first.models, *first.harnesses) + assert len({entry.id for entry in entries}) == len(entries) + assert len(first.models) == 9 + assert len(first.harnesses) == 5 + assert all(entry.sources and all(source.startswith("https://") for source in entry.sources) for entry in entries) + + +def test_every_catalog_entry_resolves_without_changing_custom_ownership() -> None: + catalog: Final = get_fuse_presets() + for entry in catalog.models: + assert resolve_fuse_profile(None, entry.id, "model") == entry.text + assert resolve_fuse_profile("Custom text", entry.id, "model") == "Custom text" + for entry in catalog.harnesses: + assert resolve_fuse_profile(None, entry.id, "harness") == entry.text + assert resolve_fuse_profile("Custom text", entry.id, "harness") == "Custom text" + assert resolve_fuse_profile("Custom text", None, "model") == "Custom text" + assert resolve_fuse_profile("Custom text", None, "harness") == "Custom text" + + +def test_cached_catalog_and_records_cannot_be_modified() -> None: + catalog: Final = get_fuse_presets() + for record, field in ((catalog, "version"), (catalog.models[0], "text"), (catalog.harnesses[0], "text")): + with pytest.raises(ValidationError, match="frozen"): + setattr(record, field, "Changed") diff --git a/tests/test_litellm/router_strategy/test_llm_v2.py b/tests/test_litellm/router_strategy/test_llm_v2.py index 5447c8b43ce..27d31cbe640 100644 --- a/tests/test_litellm/router_strategy/test_llm_v2.py +++ b/tests/test_litellm/router_strategy/test_llm_v2.py @@ -11,8 +11,10 @@ from litellm import ModelResponse, Router from litellm.caching.dual_cache import DualCache from litellm.router_strategy.complexity_router.complexity_router import ComplexityRouter from litellm.router_strategy.complexity_router.config import ComplexityRouterConfig, ComplexityTier +from litellm.router_strategy.complexity_router.fuse_presets import get_fuse_presets from litellm.router_strategy.complexity_router.llm_v2 import ( LLM_V2_PROMPT_VERSION, + LLM_V2_SYSTEM_PROMPT, LLMV2Calibration, LLMV2Config, LLMV2ProbabilityCalibration, @@ -174,6 +176,114 @@ def test_invalid_forecast_settings_are_rejected(overrides: dict[str, object]) -> LLMV2Config.model_validate({**base.model_dump(), **overrides}) +def _preset_config(**overrides: object) -> LLMV2Config: + catalog: Final = get_fuse_presets() + return LLMV2Config.model_validate( + { + "efficient_profile_preset": catalog.models[0].id, + "capable_profile_preset": catalog.models[-1].id, + "harness_preset": catalog.harnesses[-1].id, + "max_quality_gap": 0.05, + **overrides, + } + ) + + +def test_preset_roundtrip_keeps_references_without_materializing_text() -> None: + config: Final = _preset_config() + serialized: Final = config.model_dump(exclude_none=True) + assert serialized["efficient_profile_preset"] == config.efficient_profile_preset + assert serialized["capable_profile_preset"] == config.capable_profile_preset + assert serialized["harness_preset"] == config.harness_preset + assert not {"efficient_profile", "capable_profile", "harness"}.intersection(serialized) + assert LLMV2Config.model_validate(config.model_dump()) == config + assert LLMV2Config.model_validate_json(config.model_dump_json()) == config + + +@pytest.mark.parametrize("field", ("efficient_profile", "capable_profile", "harness")) +def test_preset_explicit_override_wins_and_survives_roundtrip(field: str) -> None: + config: Final = _preset_config(**{field: " Operator description "}) + roundtrip: Final = LLMV2Config.model_validate_json(config.model_dump_json()) + assert roundtrip.model_dump()[field] == "Operator description" + assert roundtrip.efficient_profile_preset == config.efficient_profile_preset + assert roundtrip.capable_profile_preset == config.capable_profile_preset + assert roundtrip.harness_preset == config.harness_preset + payload: Final = json.loads( + roundtrip.system_prompt("opaque-efficient", "opaque-capable").split("Configured solver profiles:\n")[1] + ) + if field == "harness": + assert payload["harness"] == "Operator description" + else: + assert payload[field.removesuffix("_profile")]["profile"] == "Operator description" + + +@pytest.mark.parametrize("field", ("efficient_profile", "capable_profile", "harness")) +@pytest.mark.parametrize("invalid", ("", " \n\t", "x" * 4001)) +def test_preset_does_not_bypass_supplied_text_bounds(field: str, invalid: str) -> None: + with pytest.raises(ValidationError, match=field): + _preset_config(**{field: invalid}) + + +@pytest.mark.parametrize("field", ("efficient_profile", "capable_profile", "harness")) +@pytest.mark.parametrize("override", (None, "Custom override")) +@pytest.mark.parametrize("invalid_id", ("missing-v1", "")) +def test_preset_unknown_reference_rejects_even_when_overridden( + field: str, override: str | None, invalid_id: str +) -> None: + with pytest.raises(ValidationError, match=f"{field}.*preset"): + _preset_config(**{field: override, f"{field}_preset": invalid_id}) + + +@pytest.mark.parametrize("field", ("efficient_profile", "capable_profile", "harness")) +def test_preset_missing_text_and_reference_rejects(field: str) -> None: + with pytest.raises(ValidationError, match=field): + _preset_config(**{f"{field}_preset": None}) + + +@pytest.mark.parametrize("field", ("efficient_profile", "capable_profile", "harness")) +def test_preset_reference_rejects_the_wrong_catalog_kind(field: str) -> None: + catalog: Final = get_fuse_presets() + wrong_id: Final = catalog.models[0].id if field == "harness" else catalog.harnesses[0].id + with pytest.raises(ValidationError, match=field): + _preset_config(**{f"{field}_preset": wrong_id}) + + +@pytest.mark.parametrize("mode", ("json_schema", "json_object")) +def test_custom_profile_prompt_bytes_are_unchanged(mode: str) -> None: + base: Final = _config().llm_v2_config + assert base is not None + config: Final = LLMV2Config.model_validate({**base.model_dump(), "response_format": mode}) + old_payload: Final = { + "prompt_version": LLM_V2_PROMPT_VERSION, + "harness": config.harness, + "efficient": {"model": "opaque-efficient", "profile": config.efficient_profile}, + "capable": {"model": "opaque-capable", "profile": config.capable_profile}, + } + schema: Final = ( + "\n\nResponse JSON schema:\n" + json.dumps(LLMV2Verdict.model_json_schema()) if mode == "json_object" else "" + ) + assert config.system_prompt("opaque-efficient", "opaque-capable") == ( + LLM_V2_SYSTEM_PROMPT + "\n\nConfigured solver profiles:\n" + json.dumps(old_payload) + schema + ) + + +@pytest.mark.asyncio +async def test_preset_router_passes_catalog_text_and_opaque_group_names_to_judge() -> None: + catalog: Final = get_fuse_presets() + config: Final = _config(llm_v2_config=_preset_config().model_dump()) + router, client = _router(_verdict().model_dump_json(), config) + outcome: Final = await router.aclassify("Complete the supplied task") + assert outcome.tier == ComplexityTier.SIMPLE + prompt: Final = client.acompletion.call_args.kwargs["messages"][0]["content"] + payload: Final = json.loads(prompt.split("Configured solver profiles:\n")[1]) + assert payload == { + "prompt_version": LLM_V2_PROMPT_VERSION, + "harness": catalog.harnesses[-1].text, + "efficient": {"model": "efficient", "profile": catalog.models[0].text}, + "capable": {"model": "capable", "profile": catalog.models[-1].text}, + } + + @pytest.mark.asyncio async def test_one_judge_fuses_whole_task_and_keeps_caller_text_out_of_system_prompt() -> None: router, client = _router(_verdict().model_dump_json()) diff --git a/tests/test_litellm/router_utils/test_auto_router_model_naming.py b/tests/test_litellm/router_utils/test_auto_router_model_naming.py index 3dcb8d5af94..7d59a0590f2 100644 --- a/tests/test_litellm/router_utils/test_auto_router_model_naming.py +++ b/tests/test_litellm/router_utils/test_auto_router_model_naming.py @@ -1,7 +1,10 @@ from collections.abc import Mapping +from typing import Final import pytest +from litellm.router_strategy.complexity_router.fuse_presets import get_fuse_presets + from litellm.router_utils.auto_router_model_naming import ( carries_complexity_router_settings, classify_strategy_router_model, @@ -171,6 +174,52 @@ def test_validate_accepts_loadable_complexity_config(complexity_router_config): assert validate_complexity_router_config_write(complexity_router_config=complexity_router_config) is None +def _fuse_write_config(profiles: Mapping[str, object]) -> Mapping[str, object]: + return { + "classifier_type": "llm_v2", + "classifier_llm_config": {"model": "judge"}, + "tiers": {"SIMPLE": ["opaque-efficient"], "REASONING": ["opaque-capable"]}, + "llm_v2_config": {"max_quality_gap": 0.05, **profiles}, + } + + +def test_fuse_write_accepts_presets_and_custom_text_with_the_same_entitlement() -> None: + catalog: Final = get_fuse_presets() + presets: Final = _fuse_write_config( + { + "efficient_profile_preset": catalog.models[0].id, + "capable_profile_preset": catalog.models[-1].id, + "harness_preset": catalog.harnesses[0].id, + } + ) + custom: Final = _fuse_write_config( + { + "efficient_profile": catalog.models[0].text, + "capable_profile": catalog.models[-1].text, + "harness": catalog.harnesses[0].text, + } + ) + assert validate_complexity_router_config_write(presets) is None + assert validate_complexity_router_config_write(custom) is None + assert claimed_capability(presets) is claimed_capability(custom) + assert claimed_capability(presets) is not None + + +@pytest.mark.parametrize("field", ("efficient_profile", "capable_profile", "harness")) +def test_fuse_write_rejects_unknown_preset_even_with_custom_text(field: str) -> None: + config: Final = _fuse_write_config( + { + "efficient_profile": "Custom efficient solver", + "capable_profile": "Custom capable solver", + "harness": "Custom runtime", + f"{field}_preset": "unknown-v1", + } + ) + violation: Final = validate_complexity_router_config_write(config) + assert violation is not None + assert f"{field}_preset" in violation + + def test_naming_check_ignores_the_config_entirely(): """The naming contract and the config's contents are separate questions with separate owners; a write may carry a config without naming a model, so neither can stand in for the other.""" diff --git a/ui/litellm-dashboard/src/components/add_model/ForecastClassifierConfig.integration.test.tsx b/ui/litellm-dashboard/src/components/add_model/ForecastClassifierConfig.integration.test.tsx index 4a574ac736d..f9b0edf9508 100644 --- a/ui/litellm-dashboard/src/components/add_model/ForecastClassifierConfig.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/ForecastClassifierConfig.integration.test.tsx @@ -1,7 +1,7 @@ import React, { useState } from "react"; -import { describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import userEvent from "@testing-library/user-event"; -import { fireEvent, renderWithProviders, screen } from "../../../tests/test-utils"; +import { act, fireEvent, renderWithProviders, screen, testQueryClient, waitFor } from "../../../tests/test-utils"; import ClassificationMethodConfig from "./ClassificationMethodConfig"; import AutoRouterClassifierTabs from "./AutoRouterClassifierTabs"; import ForecastClassifierConfig from "./ForecastClassifierConfig"; @@ -37,6 +37,53 @@ const fuseInitial: ComplexityRouterConfigValue = { }, }; const options = ["judge", "efficient", "capable"].map((model) => ({ value: model, label: model })); +const catalog = { + version: "catalog-v1", + models: [ + { + id: "efficient-v1", + label: "Efficient preset", + text: "Maintained efficient profile", + sources: ["https://example.com/efficient"], + model: "efficient-model", + }, + { + id: "capable-v1", + label: "Capable preset", + text: "Maintained capable profile", + sources: ["https://example.com/capable"], + model: "capable-model", + }, + ], + harnesses: [ + { + id: "runtime-v1", + label: "Runtime preset", + text: "Maintained runtime profile", + sources: ["https://example.com/runtime"], + }, + ], +}; +const presetConfig = { + efficient_profile_preset: catalog.models[0].id, + capable_profile_preset: catalog.models[1].id, + harness_preset: catalog.harnesses[0].id, + max_quality_gap: 0.05, +}; +const presetInitial = { ...fuseInitial, llm_v2_config: presetConfig }; + +beforeEach(() => { + testQueryClient.clear(); + vi.stubGlobal( + "fetch", + vi.fn().mockImplementation(async () => Response.json(catalog)), + ); +}); + +afterEach(() => { + testQueryClient.clear(); + vi.unstubAllGlobals(); +}); function Form({ initialValue = initial }: { initialValue?: ComplexityRouterConfigValue }) { const [value, setValue] = useState(initialValue); @@ -72,6 +119,118 @@ function Form({ initialValue = initial }: { initialValue?: ComplexityRouterConfi } describe("forecast classifier form", () => { + it("selects all three maintained presets, previews provenance, and saves only references", async () => { + const user = userEvent.setup(); + renderWithProviders(
); + await user.click(screen.getByRole("combobox", { name: "Efficient solver profile preset" })); + await user.click(await screen.findByRole("option", { name: /^Efficient preset/ })); + await user.click(screen.getByRole("combobox", { name: "Capable solver profile preset" })); + await user.click(screen.getByRole("option", { name: /^Capable preset/ })); + await user.click(screen.getByRole("combobox", { name: "Harness and budget preset" })); + await user.click(screen.getByRole("option", { name: /^Runtime preset/ })); + expect(screen.getByLabelText("Efficient solver profile")).toHaveValue(catalog.models[0].text); + expect(screen.getByLabelText("Efficient solver profile")).toHaveAttribute("readonly"); + expect(screen.getByLabelText("Capable solver profile")).toHaveValue(catalog.models[1].text); + expect(screen.getByLabelText("Harness and budget")).toHaveValue(catalog.harnesses[0].text); + expect(screen.getAllByText(`Catalog version: ${catalog.version}`)).toHaveLength(3); + expect(screen.getByText(`Model: ${catalog.models[0].model}`)).toBeInTheDocument(); + expect(screen.getAllByRole("link", { name: "Source 1" }).map((link) => link.getAttribute("href"))).toEqual([ + catalog.models[0].sources[0], + catalog.models[1].sources[0], + catalog.harnesses[0].sources[0], + ]); + fireEvent.click(screen.getByRole("button", { name: "Save configuration" })); + expect(JSON.parse(screen.getByRole("status", { name: "Saved configuration" }).textContent!).llm_v2_config).toEqual( + presetConfig, + ); + expect(fetch).toHaveBeenCalledTimes(1); + expect(fetch).toHaveBeenCalledWith( + expect.objectContaining({ url: expect.stringMatching(/\/public\/complexity_router\/fuse_presets$/) }), + ); + }); + + it.each([undefined, null, "Explicit override"])( + "copies effective text to Custom and clears only that reference, override=%s", + async (override) => { + const user = userEvent.setup(); + renderWithProviders( + , + ); + const effectiveText = override ?? catalog.models[0].text; + await waitFor(() => expect(screen.getByLabelText("Efficient solver profile")).toHaveValue(effectiveText)); + await user.click(screen.getByRole("combobox", { name: "Efficient solver profile preset" })); + await user.click(screen.getByRole("option", { name: "Custom", exact: true })); + expect(screen.getByLabelText("Efficient solver profile")).not.toHaveAttribute("readonly"); + expect(screen.getByLabelText("Efficient solver profile")).toHaveValue(effectiveText); + fireEvent.change(screen.getByLabelText("Efficient solver profile"), { target: { value: "Custom budget" } }); + fireEvent.click(screen.getByRole("button", { name: "Save configuration" })); + const { efficient_profile_preset: _preset, ...rest } = presetConfig; + expect( + JSON.parse(screen.getByRole("status", { name: "Saved configuration" }).textContent!).llm_v2_config, + ).toEqual({ + ...rest, + efficient_profile: "Custom budget", + }); + }, + ); + + it.each([ + { ...fuseInitial.llm_v2_config!, efficient_profile: catalog.models[0].text }, + { + ...presetConfig, + efficient_profile: "Explicit override", + capable_profile: "Capable override", + harness: "Harness override", + }, + ])("keeps existing custom ownership and references on an unchanged save: %j", async (settings) => { + renderWithProviders(); + await waitFor(() => expect(screen.queryByText(/Loading profile presets/)).not.toBeInTheDocument()); + expect(screen.getByRole("combobox", { name: "Efficient solver profile preset" })).toHaveValue("Custom"); + expect(screen.getByLabelText("Efficient solver profile")).toHaveValue(settings.efficient_profile); + fireEvent.click(screen.getByRole("button", { name: "Save configuration" })); + expect(JSON.parse(screen.getByRole("status", { name: "Saved configuration" }).textContent!).llm_v2_config).toEqual( + settings, + ); + }); + + it.each([true, false])( + "keeps edits and stored IDs while the pending catalog settles, success=%s", + async (success) => { + let resolveCatalog: (response: Response) => void = () => {}; + vi.mocked(fetch).mockReturnValue( + new Promise((resolve) => { + resolveCatalog = resolve; + }), + ); + const settings = { ...presetConfig, efficient_profile: "Original override" }; + renderWithProviders(); + expect(screen.getByText(/Loading profile presets/)).toBeInTheDocument(); + fireEvent.change(screen.getByLabelText("Efficient solver profile"), { target: { value: "Typed while loading" } }); + await act(async () => + resolveCatalog(success ? Response.json(catalog) : Response.json({ error: "unavailable" }, { status: 503 })), + ); + if (success) await screen.findAllByText(`Catalog version: ${catalog.version}`); + else expect(await screen.findByText(/Profile presets could not be loaded/)).toBeInTheDocument(); + expect(screen.getByLabelText("Efficient solver profile")).toHaveValue("Typed while loading"); + fireEvent.click(screen.getByRole("button", { name: "Save configuration" })); + expect( + JSON.parse(screen.getByRole("status", { name: "Saved configuration" }).textContent!).llm_v2_config, + ).toEqual({ ...settings, efficient_profile: "Typed while loading" }); + }, + ); + + it("keeps unknown saved IDs visible with unavailable previews rather than replacing them", async () => { + const settings = { ...presetConfig, efficient_profile_preset: "unavailable-v8" }; + renderWithProviders(); + await screen.findAllByText(`Catalog version: ${catalog.version}`); + expect(screen.getByRole("combobox", { name: "Efficient solver profile preset" })).toHaveValue("unavailable-v8"); + expect(screen.getByText("Preset preview unavailable. The saved reference is preserved")).toBeInTheDocument(); + fireEvent.click(screen.getByRole("button", { name: "Save configuration" })); + expect(JSON.parse(screen.getByRole("status", { name: "Saved configuration" }).textContent!).llm_v2_config).toEqual( + settings, + ); + }); + it("switches a populated standard router to Capability without saving hidden pools or their overrides", () => { renderWithProviders( ) : ( <> - {(["efficient_profile", "capable_profile", "harness"] as const).map((field) => { - const label = { - efficient_profile: "Efficient solver profile", - capable_profile: "Capable solver profile", - harness: "Harness and budget", - }[field]; - return ( -
- -